mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 08:32:02 +00:00
Merge remote-tracking branch 'upstream/master' into antlr-test
This commit is contained in:
Ivan Lezhankin
commit
321f65a649
@ -214,6 +214,19 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
|
||||
endif ()
|
||||
endif()
|
||||
|
||||
# Create BuildID when using lld. For other linkers it is created by default.
|
||||
if (LINKER_NAME MATCHES "lld$")
|
||||
# SHA1 is not cryptographically secure but it is the best what lld is offering.
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
|
||||
endif ()
|
||||
|
||||
# Add a section with the hash of the compiled machine code for integrity checks.
|
||||
# Only for official builds, because adding a section can be time consuming (rewrite of several GB).
|
||||
# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
|
||||
if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
|
||||
set (USE_BINARY_HASH 1)
|
||||
endif ()
|
||||
|
||||
cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
|
||||
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
# https://youtrack.jetbrains.com/issue/CPP-2659
|
||||
# https://youtrack.jetbrains.com/issue/CPP-870
|
||||
|
||||
if (NOT DEFINED ENV{CLION_IDE})
|
||||
if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
|
||||
find_program(NINJA_PATH ninja)
|
||||
if (NINJA_PATH)
|
||||
set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
|
||||
|
||||
@ -56,6 +56,9 @@
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/MemorySanitizer.h>
|
||||
#include <Common/SymbolIndex.h>
|
||||
#include <Common/getExecutablePath.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
#include <Common/Elf.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config_version.h>
|
||||
@ -80,16 +83,6 @@ static void call_default_signal_handler(int sig)
|
||||
raise(sig);
|
||||
}
|
||||
|
||||
const char * msan_strsignal(int sig)
|
||||
{
|
||||
// Apparently strsignal is not instrumented by MemorySanitizer, so we
|
||||
// have to unpoison it to avoid msan reports inside fmt library when we
|
||||
// print it.
|
||||
const char * signal_name = sys_siglist[sig];
|
||||
__msan_unpoison_string(signal_name);
|
||||
return signal_name;
|
||||
}
|
||||
|
||||
static constexpr size_t max_query_id_size = 127;
|
||||
|
||||
static const size_t signal_pipe_buf_size =
|
||||
@ -294,13 +287,13 @@ private:
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
|
||||
thread_num, msan_strsignal(sig), sig);
|
||||
thread_num, strsignal(sig), sig);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
|
||||
thread_num, query_id, msan_strsignal(sig), sig);
|
||||
thread_num, query_id, strsignal(sig), sig);
|
||||
}
|
||||
|
||||
String error_message;
|
||||
@ -328,6 +321,32 @@ private:
|
||||
/// Write symbolized stack trace line by line for better grep-ability.
|
||||
stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); });
|
||||
|
||||
#if defined(__linux__)
|
||||
/// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
|
||||
String calculated_binary_hash = getHashOfLoadedBinaryHex();
|
||||
if (daemon.stored_binary_hash.empty())
|
||||
{
|
||||
LOG_FATAL(log, "Calculated checksum of the binary: {}."
|
||||
" There is no information about the reference checksum.", calculated_binary_hash);
|
||||
}
|
||||
else if (calculated_binary_hash == daemon.stored_binary_hash)
|
||||
{
|
||||
LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond"
|
||||
" to the reference checksum stored in the binary ({1})."
|
||||
" It may indicate one of the following:"
|
||||
" - the file was changed just after startup;"
|
||||
" - the file is damaged on disk due to faulty hardware;"
|
||||
" - the loaded executable is damaged in memory due to faulty hardware;"
|
||||
" - the file was intentionally modified;"
|
||||
" - logical error in code."
|
||||
, calculated_binary_hash, daemon.stored_binary_hash);
|
||||
}
|
||||
#endif
|
||||
|
||||
/// Write crash to system.crash_log table if available.
|
||||
if (collectCrashLog)
|
||||
collectCrashLog(sig, thread_num, query_id, stack_trace);
|
||||
@ -481,8 +500,9 @@ void BaseDaemon::kill()
|
||||
{
|
||||
dumpCoverageReportIfPossible();
|
||||
pid_file.reset();
|
||||
if (::raise(SIGKILL) != 0)
|
||||
throw Poco::SystemException("cannot kill process");
|
||||
/// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL.
|
||||
/// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker).
|
||||
_exit(128 + SIGKILL);
|
||||
}
|
||||
|
||||
std::string BaseDaemon::getDefaultCorePath() const
|
||||
@ -787,6 +807,13 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
#else
|
||||
build_id_info = "no build id";
|
||||
#endif
|
||||
|
||||
#if defined(__linux__)
|
||||
std::string executable_path = getExecutablePath();
|
||||
|
||||
if (!executable_path.empty())
|
||||
stored_binary_hash = DB::Elf(executable_path).getBinaryHash();
|
||||
#endif
|
||||
}
|
||||
|
||||
void BaseDaemon::logRevision() const
|
||||
@ -846,13 +873,13 @@ void BaseDaemon::handleSignal(int signal_id)
|
||||
onInterruptSignals(signal_id);
|
||||
}
|
||||
else
|
||||
throw DB::Exception(std::string("Unsupported signal: ") + msan_strsignal(signal_id), 0);
|
||||
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
|
||||
}
|
||||
|
||||
void BaseDaemon::onInterruptSignals(int signal_id)
|
||||
{
|
||||
is_cancelled = true;
|
||||
LOG_INFO(&logger(), "Received termination signal ({})", msan_strsignal(signal_id));
|
||||
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));
|
||||
|
||||
if (sigint_signals_counter >= 2)
|
||||
{
|
||||
@ -998,3 +1025,9 @@ void BaseDaemon::setupWatchdog()
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
String BaseDaemon::getStoredBinaryHash() const
|
||||
{
|
||||
return stored_binary_hash;
|
||||
}
|
||||
|
||||
@ -60,7 +60,7 @@ public:
|
||||
static void terminate();
|
||||
|
||||
/// Forceful shutdown
|
||||
void kill();
|
||||
[[noreturn]] void kill();
|
||||
|
||||
/// Cancellation request has been received.
|
||||
bool isCancelled() const
|
||||
@ -121,6 +121,9 @@ public:
|
||||
/// argv0 is needed to change process name (consequently, it is needed for scripts involving "pgrep", "pidof" to work correctly).
|
||||
void shouldSetupWatchdog(char * argv0_);
|
||||
|
||||
/// Hash of the binary for integrity checks.
|
||||
String getStoredBinaryHash() const;
|
||||
|
||||
protected:
|
||||
virtual void logRevision() const;
|
||||
|
||||
@ -168,6 +171,7 @@ protected:
|
||||
Poco::Util::AbstractConfiguration * last_configuration = nullptr;
|
||||
|
||||
String build_id_info;
|
||||
String stored_binary_hash;
|
||||
|
||||
std::vector<int> handled_signals;
|
||||
|
||||
|
||||
125
base/glibc-compatibility/musl/strsignal.c
Normal file
125
base/glibc-compatibility/musl/strsignal.c
Normal file
@ -0,0 +1,125 @@
|
||||
#include <signal.h>
|
||||
#include <string.h>
|
||||
|
||||
#if (SIGHUP == 1) && (SIGINT == 2) && (SIGQUIT == 3) && (SIGILL == 4) \
|
||||
&& (SIGTRAP == 5) && (SIGABRT == 6) && (SIGBUS == 7) && (SIGFPE == 8) \
|
||||
&& (SIGKILL == 9) && (SIGUSR1 == 10) && (SIGSEGV == 11) && (SIGUSR2 == 12) \
|
||||
&& (SIGPIPE == 13) && (SIGALRM == 14) && (SIGTERM == 15) && (SIGSTKFLT == 16) \
|
||||
&& (SIGCHLD == 17) && (SIGCONT == 18) && (SIGSTOP == 19) && (SIGTSTP == 20) \
|
||||
&& (SIGTTIN == 21) && (SIGTTOU == 22) && (SIGURG == 23) && (SIGXCPU == 24) \
|
||||
&& (SIGXFSZ == 25) && (SIGVTALRM == 26) && (SIGPROF == 27) && (SIGWINCH == 28) \
|
||||
&& (SIGPOLL == 29) && (SIGPWR == 30) && (SIGSYS == 31)
|
||||
|
||||
#define sigmap(x) x
|
||||
|
||||
#else
|
||||
|
||||
static const char map[] = {
|
||||
[SIGHUP] = 1,
|
||||
[SIGINT] = 2,
|
||||
[SIGQUIT] = 3,
|
||||
[SIGILL] = 4,
|
||||
[SIGTRAP] = 5,
|
||||
[SIGABRT] = 6,
|
||||
[SIGBUS] = 7,
|
||||
[SIGFPE] = 8,
|
||||
[SIGKILL] = 9,
|
||||
[SIGUSR1] = 10,
|
||||
[SIGSEGV] = 11,
|
||||
[SIGUSR2] = 12,
|
||||
[SIGPIPE] = 13,
|
||||
[SIGALRM] = 14,
|
||||
[SIGTERM] = 15,
|
||||
#if defined(SIGSTKFLT)
|
||||
[SIGSTKFLT] = 16,
|
||||
#elif defined(SIGEMT)
|
||||
[SIGEMT] = 16,
|
||||
#endif
|
||||
[SIGCHLD] = 17,
|
||||
[SIGCONT] = 18,
|
||||
[SIGSTOP] = 19,
|
||||
[SIGTSTP] = 20,
|
||||
[SIGTTIN] = 21,
|
||||
[SIGTTOU] = 22,
|
||||
[SIGURG] = 23,
|
||||
[SIGXCPU] = 24,
|
||||
[SIGXFSZ] = 25,
|
||||
[SIGVTALRM] = 26,
|
||||
[SIGPROF] = 27,
|
||||
[SIGWINCH] = 28,
|
||||
[SIGPOLL] = 29,
|
||||
[SIGPWR] = 30,
|
||||
[SIGSYS] = 31
|
||||
};
|
||||
|
||||
#define sigmap(x) ((x) >= sizeof map ? (x) : map[(x)])
|
||||
|
||||
#endif
|
||||
|
||||
static const char strings[] =
|
||||
"Unknown signal\0"
|
||||
"Hangup\0"
|
||||
"Interrupt\0"
|
||||
"Quit\0"
|
||||
"Illegal instruction\0"
|
||||
"Trace/breakpoint trap\0"
|
||||
"Aborted\0"
|
||||
"Bus error\0"
|
||||
"Arithmetic exception\0"
|
||||
"Killed\0"
|
||||
"User defined signal 1\0"
|
||||
"Segmentation fault\0"
|
||||
"User defined signal 2\0"
|
||||
"Broken pipe\0"
|
||||
"Alarm clock\0"
|
||||
"Terminated\0"
|
||||
#if defined(SIGSTKFLT)
|
||||
"Stack fault\0"
|
||||
#elif defined(SIGEMT)
|
||||
"Emulator trap\0"
|
||||
#else
|
||||
"Unknown signal\0"
|
||||
#endif
|
||||
"Child process status\0"
|
||||
"Continued\0"
|
||||
"Stopped (signal)\0"
|
||||
"Stopped\0"
|
||||
"Stopped (tty input)\0"
|
||||
"Stopped (tty output)\0"
|
||||
"Urgent I/O condition\0"
|
||||
"CPU time limit exceeded\0"
|
||||
"File size limit exceeded\0"
|
||||
"Virtual timer expired\0"
|
||||
"Profiling timer expired\0"
|
||||
"Window changed\0"
|
||||
"I/O possible\0"
|
||||
"Power failure\0"
|
||||
"Bad system call\0"
|
||||
"RT32"
|
||||
"\0RT33\0RT34\0RT35\0RT36\0RT37\0RT38\0RT39\0RT40"
|
||||
"\0RT41\0RT42\0RT43\0RT44\0RT45\0RT46\0RT47\0RT48"
|
||||
"\0RT49\0RT50\0RT51\0RT52\0RT53\0RT54\0RT55\0RT56"
|
||||
"\0RT57\0RT58\0RT59\0RT60\0RT61\0RT62\0RT63\0RT64"
|
||||
#if _NSIG > 65
|
||||
"\0RT65\0RT66\0RT67\0RT68\0RT69\0RT70\0RT71\0RT72"
|
||||
"\0RT73\0RT74\0RT75\0RT76\0RT77\0RT78\0RT79\0RT80"
|
||||
"\0RT81\0RT82\0RT83\0RT84\0RT85\0RT86\0RT87\0RT88"
|
||||
"\0RT89\0RT90\0RT91\0RT92\0RT93\0RT94\0RT95\0RT96"
|
||||
"\0RT97\0RT98\0RT99\0RT100\0RT101\0RT102\0RT103\0RT104"
|
||||
"\0RT105\0RT106\0RT107\0RT108\0RT109\0RT110\0RT111\0RT112"
|
||||
"\0RT113\0RT114\0RT115\0RT116\0RT117\0RT118\0RT119\0RT120"
|
||||
"\0RT121\0RT122\0RT123\0RT124\0RT125\0RT126\0RT127\0RT128"
|
||||
#endif
|
||||
"";
|
||||
|
||||
char *strsignal(int signum)
|
||||
{
|
||||
const char *s = strings;
|
||||
|
||||
signum = sigmap(signum);
|
||||
if (signum - 1U >= _NSIG-1) signum = 0;
|
||||
|
||||
for (; signum--; s++) for (; *s; s++);
|
||||
|
||||
return (char *)s;
|
||||
}
|
||||
@ -142,7 +142,7 @@ TRAP(qecvt)
|
||||
TRAP(qfcvt)
|
||||
TRAP(register_printf_function)
|
||||
TRAP(seed48)
|
||||
TRAP(setenv)
|
||||
//TRAP(setenv)
|
||||
TRAP(setfsent)
|
||||
TRAP(setgrent)
|
||||
TRAP(sethostent)
|
||||
@ -164,7 +164,7 @@ TRAP(sigsuspend)
|
||||
TRAP(sleep)
|
||||
TRAP(srand48)
|
||||
//TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately.
|
||||
TRAP(strsignal)
|
||||
//TRAP(strsignal) // This function is imported from Musl and is thread safe.
|
||||
TRAP(strtok)
|
||||
TRAP(tcflow)
|
||||
TRAP(tcsendbreak)
|
||||
|
||||
4
contrib/CMakeLists.txt
vendored
4
contrib/CMakeLists.txt
vendored
@ -18,7 +18,11 @@ if (WITH_COVERAGE)
|
||||
set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})
|
||||
separate_arguments(WITHOUT_COVERAGE_LIST)
|
||||
# disable coverage for contib files and build with optimisations
|
||||
if (COMPILER_CLANG)
|
||||
add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST})
|
||||
else()
|
||||
add_compile_options(-O3 -DNDEBUG -finline-functions ${WITHOUT_COVERAGE_LIST})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (SANITIZE STREQUAL "undefined")
|
||||
|
||||
2
contrib/replxx
vendored
2
contrib/replxx
vendored
@ -1 +1 @@
|
||||
Subproject commit 254be98ae7f2fd92d6db768f8e11ea5a5226cbf5
|
||||
Subproject commit cdb6e3f2ce4464225daf9c8beeae7db98d590bdc
|
||||
2
contrib/rocksdb
vendored
2
contrib/rocksdb
vendored
@ -1 +1 @@
|
||||
Subproject commit 8b966f0ca298fc1475bd09d9775f32dff0fdce0a
|
||||
Subproject commit 54a0decabbcf4c0bb5cf7befa9c597f28289bff5
|
||||
@ -10,6 +10,11 @@
|
||||
<max_execution_time>
|
||||
<max>10</max>
|
||||
</max_execution_time>
|
||||
|
||||
<!-- Not ready for production -->
|
||||
<compile_expressions>
|
||||
<readonly />
|
||||
</compile_expressions>
|
||||
</constraints>
|
||||
</default>
|
||||
</profiles>
|
||||
|
||||
@ -81,12 +81,11 @@ function fuzz
|
||||
echo Server started
|
||||
|
||||
fuzzer_exit_code=0
|
||||
# SC2012: Use find instead of ls to better handle non-alphanumeric filenames.
|
||||
# They are all alphanumeric.
|
||||
# shellcheck disable=SC2012
|
||||
./clickhouse-client --query-fuzzer-runs=1000 \
|
||||
< <(for f in $(ls ch/tests/queries/0_stateless/*.sql | sort -R); do cat "$f"; echo ';'; done) \
|
||||
> >(tail -10000 > fuzzer.log) \
|
||||
# SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric.
|
||||
# SC2046: Quote this to prevent word splitting. Actually I need word splitting.
|
||||
# shellcheck disable=SC2012,SC2046
|
||||
./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \
|
||||
> >(tail -n 10000 > fuzzer.log) \
|
||||
2>&1 \
|
||||
|| fuzzer_exit_code=$?
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ Functional tests are the most simple and convenient to use. Most of ClickHouse f
|
||||
|
||||
Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference.
|
||||
|
||||
Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and not available to general public. We tend to use only `stateless` tests and avoid adding new `stateful` tests.
|
||||
Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and it is available to general public.
|
||||
|
||||
Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`.
|
||||
|
||||
@ -84,11 +84,9 @@ If you want to improve performance of ClickHouse in some scenario, and if improv
|
||||
|
||||
Some programs in `tests` directory are not prepared tests, but are test tools. For example, for `Lexer` there is a tool `src/Parsers/tests/lexer` that just do tokenization of stdin and writes colorized result to stdout. You can use these kind of tools as a code examples and for exploration and manual testing.
|
||||
|
||||
You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated.
|
||||
|
||||
## Miscellaneous Tests {#miscellaneous-tests}
|
||||
|
||||
There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests.
|
||||
There are tests for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests.
|
||||
|
||||
There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not.
|
||||
|
||||
@ -169,53 +167,55 @@ Precise query execution timings are not recorded and not compared due to high va
|
||||
|
||||
## Build Tests {#build-tests}
|
||||
|
||||
Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. Tests are located at `ci` directory. They run build from source inside Docker, Vagrant, and sometimes with `qemu-user-static` inside Docker. These tests are under development and test runs are not automated.
|
||||
Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well.
|
||||
|
||||
Motivation:
|
||||
|
||||
Normally we release and run all tests on a single variant of ClickHouse build. But there are alternative build variants that are not thoroughly tested. Examples:
|
||||
|
||||
- build on FreeBSD
|
||||
- build on Debian with libraries from system packages
|
||||
- build with shared linking of libraries
|
||||
- build on AArch64 platform
|
||||
- build on PowerPc platform
|
||||
Examples:
|
||||
- cross-compile for Darwin x86_64 (Mac OS X)
|
||||
- cross-compile for FreeBSD x86_64
|
||||
- cross-compile for Linux AArch64
|
||||
- build on Ubuntu with libraries from system packages (discouraged)
|
||||
- build with shared linking of libraries (discouraged)
|
||||
|
||||
For example, build with system packages is bad practice, because we cannot guarantee what exact version of packages a system will have. But this is really needed by Debian maintainers. For this reason we at least have to support this variant of build. Another example: shared linking is a common source of trouble, but it is needed for some enthusiasts.
|
||||
|
||||
Though we cannot run all tests on all variant of builds, we want to check at least that various build variants are not broken. For this purpose we use build tests.
|
||||
|
||||
We also test that there are no translation units that are too long to compile or require too much RAM.
|
||||
|
||||
We also test that there are no too large stack frames.
|
||||
|
||||
## Testing for Protocol Compatibility {#testing-for-protocol-compatibility}
|
||||
|
||||
When we extend ClickHouse network protocol, we test manually that old clickhouse-client works with new clickhouse-server and new clickhouse-client works with old clickhouse-server (simply by running binaries from corresponding packages).
|
||||
|
||||
We also test some cases automatically with integrational tests:
|
||||
- if data written by old version of ClickHouse can be successfully read by the new version;
|
||||
- do distributed queries work in a cluster with different ClickHouse versions.
|
||||
|
||||
## Help from the Compiler {#help-from-the-compiler}
|
||||
|
||||
Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall -Wextra -Werror` and with some additional enabled warnings. Although these options are not enabled for third-party libraries.
|
||||
|
||||
Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build.
|
||||
|
||||
For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.
|
||||
For production builds, clang is used, but we also test make gcc builds. For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.
|
||||
|
||||
## Sanitizers {#sanitizers}
|
||||
|
||||
### Address sanitizer
|
||||
We run functional and integration tests under ASan on per-commit basis.
|
||||
|
||||
### Valgrind (Memcheck)
|
||||
We run functional tests under Valgrind overnight. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse).
|
||||
|
||||
### Undefined behaviour sanitizer
|
||||
We run functional and integration tests under ASan on per-commit basis.
|
||||
We run functional, integration, stress and unit tests under ASan on per-commit basis.
|
||||
|
||||
### Thread sanitizer
|
||||
We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis.
|
||||
We run functional, integration, stress and unit tests under TSan on per-commit basis.
|
||||
|
||||
### Memory sanitizer
|
||||
Currently we still don’t use MSan.
|
||||
We run functional, integration, stress and unit tests under MSan on per-commit basis.
|
||||
|
||||
### Debug allocator
|
||||
Debug version of `jemalloc` is used for debug build.
|
||||
### Undefined behaviour sanitizer
|
||||
We run functional, integration, stress and unit tests under UBSan on per-commit basis. The code of some third-party libraries is not sanitized for UB.
|
||||
|
||||
### Valgrind (Memcheck)
|
||||
We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse).
|
||||
|
||||
## Fuzzing {#fuzzing}
|
||||
|
||||
@ -233,19 +233,62 @@ Google OSS-Fuzz can be found at `docker/fuzz`.
|
||||
We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them.
|
||||
You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer).
|
||||
|
||||
We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order.
|
||||
|
||||
## Stress test
|
||||
|
||||
Stress tests are another case of fuzzing. It runs all functional tests in parallel in random order with a single server. Results of the tests are not checked.
|
||||
|
||||
It is checked that:
|
||||
- server does not crash, no debug or sanitizer traps are triggered;
|
||||
- there are no deadlocks;
|
||||
- the database structure is consistent;
|
||||
- server can successfully stop after the test and start again without exceptions.
|
||||
|
||||
There are five variants (Debug, ASan, TSan, MSan, UBSan).
|
||||
|
||||
## Thread Fuzzer
|
||||
|
||||
Thread Fuzzer (please don't mix up with Thread Sanitizer) is another kind of fuzzing that allows to randomize thread order of execution. It helps to find even more special cases.
|
||||
|
||||
## Security Audit {#security-audit}
|
||||
|
||||
People from Yandex Security Team do some basic overview of ClickHouse capabilities from the security standpoint.
|
||||
|
||||
## Static Analyzers {#static-analyzers}
|
||||
|
||||
We run `PVS-Studio` on per-commit basis. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/).
|
||||
We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks.
|
||||
|
||||
We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/).
|
||||
|
||||
If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box.
|
||||
|
||||
We also use `shellcheck` for static analysis of shell scripts.
|
||||
|
||||
## Hardening {#hardening}
|
||||
|
||||
`FORTIFY_SOURCE` is used by default. It is almost useless, but still makes sense in rare cases and we don’t disable it.
|
||||
In debug build we are using custom allocator that does ASLR of user-level allocations.
|
||||
|
||||
We also manually protect memory regions that are expected to be readonly after allocation.
|
||||
|
||||
In debug build we also involve a customization of libc that ensures that no "harmful" (obsolete, insecure, not thread-safe) functions are called.
|
||||
|
||||
Debug assertions are used extensively.
|
||||
|
||||
In debug build, if exception with "logical error" code (implies a bug) is being thrown, the program is terminated prematurally. It allows to use exceptions in release build but make it an assertion in debug build.
|
||||
|
||||
Debug version of jemalloc is used for debug builds.
|
||||
Debug version of libc++ is used for debug builds.
|
||||
|
||||
## Runtime Integrity Checks
|
||||
|
||||
Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas.
|
||||
|
||||
It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client, bit flips on the wire). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ).
|
||||
|
||||
ClickHouse provides diagnostics that will help ops engineers to find faulty hardware.
|
||||
|
||||
\* and it is not slow.
|
||||
|
||||
## Code Style {#code-style}
|
||||
|
||||
@ -259,6 +302,8 @@ Alternatively you can try `uncrustify` tool to reformat your code. Configuration
|
||||
|
||||
`CLion` has its own code formatter that has to be tuned for our code style.
|
||||
|
||||
We also use `codespell` to find typos in code. It is automated as well.
|
||||
|
||||
## Metrica B2B Tests {#metrica-b2b-tests}
|
||||
|
||||
Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together.
|
||||
@ -267,13 +312,25 @@ These tests are automated by separate team. Due to high number of moving parts,
|
||||
|
||||
## Test Coverage {#test-coverage}
|
||||
|
||||
As of July 2018 we don’t track test coverage.
|
||||
We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis.
|
||||
|
||||
## Tests for Tests
|
||||
|
||||
There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky.
|
||||
|
||||
## Testflows
|
||||
|
||||
[Testflows](https://testflows.com/) is an enterprise-grade testing framework. It is used by Altinity for some of the tests and we run these tests in our CI.
|
||||
|
||||
## Yandex Checks (only for Yandex employees)
|
||||
|
||||
These checks are importing ClickHouse code into Yandex internal monorepository, so ClickHouse codebase can be used as a library by other products at Yandex (YT and YDB). Note that clickhouse-server itself is not being build from internal repo and unmodified open-source build is used for Yandex applications.
|
||||
|
||||
## Test Automation {#test-automation}
|
||||
|
||||
We run tests with Yandex internal CI and job automation system named “Sandbox”.
|
||||
|
||||
Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored eternally. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you.
|
||||
Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored for several months. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you.
|
||||
|
||||
We don’t use Travis CI due to the limit on time and computational power.
|
||||
We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins.
|
||||
|
||||
@ -410,3 +410,5 @@ GROUP BY yr,
|
||||
ORDER BY yr,
|
||||
mo;
|
||||
```
|
||||
|
||||
The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==).
|
||||
|
||||
@ -71,4 +71,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1"
|
||||
|
||||
[ClickHouse tutorial](../../getting-started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial.
|
||||
|
||||
Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there).
|
||||
Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hits` and `test.visits` there).
|
||||
|
||||
@ -398,6 +398,8 @@ ORDER BY c DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
You can also play with the data in Playground, [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIERheU9mV2VlaywgY291bnQoKikgQVMgYwpGUk9NIG9udGltZQpXSEVSRSBZZWFyPj0yMDAwIEFORCBZZWFyPD0yMDA4CkdST1VQIEJZIERheU9mV2VlawpPUkRFUiBCWSBjIERFU0M7Cg==).
|
||||
|
||||
This performance test was created by Vadim Tkachenko. See:
|
||||
|
||||
- https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/
|
||||
|
||||
@ -13,6 +13,7 @@ toc_title: Client Libraries
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -20,7 +20,33 @@ System tables:
|
||||
|
||||
Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start.
|
||||
|
||||
Unlike other system tables, the system tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
|
||||
Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
|
||||
|
||||
System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are:
|
||||
|
||||
- `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`.
|
||||
- `table`: table to insert data.
|
||||
- `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression.
|
||||
- `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md) expression.
|
||||
- `flush_interval_milliseconds`: interval of flushing data to disk.
|
||||
- `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit.
|
||||
|
||||
An example:
|
||||
|
||||
```
|
||||
<yandex>
|
||||
<query_log>
|
||||
<database>system</database>
|
||||
<table>query_log</table>
|
||||
<partition_by>toYYYYMM(event_date)</partition_by>
|
||||
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||
<!--
|
||||
<engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
|
||||
-->
|
||||
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||
</query_log>
|
||||
</yandex>
|
||||
```
|
||||
|
||||
By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.
|
||||
|
||||
|
||||
@ -11,6 +11,7 @@ This table contains the following columns (the column type is shown in brackets)
|
||||
- `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`.
|
||||
- `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](../../engines/table-engines/mergetree-family/replication.md).
|
||||
- `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication.
|
||||
- `supports_parallel_insert` (UInt8) — Flag that indicates if table engine supports parallel insert (see [`max_insert_threads`](../../operations/settings/settings.md#settings-max-insert-threads) setting).
|
||||
|
||||
Example:
|
||||
|
||||
@ -21,11 +22,11 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree')
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐
|
||||
│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │
|
||||
│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │
|
||||
│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │
|
||||
└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘
|
||||
┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐
|
||||
│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │
|
||||
│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │
|
||||
│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │
|
||||
└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘
|
||||
```
|
||||
|
||||
**See also**
|
||||
|
||||
@ -1290,25 +1290,68 @@ Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference
|
||||
|
||||
## arrayMin(\[func,\] arr1, …) {#array-min}
|
||||
|
||||
Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
|
||||
Returns the min of the `func` values. If the function is omitted, it just returns the min of the array elements.
|
||||
|
||||
Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
SELECT arrayMin([1, 2, 4]) AS res
|
||||
┌─res─┐
|
||||
│ 1 │
|
||||
└─────┘
|
||||
|
||||
|
||||
SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res
|
||||
┌─res─┐
|
||||
│ -4 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## arrayMax(\[func,\] arr1, …) {#array-max}
|
||||
|
||||
Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements.
|
||||
Returns the max of the `func` values. If the function is omitted, it just returns the max of the array elements.
|
||||
|
||||
Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
SELECT arrayMax([1, 2, 4]) AS res
|
||||
┌─res─┐
|
||||
│ 4 │
|
||||
└─────┘
|
||||
|
||||
|
||||
SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res
|
||||
┌─res─┐
|
||||
│ -1 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## arraySum(\[func,\] arr1, …) {#array-sum}
|
||||
|
||||
Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements.
|
||||
|
||||
Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
Examples:
|
||||
```sql
|
||||
SELECT arraySum([2,3]) AS res
|
||||
┌─res─┐
|
||||
│ 5 │
|
||||
└─────┘
|
||||
|
||||
|
||||
SELECT arraySum(x -> x*x, [2, 3]) AS res
|
||||
┌─res─┐
|
||||
│ 13 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
|
||||
## arrayAvg(\[func,\] arr1, …) {#array-avg}
|
||||
|
||||
Returns the sum of the `func` values. If the function is omitted, it just returns the average of the array elements.
|
||||
Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements.
|
||||
|
||||
Note that the `arrayAvg` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ toc_title: Client Libraries
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -15,6 +15,7 @@ toc_title: "Biblioth\xE8ques Clientes"
|
||||
- [clickhouse-chauffeur](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2 / phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov / clickhouse-PHP-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -15,6 +15,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8"
|
||||
- [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-クライアント](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -13,6 +13,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -267,7 +267,7 @@ void executeQuery(
|
||||
|
||||
**9.** 多行注释的开头和结尾不得有空行（关闭多行注释的行除外）。
|
||||
|
||||
**10.** 要注释掉代码，请使用基本注释，而不是«记录»注释。
|
||||
**10.** 要注释掉代码，请使用基本注释，而不是“文档”注释。
|
||||
|
||||
**11.** 在提交之前删除代码的无效注释部分。
|
||||
|
||||
@ -335,7 +335,7 @@ template <bool without_www>
|
||||
struct ExtractDomain
|
||||
```
|
||||
|
||||
**7.** 对于抽象类型（接口），用 `I` 前缀。
|
||||
**7.** 对于抽象类（接口），用 `I` 前缀。
|
||||
|
||||
``` cpp
|
||||
class IBlockInputStream
|
||||
@ -349,7 +349,7 @@ class IBlockInputStream
|
||||
bool info_successfully_loaded = false;
|
||||
```
|
||||
|
||||
**9.** `define` 和全局常量的名称使用带下划线的 `ALL_CAPS`。
|
||||
**9.** `define` 和全局常量的名称使用全大写带下划线的形式，如 `ALL_CAPS`。
|
||||
|
||||
``` cpp
|
||||
#define MAX_SRC_TABLE_NAMES_TO_STORE 1000
|
||||
@ -357,14 +357,14 @@ bool info_successfully_loaded = false;
|
||||
|
||||
**10.** 文件名应使用与其内容相同的样式。
|
||||
|
||||
如果文件包含单个类，则以与该类名称相同的方式命名该文件。
|
||||
如果文件包含单个类，则以与该类名称相同的方式命名该文件（CamelCase）。
|
||||
|
||||
如果文件包含单个函数，则以与函数名称相同的方式命名文件。
|
||||
如果文件包含单个函数，则以与函数名称相同的方式命名文件（camelCase）。
|
||||
|
||||
**11.** 如果名称包含缩写，则：
|
||||
|
||||
- 对于变量名，缩写应使用小写字母 `mysql_connection`（不是 `mySQL_connection` ）。
|
||||
- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection` 。
|
||||
- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection`）。
|
||||
|
||||
**12.** 仅用于初始化类成员的构造方法参数的命名方式应与类成员相同，但最后使用下划线。
|
||||
|
||||
@ -411,7 +411,7 @@ enum class CompressionMethod
|
||||
|
||||
如果缩短版本是常用的，则可以接受不完整的单词。
|
||||
|
||||
如果注释中旁边包含全名，您也可以使用缩写。
|
||||
如果旁边有注释包含全名，您也可以使用缩写。
|
||||
|
||||
**17.** C++ 源码文件名称必须为 `.cpp` 拓展名。 头文件必须为 `.h` 拓展名。
|
||||
|
||||
@ -441,7 +441,7 @@ enum class CompressionMethod
|
||||
|
||||
在离线数据处理应用程序中，通常可以接受不捕获异常。
|
||||
|
||||
在处理用户请求的服务器中，通常足以捕获连接处理程序顶层的异常。
|
||||
在处理用户请求的服务器中，捕获连接处理程序顶层的异常通常就足够了。
|
||||
|
||||
在线程函数中，你应该在 `join` 之后捕获并保留所有异常以在主线程中重新抛出它们。
|
||||
|
||||
@ -548,7 +548,7 @@ Fork不用于并行化。
|
||||
|
||||
**10.** 常量。
|
||||
|
||||
使用 const 引用，指向常量的指针，`const_iterator`和 const 指针。
|
||||
使用 const 引用、指针，指向常量、`const_iterator`和 const 方法。
|
||||
|
||||
将 `const` 视为默认值，仅在必要时使用非 `const`。
|
||||
|
||||
@ -560,7 +560,7 @@ Fork不用于并行化。
|
||||
|
||||
**12.** 数值类型。
|
||||
|
||||
使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32`， 以及 `Int64`， `size_t`， `ssize_t` 还有 `ptrdiff_t`。
|
||||
使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32` 和 `Int64`，同样还有 `size_t`， `ssize_t` 和 `ptrdiff_t`。
|
||||
|
||||
不要使用这些类型：`signed / unsigned long`，`long long`，`short`，`signed / unsigned char`，`char`。
|
||||
|
||||
@ -732,11 +732,11 @@ CPU指令集是我们服务器中支持的最小集合。 目前，它是SSE 4.2
|
||||
|
||||
**8.** 尽可能经常地进行提交，即使代码只是部分准备好了。
|
||||
|
||||
目的明确的功能，使用分支。
|
||||
为了这种目的可以创建分支。
|
||||
|
||||
如果 `master` 分支中的代码尚不可构建，请在 `push` 之前将其从构建中排除。您需要在几天内完成或删除它。
|
||||
如果您的代码在 `master` 分支中尚不可构建，在 `push` 之前需要将其从构建中排除。您需要在几天内完成或删除它。
|
||||
|
||||
**9.** 对于不重要的更改，请使用分支并在服务器上发布它们。
|
||||
**9.** 对于非一般的更改，请使用分支并在服务器上发布它们。
|
||||
|
||||
**10.** 未使用的代码将从 repo 中删除。
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试
|
||||
- [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver)
|
||||
- [clickhouse-client](https://github.com/yurial/clickhouse-client)
|
||||
- [aiochclient](https://github.com/maximdanilchenko/aiochclient)
|
||||
- [asynch](https://github.com/long2ice/asynch)
|
||||
- PHP
|
||||
- [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse)
|
||||
- [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client)
|
||||
|
||||
@ -22,9 +22,35 @@ toc_title: "\u7CFB\u7EDF\u8868"
|
||||
|
||||
大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。
|
||||
|
||||
与其他系统表不同，系统表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) 由 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。
|
||||
与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。
|
||||
|
||||
默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。
|
||||
用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下:
|
||||
|
||||
- `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。
|
||||
- `table`: 系统日志表名。
|
||||
- `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。
|
||||
- `ttl`: 指定系统日志表TTL选项。
|
||||
- `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。
|
||||
- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。
|
||||
|
||||
一个配置定义的例子如下：
|
||||
|
||||
```
|
||||
<yandex>
|
||||
<query_log>
|
||||
<database>system</database>
|
||||
<table>query_log</table>
|
||||
<partition_by>toYYYYMM(event_date)</partition_by>
|
||||
<ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
|
||||
<!--
|
||||
<engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
|
||||
-->
|
||||
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||
</query_log>
|
||||
</yandex>
|
||||
```
|
||||
|
||||
默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。
|
||||
|
||||
## 系统指标的来源 {#system-tables-sources-of-system-metrics}
|
||||
|
||||
|
||||
@ -318,6 +318,10 @@ else ()
|
||||
if (USE_GDB_ADD_INDEX)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
|
||||
endif()
|
||||
|
||||
if (USE_BINARY_HASH)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM)
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if (ENABLE_TESTS AND USE_GTEST)
|
||||
|
||||
@ -59,7 +59,9 @@
|
||||
#include <DataStreams/AsynchronousBlockInputStream.h>
|
||||
#include <DataStreams/AddingDefaultsBlockInputStream.h>
|
||||
#include <DataStreams/InternalTextLogsRowOutputStream.h>
|
||||
#include <DataStreams/NullBlockOutputStream.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTDropQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTUseQuery.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
@ -110,6 +112,7 @@ namespace ErrorCodes
|
||||
extern const int INVALID_USAGE_OF_INPUT;
|
||||
extern const int DEADLOCK_AVOIDED;
|
||||
extern const int UNRECOGNIZED_ARGUMENTS;
|
||||
extern const int SYNTAX_ERROR;
|
||||
}
|
||||
|
||||
|
||||
@ -136,6 +139,9 @@ private:
|
||||
bool stdin_is_a_tty = false; /// stdin is a terminal.
|
||||
bool stdout_is_a_tty = false; /// stdout is a terminal.
|
||||
|
||||
/// If not empty, queries will be read from these files
|
||||
std::vector<std::string> queries_files;
|
||||
|
||||
std::unique_ptr<Connection> connection; /// Connection to DB.
|
||||
String full_query; /// Current query as it was given to the client.
|
||||
|
||||
@ -478,10 +484,10 @@ private:
|
||||
/// - stdin is not a terminal. In this case queries are read from it.
|
||||
/// - -qf (--queries-file) command line option is present.
|
||||
/// The value of the option is used as file with query (or of multiple queries) to execute.
|
||||
if (!stdin_is_a_tty || config().has("query") || config().has("queries-file"))
|
||||
if (!stdin_is_a_tty || config().has("query") || !queries_files.empty())
|
||||
is_interactive = false;
|
||||
|
||||
if (config().has("query") && config().has("queries-file"))
|
||||
if (config().has("query") && !queries_files.empty())
|
||||
{
|
||||
throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
@ -696,14 +702,8 @@ private:
|
||||
auto query_id = config().getString("query_id", "");
|
||||
if (!query_id.empty())
|
||||
context.setCurrentQueryId(query_id);
|
||||
if (query_fuzzer_runs)
|
||||
{
|
||||
nonInteractiveWithFuzzing();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
nonInteractive();
|
||||
}
|
||||
|
||||
/// If exception code isn't zero, we should return non-zero return code anyway.
|
||||
if (last_exception_received_from_server)
|
||||
@ -794,15 +794,21 @@ private:
|
||||
{
|
||||
String text;
|
||||
|
||||
if (config().has("queries-file"))
|
||||
if (!queries_files.empty())
|
||||
{
|
||||
ReadBufferFromFile in(config().getString("queries-file"));
|
||||
for (const auto & queries_file : queries_files)
|
||||
{
|
||||
connection->setDefaultDatabase(connection_parameters.default_database);
|
||||
ReadBufferFromFile in(queries_file);
|
||||
readStringUntilEOF(text, in);
|
||||
processMultiQuery(text);
|
||||
}
|
||||
return;
|
||||
}
|
||||
else if (config().has("query"))
|
||||
{
|
||||
text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query.
|
||||
}
|
||||
else
|
||||
{
|
||||
/// If 'query' parameter is not set, read a query from stdin.
|
||||
@ -811,113 +817,10 @@ private:
|
||||
readStringUntilEOF(text, in);
|
||||
}
|
||||
|
||||
processQueryText(text);
|
||||
}
|
||||
|
||||
void nonInteractiveWithFuzzing()
|
||||
{
|
||||
if (config().has("query"))
|
||||
{
|
||||
// Poco configuration should not process substitutions in form of
|
||||
// ${...} inside query
|
||||
processWithFuzzing(config().getRawString("query"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to stream the queries from stdin, without reading all of them
|
||||
// into memory. The interface of the parser does not support streaming,
|
||||
// in particular, it can't distinguish the end of partial input buffer
|
||||
// and the final end of input file. This means we have to try to split
|
||||
// the input into separate queries here. Two patterns of input are
|
||||
// especially interesting:
|
||||
// 1) multiline query:
|
||||
// select 1
|
||||
// from system.numbers;
|
||||
//
|
||||
// 2) csv insert with in-place data:
|
||||
// insert into t format CSV 1;2
|
||||
//
|
||||
// (1) means we can't split on new line, and (2) means we can't split on
|
||||
// semicolon. Solution: split on ';\n'. This sequence is frequent enough
|
||||
// in the SQL tests which are our principal input for fuzzing. Now we
|
||||
// have another interesting case:
|
||||
// 3) escaped semicolon followed by newline, e.g.
|
||||
// select ';
|
||||
// '
|
||||
//
|
||||
// To handle (3), parse until we can, and read more data if the parser
|
||||
// complains. Hopefully this should be enough...
|
||||
ReadBufferFromFileDescriptor in(STDIN_FILENO);
|
||||
std::string text;
|
||||
while (!in.eof())
|
||||
{
|
||||
// Read until separator.
|
||||
while (!in.eof())
|
||||
{
|
||||
char * next_separator = find_first_symbols<';'>(in.position(),
|
||||
in.buffer().end());
|
||||
|
||||
if (next_separator < in.buffer().end())
|
||||
{
|
||||
next_separator++;
|
||||
if (next_separator < in.buffer().end()
|
||||
&& *next_separator == '\n')
|
||||
{
|
||||
// Found ';\n', append it to the query text and try to
|
||||
// parse.
|
||||
next_separator++;
|
||||
text.append(in.position(), next_separator - in.position());
|
||||
in.position() = next_separator;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Didn't find the semicolon and reached the end of buffer.
|
||||
text.append(in.position(), next_separator - in.position());
|
||||
in.position() = next_separator;
|
||||
|
||||
if (text.size() > 1024 * 1024)
|
||||
{
|
||||
// We've read a lot of text and still haven't seen a separator.
|
||||
// Likely some pathological input, just fall through to prevent
|
||||
// too long loops.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse and execute what we've read.
|
||||
const auto * new_end = processWithFuzzing(text);
|
||||
|
||||
if (new_end > &text[0])
|
||||
{
|
||||
const auto rest_size = text.size() - (new_end - &text[0]);
|
||||
|
||||
memcpy(&text[0], new_end, rest_size);
|
||||
text.resize(rest_size);
|
||||
}
|
||||
if (query_fuzzer_runs)
|
||||
processWithFuzzing(text);
|
||||
else
|
||||
{
|
||||
// We didn't read enough text to parse a query. Will read more.
|
||||
}
|
||||
|
||||
// Ensure that we're still connected to the server. If the server died,
|
||||
// the reconnect is going to fail with an exception, and the fuzzer
|
||||
// will exit. The ping() would be the best match here, but it's
|
||||
// private, probably for a good reason that the protocol doesn't allow
|
||||
// pings at any possible moment.
|
||||
// Don't forget to reset the default database which might have changed.
|
||||
connection->setDefaultDatabase("");
|
||||
connection->forceConnected(connection_parameters.timeouts);
|
||||
|
||||
if (text.size() > 4 * 1024)
|
||||
{
|
||||
// Some pathological situation where the text is larger than 4kB
|
||||
// and we still cannot parse a single query in it. Abort.
|
||||
std::cerr << "Read too much text and still can't parse a query."
|
||||
" Aborting." << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
processQueryText(text);
|
||||
}
|
||||
|
||||
bool processQueryText(const String & text)
|
||||
@ -945,7 +848,8 @@ private:
|
||||
{
|
||||
const bool test_mode = config().has("testmode");
|
||||
|
||||
{ /// disable logs if expects errors
|
||||
{
|
||||
/// disable logs if expects errors
|
||||
TestHint test_hint(test_mode, all_queries_text);
|
||||
if (test_hint.clientError() || test_hint.serverError())
|
||||
processTextAsSingleQuery("SET send_logs_level = 'fatal'");
|
||||
@ -1019,7 +923,7 @@ private:
|
||||
if (hint.clientError() != e.code())
|
||||
{
|
||||
if (hint.clientError())
|
||||
e.addMessage("\nExpected clinet error: " + std::to_string(hint.clientError()));
|
||||
e.addMessage("\nExpected client error: " + std::to_string(hint.clientError()));
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -1078,6 +982,12 @@ private:
|
||||
expected_client_error = test_hint.clientError();
|
||||
expected_server_error = test_hint.serverError();
|
||||
|
||||
if (query_fuzzer_runs)
|
||||
{
|
||||
processWithFuzzing(full_query);
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
processParsedSingleQuery();
|
||||
@ -1102,8 +1012,11 @@ private:
|
||||
received_exception_from_server = true;
|
||||
}
|
||||
|
||||
if (!test_hint.checkActual(actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server))
|
||||
if (!test_hint.checkActual(
|
||||
actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server))
|
||||
{
|
||||
connection->forceConnected(connection_parameters.timeouts);
|
||||
}
|
||||
|
||||
if (received_exception_from_server && !ignore_error)
|
||||
{
|
||||
@ -1112,6 +1025,7 @@ private:
|
||||
else
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
this_query_begin = this_query_end;
|
||||
}
|
||||
@ -1120,62 +1034,45 @@ private:
|
||||
}
|
||||
|
||||
|
||||
// Returns the last position we could parse.
|
||||
const char * processWithFuzzing(const String & text)
|
||||
void processWithFuzzing(const String & text)
|
||||
{
|
||||
/// Several queries separated by ';'.
|
||||
/// INSERT data is ended by the end of line, not ';'.
|
||||
ASTPtr orig_ast;
|
||||
|
||||
try
|
||||
{
|
||||
const char * begin = text.data();
|
||||
const char * end = begin + text.size();
|
||||
|
||||
while (begin < end)
|
||||
{
|
||||
// Skip whitespace before the query
|
||||
while (isWhitespaceASCII(*begin) || *begin == ';')
|
||||
{
|
||||
++begin;
|
||||
orig_ast = parseQuery(begin, begin + text.size(), true);
|
||||
}
|
||||
catch (const Exception & e)
|
||||
{
|
||||
if (e.code() != ErrorCodes::SYNTAX_ERROR)
|
||||
throw;
|
||||
}
|
||||
|
||||
const auto * this_query_begin = begin;
|
||||
ASTPtr orig_ast = parseQuery(begin, end, true);
|
||||
|
||||
if (!orig_ast)
|
||||
{
|
||||
// Can't continue after a parsing error
|
||||
return begin;
|
||||
return;
|
||||
}
|
||||
|
||||
auto * as_insert = orig_ast->as<ASTInsertQuery>();
|
||||
if (as_insert && as_insert->data)
|
||||
{
|
||||
// INSERT data is ended by newline
|
||||
as_insert->end = find_first_symbols<'\n'>(as_insert->data, end);
|
||||
begin = as_insert->end;
|
||||
}
|
||||
|
||||
full_query = text.substr(this_query_begin - text.data(),
|
||||
begin - text.data());
|
||||
|
||||
// Don't repeat inserts, the tables grow too big. Also don't repeat
|
||||
// creates because first we run the unmodified query, it will succeed,
|
||||
// and the subsequent queries will fail. When we run out of fuzzer
|
||||
// errors, it may be interesting to add fuzzing of create queries that
|
||||
// wraps columns into LowCardinality or Nullable. Also there are other
|
||||
// kinds of create queries such as CREATE DICTIONARY, we could fuzz
|
||||
// them as well.
|
||||
int this_query_runs = query_fuzzer_runs;
|
||||
if (as_insert
|
||||
|| orig_ast->as<ASTCreateQuery>())
|
||||
// them as well. Also there is no point fuzzing DROP queries.
|
||||
size_t this_query_runs = query_fuzzer_runs;
|
||||
if (orig_ast->as<ASTInsertQuery>() || orig_ast->as<ASTCreateQuery>() || orig_ast->as<ASTDropQuery>())
|
||||
{
|
||||
this_query_runs = 1;
|
||||
}
|
||||
|
||||
ASTPtr fuzz_base = orig_ast;
|
||||
for (int fuzz_step = 0; fuzz_step < this_query_runs; fuzz_step++)
|
||||
for (size_t fuzz_step = 0; fuzz_step < this_query_runs; ++fuzz_step)
|
||||
{
|
||||
fprintf(stderr, "fuzzing step %d out of %d for query at pos %zd\n",
|
||||
fuzz_step, this_query_runs, this_query_begin - text.data());
|
||||
fmt::print(stderr, "Fuzzing step {} out of {}\n",
|
||||
fuzz_step, this_query_runs);
|
||||
|
||||
ASTPtr ast_to_process;
|
||||
try
|
||||
@ -1200,14 +1097,14 @@ private:
|
||||
// Debug AST cloning errors.
|
||||
if (base_before_fuzz != base_after_fuzz)
|
||||
{
|
||||
fprintf(stderr, "base before fuzz: %s\n"
|
||||
"base after fuzz: %s\n", base_before_fuzz.c_str(),
|
||||
base_after_fuzz.c_str());
|
||||
fprintf(stderr, "dump before fuzz:\n%s\n",
|
||||
dump_before_fuzz.str().c_str());
|
||||
fprintf(stderr, "dump of cloned ast:\n%s\n",
|
||||
dump_of_cloned_ast.str().c_str());
|
||||
fprintf(stderr, "dump after fuzz:\n");
|
||||
fmt::print(stderr,
|
||||
"Base before fuzz: {}\n"
|
||||
"Base after fuzz: {}\n",
|
||||
base_before_fuzz, base_after_fuzz);
|
||||
fmt::print(stderr, "Dump before fuzz:\n{}\n", dump_before_fuzz.str());
|
||||
fmt::print(stderr, "Dump of cloned AST:\n{}\n", dump_of_cloned_ast.str());
|
||||
fmt::print(stderr, "Dump after fuzz:\n");
|
||||
|
||||
WriteBufferFromOStream cerr_buf(std::cerr, 4096);
|
||||
fuzz_base->dumpTree(cerr_buf);
|
||||
cerr_buf.next();
|
||||
@ -1220,7 +1117,7 @@ private:
|
||||
auto fuzzed_text = ast_to_process->formatForErrorMessage();
|
||||
if (fuzz_step > 0 && fuzzed_text == base_before_fuzz)
|
||||
{
|
||||
fprintf(stderr, "got boring ast\n");
|
||||
fmt::print(stderr, "Got boring AST\n");
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -1250,7 +1147,7 @@ private:
|
||||
// Probably the server is dead because we found an assertion
|
||||
// failure. Fail fast.
|
||||
fmt::print(stderr, "Lost connection to the server\n");
|
||||
return begin;
|
||||
return;
|
||||
}
|
||||
|
||||
// The server is still alive so we're going to continue fuzzing.
|
||||
@ -1266,21 +1163,18 @@ private:
|
||||
else if (ast_to_process->formatForErrorMessage().size() > 500)
|
||||
{
|
||||
// ast too long, start from original ast
|
||||
fprintf(stderr, "Current AST is too long, discarding it and using the original AST as a start\n");
|
||||
fmt::print(stderr, "Current AST is too long, discarding it and using the original AST as a start\n");
|
||||
fuzz_base = orig_ast;
|
||||
}
|
||||
else
|
||||
{
|
||||
// fuzz starting from this successful query
|
||||
fprintf(stderr, "Query succeeded, using this AST as a start\n");
|
||||
fmt::print(stderr, "Query succeeded, using this AST as a start\n");
|
||||
fuzz_base = ast_to_process;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return begin;
|
||||
}
|
||||
|
||||
void processTextAsSingleQuery(const String & text_)
|
||||
{
|
||||
full_query = text_;
|
||||
@ -1891,6 +1785,13 @@ private:
|
||||
{
|
||||
if (!block_out_stream)
|
||||
{
|
||||
/// Ignore all results when fuzzing as they can be huge.
|
||||
if (query_fuzzer_runs)
|
||||
{
|
||||
block_out_stream = std::make_shared<NullBlockOutputStream>(block);
|
||||
return;
|
||||
}
|
||||
|
||||
WriteBuffer * out_buf = nullptr;
|
||||
String pager = config().getString("pager", "");
|
||||
if (!pager.empty())
|
||||
@ -2348,7 +2249,8 @@ public:
|
||||
"Suggestion limit for how many databases, tables and columns to fetch.")
|
||||
("multiline,m", "multiline")
|
||||
("multiquery,n", "multiquery")
|
||||
("queries-file", po::value<std::string>(), "file path with queries to execute")
|
||||
("queries-file", po::value<std::vector<std::string>>()->multitoken(),
|
||||
"file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)")
|
||||
("format,f", po::value<std::string>(), "default output format")
|
||||
("testmode,T", "enable test hints in comments")
|
||||
("ignore-error", "do not stop processing in multiquery mode")
|
||||
@ -2478,12 +2380,11 @@ public:
|
||||
if (options.count("query"))
|
||||
config().setString("query", options["query"].as<std::string>());
|
||||
if (options.count("queries-file"))
|
||||
config().setString("queries-file", options["queries-file"].as<std::string>());
|
||||
queries_files = options["queries-file"].as<std::vector<std::string>>();
|
||||
if (options.count("database"))
|
||||
config().setString("database", options["database"].as<std::string>());
|
||||
if (options.count("pager"))
|
||||
config().setString("pager", options["pager"].as<std::string>());
|
||||
|
||||
if (options.count("port") && !options["port"].defaulted())
|
||||
config().setInt("port", options["port"].as<int>());
|
||||
if (options.count("secure"))
|
||||
@ -2537,7 +2438,6 @@ public:
|
||||
config().setBool("multiquery", true);
|
||||
|
||||
// Ignore errors in parsing queries.
|
||||
// TODO stop using parseQuery.
|
||||
config().setBool("ignore-error", true);
|
||||
ignore_error = true;
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
#endif
|
||||
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
|
||||
#include <common/phdr_cache.h>
|
||||
#include <ext/scope_guard.h>
|
||||
@ -62,6 +63,14 @@ int mainEntryClickHouseStatus(int argc, char ** argv);
|
||||
int mainEntryClickHouseRestart(int argc, char ** argv);
|
||||
#endif
|
||||
|
||||
int mainEntryClickHouseHashBinary(int, char **)
|
||||
{
|
||||
/// Intentionally without newline. So you can run:
|
||||
/// objcopy --add-section .note.ClickHouse.hash=<(./clickhouse hash-binary) clickhouse
|
||||
std::cout << getHashOfLoadedBinaryHex();
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
|
||||
|
||||
namespace
|
||||
@ -110,6 +119,7 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
|
||||
{"status", mainEntryClickHouseStatus},
|
||||
{"restart", mainEntryClickHouseRestart},
|
||||
#endif
|
||||
{"hash-binary", mainEntryClickHouseHashBinary},
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -65,6 +65,8 @@
|
||||
#include <Server/TCPHandlerFactory.h>
|
||||
#include <Common/SensitiveDataMasker.h>
|
||||
#include <Common/ThreadFuzzer.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
#include <Common/Elf.h>
|
||||
#include <Server/MySQLHandlerFactory.h>
|
||||
#include <Server/PostgreSQLHandlerFactory.h>
|
||||
#include <Server/ProtocolServerAdapter.h>
|
||||
@ -184,6 +186,7 @@ namespace ErrorCodes
|
||||
extern const int FAILED_TO_GETPWUID;
|
||||
extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
|
||||
extern const int NETWORK_ERROR;
|
||||
extern const int CORRUPTED_DATA;
|
||||
}
|
||||
|
||||
|
||||
@ -436,7 +439,44 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
std::string executable_path = getExecutablePath();
|
||||
if (executable_path.empty())
|
||||
|
||||
if (!executable_path.empty())
|
||||
{
|
||||
/// Integrity check based on checksum of the executable code.
|
||||
/// Note: it is not intended to protect from malicious party,
|
||||
/// because the reference checksum can be easily modified as well.
|
||||
/// And we don't involve asymmetric encryption with PKI yet.
|
||||
/// It's only intended to protect from faulty hardware.
|
||||
/// Note: it is only based on machine code.
|
||||
/// But there are other sections of the binary (e.g. exception handling tables)
|
||||
/// that are interpreted (not executed) but can alter the behaviour of the program as well.
|
||||
|
||||
String calculated_binary_hash = getHashOfLoadedBinaryHex();
|
||||
|
||||
if (stored_binary_hash.empty())
|
||||
{
|
||||
LOG_WARNING(log, "Calculated checksum of the binary: {}."
|
||||
" There is no information about the reference checksum.", calculated_binary_hash);
|
||||
}
|
||||
else if (calculated_binary_hash == stored_binary_hash)
|
||||
{
|
||||
LOG_INFO(log, "Calculated checksum of the binary: {}, integrity check passed.", calculated_binary_hash);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::CORRUPTED_DATA,
|
||||
"Calculated checksum of the ClickHouse binary ({0}) does not correspond"
|
||||
" to the reference checksum stored in the binary ({1})."
|
||||
" It may indicate one of the following:"
|
||||
" - the file {2} was changed just after startup;"
|
||||
" - the file {2} is damaged on disk due to faulty hardware;"
|
||||
" - the loaded executable is damaged in memory due to faulty hardware;"
|
||||
" - the file {2} was intentionally modified;"
|
||||
" - logical error in code."
|
||||
, calculated_binary_hash, stored_binary_hash, executable_path);
|
||||
}
|
||||
}
|
||||
else
|
||||
executable_path = "/usr/bin/clickhouse"; /// It is used for information messages.
|
||||
|
||||
/// After full config loaded
|
||||
|
||||
@ -676,7 +676,7 @@
|
||||
<database>system</database>
|
||||
<table>query_log</table>
|
||||
<!--
|
||||
PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/
|
||||
PARTITION BY expr: https://clickhouse.yandex/docs/en/table_engines/mergetree-family/custom_partitioning_key/
|
||||
Example:
|
||||
event_date
|
||||
toMonday(event_date)
|
||||
|
||||
@ -53,17 +53,35 @@ class AggregateFunctionIfNullUnary final
|
||||
private:
|
||||
size_t num_arguments;
|
||||
|
||||
/// The name of the nested function, including combinators (i.e. *If)
|
||||
///
|
||||
/// getName() from the nested_function cannot be used because in case of *If combinator
|
||||
/// with Nullable argument nested_function will point to the function w/o combinator.
|
||||
/// (I.e. sumIf(Nullable, 1) -> sum()), and distributed query processing will fail.
|
||||
///
|
||||
/// And nested_function cannot point to the function with *If since
|
||||
/// due to optimization in the add() which pass only one column with the result,
|
||||
/// and so AggregateFunctionIf::add() cannot be called this way
|
||||
/// (it write to the last argument -- num_arguments-1).
|
||||
///
|
||||
/// And to avoid extra level of indirection, the name of function is cached:
|
||||
///
|
||||
/// AggregateFunctionIfNullUnary::add -> [ AggregateFunctionIf::add -> ] AggregateFunctionSum::add
|
||||
String name;
|
||||
|
||||
using Base = AggregateFunctionNullBase<result_is_nullable, serialize_flag,
|
||||
AggregateFunctionIfNullUnary<result_is_nullable, serialize_flag>>;
|
||||
public:
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return Base::getName();
|
||||
return name;
|
||||
}
|
||||
|
||||
AggregateFunctionIfNullUnary(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
|
||||
: Base(std::move(nested_function_), arguments, params), num_arguments(arguments.size())
|
||||
AggregateFunctionIfNullUnary(const String & name_, AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params)
|
||||
: Base(std::move(nested_function_), arguments, params)
|
||||
, num_arguments(arguments.size())
|
||||
, name(name_)
|
||||
{
|
||||
if (num_arguments == 0)
|
||||
throw Exception("Aggregate function " + getName() + " require at least one argument",
|
||||
@ -174,14 +192,14 @@ AggregateFunctionPtr AggregateFunctionIf::getOwnNullAdapter(
|
||||
{
|
||||
if (return_type_is_nullable)
|
||||
{
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<true, true>>(nested_func, arguments, params);
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<true, true>>(nested_function->getName(), nested_func, arguments, params);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (serialize_flag)
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<false, true>>(nested_func, arguments, params);
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<false, true>>(nested_function->getName(), nested_func, arguments, params);
|
||||
else
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<false, false>>(nested_func, arguments, params);
|
||||
return std::make_shared<AggregateFunctionIfNullUnary<false, false>>(nested_function->getName(), nested_func, arguments, params);
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
41
src/Common/DirectorySyncGuard.cpp
Normal file
41
src/Common/DirectorySyncGuard.cpp
Normal file
@ -0,0 +1,41 @@
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Disks/IDisk.h>
|
||||
#include <fcntl.h> // O_RDWR
|
||||
|
||||
/// OSX does not have O_DIRECTORY
|
||||
#ifndef O_DIRECTORY
|
||||
#define O_DIRECTORY O_RDWR
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_FSYNC;
|
||||
}
|
||||
|
||||
DirectorySyncGuard::DirectorySyncGuard(const DiskPtr & disk_, const String & path)
|
||||
: disk(disk_)
|
||||
, fd(disk_->open(path, O_DIRECTORY))
|
||||
{}
|
||||
|
||||
DirectorySyncGuard::~DirectorySyncGuard()
|
||||
{
|
||||
try
|
||||
{
|
||||
#if defined(OS_DARWIN)
|
||||
if (fcntl(fd, F_FULLFSYNC, 0))
|
||||
throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
|
||||
#endif
|
||||
disk->sync(fd);
|
||||
disk->close(fd);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,36 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include <Disks/IDisk.h>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class IDisk;
|
||||
using DiskPtr = std::shared_ptr<IDisk>;
|
||||
|
||||
/// Helper class, that receives file descriptor and does fsync for it in destructor.
|
||||
/// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end.
|
||||
/// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version.
|
||||
/// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496
|
||||
class FileSyncGuard
|
||||
class DirectorySyncGuard
|
||||
{
|
||||
public:
|
||||
/// NOTE: If you have already opened descriptor, it's preferred to use
|
||||
/// this constructor instead of constructor with path.
|
||||
FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {}
|
||||
|
||||
FileSyncGuard(const DiskPtr & disk_, const String & path)
|
||||
: disk(disk_), fd(disk_->open(path, O_RDWR)) {}
|
||||
|
||||
~FileSyncGuard()
|
||||
{
|
||||
try
|
||||
{
|
||||
disk->sync(fd);
|
||||
disk->close(fd);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
DirectorySyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {}
|
||||
DirectorySyncGuard(const DiskPtr & disk_, const std::string & path);
|
||||
~DirectorySyncGuard();
|
||||
|
||||
private:
|
||||
DiskPtr disk;
|
||||
@ -151,6 +151,15 @@ String Elf::getBuildID(const char * nhdr_pos, size_t size)
|
||||
}
|
||||
|
||||
|
||||
String Elf::getBinaryHash() const
|
||||
{
|
||||
if (auto section = findSectionByName(".note.ClickHouse.hash"))
|
||||
return {section->begin(), section->end()};
|
||||
else
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
const char * Elf::Section::name() const
|
||||
{
|
||||
if (!elf.section_names)
|
||||
|
||||
@ -59,6 +59,9 @@ public:
|
||||
String getBuildID() const;
|
||||
static String getBuildID(const char * nhdr_pos, size_t size);
|
||||
|
||||
/// Hash of the binary for integrity checks.
|
||||
String getBinaryHash() const;
|
||||
|
||||
private:
|
||||
MMapReadBufferFromFile in;
|
||||
size_t elf_size;
|
||||
|
||||
@ -531,6 +531,7 @@
|
||||
M(562, TLD_LIST_NOT_FOUND) \
|
||||
M(563, CANNOT_READ_MAP_FROM_TEXT) \
|
||||
M(564, INTERSERVER_SCHEME_DOESNT_MATCH) \
|
||||
M(565, TOO_MANY_PARTITIONS) \
|
||||
\
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
|
||||
@ -234,7 +234,12 @@ void MemoryTracker::updatePeak(Int64 will_be)
|
||||
void MemoryTracker::free(Int64 size)
|
||||
{
|
||||
if (BlockerInThread::isBlocked(level))
|
||||
{
|
||||
/// Since the BlockerInThread should respect the level, we should go to the next parent.
|
||||
if (auto * loaded_next = parent.load(std::memory_order_relaxed))
|
||||
loaded_next->free(size);
|
||||
return;
|
||||
}
|
||||
|
||||
std::bernoulli_distribution sample(sample_probability);
|
||||
if (unlikely(sample_probability && sample(thread_local_rng)))
|
||||
|
||||
@ -25,7 +25,7 @@ thread_local ThreadStatus * current_thread = nullptr;
|
||||
thread_local ThreadStatus * main_thread = nullptr;
|
||||
|
||||
#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
|
||||
alignas(4096) static thread_local char alt_stack[4096];
|
||||
alignas(4096) static thread_local char alt_stack[std::max<size_t>(MINSIGSTKSZ, 4096)];
|
||||
static thread_local bool has_alt_stack = false;
|
||||
#endif
|
||||
|
||||
|
||||
58
src/Common/getHashOfLoadedBinary.cpp
Normal file
58
src/Common/getHashOfLoadedBinary.cpp
Normal file
@ -0,0 +1,58 @@
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
|
||||
#if defined(__linux__)
|
||||
|
||||
#include <link.h>
|
||||
#include <array>
|
||||
#include <Common/hex.h>
|
||||
|
||||
|
||||
static int callback(dl_phdr_info * info, size_t, void * data)
|
||||
{
|
||||
SipHash & hash = *reinterpret_cast<SipHash*>(data);
|
||||
|
||||
for (size_t header_index = 0; header_index < info->dlpi_phnum; ++header_index)
|
||||
{
|
||||
const auto & phdr = info->dlpi_phdr[header_index];
|
||||
|
||||
if (phdr.p_type == PT_LOAD && (phdr.p_flags & PF_X))
|
||||
{
|
||||
hash.update(phdr.p_filesz);
|
||||
hash.update(reinterpret_cast<const char *>(info->dlpi_addr + phdr.p_vaddr), phdr.p_filesz);
|
||||
}
|
||||
}
|
||||
|
||||
return 1; /// Do not continue iterating.
|
||||
}
|
||||
|
||||
|
||||
SipHash getHashOfLoadedBinary()
|
||||
{
|
||||
SipHash hash;
|
||||
dl_iterate_phdr(callback, &hash);
|
||||
return hash;
|
||||
}
|
||||
|
||||
|
||||
std::string getHashOfLoadedBinaryHex()
|
||||
{
|
||||
SipHash hash = getHashOfLoadedBinary();
|
||||
std::array<UInt64, 2> checksum;
|
||||
hash.get128(checksum[0], checksum[1]);
|
||||
return getHexUIntUppercase(checksum);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
SipHash getHashOfLoadedBinary()
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
std::string getHashOfLoadedBinaryHex()
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
#endif
|
||||
15
src/Common/getHashOfLoadedBinary.h
Normal file
15
src/Common/getHashOfLoadedBinary.h
Normal file
@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
/** Calculate hash of the executable loaded segments of the first loaded object.
|
||||
* It can be used for integrity checks.
|
||||
* Does not work when ClickHouse is build as multiple shared libraries.
|
||||
* Note: we don't hash all loaded readonly segments, because some of them are modified by 'strip'
|
||||
* and we want something that survives 'strip'.
|
||||
* Note: program behaviour can be affected not only by machine code but also by the data in these segments,
|
||||
* so the integrity check is going to be incomplete.
|
||||
*/
|
||||
SipHash getHashOfLoadedBinary();
|
||||
std::string getHashOfLoadedBinaryHex();
|
||||
@ -37,6 +37,7 @@ SRCS(
|
||||
CurrentMetrics.cpp
|
||||
CurrentThread.cpp
|
||||
DNSResolver.cpp
|
||||
DirectorySyncGuard.cpp
|
||||
Dwarf.cpp
|
||||
Elf.cpp
|
||||
ErrorCodes.cpp
|
||||
@ -98,6 +99,7 @@ SRCS(
|
||||
formatIPv6.cpp
|
||||
formatReadable.cpp
|
||||
getExecutablePath.cpp
|
||||
getHashOfLoadedBinary.cpp
|
||||
getMappedArea.cpp
|
||||
getMultipleKeysFromConfig.cpp
|
||||
getNumberOfPhysicalCPUCores.cpp
|
||||
|
||||
@ -16,8 +16,8 @@ private:
|
||||
bool nextImpl() override;
|
||||
|
||||
public:
|
||||
CompressedReadBuffer(ReadBuffer & in_)
|
||||
: CompressedReadBufferBase(&in_), BufferWithOwnMemory<ReadBuffer>(0)
|
||||
CompressedReadBuffer(ReadBuffer & in_, bool allow_different_codecs_ = false)
|
||||
: CompressedReadBufferBase(&in_, allow_different_codecs_), BufferWithOwnMemory<ReadBuffer>(0)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -67,6 +67,10 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
|
||||
buf[pos / 8] ^= 1 << pos % 8;
|
||||
};
|
||||
|
||||
/// If size is too huge, then this may be caused by corruption.
|
||||
/// And anyway this is pretty heavy, so avoid burning too much CPU here.
|
||||
if (size < (1ULL << 20))
|
||||
{
|
||||
/// Check if the difference caused by single bit flip in data.
|
||||
for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos)
|
||||
{
|
||||
@ -82,6 +86,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
|
||||
|
||||
flip_bit(data, bit_pos); /// Restore
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the difference caused by single bit flip in stored checksum.
|
||||
size_t difference = __builtin_popcountll(expected_checksum.first ^ calculated_checksum.first)
|
||||
|
||||
@ -46,6 +46,9 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std
|
||||
|
||||
void CompressionCodecFactory::validateCodec(const String & family_name, std::optional<int> level, bool sanity_check) const
|
||||
{
|
||||
if (family_name.empty())
|
||||
throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (level)
|
||||
{
|
||||
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
#include "BackgroundSchedulePool.h"
|
||||
#include <Common/MemoryTracker.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Access/AccessControlManager.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <Common/MemoryTracker.h>
|
||||
#include <Common/OpenSSLHelpers.h>
|
||||
|
||||
#include <ext/scope_guard.h>
|
||||
|
||||
@ -353,6 +353,7 @@ class IColumn;
|
||||
M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \
|
||||
\
|
||||
M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \
|
||||
M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \
|
||||
M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \
|
||||
M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \
|
||||
\
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
#include <DataStreams/PushingToViewsBlockOutputStream.h>
|
||||
#include <DataStreams/SquashingBlockInputStream.h>
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <DataStreams/MaterializingBlockInputStream.h>
|
||||
#include <DataStreams/copyData.h>
|
||||
#include <DataTypes/NestedUtils.h>
|
||||
#include <Interpreters/InterpreterSelectQuery.h>
|
||||
#include <Interpreters/InterpreterInsertQuery.h>
|
||||
@ -14,6 +16,7 @@
|
||||
#include <Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h>
|
||||
#include <Storages/StorageValues.h>
|
||||
#include <Storages/LiveView/StorageLiveView.h>
|
||||
#include <Storages/StorageMaterializedView.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataStreams/copyData.h>
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
#include <DataStreams/MaterializingBlockInputStream.h>
|
||||
#include <Storages/StorageMaterializedView.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <Storages/IStorage.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ReplicatedMergeTreeBlockOutputStream;
|
||||
class Context;
|
||||
|
||||
|
||||
/** Writes data to the specified table and to all dependent materialized views.
|
||||
|
||||
@ -225,7 +225,7 @@ std::variant<Block, int> RemoteQueryExecutor::read(std::unique_ptr<ReadContext>
|
||||
if (!read_context->resumeRoutine())
|
||||
return Block();
|
||||
|
||||
if (read_context->is_read_in_progress)
|
||||
if (read_context->is_read_in_progress.load(std::memory_order_relaxed))
|
||||
{
|
||||
read_context->setTimer();
|
||||
return read_context->epoll_fd;
|
||||
|
||||
@ -22,7 +22,7 @@ class RemoteQueryExecutorReadContext
|
||||
public:
|
||||
using Self = RemoteQueryExecutorReadContext;
|
||||
|
||||
bool is_read_in_progress = false;
|
||||
std::atomic_bool is_read_in_progress = false;
|
||||
Packet packet;
|
||||
|
||||
std::exception_ptr exception;
|
||||
@ -162,7 +162,7 @@ public:
|
||||
|
||||
bool resumeRoutine()
|
||||
{
|
||||
if (is_read_in_progress && !checkTimeout())
|
||||
if (is_read_in_progress.load(std::memory_order_relaxed) && !checkTimeout())
|
||||
return false;
|
||||
|
||||
{
|
||||
@ -226,9 +226,9 @@ public:
|
||||
throw;
|
||||
}
|
||||
|
||||
read_context.is_read_in_progress = true;
|
||||
read_context.is_read_in_progress.store(true, std::memory_order_relaxed);
|
||||
fiber = std::move(fiber).resume();
|
||||
read_context.is_read_in_progress = false;
|
||||
read_context.is_read_in_progress.store(false, std::memory_order_relaxed);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -18,6 +18,7 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
|
||||
extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
}
|
||||
|
||||
// FunctionStringHash
|
||||
@ -30,6 +31,8 @@ public:
|
||||
static constexpr auto name = Name::name;
|
||||
static constexpr size_t default_shingle_size = 3;
|
||||
static constexpr size_t default_num_hashes = 6;
|
||||
static constexpr size_t max_shingle_size = 25;
|
||||
static constexpr size_t max_num_hashes = 25;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionsStringHash>(); }
|
||||
|
||||
@ -100,10 +103,14 @@ public:
|
||||
}
|
||||
|
||||
if (shingle_size == 0)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument (shingle size) of function {} cannot be zero", getName());
|
||||
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Second argument (shingle size) of function {} cannot be zero", getName());
|
||||
if (num_hashes == 0)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Third argument (num hashes) of function {} cannot be zero", getName());
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Third argument (num hashes) of function {} cannot be zero", getName());
|
||||
|
||||
if (shingle_size > max_shingle_size)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Second argument (shingle size) of function {} cannot be greater then {}", getName(), max_shingle_size);
|
||||
if (num_hashes > max_num_hashes)
|
||||
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Third argument (num hashes) of function {} cannot be greater then {}", getName(), max_num_hashes);
|
||||
|
||||
auto type = std::make_shared<DataTypeUInt64>();
|
||||
if constexpr (is_simhash)
|
||||
|
||||
@ -50,11 +50,6 @@ public:
|
||||
/// Close file before destruction of object.
|
||||
void close();
|
||||
|
||||
void finalize() override
|
||||
{
|
||||
close();
|
||||
}
|
||||
|
||||
std::string getFileName() const override
|
||||
{
|
||||
return file_name;
|
||||
|
||||
@ -847,6 +847,13 @@ bool DDLWorker::taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, const Storage
|
||||
if (!ast_ddl->as<ASTAlterQuery>() && !ast_ddl->as<ASTOptimizeQuery>() && !ast_ddl->as<ASTDropQuery>())
|
||||
return false;
|
||||
|
||||
if (auto * alter = ast_ddl->as<ASTAlterQuery>())
|
||||
{
|
||||
// Setting alters should be executed on all replicas
|
||||
if (alter->isSettingsAlter())
|
||||
return false;
|
||||
}
|
||||
|
||||
return storage->supportsReplication();
|
||||
}
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
#include <Processors/Sources/SourceFromInputStream.h>
|
||||
#include <Processors/Transforms/ExpressionTransform.h>
|
||||
#include <Storages/StorageDistributed.h>
|
||||
#include <Storages/StorageMaterializedView.h>
|
||||
#include <TableFunctions/TableFunctionFactory.h>
|
||||
#include <Common/checkStackSize.h>
|
||||
#include <Interpreters/QueryLog.h>
|
||||
|
||||
@ -543,7 +543,6 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
if (analysis_result.prewhere_info)
|
||||
{
|
||||
ExpressionActions(analysis_result.prewhere_info->prewhere_actions).execute(header);
|
||||
header = materializeBlock(header);
|
||||
if (analysis_result.prewhere_info->remove_prewhere_column)
|
||||
header.erase(analysis_result.prewhere_info->prewhere_column_name);
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Common/SymbolIndex.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Interpreters/ExternalDictionariesLoader.h>
|
||||
@ -221,21 +222,43 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
switch (query.type)
|
||||
{
|
||||
case Type::SHUTDOWN:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_SHUTDOWN);
|
||||
if (kill(0, SIGTERM))
|
||||
throwFromErrno("System call kill(0, SIGTERM) failed", ErrorCodes::CANNOT_KILL);
|
||||
break;
|
||||
}
|
||||
case Type::KILL:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_SHUTDOWN);
|
||||
if (kill(0, SIGKILL))
|
||||
throwFromErrno("System call kill(0, SIGKILL) failed", ErrorCodes::CANNOT_KILL);
|
||||
/// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL.
|
||||
/// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker).
|
||||
LOG_INFO(log, "Exit immediately as the SYSTEM KILL command has been issued.");
|
||||
_exit(128 + SIGKILL);
|
||||
// break; /// unreachable
|
||||
}
|
||||
case Type::SUSPEND:
|
||||
{
|
||||
auto command = fmt::format("kill -STOP {0} && sleep {1} && kill -CONT {0}", getpid(), query.seconds);
|
||||
LOG_DEBUG(log, "Will run {}", command);
|
||||
auto res = ShellCommand::execute(command);
|
||||
res->in.close();
|
||||
WriteBufferFromOwnString out;
|
||||
copyData(res->out, out);
|
||||
copyData(res->err, out);
|
||||
if (!out.str().empty())
|
||||
LOG_DEBUG(log, "The command returned output: {}", command, out.str());
|
||||
res->wait();
|
||||
break;
|
||||
}
|
||||
case Type::DROP_DNS_CACHE:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_DROP_DNS_CACHE);
|
||||
DNSResolver::instance().dropCache();
|
||||
/// Reinitialize clusters to update their resolved_addresses
|
||||
system_context.reloadClusterConfig();
|
||||
break;
|
||||
}
|
||||
case Type::DROP_MARK_CACHE:
|
||||
context.checkAccess(AccessType::SYSTEM_DROP_MARK_CACHE);
|
||||
system_context.dropMarkCache();
|
||||
@ -251,12 +274,15 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
break;
|
||||
#endif
|
||||
case Type::RELOAD_DICTIONARY:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY);
|
||||
system_context.getExternalDictionariesLoader().loadOrReload(
|
||||
DatabaseCatalog::instance().resolveDictionaryName(query.target_dictionary));
|
||||
ExternalDictionariesLoader::resetAll();
|
||||
break;
|
||||
}
|
||||
case Type::RELOAD_DICTIONARIES:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY);
|
||||
executeCommandsAndThrowIfError(
|
||||
[&] () { system_context.getExternalDictionariesLoader().reloadAllTriedToLoad(); },
|
||||
@ -264,6 +290,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
);
|
||||
ExternalDictionariesLoader::resetAll();
|
||||
break;
|
||||
}
|
||||
case Type::RELOAD_EMBEDDED_DICTIONARIES:
|
||||
context.checkAccess(AccessType::SYSTEM_RELOAD_EMBEDDED_DICTIONARIES);
|
||||
system_context.getEmbeddedDictionaries().reload();
|
||||
@ -273,6 +300,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
system_context.reloadConfig();
|
||||
break;
|
||||
case Type::RELOAD_SYMBOLS:
|
||||
{
|
||||
#if defined(__ELF__) && !defined(__FreeBSD__)
|
||||
context.checkAccess(AccessType::SYSTEM_RELOAD_SYMBOLS);
|
||||
(void)SymbolIndex::instance(true);
|
||||
@ -280,6 +308,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
#else
|
||||
throw Exception("SYSTEM RELOAD SYMBOLS is not supported on current platform", ErrorCodes::NOT_IMPLEMENTED);
|
||||
#endif
|
||||
}
|
||||
case Type::STOP_MERGES:
|
||||
startStopAction(ActionLocks::PartsMerge, false);
|
||||
break;
|
||||
@ -340,6 +369,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
break;
|
||||
case Type::FLUSH_LOGS:
|
||||
{
|
||||
context.checkAccess(AccessType::SYSTEM_FLUSH_LOGS);
|
||||
executeCommandsAndThrowIfError(
|
||||
[&] () { if (auto query_log = context.getQueryLog()) query_log->flush(true); },
|
||||
@ -352,6 +382,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
[&] () { if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); }
|
||||
);
|
||||
break;
|
||||
}
|
||||
case Type::STOP_LISTEN_QUERIES:
|
||||
case Type::START_LISTEN_QUERIES:
|
||||
throw Exception(String(ASTSystemQuery::typeToString(query.type)) + " is not supported yet", ErrorCodes::NOT_IMPLEMENTED);
|
||||
@ -586,7 +617,8 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
|
||||
switch (query.type)
|
||||
{
|
||||
case Type::SHUTDOWN: [[fallthrough]];
|
||||
case Type::KILL:
|
||||
case Type::KILL: [[fallthrough]];
|
||||
case Type::SUSPEND:
|
||||
{
|
||||
required_access.emplace_back(AccessType::SYSTEM_SHUTDOWN);
|
||||
break;
|
||||
|
||||
@ -344,6 +344,22 @@ void ASTAlterCommand::formatImpl(
|
||||
throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
||||
}
|
||||
|
||||
bool ASTAlterQuery::isSettingsAlter() const
|
||||
{
|
||||
if (command_list)
|
||||
{
|
||||
if (command_list->children.empty())
|
||||
return false;
|
||||
for (const auto & child : command_list->children)
|
||||
{
|
||||
const auto & command = child->as<const ASTAlterCommand &>();
|
||||
if (command.type != ASTAlterCommand::MODIFY_SETTING)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Get the text that identifies this element. */
|
||||
String ASTAlterQuery::getID(char delim) const
|
||||
|
||||
@ -187,6 +187,8 @@ public:
|
||||
|
||||
ASTExpressionList * command_list = nullptr;
|
||||
|
||||
bool isSettingsAlter() const;
|
||||
|
||||
String getID(char) const override;
|
||||
|
||||
ASTPtr clone() const override;
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <re2/re2.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -14,6 +15,7 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NO_SUCH_COLUMN_IN_TABLE;
|
||||
extern const int CANNOT_COMPILE_REGEXP;
|
||||
}
|
||||
|
||||
void IASTColumnsTransformer::transform(const ASTPtr & transformer, ASTs & nodes)
|
||||
@ -86,6 +88,9 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo
|
||||
(*it)->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
if (!original_pattern.empty())
|
||||
settings.ostr << quoteString(original_pattern);
|
||||
|
||||
if (children.size() > 1)
|
||||
settings.ostr << ")";
|
||||
}
|
||||
@ -93,6 +98,8 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo
|
||||
void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
|
||||
{
|
||||
std::set<String> expected_columns;
|
||||
if (original_pattern.empty())
|
||||
{
|
||||
for (const auto & child : children)
|
||||
expected_columns.insert(child->as<const ASTIdentifier &>().name());
|
||||
|
||||
@ -105,13 +112,27 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
|
||||
{
|
||||
expected_columns.erase(expected_column);
|
||||
it = nodes.erase(it);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (auto it = nodes.begin(); it != nodes.end();)
|
||||
{
|
||||
if (const auto * id = it->get()->as<ASTIdentifier>())
|
||||
{
|
||||
if (isColumnMatching(id->shortName()))
|
||||
{
|
||||
it = nodes.erase(it);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
if (is_strict && !expected_columns.empty())
|
||||
{
|
||||
@ -125,6 +146,21 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
|
||||
}
|
||||
}
|
||||
|
||||
void ASTColumnsExceptTransformer::setPattern(String pattern)
|
||||
{
|
||||
original_pattern = std::move(pattern);
|
||||
column_matcher = std::make_shared<RE2>(original_pattern, RE2::Quiet);
|
||||
if (!column_matcher->ok())
|
||||
throw DB::Exception(
|
||||
"COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(),
|
||||
DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
|
||||
}
|
||||
|
||||
bool ASTColumnsExceptTransformer::isColumnMatching(const String & column_name) const
|
||||
{
|
||||
return RE2::PartialMatch(column_name, *column_matcher);
|
||||
}
|
||||
|
||||
void ASTColumnsReplaceTransformer::Replacement::formatImpl(
|
||||
const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||
{
|
||||
|
||||
@ -2,6 +2,11 @@
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
namespace re2
|
||||
{
|
||||
class RE2;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class IASTColumnsTransformer : public IAST
|
||||
@ -43,9 +48,13 @@ public:
|
||||
return clone;
|
||||
}
|
||||
void transform(ASTs & nodes) const override;
|
||||
void setPattern(String pattern);
|
||||
bool isColumnMatching(const String & column_name) const;
|
||||
|
||||
protected:
|
||||
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
|
||||
std::shared_ptr<re2::RE2> column_matcher;
|
||||
String original_pattern;
|
||||
};
|
||||
|
||||
class ASTColumnsReplaceTransformer : public IASTColumnsTransformer
|
||||
|
||||
@ -22,6 +22,8 @@ const char * ASTSystemQuery::typeToString(Type type)
|
||||
return "SHUTDOWN";
|
||||
case Type::KILL:
|
||||
return "KILL";
|
||||
case Type::SUSPEND:
|
||||
return "SUSPEND";
|
||||
case Type::DROP_DNS_CACHE:
|
||||
return "DROP DNS CACHE";
|
||||
case Type::DROP_MARK_CACHE:
|
||||
@ -146,7 +148,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
|
||||
|
||||
auto print_on_volume = [&]
|
||||
{
|
||||
settings.ostr << " ON VOLUME "
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON VOLUME "
|
||||
<< (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(storage_policy)
|
||||
<< (settings.hilite ? hilite_none : "")
|
||||
<< "."
|
||||
@ -182,10 +184,21 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
|
||||
print_database_table();
|
||||
}
|
||||
else if (type == Type::RELOAD_DICTIONARY)
|
||||
{
|
||||
print_database_dictionary();
|
||||
}
|
||||
else if (type == Type::DROP_REPLICA)
|
||||
{
|
||||
print_drop_replica();
|
||||
}
|
||||
else if (type == Type::SUSPEND)
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR "
|
||||
<< (settings.hilite ? hilite_none : "") << seconds
|
||||
<< (settings.hilite ? hilite_keyword : "") << " SECOND"
|
||||
<< (settings.hilite ? hilite_none : "");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ public:
|
||||
UNKNOWN,
|
||||
SHUTDOWN,
|
||||
KILL,
|
||||
SUSPEND,
|
||||
DROP_DNS_CACHE,
|
||||
DROP_MARK_CACHE,
|
||||
DROP_UNCOMPRESSED_CACHE,
|
||||
@ -65,9 +66,10 @@ public:
|
||||
String table;
|
||||
String replica;
|
||||
String replica_zk_path;
|
||||
bool is_drop_whole_replica;
|
||||
bool is_drop_whole_replica{};
|
||||
String storage_policy;
|
||||
String volume;
|
||||
UInt64 seconds{};
|
||||
|
||||
String getID(char) const override { return "SYSTEM query"; }
|
||||
|
||||
|
||||
@ -343,6 +343,26 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
throw Exception("Argument of function toDate is unquoted: toDate(" + contents_str + "), must be: toDate('" + contents_str + "')"
|
||||
, ErrorCodes::SYNTAX_ERROR);
|
||||
}
|
||||
else if (Poco::toLower(getIdentifierName(identifier)) == "position")
|
||||
{
|
||||
/// POSITION(needle IN haystack) is equivalent to function position(haystack, needle)
|
||||
if (const auto * list = expr_list_args->as<ASTExpressionList>())
|
||||
{
|
||||
if (list->children.size() == 1)
|
||||
{
|
||||
if (const auto * in_func = list->children[0]->as<ASTFunction>())
|
||||
{
|
||||
if (in_func->name == "in")
|
||||
{
|
||||
// switch the two arguments
|
||||
const auto & arg_list = in_func->arguments->as<ASTExpressionList &>();
|
||||
if (arg_list.children.size() == 2)
|
||||
expr_list_args->children = {arg_list.children[1], arg_list.children[0]};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The parametric aggregate function has two lists (parameters and arguments) in parentheses. Example: quantile(0.9)(x).
|
||||
if (allow_function_parameters && pos->type == TokenType::OpeningRoundBracket)
|
||||
@ -1427,6 +1447,8 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
|
||||
is_strict = true;
|
||||
|
||||
ASTs identifiers;
|
||||
ASTPtr regex_node;
|
||||
ParserStringLiteral regex;
|
||||
auto parse_id = [&identifiers, &pos, &expected]
|
||||
{
|
||||
ASTPtr identifier;
|
||||
@ -1441,7 +1463,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
|
||||
{
|
||||
// support one or more parameter
|
||||
++pos;
|
||||
if (!ParserList::parseUtil(pos, expected, parse_id, false))
|
||||
if (!ParserList::parseUtil(pos, expected, parse_id, false) && !regex.parse(pos, regex_node, expected))
|
||||
return false;
|
||||
|
||||
if (pos->type != TokenType::ClosingRoundBracket)
|
||||
@ -1451,11 +1473,14 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
|
||||
else
|
||||
{
|
||||
// only one parameter
|
||||
if (!parse_id())
|
||||
if (!parse_id() && !regex.parse(pos, regex_node, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
auto res = std::make_shared<ASTColumnsExceptTransformer>();
|
||||
if (regex_node)
|
||||
res->setPattern(regex_node->as<ASTLiteral &>().value.get<String>());
|
||||
else
|
||||
res->children = std::move(identifiers);
|
||||
res->is_strict = is_strict;
|
||||
node = std::move(res);
|
||||
|
||||
@ -169,6 +169,20 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
|
||||
parseDatabaseAndTableName(pos, expected, res->database, res->table);
|
||||
break;
|
||||
|
||||
case Type::SUSPEND:
|
||||
{
|
||||
ASTPtr seconds;
|
||||
if (!(ParserKeyword{"FOR"}.ignore(pos, expected)
|
||||
&& ParserUnsignedInteger().parse(pos, seconds, expected)
|
||||
&& ParserKeyword{"SECOND"}.ignore(pos, expected))) /// SECOND, not SECONDS to be consistent with INTERVAL parsing in SQL
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
res->seconds = seconds->as<ASTLiteral>()->value.get<UInt64>();
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
/// There are no [db.table] after COMMAND NAME
|
||||
break;
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#include <Processors/Merges/MergingSortedTransform.h>
|
||||
#include <DataStreams/ColumnGathererStream.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <DataStreams/materializeBlock.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
|
||||
@ -1181,7 +1181,7 @@ void TCPHandler::receiveUnexpectedData()
|
||||
std::shared_ptr<ReadBuffer> maybe_compressed_in;
|
||||
|
||||
if (last_block_in.compression == Protocol::Compression::Enable)
|
||||
maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
|
||||
maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in, /* allow_different_codecs */ true);
|
||||
else
|
||||
maybe_compressed_in = in;
|
||||
|
||||
@ -1198,8 +1198,11 @@ void TCPHandler::initBlockInput()
|
||||
{
|
||||
if (!state.block_in)
|
||||
{
|
||||
/// 'allow_different_codecs' is set to true, because some parts of compressed data can be precompressed in advance
|
||||
/// with another codec that the rest of the data. Example: data sent by Distributed tables.
|
||||
|
||||
if (state.compression == Protocol::Compression::Enable)
|
||||
state.maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
|
||||
state.maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in, /* allow_different_codecs */ true);
|
||||
else
|
||||
state.maybe_compressed_in = in;
|
||||
|
||||
|
||||
@ -299,6 +299,10 @@ DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobRep
|
||||
const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block;
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
|
||||
/// Do not initiate INSERT for empty block.
|
||||
if (shard_block.rows() == 0)
|
||||
return;
|
||||
|
||||
if (!job.is_local_job || !settings.prefer_localhost_replica)
|
||||
{
|
||||
if (!job.stream)
|
||||
@ -368,7 +372,8 @@ void DistributedBlockOutputStream::writeSync(const Block & block)
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
const auto & shards_info = cluster->getShardsInfo();
|
||||
bool random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key;
|
||||
size_t start = 0, end = shards_info.size();
|
||||
size_t start = 0;
|
||||
size_t end = shards_info.size();
|
||||
if (random_shard_insert)
|
||||
{
|
||||
start = storage.getRandomShardIndex(shards_info);
|
||||
@ -582,6 +587,17 @@ void DistributedBlockOutputStream::writeToLocal(const Block & block, const size_
|
||||
|
||||
void DistributedBlockOutputStream::writeToShard(const Block & block, const std::vector<std::string> & dir_names)
|
||||
{
|
||||
const auto & settings = context.getSettingsRef();
|
||||
|
||||
std::string compression_method = Poco::toUpper(settings.network_compression_method.toString());
|
||||
std::optional<int> compression_level;
|
||||
|
||||
if (compression_method == "ZSTD")
|
||||
compression_level = settings.network_zstd_compression_level;
|
||||
|
||||
CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs);
|
||||
CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level);
|
||||
|
||||
/// tmp directory is used to ensure atomicity of transactions
|
||||
/// and keep monitor thread out from reading incomplete data
|
||||
std::string first_file_tmp_path{};
|
||||
@ -607,7 +623,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::
|
||||
/// Write batch to temporary location
|
||||
{
|
||||
WriteBufferFromFile out{first_file_tmp_path};
|
||||
CompressedWriteBuffer compress{out};
|
||||
CompressedWriteBuffer compress{out, compression_codec};
|
||||
NativeBlockOutputStream stream{compress, DBMS_TCP_PROTOCOL_VERSION, block.cloneEmpty()};
|
||||
|
||||
/// Prepare the header.
|
||||
|
||||
@ -120,9 +120,6 @@ public:
|
||||
/// Returns true if the storage supports deduplication of inserted data blocks.
|
||||
virtual bool supportsDeduplication() const { return false; }
|
||||
|
||||
/// Returns true if the storage supports settings.
|
||||
virtual bool supportsSettings() const { return false; }
|
||||
|
||||
/// Returns true if the blocks shouldn't be pushed to associated views on insert.
|
||||
virtual bool noPushingToViews() const { return false; }
|
||||
|
||||
|
||||
@ -36,7 +36,6 @@ class StorageKafka final : public ext::shared_ptr_helper<StorageKafka>, public I
|
||||
public:
|
||||
std::string getName() const override { return "Kafka"; }
|
||||
|
||||
bool supportsSettings() const override { return true; }
|
||||
bool noPushingToViews() const override { return true; }
|
||||
|
||||
void startup() override;
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include <Disks/SingleDiskVolume.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/FileSyncGuard.h>
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
#include <DataStreams/NativeBlockOutputStream.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <ext/scope_guard.h>
|
||||
@ -398,7 +398,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
|
||||
|
||||
disk->createDirectories(part_download_path);
|
||||
|
||||
std::optional<FileSyncGuard> sync_guard;
|
||||
std::optional<DirectorySyncGuard> sync_guard;
|
||||
if (data.getSettings()->fsync_part_directory)
|
||||
sync_guard.emplace(disk, part_download_path);
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
#include <Storages/MergeTree/checkDataPart.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/FileSyncGuard.h>
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
#include <common/JSON.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Compression/getCompressionCodecForFile.h>
|
||||
@ -835,12 +835,8 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_
|
||||
String from = getFullRelativePath();
|
||||
String to = storage.relative_data_path + new_relative_path + "/";
|
||||
|
||||
std::optional<FileSyncGuard> sync_guard;
|
||||
if (storage.getSettings()->fsync_part_directory)
|
||||
sync_guard.emplace(volume->getDisk(), to);
|
||||
|
||||
if (!volume->getDisk()->exists(from))
|
||||
throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST);
|
||||
throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST);
|
||||
|
||||
if (volume->getDisk()->exists(to))
|
||||
{
|
||||
@ -862,6 +858,10 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_
|
||||
volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr)));
|
||||
volume->getDisk()->moveFile(from, to);
|
||||
relative_path = new_relative_path;
|
||||
|
||||
std::optional<DirectorySyncGuard> sync_guard;
|
||||
if (storage.getSettings()->fsync_part_directory)
|
||||
sync_guard.emplace(volume->getDisk(), to);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -357,7 +357,6 @@ public:
|
||||
|| merging_params.mode == MergingParams::VersionedCollapsing;
|
||||
}
|
||||
|
||||
bool supportsSettings() const override { return true; }
|
||||
NamesAndTypesList getVirtuals() const override;
|
||||
|
||||
bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context &, const StorageMetadataPtr & metadata_snapshot) const override;
|
||||
|
||||
@ -29,7 +29,7 @@
|
||||
#include <Common/interpolate.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/FileSyncGuard.h>
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
#include <cmath>
|
||||
@ -780,7 +780,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
gathering_column_names.clear();
|
||||
}
|
||||
|
||||
std::optional<FileSyncGuard> sync_guard;
|
||||
std::optional<DirectorySyncGuard> sync_guard;
|
||||
if (data.getSettings()->fsync_part_directory)
|
||||
sync_guard.emplace(disk, new_part_tmp_path);
|
||||
|
||||
@ -1182,7 +1182,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
||||
|
||||
disk->createDirectories(new_part_tmp_path);
|
||||
|
||||
std::optional<FileSyncGuard> sync_guard;
|
||||
std::optional<DirectorySyncGuard> sync_guard;
|
||||
if (data.getSettings()->fsync_part_directory)
|
||||
sync_guard.emplace(disk, new_part_tmp_path);
|
||||
|
||||
|
||||
@ -59,6 +59,7 @@ namespace ErrorCodes
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int TOO_MANY_ROWS;
|
||||
extern const int CANNOT_PARSE_TEXT;
|
||||
extern const int TOO_MANY_PARTITIONS;
|
||||
}
|
||||
|
||||
|
||||
@ -706,6 +707,21 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts(
|
||||
if (parts_with_ranges.empty())
|
||||
return std::make_unique<QueryPlan>();
|
||||
|
||||
auto max_partitions_to_read
|
||||
= settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data.getSettings()->max_partitions_to_read;
|
||||
if (max_partitions_to_read > 0)
|
||||
{
|
||||
std::set<String> partitions;
|
||||
for (auto & part_with_ranges : parts_with_ranges)
|
||||
partitions.insert(part_with_ranges.data_part->info.partition_id);
|
||||
if (partitions.size() > size_t(max_partitions_to_read))
|
||||
throw Exception(
|
||||
ErrorCodes::TOO_MANY_PARTITIONS,
|
||||
"Too many partitions to read. Current {}, max {}",
|
||||
partitions.size(),
|
||||
max_partitions_to_read);
|
||||
}
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::SelectedParts, parts_with_ranges.size());
|
||||
ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges);
|
||||
ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks);
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Poco/File.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/FileSyncGuard.h>
|
||||
#include <Common/DirectorySyncGuard.h>
|
||||
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
@ -362,7 +362,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
|
||||
new_data_part->minmax_idx = std::move(minmax_idx);
|
||||
new_data_part->is_temp = true;
|
||||
|
||||
std::optional<FileSyncGuard> sync_guard;
|
||||
std::optional<DirectorySyncGuard> sync_guard;
|
||||
if (new_data_part->isStoredOnDisk())
|
||||
{
|
||||
/// The name could be non-unique in case of stale files from previous runs.
|
||||
|
||||
@ -110,6 +110,7 @@ struct Settings;
|
||||
M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \
|
||||
M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \
|
||||
M(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \
|
||||
M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \
|
||||
\
|
||||
/** Obsolete settings. Kept for backward compatibility only. */ \
|
||||
M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \
|
||||
|
||||
@ -3,9 +3,9 @@
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <Storages/MergeTree/BackgroundProcessList.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/MemoryTracker.h>
|
||||
#include <Poco/URI.h>
|
||||
|
||||
|
||||
namespace CurrentMetrics
|
||||
{
|
||||
extern const Metric ReplicatedFetch;
|
||||
|
||||
@ -756,6 +756,7 @@ void registerStorageMergeTree(StorageFactory & factory)
|
||||
.supports_skipping_indices = true,
|
||||
.supports_sort_order = true,
|
||||
.supports_ttl = true,
|
||||
.supports_parallel_insert = true,
|
||||
};
|
||||
|
||||
factory.registerStorage("MergeTree", create, features);
|
||||
|
||||
@ -29,7 +29,6 @@ class StorageRabbitMQ final: public ext::shared_ptr_helper<StorageRabbitMQ>, pub
|
||||
public:
|
||||
std::string getName() const override { return "RabbitMQ"; }
|
||||
|
||||
bool supportsSettings() const override { return true; }
|
||||
bool noPushingToViews() const override { return true; }
|
||||
|
||||
void startup() override;
|
||||
|
||||
@ -367,6 +367,7 @@ void registerStorageEmbeddedRocksDB(StorageFactory & factory)
|
||||
{
|
||||
StorageFactory::StorageFeatures features{
|
||||
.supports_sort_order = true,
|
||||
.supports_parallel_insert = true,
|
||||
};
|
||||
|
||||
factory.registerStorage("EmbeddedRocksDB", create, features);
|
||||
|
||||
@ -996,6 +996,9 @@ void registerStorageBuffer(StorageFactory & factory)
|
||||
StorageBuffer::Thresholds{max_time, max_rows, max_bytes},
|
||||
destination_id,
|
||||
static_cast<bool>(args.local_context.getSettingsRef().insert_allow_materialized_columns));
|
||||
},
|
||||
{
|
||||
.supports_parallel_insert = true,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -540,7 +540,7 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const StorageMeta
|
||||
/// Ban an attempt to make async insert into the table belonging to DatabaseMemory
|
||||
if (!storage_policy && !owned_cluster && !settings.insert_distributed_sync)
|
||||
{
|
||||
throw Exception("Storage " + getName() + " must has own data directory to enable asynchronous inserts",
|
||||
throw Exception("Storage " + getName() + " must have own data directory to enable asynchronous inserts",
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
@ -558,8 +558,10 @@ BlockOutputStreamPtr StorageDistributed::write(const ASTPtr &, const StorageMeta
|
||||
|
||||
/// DistributedBlockOutputStream will not own cluster, but will own ConnectionPools of the cluster
|
||||
return std::make_shared<DistributedBlockOutputStream>(
|
||||
context, *this, metadata_snapshot, createInsertToRemoteTableQuery(remote_database, remote_table, metadata_snapshot->getSampleBlockNonMaterialized()), cluster,
|
||||
insert_sync, timeout);
|
||||
context, *this, metadata_snapshot,
|
||||
createInsertToRemoteTableQuery(
|
||||
remote_database, remote_table, metadata_snapshot->getSampleBlockNonMaterialized()),
|
||||
cluster, insert_sync, timeout);
|
||||
}
|
||||
|
||||
|
||||
@ -1003,6 +1005,7 @@ void registerStorageDistributed(StorageFactory & factory)
|
||||
args.attach);
|
||||
},
|
||||
{
|
||||
.supports_parallel_insert = true,
|
||||
.source_access_type = AccessType::REMOTE,
|
||||
});
|
||||
}
|
||||
|
||||
@ -47,14 +47,20 @@ public:
|
||||
bool has_force_restore_data_flag;
|
||||
};
|
||||
|
||||
/// Analog of the IStorage::supports*() helpers
|
||||
/// (But the former cannot be replaced with StorageFeatures due to nesting)
|
||||
struct StorageFeatures
|
||||
{
|
||||
bool supports_settings = false;
|
||||
bool supports_skipping_indices = false;
|
||||
bool supports_sort_order = false;
|
||||
bool supports_ttl = false;
|
||||
/// See also IStorage::supportsReplication()
|
||||
bool supports_replication = false;
|
||||
/// See also IStorage::supportsDeduplication()
|
||||
bool supports_deduplication = false;
|
||||
/// See also IStorage::supportsParallelInsert()
|
||||
bool supports_parallel_insert = false;
|
||||
AccessType source_access_type = AccessType::NONE;
|
||||
};
|
||||
|
||||
@ -85,6 +91,7 @@ public:
|
||||
.supports_ttl = false,
|
||||
.supports_replication = false,
|
||||
.supports_deduplication = false,
|
||||
.supports_parallel_insert = false,
|
||||
.source_access_type = AccessType::NONE,
|
||||
});
|
||||
|
||||
|
||||
@ -303,6 +303,9 @@ void registerStorageMemory(StorageFactory & factory)
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
return StorageMemory::create(args.table_id, args.columns, args.constraints);
|
||||
},
|
||||
{
|
||||
.supports_parallel_insert = true,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#include <DataStreams/narrowBlockInputStreams.h>
|
||||
#include <DataStreams/OneBlockInputStream.h>
|
||||
#include <DataStreams/materializeBlock.h>
|
||||
#include <Storages/StorageMerge.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
#include <Storages/VirtualColumnUtils.h>
|
||||
|
||||
@ -29,6 +29,9 @@ void registerStorageNull(StorageFactory & factory)
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
return StorageNull::create(args.table_id, args.columns, args.constraints);
|
||||
},
|
||||
{
|
||||
.supports_parallel_insert = true,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -25,7 +25,6 @@ public:
|
||||
bool supportsReplication() const override { return getNested()->supportsReplication(); }
|
||||
bool supportsParallelInsert() const override { return getNested()->supportsParallelInsert(); }
|
||||
bool supportsDeduplication() const override { return getNested()->supportsDeduplication(); }
|
||||
bool supportsSettings() const override { return getNested()->supportsSettings(); }
|
||||
bool noPushingToViews() const override { return getNested()->noPushingToViews(); }
|
||||
bool hasEvenlyDistributedRead() const override { return getNested()->hasEvenlyDistributedRead(); }
|
||||
|
||||
|
||||
@ -4958,8 +4958,13 @@ void StorageReplicatedMergeTree::fetchPartition(
|
||||
const String & from_,
|
||||
const Context & query_context)
|
||||
{
|
||||
String auxiliary_zookeeper_name = extractZooKeeperName(from_);
|
||||
String from = extractZooKeeperPath(from_);
|
||||
Macros::MacroExpansionInfo info;
|
||||
info.expand_special_macros_only = false;
|
||||
info.table_id = getStorageID();
|
||||
info.table_id.uuid = UUIDHelpers::Nil;
|
||||
auto expand_from = query_context.getMacros()->expand(from_, info);
|
||||
String auxiliary_zookeeper_name = extractZooKeeperName(expand_from);
|
||||
String from = extractZooKeeperPath(expand_from);
|
||||
if (from.empty())
|
||||
throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
|
||||
@ -8,26 +8,31 @@ namespace DB
|
||||
|
||||
NamesAndTypesList StorageSystemTableEngines::getNamesAndTypes()
|
||||
{
|
||||
return {{"name", std::make_shared<DataTypeString>()},
|
||||
return {
|
||||
{"name", std::make_shared<DataTypeString>()},
|
||||
{"supports_settings", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_skipping_indices", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_sort_order", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_ttl", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_replication", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_deduplication", std::make_shared<DataTypeUInt8>()}};
|
||||
{"supports_deduplication", std::make_shared<DataTypeUInt8>()},
|
||||
{"supports_parallel_insert", std::make_shared<DataTypeUInt8>()},
|
||||
};
|
||||
}
|
||||
|
||||
void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const
|
||||
{
|
||||
for (const auto & pair : StorageFactory::instance().getAllStorages())
|
||||
{
|
||||
res_columns[0]->insert(pair.first);
|
||||
res_columns[1]->insert(pair.second.features.supports_settings);
|
||||
res_columns[2]->insert(pair.second.features.supports_skipping_indices);
|
||||
res_columns[3]->insert(pair.second.features.supports_sort_order);
|
||||
res_columns[4]->insert(pair.second.features.supports_ttl);
|
||||
res_columns[5]->insert(pair.second.features.supports_replication);
|
||||
res_columns[6]->insert(pair.second.features.supports_deduplication);
|
||||
int i = 0;
|
||||
res_columns[i++]->insert(pair.first);
|
||||
res_columns[i++]->insert(pair.second.features.supports_settings);
|
||||
res_columns[i++]->insert(pair.second.features.supports_skipping_indices);
|
||||
res_columns[i++]->insert(pair.second.features.supports_sort_order);
|
||||
res_columns[i++]->insert(pair.second.features.supports_ttl);
|
||||
res_columns[i++]->insert(pair.second.features.supports_replication);
|
||||
res_columns[i++]->insert(pair.second.features.supports_deduplication);
|
||||
res_columns[i++]->insert(pair.second.features.supports_parallel_insert);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,17 @@
|
||||
<yandex>
|
||||
<remote_servers>
|
||||
<cluster>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>ch1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>ch2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</cluster>
|
||||
</remote_servers>
|
||||
</yandex>
|
||||
@ -0,0 +1,5 @@
|
||||
<yandex>
|
||||
<distributed_ddl>
|
||||
<path>/clickhouse/task_queue/ddl</path>
|
||||
</distributed_ddl>
|
||||
</yandex>
|
||||
54
tests/integration/test_alter_settings_on_cluster/test.py
Normal file
54
tests/integration/test_alter_settings_on_cluster/test.py
Normal file
@ -0,0 +1,54 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
ch1 = cluster.add_instance(
|
||||
"ch1",
|
||||
main_configs=[
|
||||
"configs/config.d/clusters.xml",
|
||||
"configs/config.d/distributed_ddl.xml",
|
||||
],
|
||||
with_zookeeper=True,
|
||||
)
|
||||
ch2 = cluster.add_instance(
|
||||
"ch2",
|
||||
main_configs=[
|
||||
"configs/config.d/clusters.xml",
|
||||
"configs/config.d/distributed_ddl.xml",
|
||||
],
|
||||
with_zookeeper=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
ch1.query("CREATE DATABASE test_default_database ON CLUSTER 'cluster';")
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_default_database_on_cluster(started_cluster):
|
||||
ch1.query(
|
||||
database="test_default_database",
|
||||
sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r1') ORDER BY tuple();",
|
||||
)
|
||||
|
||||
ch2.query(
|
||||
database="test_default_database",
|
||||
sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r2') ORDER BY tuple();",
|
||||
)
|
||||
|
||||
ch1.query(
|
||||
database="test_default_database",
|
||||
sql="ALTER TABLE test_local_table ON CLUSTER 'cluster' MODIFY SETTING old_parts_lifetime = 100;",
|
||||
)
|
||||
|
||||
for node in [ch1, ch2]:
|
||||
assert node.query(
|
||||
database="test_default_database",
|
||||
sql="SHOW CREATE test_local_table FORMAT TSV",
|
||||
).endswith("old_parts_lifetime = 100\n")
|
||||
@ -0,0 +1,28 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
ch1 = cluster.add_instance("ch1")
|
||||
ch2 = cluster.add_instance("ch2")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
ch1.query("CREATE DATABASE test_default_database;")
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_default_database_on_cluster(started_cluster):
|
||||
ch1.query(
|
||||
database="test_default_database",
|
||||
sql="CREATE TABLE test_local_table ENGINE MergeTree PARTITION BY i ORDER BY tuple() SETTINGS max_partitions_to_read = 1 AS SELECT arrayJoin([1, 2]) i;",
|
||||
)
|
||||
|
||||
assert ch2.query(
|
||||
sql="SELECT * FROM remote('ch1:9000', test_default_database, test_local_table) ORDER BY i FORMAT TSV SETTINGS max_partitions_to_read = 0;",
|
||||
) == "1\n2\n"
|
||||
@ -0,0 +1,13 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
@ -0,0 +1,16 @@
|
||||
SET send_logs_level = 'fatal';
|
||||
select 1 = position('' in '');
|
||||
select 1 = position('' in 'abc');
|
||||
select 0 = position('abc' in '');
|
||||
select 1 = position('abc' in 'abc');
|
||||
select 2 = position('bc' in 'abc');
|
||||
select 3 = position('c' in 'abc');
|
||||
|
||||
select 1 = position('' in '');
|
||||
select 1 = position('' in 'абв');
|
||||
select 0 = position('абв' in '');
|
||||
select 1 = position('абв' in 'абв');
|
||||
select 3 = position('бв' in 'абв');
|
||||
select 5 = position('в' in 'абв');
|
||||
|
||||
select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s);
|
||||
@ -108,4 +108,8 @@ SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinH
|
||||
SELECT 'wordShingleMinHashCaseInsensitiveUTF8';
|
||||
SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitiveUTF8(s, 2, 3) as h FROM defaults GROUP BY h;
|
||||
|
||||
SELECT wordShingleSimHash('foobar', 9223372036854775807); -- { serverError 69 }
|
||||
SELECT wordShingleSimHash('foobar', 1001); -- { serverError 69 }
|
||||
SELECT wordShingleSimHash('foobar', 0); -- { serverError 69 }
|
||||
|
||||
DROP TABLE defaults;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user