diff --git a/CMakeLists.txt b/CMakeLists.txt index 7716fe82677..853b2df7aca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -214,6 +214,19 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") endif () endif() +# Create BuildID when using lld. For other linkers it is created by default. +if (LINKER_NAME MATCHES "lld$") + # SHA1 is not cryptographically secure but it is the best what lld is offering. + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") +endif () + +# Add a section with the hash of the compiled machine code for integrity checks. +# Only for official builds, because adding a section can be time consuming (rewrite of several GB). +# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary) +if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE)) + set (USE_BINARY_HASH 1) +endif () + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd diff --git a/PreLoad.cmake b/PreLoad.cmake index d66bcd68a10..0a25a55e7bf 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -12,7 +12,7 @@ # https://youtrack.jetbrains.com/issue/CPP-2659 # https://youtrack.jetbrains.com/issue/CPP-870 -if (NOT DEFINED ENV{CLION_IDE}) +if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE}) find_program(NINJA_PATH ninja) if (NINJA_PATH) set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 4d29576562b..830e7857a1f 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -80,16 +83,6 @@ static void call_default_signal_handler(int sig) raise(sig); } -const char * msan_strsignal(int sig) -{ - // Apparently strsignal is not instrumented by MemorySanitizer, so we - // have to unpoison it to avoid msan reports inside fmt library when we - // print it. - const char * signal_name = sys_siglist[sig]; - __msan_unpoison_string(signal_name); - return signal_name; -} - static constexpr size_t max_query_id_size = 127; static const size_t signal_pipe_buf_size = @@ -294,13 +287,13 @@ private: { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, msan_strsignal(sig), sig); + thread_num, strsignal(sig), sig); } else { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, query_id, msan_strsignal(sig), sig); + thread_num, query_id, strsignal(sig), sig); } String error_message; @@ -328,6 +321,32 @@ private: /// Write symbolized stack trace line by line for better grep-ability. stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); +#if defined(__linux__) + /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + if (daemon.stored_binary_hash.empty()) + { + LOG_FATAL(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == daemon.stored_binary_hash) + { + LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file was changed just after startup;" + " - the file is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, daemon.stored_binary_hash); + } +#endif + /// Write crash to system.crash_log table if available. if (collectCrashLog) collectCrashLog(sig, thread_num, query_id, stack_trace); @@ -481,8 +500,9 @@ void BaseDaemon::kill() { dumpCoverageReportIfPossible(); pid_file.reset(); - if (::raise(SIGKILL) != 0) - throw Poco::SystemException("cannot kill process"); + /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL. + /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker). + _exit(128 + SIGKILL); } std::string BaseDaemon::getDefaultCorePath() const @@ -787,6 +807,13 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() #else build_id_info = "no build id"; #endif + +#if defined(__linux__) + std::string executable_path = getExecutablePath(); + + if (!executable_path.empty()) + stored_binary_hash = DB::Elf(executable_path).getBinaryHash(); +#endif } void BaseDaemon::logRevision() const @@ -846,13 +873,13 @@ void BaseDaemon::handleSignal(int signal_id) onInterruptSignals(signal_id); } else - throw DB::Exception(std::string("Unsupported signal: ") + msan_strsignal(signal_id), 0); + throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); } void BaseDaemon::onInterruptSignals(int signal_id) { is_cancelled = true; - LOG_INFO(&logger(), "Received termination signal ({})", msan_strsignal(signal_id)); + LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); if (sigint_signals_counter >= 2) { @@ -998,3 +1025,9 @@ void BaseDaemon::setupWatchdog() #endif } } + + +String BaseDaemon::getStoredBinaryHash() const +{ + return stored_binary_hash; +} diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 090d4997606..42d94629ae9 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -60,7 +60,7 @@ public: static void terminate(); /// Forceful shutdown - void kill(); + [[noreturn]] void kill(); /// Cancellation request has been received. bool isCancelled() const @@ -121,6 +121,9 @@ public: /// argv0 is needed to change process name (consequently, it is needed for scripts involving "pgrep", "pidof" to work correctly). void shouldSetupWatchdog(char * argv0_); + /// Hash of the binary for integrity checks. + String getStoredBinaryHash() const; + protected: virtual void logRevision() const; @@ -168,6 +171,7 @@ protected: Poco::Util::AbstractConfiguration * last_configuration = nullptr; String build_id_info; + String stored_binary_hash; std::vector handled_signals; diff --git a/base/glibc-compatibility/musl/strsignal.c b/base/glibc-compatibility/musl/strsignal.c new file mode 100644 index 00000000000..fee894e8550 --- /dev/null +++ b/base/glibc-compatibility/musl/strsignal.c @@ -0,0 +1,125 @@ +#include +#include + +#if (SIGHUP == 1) && (SIGINT == 2) && (SIGQUIT == 3) && (SIGILL == 4) \ + && (SIGTRAP == 5) && (SIGABRT == 6) && (SIGBUS == 7) && (SIGFPE == 8) \ + && (SIGKILL == 9) && (SIGUSR1 == 10) && (SIGSEGV == 11) && (SIGUSR2 == 12) \ + && (SIGPIPE == 13) && (SIGALRM == 14) && (SIGTERM == 15) && (SIGSTKFLT == 16) \ + && (SIGCHLD == 17) && (SIGCONT == 18) && (SIGSTOP == 19) && (SIGTSTP == 20) \ + && (SIGTTIN == 21) && (SIGTTOU == 22) && (SIGURG == 23) && (SIGXCPU == 24) \ + && (SIGXFSZ == 25) && (SIGVTALRM == 26) && (SIGPROF == 27) && (SIGWINCH == 28) \ + && (SIGPOLL == 29) && (SIGPWR == 30) && (SIGSYS == 31) + +#define sigmap(x) x + +#else + +static const char map[] = { + [SIGHUP] = 1, + [SIGINT] = 2, + [SIGQUIT] = 3, + [SIGILL] = 4, + [SIGTRAP] = 5, + [SIGABRT] = 6, + [SIGBUS] = 7, + [SIGFPE] = 8, + [SIGKILL] = 9, + [SIGUSR1] = 10, + [SIGSEGV] = 11, + [SIGUSR2] = 12, + [SIGPIPE] = 13, + [SIGALRM] = 14, + [SIGTERM] = 15, +#if defined(SIGSTKFLT) + [SIGSTKFLT] = 16, +#elif defined(SIGEMT) + [SIGEMT] = 16, +#endif + [SIGCHLD] = 17, + [SIGCONT] = 18, + [SIGSTOP] = 19, + [SIGTSTP] = 20, + [SIGTTIN] = 21, + [SIGTTOU] = 22, + [SIGURG] = 23, + [SIGXCPU] = 24, + [SIGXFSZ] = 25, + [SIGVTALRM] = 26, + [SIGPROF] = 27, + [SIGWINCH] = 28, + [SIGPOLL] = 29, + [SIGPWR] = 30, + [SIGSYS] = 31 +}; + +#define sigmap(x) ((x) >= sizeof map ? (x) : map[(x)]) + +#endif + +static const char strings[] = + "Unknown signal\0" + "Hangup\0" + "Interrupt\0" + "Quit\0" + "Illegal instruction\0" + "Trace/breakpoint trap\0" + "Aborted\0" + "Bus error\0" + "Arithmetic exception\0" + "Killed\0" + "User defined signal 1\0" + "Segmentation fault\0" + "User defined signal 2\0" + "Broken pipe\0" + "Alarm clock\0" + "Terminated\0" +#if defined(SIGSTKFLT) + "Stack fault\0" +#elif defined(SIGEMT) + "Emulator trap\0" +#else + "Unknown signal\0" +#endif + "Child process status\0" + "Continued\0" + "Stopped (signal)\0" + "Stopped\0" + "Stopped (tty input)\0" + "Stopped (tty output)\0" + "Urgent I/O condition\0" + "CPU time limit exceeded\0" + "File size limit exceeded\0" + "Virtual timer expired\0" + "Profiling timer expired\0" + "Window changed\0" + "I/O possible\0" + "Power failure\0" + "Bad system call\0" + "RT32" + "\0RT33\0RT34\0RT35\0RT36\0RT37\0RT38\0RT39\0RT40" + "\0RT41\0RT42\0RT43\0RT44\0RT45\0RT46\0RT47\0RT48" + "\0RT49\0RT50\0RT51\0RT52\0RT53\0RT54\0RT55\0RT56" + "\0RT57\0RT58\0RT59\0RT60\0RT61\0RT62\0RT63\0RT64" +#if _NSIG > 65 + "\0RT65\0RT66\0RT67\0RT68\0RT69\0RT70\0RT71\0RT72" + "\0RT73\0RT74\0RT75\0RT76\0RT77\0RT78\0RT79\0RT80" + "\0RT81\0RT82\0RT83\0RT84\0RT85\0RT86\0RT87\0RT88" + "\0RT89\0RT90\0RT91\0RT92\0RT93\0RT94\0RT95\0RT96" + "\0RT97\0RT98\0RT99\0RT100\0RT101\0RT102\0RT103\0RT104" + "\0RT105\0RT106\0RT107\0RT108\0RT109\0RT110\0RT111\0RT112" + "\0RT113\0RT114\0RT115\0RT116\0RT117\0RT118\0RT119\0RT120" + "\0RT121\0RT122\0RT123\0RT124\0RT125\0RT126\0RT127\0RT128" +#endif + ""; + +char *strsignal(int signum) +{ + const char *s = strings; + + signum = sigmap(signum); + if (signum - 1U >= _NSIG-1) signum = 0; + + for (; signum--; s++) for (; *s; s++); + + return (char *)s; +} diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 4d08174170f..df625a3e4d6 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -142,7 +142,7 @@ TRAP(qecvt) TRAP(qfcvt) TRAP(register_printf_function) TRAP(seed48) -TRAP(setenv) +//TRAP(setenv) TRAP(setfsent) TRAP(setgrent) TRAP(sethostent) @@ -164,7 +164,7 @@ TRAP(sigsuspend) TRAP(sleep) TRAP(srand48) //TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately. -TRAP(strsignal) +//TRAP(strsignal) // This function is imported from Musl and is thread safe. TRAP(strtok) TRAP(tcflow) TRAP(tcsendbreak) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index be3d3f86348..12078cea263 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -18,7 +18,11 @@ if (WITH_COVERAGE) set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) separate_arguments(WITHOUT_COVERAGE_LIST) # disable coverage for contib files and build with optimisations - add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + if (COMPILER_CLANG) + add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + else() + add_compile_options(-O3 -DNDEBUG -finline-functions ${WITHOUT_COVERAGE_LIST}) + endif() endif() if (SANITIZE STREQUAL "undefined") diff --git a/contrib/replxx b/contrib/replxx index 254be98ae7f..cdb6e3f2ce4 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 254be98ae7f2fd92d6db768f8e11ea5a5226cbf5 +Subproject commit cdb6e3f2ce4464225daf9c8beeae7db98d590bdc diff --git a/contrib/rocksdb b/contrib/rocksdb index 8b966f0ca29..54a0decabbc 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 8b966f0ca298fc1475bd09d9775f32dff0fdce0a +Subproject commit 54a0decabbcf4c0bb5cf7befa9c597f28289bff5 diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml index 356d3212932..542f7a673cd 100644 --- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml +++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml @@ -10,6 +10,11 @@ 10 + + + + + diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 26db6455fd5..0488cdce155 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -81,12 +81,11 @@ function fuzz echo Server started fuzzer_exit_code=0 - # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. - # They are all alphanumeric. - # shellcheck disable=SC2012 - ./clickhouse-client --query-fuzzer-runs=1000 \ - < <(for f in $(ls ch/tests/queries/0_stateless/*.sql | sort -R); do cat "$f"; echo ';'; done) \ - > >(tail -10000 > fuzzer.log) \ + # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric. + # SC2046: Quote this to prevent word splitting. Actually I need word splitting. + # shellcheck disable=SC2012,SC2046 + ./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \ + > >(tail -n 10000 > fuzzer.log) \ 2>&1 \ || fuzzer_exit_code=$? diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 5b096bcc5fa..fb453e55417 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -11,7 +11,7 @@ Functional tests are the most simple and convenient to use. Most of ClickHouse f Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference. -Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and not available to general public. We tend to use only `stateless` tests and avoid adding new `stateful` tests. +Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and it is available to general public. Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`. @@ -84,11 +84,9 @@ If you want to improve performance of ClickHouse in some scenario, and if improv Some programs in `tests` directory are not prepared tests, but are test tools. For example, for `Lexer` there is a tool `src/Parsers/tests/lexer` that just do tokenization of stdin and writes colorized result to stdout. You can use these kind of tools as a code examples and for exploration and manual testing. -You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated. - ## Miscellaneous Tests {#miscellaneous-tests} -There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. +There are tests for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not. @@ -169,53 +167,55 @@ Precise query execution timings are not recorded and not compared due to high va ## Build Tests {#build-tests} -Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. Tests are located at `ci` directory. They run build from source inside Docker, Vagrant, and sometimes with `qemu-user-static` inside Docker. These tests are under development and test runs are not automated. +Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well. -Motivation: - -Normally we release and run all tests on a single variant of ClickHouse build. But there are alternative build variants that are not thoroughly tested. Examples: - -- build on FreeBSD -- build on Debian with libraries from system packages -- build with shared linking of libraries -- build on AArch64 platform -- build on PowerPc platform +Examples: +- cross-compile for Darwin x86_64 (Mac OS X) +- cross-compile for FreeBSD x86_64 +- cross-compile for Linux AArch64 +- build on Ubuntu with libraries from system packages (discouraged) +- build with shared linking of libraries (discouraged) For example, build with system packages is bad practice, because we cannot guarantee what exact version of packages a system will have. But this is really needed by Debian maintainers. For this reason we at least have to support this variant of build. Another example: shared linking is a common source of trouble, but it is needed for some enthusiasts. Though we cannot run all tests on all variant of builds, we want to check at least that various build variants are not broken. For this purpose we use build tests. +We also test that there are no translation units that are too long to compile or require too much RAM. + +We also test that there are no too large stack frames. + ## Testing for Protocol Compatibility {#testing-for-protocol-compatibility} When we extend ClickHouse network protocol, we test manually that old clickhouse-client works with new clickhouse-server and new clickhouse-client works with old clickhouse-server (simply by running binaries from corresponding packages). +We also test some cases automatically with integrational tests: +- if data written by old version of ClickHouse can be successfully read by the new version; +- do distributed queries work in a cluster with different ClickHouse versions. + ## Help from the Compiler {#help-from-the-compiler} Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall -Wextra -Werror` and with some additional enabled warnings. Although these options are not enabled for third-party libraries. Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build. -For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime. +For production builds, clang is used, but we also test make gcc builds. For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime. ## Sanitizers {#sanitizers} ### Address sanitizer -We run functional and integration tests under ASan on per-commit basis. - -### Valgrind (Memcheck) -We run functional tests under Valgrind overnight. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). - -### Undefined behaviour sanitizer -We run functional and integration tests under ASan on per-commit basis. +We run functional, integration, stress and unit tests under ASan on per-commit basis. ### Thread sanitizer -We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis. +We run functional, integration, stress and unit tests under TSan on per-commit basis. ### Memory sanitizer -Currently we still don’t use MSan. +We run functional, integration, stress and unit tests under MSan on per-commit basis. -### Debug allocator -Debug version of `jemalloc` is used for debug build. +### Undefined behaviour sanitizer +We run functional, integration, stress and unit tests under UBSan on per-commit basis. The code of some third-party libraries is not sanitized for UB. + +### Valgrind (Memcheck) +We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). ## Fuzzing {#fuzzing} @@ -233,19 +233,62 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. + +## Stress test + +Stress tests are another case of fuzzing. It runs all functional tests in parallel in random order with a single server. Results of the tests are not checked. + +It is checked that: +- server does not crash, no debug or sanitizer traps are triggered; +- there are no deadlocks; +- the database structure is consistent; +- server can successfully stop after the test and start again without exceptions. + +There are five variants (Debug, ASan, TSan, MSan, UBSan). + +## Thread Fuzzer + +Thread Fuzzer (please don't mix up with Thread Sanitizer) is another kind of fuzzing that allows to randomize thread order of execution. It helps to find even more special cases. + ## Security Audit {#security-audit} People from Yandex Security Team do some basic overview of ClickHouse capabilities from the security standpoint. ## Static Analyzers {#static-analyzers} -We run `PVS-Studio` on per-commit basis. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). +We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. + +We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box. +We also use `shellcheck` for static analysis of shell scripts. + ## Hardening {#hardening} -`FORTIFY_SOURCE` is used by default. It is almost useless, but still makes sense in rare cases and we don’t disable it. +In debug build we are using custom allocator that does ASLR of user-level allocations. + +We also manually protect memory regions that are expected to be readonly after allocation. + +In debug build we also involve a customization of libc that ensures that no "harmful" (obsolete, insecure, not thread-safe) functions are called. + +Debug assertions are used extensively. + +In debug build, if exception with "logical error" code (implies a bug) is being thrown, the program is terminated prematurally. It allows to use exceptions in release build but make it an assertion in debug build. + +Debug version of jemalloc is used for debug builds. +Debug version of libc++ is used for debug builds. + +## Runtime Integrity Checks + +Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. + +It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client, bit flips on the wire). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). + +ClickHouse provides diagnostics that will help ops engineers to find faulty hardware. + +\* and it is not slow. ## Code Style {#code-style} @@ -259,6 +302,8 @@ Alternatively you can try `uncrustify` tool to reformat your code. Configuration `CLion` has its own code formatter that has to be tuned for our code style. +We also use `codespell` to find typos in code. It is automated as well. + ## Metrica B2B Tests {#metrica-b2b-tests} Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together. @@ -267,13 +312,25 @@ These tests are automated by separate team. Due to high number of moving parts, ## Test Coverage {#test-coverage} -As of July 2018 we don’t track test coverage. +We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis. + +## Tests for Tests + +There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky. + +## Testflows + +[Testflows](https://testflows.com/) is an enterprise-grade testing framework. It is used by Altinity for some of the tests and we run these tests in our CI. + +## Yandex Checks (only for Yandex employees) + +These checks are importing ClickHouse code into Yandex internal monorepository, so ClickHouse codebase can be used as a library by other products at Yandex (YT and YDB). Note that clickhouse-server itself is not being build from internal repo and unmodified open-source build is used for Yandex applications. ## Test Automation {#test-automation} We run tests with Yandex internal CI and job automation system named “Sandbox”. -Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored eternally. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. +Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored for several months. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. We don’t use Travis CI due to the limit on time and computational power. We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins. diff --git a/docs/en/getting-started/example-datasets/brown-benchmark.md b/docs/en/getting-started/example-datasets/brown-benchmark.md index 80dbd0c1b6a..b5ca23eddb9 100644 --- a/docs/en/getting-started/example-datasets/brown-benchmark.md +++ b/docs/en/getting-started/example-datasets/brown-benchmark.md @@ -410,3 +410,5 @@ GROUP BY yr, ORDER BY yr, mo; ``` + +The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==). diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index cdbb9b56eeb..159c99b15a0 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -71,4 +71,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" [ClickHouse tutorial](../../getting-started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. -Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there). +Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hits` and `test.visits` there). diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index 5e499cafb2a..6e46cddba52 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -398,6 +398,8 @@ ORDER BY c DESC LIMIT 10; ``` +You can also play with the data in Playground, [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIERheU9mV2VlaywgY291bnQoKikgQVMgYwpGUk9NIG9udGltZQpXSEVSRSBZZWFyPj0yMDAwIEFORCBZZWFyPD0yMDA4CkdST1VQIEJZIERheU9mV2VlawpPUkRFUiBCWSBjIERFU0M7Cg==). + This performance test was created by Vadim Tkachenko. See: - https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index f3a6381aeca..c08eec61b1c 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 7a9e386d419..5dc23aee686 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,33 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. + +System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: + +- `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`. +- `table`: table to insert data. +- `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression. +- `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md) expression. +- `flush_interval_milliseconds`: interval of flushing data to disk. +- `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit. + +An example: + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables. diff --git a/docs/en/operations/system-tables/table_engines.md b/docs/en/operations/system-tables/table_engines.md index 4ca1fc657ee..30122cb133e 100644 --- a/docs/en/operations/system-tables/table_engines.md +++ b/docs/en/operations/system-tables/table_engines.md @@ -11,6 +11,7 @@ This table contains the following columns (the column type is shown in brackets) - `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`. - `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](../../engines/table-engines/mergetree-family/replication.md). - `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. +- `supports_parallel_insert` (UInt8) — Flag that indicates if table engine supports parallel insert (see [`max_insert_threads`](../../operations/settings/settings.md#settings-max-insert-threads) setting). Example: @@ -21,11 +22,11 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') ``` ``` text -┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ -│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ -└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ ``` **See also** diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index c4b7156ad95..dc7727bdfd8 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1290,25 +1290,68 @@ Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference ## arrayMin(\[func,\] arr1, …) {#array-min} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the min of the `func` values. If the function is omitted, it just returns the min of the array elements. Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMin([1, 2, 4]) AS res +┌─res─┐ +│ 1 │ +└─────┘ + + +SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -4 │ +└─────┘ +``` + ## arrayMax(\[func,\] arr1, …) {#array-max} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the max of the `func` values. If the function is omitted, it just returns the max of the array elements. Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMax([1, 2, 4]) AS res +┌─res─┐ +│ 4 │ +└─────┘ + + +SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -1 │ +└─────┘ +``` + ## arraySum(\[func,\] arr1, …) {#array-sum} Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements. Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arraySum([2,3]) AS res +┌─res─┐ +│ 5 │ +└─────┘ + + +SELECT arraySum(x -> x*x, [2, 3]) AS res +┌─res─┐ +│ 13 │ +└─────┘ +``` + + ## arrayAvg(\[func,\] arr1, …) {#array-avg} -Returns the sum of the `func` values. If the function is omitted, it just returns the average of the array elements. +Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements. Note that the `arrayAvg` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. diff --git a/docs/es/interfaces/third-party/client-libraries.md b/docs/es/interfaces/third-party/client-libraries.md index 818bdbbc6f0..b61ab1a5d9c 100644 --- a/docs/es/interfaces/third-party/client-libraries.md +++ b/docs/es/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/fr/interfaces/third-party/client-libraries.md b/docs/fr/interfaces/third-party/client-libraries.md index 5a86d12a09c..7949aa1d7cf 100644 --- a/docs/fr/interfaces/third-party/client-libraries.md +++ b/docs/fr/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [clickhouse-chauffeur](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2 / phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov / clickhouse-PHP-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/ja/interfaces/third-party/client-libraries.md b/docs/ja/interfaces/third-party/client-libraries.md index ffe7b641c38..c7bd368bc4c 100644 --- a/docs/ja/interfaces/third-party/client-libraries.md +++ b/docs/ja/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" - [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-クライアント](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index c07aab5826c..26e05b02509 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md index 36e4acb6a24..8f104e3a7d8 100644 --- a/docs/zh/development/style.md +++ b/docs/zh/development/style.md @@ -267,7 +267,7 @@ void executeQuery( **9.** 多行注释的开头和结尾不得有空行（关闭多行注释的行除外）。 -**10.** 要注释掉代码，请使用基本注释，而不是«记录»注释。 +**10.** 要注释掉代码，请使用基本注释，而不是“文档”注释。 **11.** 在提交之前删除代码的无效注释部分。 @@ -335,7 +335,7 @@ template struct ExtractDomain ``` -**7.** 对于抽象类型（接口），用 `I` 前缀。 +**7.** 对于抽象类（接口），用 `I` 前缀。 ``` cpp class IBlockInputStream @@ -349,7 +349,7 @@ class IBlockInputStream bool info_successfully_loaded = false; ``` -**9.** `define` 和全局常量的名称使用带下划线的 `ALL_CAPS`。 +**9.** `define` 和全局常量的名称使用全大写带下划线的形式，如 `ALL_CAPS`。 ``` cpp #define MAX_SRC_TABLE_NAMES_TO_STORE 1000 @@ -357,14 +357,14 @@ bool info_successfully_loaded = false; **10.** 文件名应使用与其内容相同的样式。 -如果文件包含单个类，则以与该类名称相同的方式命名该文件。 +如果文件包含单个类，则以与该类名称相同的方式命名该文件（CamelCase）。 -如果文件包含单个函数，则以与函数名称相同的方式命名文件。 +如果文件包含单个函数，则以与函数名称相同的方式命名文件（camelCase）。 **11.** 如果名称包含缩写，则： - 对于变量名，缩写应使用小写字母 `mysql_connection`（不是 `mySQL_connection` ）。 -- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection` 。 +- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection`）。 **12.** 仅用于初始化类成员的构造方法参数的命名方式应与类成员相同，但最后使用下划线。 @@ -411,7 +411,7 @@ enum class CompressionMethod 如果缩短版本是常用的，则可以接受不完整的单词。 -如果注释中旁边包含全名，您也可以使用缩写。 +如果旁边有注释包含全名，您也可以使用缩写。 **17.** C++ 源码文件名称必须为 `.cpp` 拓展名。 头文件必须为 `.h` 拓展名。 @@ -441,7 +441,7 @@ enum class CompressionMethod 在离线数据处理应用程序中，通常可以接受不捕获异常。 -在处理用户请求的服务器中，通常足以捕获连接处理程序顶层的异常。 +在处理用户请求的服务器中，捕获连接处理程序顶层的异常通常就足够了。 在线程函数中，你应该在 `join` 之后捕获并保留所有异常以在主线程中重新抛出它们。 @@ -548,7 +548,7 @@ Fork不用于并行化。 **10.** 常量。 -使用 const 引用，指向常量的指针，`const_iterator`和 const 指针。 +使用 const 引用、指针，指向常量、`const_iterator`和 const 方法。 将 `const` 视为默认值，仅在必要时使用非 `const`。 @@ -560,7 +560,7 @@ Fork不用于并行化。 **12.** 数值类型。 -使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32`， 以及 `Int64`， `size_t`， `ssize_t` 还有 `ptrdiff_t`。 +使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32` 和 `Int64`，同样还有 `size_t`， `ssize_t` 和 `ptrdiff_t`。 不要使用这些类型：`signed / unsigned long`，`long long`，`short`，`signed / unsigned char`，`char`。 @@ -732,11 +732,11 @@ CPU指令集是我们服务器中支持的最小集合。 目前，它是SSE 4.2 **8.** 尽可能经常地进行提交，即使代码只是部分准备好了。 -目的明确的功能，使用分支。 +为了这种目的可以创建分支。 -如果 `master` 分支中的代码尚不可构建，请在 `push` 之前将其从构建中排除。您需要在几天内完成或删除它。 +如果您的代码在 `master` 分支中尚不可构建，在 `push` 之前需要将其从构建中排除。您需要在几天内完成或删除它。 -**9.** 对于不重要的更改，请使用分支并在服务器上发布它们。 +**9.** 对于非一般的更改，请使用分支并在服务器上发布它们。 **10.** 未使用的代码将从 repo 中删除。 diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index e94eb8bcfc0..e2412f2b8de 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试 - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index fcf6741761b..56067bc5057 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -22,9 +22,35 @@ toc_title: "\u7CFB\u7EDF\u8868" 大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。 -与其他系统表不同，系统表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) 由 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 +与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 -默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 +用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下: + +- `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。 +- `table`: 系统日志表名。 +- `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。 +- `ttl`: 指定系统日志表TTL选项。 +- `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。 +- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。 + +一个配置定义的例子如下： + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` + +默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 ## 系统指标的来源 {#system-tables-sources-of-system-metrics} diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a1b5467f234..9adca58b55a 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -318,6 +318,10 @@ else () if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) endif() + + if (USE_BINARY_HASH) + add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM) + endif() endif () if (ENABLE_TESTS AND USE_GTEST) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 16e90816443..80ad8da837c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -59,7 +59,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -110,6 +112,7 @@ namespace ErrorCodes extern const int INVALID_USAGE_OF_INPUT; extern const int DEADLOCK_AVOIDED; extern const int UNRECOGNIZED_ARGUMENTS; + extern const int SYNTAX_ERROR; } @@ -136,6 +139,9 @@ private: bool stdin_is_a_tty = false; /// stdin is a terminal. bool stdout_is_a_tty = false; /// stdout is a terminal. + /// If not empty, queries will be read from these files + std::vector queries_files; + std::unique_ptr connection; /// Connection to DB. String full_query; /// Current query as it was given to the client. @@ -478,10 +484,10 @@ private: /// - stdin is not a terminal. In this case queries are read from it. /// - -qf (--queries-file) command line option is present. /// The value of the option is used as file with query (or of multiple queries) to execute. - if (!stdin_is_a_tty || config().has("query") || config().has("queries-file")) + if (!stdin_is_a_tty || config().has("query") || !queries_files.empty()) is_interactive = false; - if (config().has("query") && config().has("queries-file")) + if (config().has("query") && !queries_files.empty()) { throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS); } @@ -696,14 +702,8 @@ private: auto query_id = config().getString("query_id", ""); if (!query_id.empty()) context.setCurrentQueryId(query_id); - if (query_fuzzer_runs) - { - nonInteractiveWithFuzzing(); - } - else - { - nonInteractive(); - } + + nonInteractive(); /// If exception code isn't zero, we should return non-zero return code anyway. if (last_exception_received_from_server) @@ -794,15 +794,21 @@ private: { String text; - if (config().has("queries-file")) + if (!queries_files.empty()) { - ReadBufferFromFile in(config().getString("queries-file")); - readStringUntilEOF(text, in); - processMultiQuery(text); + for (const auto & queries_file : queries_files) + { + connection->setDefaultDatabase(connection_parameters.default_database); + ReadBufferFromFile in(queries_file); + readStringUntilEOF(text, in); + processMultiQuery(text); + } return; } else if (config().has("query")) + { text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. + } else { /// If 'query' parameter is not set, read a query from stdin. @@ -811,113 +817,10 @@ private: readStringUntilEOF(text, in); } - processQueryText(text); - } - - void nonInteractiveWithFuzzing() - { - if (config().has("query")) - { - // Poco configuration should not process substitutions in form of - // ${...} inside query - processWithFuzzing(config().getRawString("query")); - return; - } - - // Try to stream the queries from stdin, without reading all of them - // into memory. The interface of the parser does not support streaming, - // in particular, it can't distinguish the end of partial input buffer - // and the final end of input file. This means we have to try to split - // the input into separate queries here. Two patterns of input are - // especially interesting: - // 1) multiline query: - // select 1 - // from system.numbers; - // - // 2) csv insert with in-place data: - // insert into t format CSV 1;2 - // - // (1) means we can't split on new line, and (2) means we can't split on - // semicolon. Solution: split on ';\n'. This sequence is frequent enough - // in the SQL tests which are our principal input for fuzzing. Now we - // have another interesting case: - // 3) escaped semicolon followed by newline, e.g. - // select '; - // ' - // - // To handle (3), parse until we can, and read more data if the parser - // complains. Hopefully this should be enough... - ReadBufferFromFileDescriptor in(STDIN_FILENO); - std::string text; - while (!in.eof()) - { - // Read until separator. - while (!in.eof()) - { - char * next_separator = find_first_symbols<';'>(in.position(), - in.buffer().end()); - - if (next_separator < in.buffer().end()) - { - next_separator++; - if (next_separator < in.buffer().end() - && *next_separator == '\n') - { - // Found ';\n', append it to the query text and try to - // parse. - next_separator++; - text.append(in.position(), next_separator - in.position()); - in.position() = next_separator; - break; - } - } - - // Didn't find the semicolon and reached the end of buffer. - text.append(in.position(), next_separator - in.position()); - in.position() = next_separator; - - if (text.size() > 1024 * 1024) - { - // We've read a lot of text and still haven't seen a separator. - // Likely some pathological input, just fall through to prevent - // too long loops. - break; - } - } - - // Parse and execute what we've read. - const auto * new_end = processWithFuzzing(text); - - if (new_end > &text[0]) - { - const auto rest_size = text.size() - (new_end - &text[0]); - - memcpy(&text[0], new_end, rest_size); - text.resize(rest_size); - } - else - { - // We didn't read enough text to parse a query. Will read more. - } - - // Ensure that we're still connected to the server. If the server died, - // the reconnect is going to fail with an exception, and the fuzzer - // will exit. The ping() would be the best match here, but it's - // private, probably for a good reason that the protocol doesn't allow - // pings at any possible moment. - // Don't forget to reset the default database which might have changed. - connection->setDefaultDatabase(""); - connection->forceConnected(connection_parameters.timeouts); - - if (text.size() > 4 * 1024) - { - // Some pathological situation where the text is larger than 4kB - // and we still cannot parse a single query in it. Abort. - std::cerr << "Read too much text and still can't parse a query." - " Aborting." << std::endl; - exit(1); - } - } + if (query_fuzzer_runs) + processWithFuzzing(text); + else + processQueryText(text); } bool processQueryText(const String & text) @@ -945,7 +848,8 @@ private: { const bool test_mode = config().has("testmode"); - { /// disable logs if expects errors + { + /// disable logs if expects errors TestHint test_hint(test_mode, all_queries_text); if (test_hint.clientError() || test_hint.serverError()) processTextAsSingleQuery("SET send_logs_level = 'fatal'"); @@ -1019,7 +923,7 @@ private: if (hint.clientError() != e.code()) { if (hint.clientError()) - e.addMessage("\nExpected clinet error: " + std::to_string(hint.clientError())); + e.addMessage("\nExpected client error: " + std::to_string(hint.clientError())); throw; } @@ -1078,39 +982,49 @@ private: expected_client_error = test_hint.clientError(); expected_server_error = test_hint.serverError(); - try + if (query_fuzzer_runs) { - processParsedSingleQuery(); - - if (insert_ast && insert_ast->data) + processWithFuzzing(full_query); + } + else + { + try { - // For VALUES format: use the end of inline data as reported - // by the format parser (it is saved in sendData()). This - // allows us to handle queries like: - // insert into t values (1); select 1 - //, where the inline data is delimited by semicolon and not - // by a newline. - this_query_end = parsed_query->as()->end; + processParsedSingleQuery(); + + if (insert_ast && insert_ast->data) + { + // For VALUES format: use the end of inline data as reported + // by the format parser (it is saved in sendData()). This + // allows us to handle queries like: + // insert into t values (1); select 1 + //, where the inline data is delimited by semicolon and not + // by a newline. + this_query_end = parsed_query->as()->end; + } + } + catch (...) + { + last_exception_received_from_server = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); + actual_client_error = last_exception_received_from_server->code(); + if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error)) + std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message(); + received_exception_from_server = true; } - } - catch (...) - { - last_exception_received_from_server = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); - actual_client_error = last_exception_received_from_server->code(); - if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error)) - std::cerr << "Error on processing query: " << full_query << std::endl << last_exception_received_from_server->message(); - received_exception_from_server = true; - } - if (!test_hint.checkActual(actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server)) - connection->forceConnected(connection_parameters.timeouts); + if (!test_hint.checkActual( + actual_server_error, actual_client_error, received_exception_from_server, last_exception_received_from_server)) + { + connection->forceConnected(connection_parameters.timeouts); + } - if (received_exception_from_server && !ignore_error) - { - if (is_interactive) - break; - else - return false; + if (received_exception_from_server && !ignore_error) + { + if (is_interactive) + break; + else + return false; + } } this_query_begin = this_query_end; @@ -1120,165 +1034,145 @@ private: } - // Returns the last position we could parse. - const char * processWithFuzzing(const String & text) + void processWithFuzzing(const String & text) { - /// Several queries separated by ';'. - /// INSERT data is ended by the end of line, not ';'. + ASTPtr orig_ast; - const char * begin = text.data(); - const char * end = begin + text.size(); - - while (begin < end) + try { - // Skip whitespace before the query - while (isWhitespaceASCII(*begin) || *begin == ';') - { - ++begin; - } - - const auto * this_query_begin = begin; - ASTPtr orig_ast = parseQuery(begin, end, true); - - if (!orig_ast) - { - // Can't continue after a parsing error - return begin; - } - - auto * as_insert = orig_ast->as(); - if (as_insert && as_insert->data) - { - // INSERT data is ended by newline - as_insert->end = find_first_symbols<'\n'>(as_insert->data, end); - begin = as_insert->end; - } - - full_query = text.substr(this_query_begin - text.data(), - begin - text.data()); - - // Don't repeat inserts, the tables grow too big. Also don't repeat - // creates because first we run the unmodified query, it will succeed, - // and the subsequent queries will fail. When we run out of fuzzer - // errors, it may be interesting to add fuzzing of create queries that - // wraps columns into LowCardinality or Nullable. Also there are other - // kinds of create queries such as CREATE DICTIONARY, we could fuzz - // them as well. - int this_query_runs = query_fuzzer_runs; - if (as_insert - || orig_ast->as()) - { - this_query_runs = 1; - } - - ASTPtr fuzz_base = orig_ast; - for (int fuzz_step = 0; fuzz_step < this_query_runs; fuzz_step++) - { - fprintf(stderr, "fuzzing step %d out of %d for query at pos %zd\n", - fuzz_step, this_query_runs, this_query_begin - text.data()); - - ASTPtr ast_to_process; - try - { - WriteBufferFromOwnString dump_before_fuzz; - fuzz_base->dumpTree(dump_before_fuzz); - auto base_before_fuzz = fuzz_base->formatForErrorMessage(); - - ast_to_process = fuzz_base->clone(); - - WriteBufferFromOwnString dump_of_cloned_ast; - ast_to_process->dumpTree(dump_of_cloned_ast); - - // Run the original query as well. - if (fuzz_step > 0) - { - fuzzer.fuzzMain(ast_to_process); - } - - auto base_after_fuzz = fuzz_base->formatForErrorMessage(); - - // Debug AST cloning errors. - if (base_before_fuzz != base_after_fuzz) - { - fprintf(stderr, "base before fuzz: %s\n" - "base after fuzz: %s\n", base_before_fuzz.c_str(), - base_after_fuzz.c_str()); - fprintf(stderr, "dump before fuzz:\n%s\n", - dump_before_fuzz.str().c_str()); - fprintf(stderr, "dump of cloned ast:\n%s\n", - dump_of_cloned_ast.str().c_str()); - fprintf(stderr, "dump after fuzz:\n"); - WriteBufferFromOStream cerr_buf(std::cerr, 4096); - fuzz_base->dumpTree(cerr_buf); - cerr_buf.next(); - - fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); - - assert(false); - } - - auto fuzzed_text = ast_to_process->formatForErrorMessage(); - if (fuzz_step > 0 && fuzzed_text == base_before_fuzz) - { - fprintf(stderr, "got boring ast\n"); - continue; - } - - parsed_query = ast_to_process; - query_to_send = parsed_query->formatForErrorMessage(); - - processParsedSingleQuery(); - } - catch (...) - { - // Some functions (e.g. protocol parsers) don't throw, but - // set last_exception instead, so we'll also do it here for - // uniformity. - last_exception_received_from_server = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); - received_exception_from_server = true; - } - - if (received_exception_from_server) - { - fmt::print(stderr, "Error on processing query '{}': {}\n", - ast_to_process->formatForErrorMessage(), - last_exception_received_from_server->message()); - } - - if (!connection->isConnected()) - { - // Probably the server is dead because we found an assertion - // failure. Fail fast. - fmt::print(stderr, "Lost connection to the server\n"); - return begin; - } - - // The server is still alive so we're going to continue fuzzing. - // Determine what we're going to use as the starting AST. - if (received_exception_from_server) - { - // Query completed with error, keep the previous starting AST. - // Also discard the exception that we now know to be non-fatal, - // so that it doesn't influence the exit code. - last_exception_received_from_server.reset(nullptr); - received_exception_from_server = false; - } - else if (ast_to_process->formatForErrorMessage().size() > 500) - { - // ast too long, start from original ast - fprintf(stderr, "Current AST is too long, discarding it and using the original AST as a start\n"); - fuzz_base = orig_ast; - } - else - { - // fuzz starting from this successful query - fprintf(stderr, "Query succeeded, using this AST as a start\n"); - fuzz_base = ast_to_process; - } - } + const char * begin = text.data(); + orig_ast = parseQuery(begin, begin + text.size(), true); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::SYNTAX_ERROR) + throw; } - return begin; + if (!orig_ast) + { + // Can't continue after a parsing error + return; + } + + // Don't repeat inserts, the tables grow too big. Also don't repeat + // creates because first we run the unmodified query, it will succeed, + // and the subsequent queries will fail. When we run out of fuzzer + // errors, it may be interesting to add fuzzing of create queries that + // wraps columns into LowCardinality or Nullable. Also there are other + // kinds of create queries such as CREATE DICTIONARY, we could fuzz + // them as well. Also there is no point fuzzing DROP queries. + size_t this_query_runs = query_fuzzer_runs; + if (orig_ast->as() || orig_ast->as() || orig_ast->as()) + { + this_query_runs = 1; + } + + ASTPtr fuzz_base = orig_ast; + for (size_t fuzz_step = 0; fuzz_step < this_query_runs; ++fuzz_step) + { + fmt::print(stderr, "Fuzzing step {} out of {}\n", + fuzz_step, this_query_runs); + + ASTPtr ast_to_process; + try + { + WriteBufferFromOwnString dump_before_fuzz; + fuzz_base->dumpTree(dump_before_fuzz); + auto base_before_fuzz = fuzz_base->formatForErrorMessage(); + + ast_to_process = fuzz_base->clone(); + + WriteBufferFromOwnString dump_of_cloned_ast; + ast_to_process->dumpTree(dump_of_cloned_ast); + + // Run the original query as well. + if (fuzz_step > 0) + { + fuzzer.fuzzMain(ast_to_process); + } + + auto base_after_fuzz = fuzz_base->formatForErrorMessage(); + + // Debug AST cloning errors. + if (base_before_fuzz != base_after_fuzz) + { + fmt::print(stderr, + "Base before fuzz: {}\n" + "Base after fuzz: {}\n", + base_before_fuzz, base_after_fuzz); + fmt::print(stderr, "Dump before fuzz:\n{}\n", dump_before_fuzz.str()); + fmt::print(stderr, "Dump of cloned AST:\n{}\n", dump_of_cloned_ast.str()); + fmt::print(stderr, "Dump after fuzz:\n"); + + WriteBufferFromOStream cerr_buf(std::cerr, 4096); + fuzz_base->dumpTree(cerr_buf); + cerr_buf.next(); + + fmt::print(stderr, "IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly."); + + assert(false); + } + + auto fuzzed_text = ast_to_process->formatForErrorMessage(); + if (fuzz_step > 0 && fuzzed_text == base_before_fuzz) + { + fmt::print(stderr, "Got boring AST\n"); + continue; + } + + parsed_query = ast_to_process; + query_to_send = parsed_query->formatForErrorMessage(); + + processParsedSingleQuery(); + } + catch (...) + { + // Some functions (e.g. protocol parsers) don't throw, but + // set last_exception instead, so we'll also do it here for + // uniformity. + last_exception_received_from_server = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); + received_exception_from_server = true; + } + + if (received_exception_from_server) + { + fmt::print(stderr, "Error on processing query '{}': {}\n", + ast_to_process->formatForErrorMessage(), + last_exception_received_from_server->message()); + } + + if (!connection->isConnected()) + { + // Probably the server is dead because we found an assertion + // failure. Fail fast. + fmt::print(stderr, "Lost connection to the server\n"); + return; + } + + // The server is still alive so we're going to continue fuzzing. + // Determine what we're going to use as the starting AST. + if (received_exception_from_server) + { + // Query completed with error, keep the previous starting AST. + // Also discard the exception that we now know to be non-fatal, + // so that it doesn't influence the exit code. + last_exception_received_from_server.reset(nullptr); + received_exception_from_server = false; + } + else if (ast_to_process->formatForErrorMessage().size() > 500) + { + // ast too long, start from original ast + fmt::print(stderr, "Current AST is too long, discarding it and using the original AST as a start\n"); + fuzz_base = orig_ast; + } + else + { + // fuzz starting from this successful query + fmt::print(stderr, "Query succeeded, using this AST as a start\n"); + fuzz_base = ast_to_process; + } + } } void processTextAsSingleQuery(const String & text_) @@ -1891,6 +1785,13 @@ private: { if (!block_out_stream) { + /// Ignore all results when fuzzing as they can be huge. + if (query_fuzzer_runs) + { + block_out_stream = std::make_shared(block); + return; + } + WriteBuffer * out_buf = nullptr; String pager = config().getString("pager", ""); if (!pager.empty()) @@ -2348,7 +2249,8 @@ public: "Suggestion limit for how many databases, tables and columns to fetch.") ("multiline,m", "multiline") ("multiquery,n", "multiquery") - ("queries-file", po::value(), "file path with queries to execute") + ("queries-file", po::value>()->multitoken(), + "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)") ("format,f", po::value(), "default output format") ("testmode,T", "enable test hints in comments") ("ignore-error", "do not stop processing in multiquery mode") @@ -2478,12 +2380,11 @@ public: if (options.count("query")) config().setString("query", options["query"].as()); if (options.count("queries-file")) - config().setString("queries-file", options["queries-file"].as()); + queries_files = options["queries-file"].as>(); if (options.count("database")) config().setString("database", options["database"].as()); if (options.count("pager")) config().setString("pager", options["pager"].as()); - if (options.count("port") && !options["port"].defaulted()) config().setInt("port", options["port"].as()); if (options.count("secure")) @@ -2537,7 +2438,6 @@ public: config().setBool("multiquery", true); // Ignore errors in parsing queries. - // TODO stop using parseQuery. config().setBool("ignore-error", true); ignore_error = true; } diff --git a/programs/main.cpp b/programs/main.cpp index dee02c55832..cbb22b7a87b 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -18,6 +18,7 @@ #endif #include +#include #include #include @@ -62,6 +63,14 @@ int mainEntryClickHouseStatus(int argc, char ** argv); int mainEntryClickHouseRestart(int argc, char ** argv); #endif +int mainEntryClickHouseHashBinary(int, char **) +{ + /// Intentionally without newline. So you can run: + /// objcopy --add-section .note.ClickHouse.hash=<(./clickhouse hash-binary) clickhouse + std::cout << getHashOfLoadedBinaryHex(); + return 0; +} + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) namespace @@ -110,6 +119,7 @@ std::pair clickhouse_applications[] = {"status", mainEntryClickHouseStatus}, {"restart", mainEntryClickHouseRestart}, #endif + {"hash-binary", mainEntryClickHouseHashBinary}, }; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 76765c0374c..2f8029fc39c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -65,6 +65,8 @@ #include #include #include +#include +#include #include #include #include @@ -184,6 +186,7 @@ namespace ErrorCodes extern const int FAILED_TO_GETPWUID; extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; extern const int NETWORK_ERROR; + extern const int CORRUPTED_DATA; } @@ -436,7 +439,44 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) std::string executable_path = getExecutablePath(); - if (executable_path.empty()) + + if (!executable_path.empty()) + { + /// Integrity check based on checksum of the executable code. + /// Note: it is not intended to protect from malicious party, + /// because the reference checksum can be easily modified as well. + /// And we don't involve asymmetric encryption with PKI yet. + /// It's only intended to protect from faulty hardware. + /// Note: it is only based on machine code. + /// But there are other sections of the binary (e.g. exception handling tables) + /// that are interpreted (not executed) but can alter the behaviour of the program as well. + + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + + if (stored_binary_hash.empty()) + { + LOG_WARNING(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == stored_binary_hash) + { + LOG_INFO(log, "Calculated checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file {2} was changed just after startup;" + " - the file {2} is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file {2} was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, stored_binary_hash, executable_path); + } + } + else executable_path = "/usr/bin/clickhouse"; /// It is used for information messages. /// After full config loaded diff --git a/programs/server/config.xml b/programs/server/config.xml index a4c13be493e..372315c7922 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -676,7 +676,7 @@ system query_log