diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/10_question.md similarity index 93% rename from .github/ISSUE_TEMPLATE/question.md rename to .github/ISSUE_TEMPLATE/10_question.md index a5015de8217..6e23fbdc605 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/10_question.md @@ -1,6 +1,6 @@ --- name: Question -about: Ask question about ClickHouse +about: Ask a question about ClickHouse title: '' labels: question assignees: '' diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/20_feature-request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature-request.md rename to .github/ISSUE_TEMPLATE/20_feature-request.md diff --git a/.github/ISSUE_TEMPLATE/unexpected-behaviour.md b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md similarity index 94% rename from .github/ISSUE_TEMPLATE/unexpected-behaviour.md rename to .github/ISSUE_TEMPLATE/30_unexpected-behaviour.md index 27ab217ca33..3630d95ba33 100644 --- a/.github/ISSUE_TEMPLATE/unexpected-behaviour.md +++ b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md @@ -1,6 +1,6 @@ --- name: Unexpected behaviour -about: Create a report to help us improve ClickHouse +about: Some feature is working in non-obvious way title: '' labels: unexpected behaviour assignees: '' diff --git a/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md new file mode 100644 index 00000000000..6a014ce3c29 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md @@ -0,0 +1,30 @@ +--- +name: Incomplete implementation +about: Implementation of existing feature is not finished +title: '' +labels: unfinished code +assignees: '' + +--- + +(you don't have to strictly follow this form) + +**Describe the unexpected behaviour** +A clear and concise description of what works not as it is supposed to. + +**How to reproduce** +* Which ClickHouse server version to use +* Which interface to use, if matters +* Non-default settings, if any +* `CREATE TABLE` statements for all tables involved +* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary +* Queries to run that lead to unexpected result + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Error message and/or stacktrace** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug-report.md rename to .github/ISSUE_TEMPLATE/40_bug-report.md diff --git a/.github/ISSUE_TEMPLATE/usability-issue.md b/.github/ISSUE_TEMPLATE/45_usability-issue.md similarity index 93% rename from .github/ISSUE_TEMPLATE/usability-issue.md rename to .github/ISSUE_TEMPLATE/45_usability-issue.md index 6a084a72619..b03b11606c1 100644 --- a/.github/ISSUE_TEMPLATE/usability-issue.md +++ b/.github/ISSUE_TEMPLATE/45_usability-issue.md @@ -1,6 +1,6 @@ --- name: Usability issue -about: Create a report to help us improve ClickHouse +about: Report something can be made more convenient to use title: '' labels: usability assignees: '' diff --git a/.github/ISSUE_TEMPLATE/build-issue.md b/.github/ISSUE_TEMPLATE/50_build-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/build-issue.md rename to .github/ISSUE_TEMPLATE/50_build-issue.md diff --git a/.github/ISSUE_TEMPLATE/documentation-issue.md b/.github/ISSUE_TEMPLATE/60_documentation-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/documentation-issue.md rename to .github/ISSUE_TEMPLATE/60_documentation-issue.md diff --git a/.github/ISSUE_TEMPLATE/performance-issue.md b/.github/ISSUE_TEMPLATE/70_performance-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/performance-issue.md rename to .github/ISSUE_TEMPLATE/70_performance-issue.md diff --git a/.github/ISSUE_TEMPLATE/backward-compatibility.md b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md similarity index 90% rename from .github/ISSUE_TEMPLATE/backward-compatibility.md rename to .github/ISSUE_TEMPLATE/80_backward-compatibility.md index 8f87197e73d..a13e9508f70 100644 --- a/.github/ISSUE_TEMPLATE/backward-compatibility.md +++ b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md @@ -1,6 +1,6 @@ --- name: Backward compatibility issue -about: Create a report to help us improve ClickHouse +about: Report the case when the behaviour of a new version can break existing use cases title: '' labels: backward compatibility assignees: '' diff --git a/.github/ISSUE_TEMPLATE/90_fuzzing-report.md b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md new file mode 100644 index 00000000000..1d9a8a75d28 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md @@ -0,0 +1,16 @@ +--- +name: Assertion found via fuzzing +about: Potential issue has been found via Fuzzer or Stress tests +title: '' +labels: fuzz +assignees: '' + +--- + +(you don't have to strictly follow this form) + +**Describe the bug** +A link to the report + +**How to reproduce** +Try to reproduce the report and copy the tables and queries involved. diff --git a/CMakeLists.txt b/CMakeLists.txt index 79bfbe55a98..9002f1df140 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,6 +220,13 @@ if (LINKER_NAME MATCHES "lld$") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") endif () +# Add a section with the hash of the compiled machine code for integrity checks. +# Only for official builds, because adding a section can be time consuming (rewrite of several GB). +# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary) +if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE)) + set (USE_BINARY_HASH 1) +endif () + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 1224d9171ea..4cf8a8d7ce9 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -80,28 +83,6 @@ static void call_default_signal_handler(int sig) raise(sig); } -// Apparently strsignal is not instrumented by MemorySanitizer, so we -// have to unpoison it to avoid msan reports inside fmt library when we -// print it. -const char * msan_strsignal(int sig) -{ - // no glibc in osx/freebsd -#if !defined(__GLIBC_PREREQ) -#define __GLIBC_PREREQ(x, y) 0 -#endif - - // glibc 2.32+ deprecates sys_siglist[] - // newer glibc is a problem only for unbundled build. -#if __GLIBC_PREREQ(2, 32) - const char * signal_name = sigdescr_np(sig); -#else - const char * signal_name = sys_siglist[sig]; -#endif - - __msan_unpoison_string(signal_name); - return signal_name; -} - static constexpr size_t max_query_id_size = 127; static const size_t signal_pipe_buf_size = @@ -131,11 +112,13 @@ static void writeSignalIDtoSignalPipe(int sig) /** Signal handler for HUP / USR1 */ static void closeLogsSignalHandler(int sig, siginfo_t *, void *) { + DENY_ALLOCATIONS_IN_SCOPE; writeSignalIDtoSignalPipe(sig); } static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *) { + DENY_ALLOCATIONS_IN_SCOPE; writeSignalIDtoSignalPipe(sig); } @@ -144,6 +127,7 @@ static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *) */ static void signalHandler(int sig, siginfo_t * info, void * context) { + DENY_ALLOCATIONS_IN_SCOPE; auto saved_errno = errno; /// We must restore previous value of errno in signal handler. char buf[signal_pipe_buf_size]; @@ -306,13 +290,13 @@ private: { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, msan_strsignal(sig), sig); + thread_num, strsignal(sig), sig); } else { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, query_id, msan_strsignal(sig), sig); + thread_num, query_id, strsignal(sig), sig); } String error_message; @@ -340,6 +324,32 @@ private: /// Write symbolized stack trace line by line for better grep-ability. stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); +#if defined(__linux__) + /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + if (daemon.stored_binary_hash.empty()) + { + LOG_FATAL(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == daemon.stored_binary_hash) + { + LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file was changed just after startup;" + " - the file is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, daemon.stored_binary_hash); + } +#endif + /// Write crash to system.crash_log table if available. if (collectCrashLog) collectCrashLog(sig, thread_num, query_id, stack_trace); @@ -493,8 +503,9 @@ void BaseDaemon::kill() { dumpCoverageReportIfPossible(); pid_file.reset(); - if (::raise(SIGKILL) != 0) - throw Poco::SystemException("cannot kill process"); + /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL. + /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker). + _exit(128 + SIGKILL); } std::string BaseDaemon::getDefaultCorePath() const @@ -799,6 +810,13 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() #else build_id_info = "no build id"; #endif + +#if defined(__linux__) + std::string executable_path = getExecutablePath(); + + if (!executable_path.empty()) + stored_binary_hash = DB::Elf(executable_path).getBinaryHash(); +#endif } void BaseDaemon::logRevision() const @@ -858,13 +876,13 @@ void BaseDaemon::handleSignal(int signal_id) onInterruptSignals(signal_id); } else - throw DB::Exception(std::string("Unsupported signal: ") + msan_strsignal(signal_id), 0); + throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); } void BaseDaemon::onInterruptSignals(int signal_id) { is_cancelled = true; - LOG_INFO(&logger(), "Received termination signal ({})", msan_strsignal(signal_id)); + LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); if (sigint_signals_counter >= 2) { @@ -1010,3 +1028,9 @@ void BaseDaemon::setupWatchdog() #endif } } + + +String BaseDaemon::getStoredBinaryHash() const +{ + return stored_binary_hash; +} diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 090d4997606..42d94629ae9 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -60,7 +60,7 @@ public: static void terminate(); /// Forceful shutdown - void kill(); + [[noreturn]] void kill(); /// Cancellation request has been received. bool isCancelled() const @@ -121,6 +121,9 @@ public: /// argv0 is needed to change process name (consequently, it is needed for scripts involving "pgrep", "pidof" to work correctly). void shouldSetupWatchdog(char * argv0_); + /// Hash of the binary for integrity checks. + String getStoredBinaryHash() const; + protected: virtual void logRevision() const; @@ -168,6 +171,7 @@ protected: Poco::Util::AbstractConfiguration * last_configuration = nullptr; String build_id_info; + String stored_binary_hash; std::vector handled_signals; diff --git a/base/glibc-compatibility/musl/strsignal.c b/base/glibc-compatibility/musl/strsignal.c new file mode 100644 index 00000000000..fee894e8550 --- /dev/null +++ b/base/glibc-compatibility/musl/strsignal.c @@ -0,0 +1,125 @@ +#include +#include + +#if (SIGHUP == 1) && (SIGINT == 2) && (SIGQUIT == 3) && (SIGILL == 4) \ + && (SIGTRAP == 5) && (SIGABRT == 6) && (SIGBUS == 7) && (SIGFPE == 8) \ + && (SIGKILL == 9) && (SIGUSR1 == 10) && (SIGSEGV == 11) && (SIGUSR2 == 12) \ + && (SIGPIPE == 13) && (SIGALRM == 14) && (SIGTERM == 15) && (SIGSTKFLT == 16) \ + && (SIGCHLD == 17) && (SIGCONT == 18) && (SIGSTOP == 19) && (SIGTSTP == 20) \ + && (SIGTTIN == 21) && (SIGTTOU == 22) && (SIGURG == 23) && (SIGXCPU == 24) \ + && (SIGXFSZ == 25) && (SIGVTALRM == 26) && (SIGPROF == 27) && (SIGWINCH == 28) \ + && (SIGPOLL == 29) && (SIGPWR == 30) && (SIGSYS == 31) + +#define sigmap(x) x + +#else + +static const char map[] = { + [SIGHUP] = 1, + [SIGINT] = 2, + [SIGQUIT] = 3, + [SIGILL] = 4, + [SIGTRAP] = 5, + [SIGABRT] = 6, + [SIGBUS] = 7, + [SIGFPE] = 8, + [SIGKILL] = 9, + [SIGUSR1] = 10, + [SIGSEGV] = 11, + [SIGUSR2] = 12, + [SIGPIPE] = 13, + [SIGALRM] = 14, + [SIGTERM] = 15, +#if defined(SIGSTKFLT) + [SIGSTKFLT] = 16, +#elif defined(SIGEMT) + [SIGEMT] = 16, +#endif + [SIGCHLD] = 17, + [SIGCONT] = 18, + [SIGSTOP] = 19, + [SIGTSTP] = 20, + [SIGTTIN] = 21, + [SIGTTOU] = 22, + [SIGURG] = 23, + [SIGXCPU] = 24, + [SIGXFSZ] = 25, + [SIGVTALRM] = 26, + [SIGPROF] = 27, + [SIGWINCH] = 28, + [SIGPOLL] = 29, + [SIGPWR] = 30, + [SIGSYS] = 31 +}; + +#define sigmap(x) ((x) >= sizeof map ? (x) : map[(x)]) + +#endif + +static const char strings[] = + "Unknown signal\0" + "Hangup\0" + "Interrupt\0" + "Quit\0" + "Illegal instruction\0" + "Trace/breakpoint trap\0" + "Aborted\0" + "Bus error\0" + "Arithmetic exception\0" + "Killed\0" + "User defined signal 1\0" + "Segmentation fault\0" + "User defined signal 2\0" + "Broken pipe\0" + "Alarm clock\0" + "Terminated\0" +#if defined(SIGSTKFLT) + "Stack fault\0" +#elif defined(SIGEMT) + "Emulator trap\0" +#else + "Unknown signal\0" +#endif + "Child process status\0" + "Continued\0" + "Stopped (signal)\0" + "Stopped\0" + "Stopped (tty input)\0" + "Stopped (tty output)\0" + "Urgent I/O condition\0" + "CPU time limit exceeded\0" + "File size limit exceeded\0" + "Virtual timer expired\0" + "Profiling timer expired\0" + "Window changed\0" + "I/O possible\0" + "Power failure\0" + "Bad system call\0" + "RT32" + "\0RT33\0RT34\0RT35\0RT36\0RT37\0RT38\0RT39\0RT40" + "\0RT41\0RT42\0RT43\0RT44\0RT45\0RT46\0RT47\0RT48" + "\0RT49\0RT50\0RT51\0RT52\0RT53\0RT54\0RT55\0RT56" + "\0RT57\0RT58\0RT59\0RT60\0RT61\0RT62\0RT63\0RT64" +#if _NSIG > 65 + "\0RT65\0RT66\0RT67\0RT68\0RT69\0RT70\0RT71\0RT72" + "\0RT73\0RT74\0RT75\0RT76\0RT77\0RT78\0RT79\0RT80" + "\0RT81\0RT82\0RT83\0RT84\0RT85\0RT86\0RT87\0RT88" + "\0RT89\0RT90\0RT91\0RT92\0RT93\0RT94\0RT95\0RT96" + "\0RT97\0RT98\0RT99\0RT100\0RT101\0RT102\0RT103\0RT104" + "\0RT105\0RT106\0RT107\0RT108\0RT109\0RT110\0RT111\0RT112" + "\0RT113\0RT114\0RT115\0RT116\0RT117\0RT118\0RT119\0RT120" + "\0RT121\0RT122\0RT123\0RT124\0RT125\0RT126\0RT127\0RT128" +#endif + ""; + +char *strsignal(int signum) +{ + const char *s = strings; + + signum = sigmap(signum); + if (signum - 1U >= _NSIG-1) signum = 0; + + for (; signum--; s++) for (; *s; s++); + + return (char *)s; +} diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 4d08174170f..df625a3e4d6 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -142,7 +142,7 @@ TRAP(qecvt) TRAP(qfcvt) TRAP(register_printf_function) TRAP(seed48) -TRAP(setenv) +//TRAP(setenv) TRAP(setfsent) TRAP(setgrent) TRAP(sethostent) @@ -164,7 +164,7 @@ TRAP(sigsuspend) TRAP(sleep) TRAP(srand48) //TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately. -TRAP(strsignal) +//TRAP(strsignal) // This function is imported from Musl and is thread safe. TRAP(strtok) TRAP(tcflow) TRAP(tcsendbreak) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 87a30c9effc..bc06286a1ad 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -1,9 +1,9 @@ # This strings autochanged from release_lib.sh: -SET(VERSION_REVISION 54444) -SET(VERSION_MAJOR 20) -SET(VERSION_MINOR 13) +SET(VERSION_REVISION 54445) +SET(VERSION_MAJOR 21) +SET(VERSION_MINOR 1) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH e581f9ccfc5c64867b0f488cce72412fd2966471) -SET(VERSION_DESCRIBE v20.13.1.1-prestable) -SET(VERSION_STRING 20.13.1.1) +SET(VERSION_GITHASH 667dd0cf0ccecdaa6f334177b7ece2f53bd196a1) +SET(VERSION_DESCRIBE v21.1.1.5646-prestable) +SET(VERSION_STRING 21.1.1.5646) # end of autochange diff --git a/cmake/find/ccache.cmake b/cmake/find/ccache.cmake index 2cdfed56de8..d8e9cf9588d 100644 --- a/cmake/find/ccache.cmake +++ b/cmake/find/ccache.cmake @@ -32,12 +32,21 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE) if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang") message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}") - # 4+ ccache respect SOURCE_DATE_EPOCH (always includes it into the hash - # of the manifest) and debian will extract these from d/changelog, and - # makes cache of ccache unusable + # debian (debhlpers) set SOURCE_DATE_EPOCH environment variable, that is + # filled from the debian/changelog or current time. # - # FIXME: once sloppiness will be introduced for this this can be removed. - if (CCACHE_VERSION VERSION_GREATER "4.0") + # - 4.0+ ccache always includes this environment variable into the hash + # of the manifest, which do not allow to use previous cache, + # - 4.2+ ccache ignores SOURCE_DATE_EPOCH under time_macros sloppiness. + # + # So for: + # - 4.2+ time_macros sloppiness is used, + # - 4.0+ will ignore SOURCE_DATE_EPOCH environment variable. + if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.2") + message(STATUS "Use time_macros sloppiness for ccache") + set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_FOUND} --set-config=sloppiness=time_macros") + set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE_FOUND} --set-config=sloppiness=time_macros") + elseif (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0") message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache") set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE "env -u SOURCE_DATE_EPOCH ${CCACHE_FOUND}") set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK "env -u SOURCE_DATE_EPOCH ${CCACHE_FOUND}") diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index ac11322f408..26005acc4d4 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -1,5 +1,4 @@ -# Freebsd: contrib/cppkafka/include/cppkafka/detail/endianness.h:53:23: error: 'betoh16' was not declared in this scope -if (NOT ARCH_ARM AND NOT OS_FREEBSD AND OPENSSL_FOUND) +if (NOT ARCH_ARM AND OPENSSL_FOUND) option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES}) elseif(ENABLE_RDKAFKA AND NOT OPENSSL_FOUND) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use librdkafka without SSL") diff --git a/cmake/toolchain/darwin-x86_64/README.txt b/cmake/toolchain/darwin-x86_64/README.txt index 0626c9e886c..65c9aba5be6 100644 --- a/cmake/toolchain/darwin-x86_64/README.txt +++ b/cmake/toolchain/darwin-x86_64/README.txt @@ -1,2 +1,2 @@ -wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz -tar xJf MacOSX10.14.sdk.tar.xz --strip-components=1 +wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz +tar xJf MacOSX10.15.sdk.tar.xz --strip-components=1 diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 22543882521..20818767b99 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -18,7 +18,11 @@ if (WITH_COVERAGE) set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) separate_arguments(WITHOUT_COVERAGE_LIST) # disable coverage for contib files and build with optimisations - add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + if (COMPILER_CLANG) + add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + else() + add_compile_options(-O3 -DNDEBUG -finline-functions ${WITHOUT_COVERAGE_LIST}) + endif() endif() if (SANITIZE STREQUAL "undefined") diff --git a/contrib/libcxx b/contrib/libcxx index 95650a0db43..8b80a151d12 160000 --- a/contrib/libcxx +++ b/contrib/libcxx @@ -1 +1 @@ -Subproject commit 95650a0db4399ee871d5fd698ad12384fe9fa964 +Subproject commit 8b80a151d12b98ffe2d0c22f7cec12c3b9ff88d7 diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt index 5322be347ff..3b5d53cd1c0 100644 --- a/contrib/libcxx-cmake/CMakeLists.txt +++ b/contrib/libcxx-cmake/CMakeLists.txt @@ -5,6 +5,8 @@ set(LIBCXX_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx) set(SRCS ${LIBCXX_SOURCE_DIR}/src/algorithm.cpp ${LIBCXX_SOURCE_DIR}/src/any.cpp +${LIBCXX_SOURCE_DIR}/src/atomic.cpp +${LIBCXX_SOURCE_DIR}/src/barrier.cpp ${LIBCXX_SOURCE_DIR}/src/bind.cpp ${LIBCXX_SOURCE_DIR}/src/charconv.cpp ${LIBCXX_SOURCE_DIR}/src/chrono.cpp @@ -20,6 +22,7 @@ ${LIBCXX_SOURCE_DIR}/src/functional.cpp ${LIBCXX_SOURCE_DIR}/src/future.cpp ${LIBCXX_SOURCE_DIR}/src/hash.cpp ${LIBCXX_SOURCE_DIR}/src/ios.cpp +${LIBCXX_SOURCE_DIR}/src/ios.instantiations.cpp ${LIBCXX_SOURCE_DIR}/src/iostream.cpp ${LIBCXX_SOURCE_DIR}/src/locale.cpp ${LIBCXX_SOURCE_DIR}/src/memory.cpp @@ -28,6 +31,7 @@ ${LIBCXX_SOURCE_DIR}/src/mutex_destructor.cpp ${LIBCXX_SOURCE_DIR}/src/new.cpp ${LIBCXX_SOURCE_DIR}/src/optional.cpp ${LIBCXX_SOURCE_DIR}/src/random.cpp +${LIBCXX_SOURCE_DIR}/src/random_shuffle.cpp ${LIBCXX_SOURCE_DIR}/src/regex.cpp ${LIBCXX_SOURCE_DIR}/src/shared_mutex.cpp ${LIBCXX_SOURCE_DIR}/src/stdexcept.cpp diff --git a/contrib/libcxxabi b/contrib/libcxxabi index 1ebc83af4c0..df8f1e727db 160000 --- a/contrib/libcxxabi +++ b/contrib/libcxxabi @@ -1 +1 @@ -Subproject commit 1ebc83af4c06dbcd56b4d166c1314a7d4c1173f9 +Subproject commit df8f1e727dbc9e2bedf2282096fa189dc3fe0076 diff --git a/contrib/libcxxabi-cmake/CMakeLists.txt b/contrib/libcxxabi-cmake/CMakeLists.txt index 3a81fe46321..9d8b94dabf0 100644 --- a/contrib/libcxxabi-cmake/CMakeLists.txt +++ b/contrib/libcxxabi-cmake/CMakeLists.txt @@ -11,7 +11,6 @@ ${LIBCXXABI_SOURCE_DIR}/src/cxa_personality.cpp ${LIBCXXABI_SOURCE_DIR}/src/stdlib_exception.cpp ${LIBCXXABI_SOURCE_DIR}/src/abort_message.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_demangle.cpp -${LIBCXXABI_SOURCE_DIR}/src/cxa_unexpected.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_exception.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_handlers.cpp ${LIBCXXABI_SOURCE_DIR}/src/cxa_exception_storage.cpp diff --git a/contrib/librdkafka-cmake/config.h.in b/contrib/librdkafka-cmake/config.h.in index c0f2e41fb9e..e5b8ba919cf 100644 --- a/contrib/librdkafka-cmake/config.h.in +++ b/contrib/librdkafka-cmake/config.h.in @@ -83,7 +83,8 @@ #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ <= 101400) #define _TTHREAD_EMULATE_TIMESPEC_GET_ #endif - +#elif defined(__FreeBSD__) +#define HAVE_PTHREAD_SETNAME_FREEBSD 1 #else // pthread_setname_gnu #define HAVE_PTHREAD_SETNAME_GNU 1 diff --git a/contrib/rocksdb b/contrib/rocksdb index 8b966f0ca29..54a0decabbc 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 8b966f0ca298fc1475bd09d9775f32dff0fdce0a +Subproject commit 54a0decabbcf4c0bb5cf7befa9c597f28289bff5 diff --git a/debian/changelog b/debian/changelog index 5ea6b472e46..3a267a83c69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (20.13.1.1) unstable; urgency=low +clickhouse (21.1.0) unstable; urgency=low * Modified source code - -- clickhouse-release Mon, 23 Nov 2020 10:29:24 +0300 + -- Alexey Milovidov Mon, 11 Jan 2021 03:51:08 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 3ef6b8c8b32..ddfe3cd177b 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.13.1.* +ARG version=21.1.0 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/images.json b/docker/images.json index e7300972b6d..6ab2d287b74 100644 --- a/docker/images.json +++ b/docker/images.json @@ -45,7 +45,8 @@ "name": "yandex/clickhouse-stateless-test", "dependent": [ "docker/test/stateful", - "docker/test/coverage" + "docker/test/coverage", + "docker/test/unit" ] }, "docker/test/stateless_pytest": { @@ -134,7 +135,9 @@ "name": "yandex/clickhouse-test-base", "dependent": [ "docker/test/stateless", - "docker/test/stateless_pytest" + "docker/test/stateless_unbundled", + "docker/test/stateless_pytest", + "docker/test/integration/base" ] }, "docker/packager/unbundled": { @@ -151,5 +154,9 @@ "docker/test/integration/kerberized_hadoop": { "name": "yandex/clickhouse-kerberized-hadoop", "dependent": [] + }, + "docker/test/sqlancer": { + "name": "yandex/clickhouse-sqlancer-test", + "dependent": [] } } diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 168ea895eec..74de1a3e9bd 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -82,7 +82,7 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \ && rm -rf cctools-port # Download toolchain for Darwin -RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz +RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz # Download toolchain for ARM # It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling. diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 85e2e8b0f04..a42789c6186 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -3,7 +3,7 @@ set -x -e mkdir -p build/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 +tar xJf MacOSX10.15.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 mkdir -p build/cmake/toolchain/linux-aarch64 tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1 diff --git a/docker/server/.dockerignore b/docker/server/.dockerignore index 468a8cafb00..d360712c18f 100644 --- a/docker/server/.dockerignore +++ b/docker/server/.dockerignore @@ -4,5 +4,5 @@ alpine-root/install/* # docs (looks useless) alpine-root/usr/share/doc/* -# packages, etc. (used by prepare.sh) -alpine-root/tgz-packages/* \ No newline at end of file +# packages, etc. (used by alpine-build.sh) +tgz-packages/* diff --git a/docker/server/.gitignore b/docker/server/.gitignore index 4081b5f124c..692758d55aa 100644 --- a/docker/server/.gitignore +++ b/docker/server/.gitignore @@ -1 +1,2 @@ -alpine-root/* \ No newline at end of file +alpine-root/* +tgz-packages/* diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index f7e107a2fc9..890aa35fe92 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.13.1.* +ARG version=21.1.0 ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index fc2756eac8c..2de834e7b9c 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -16,7 +16,7 @@ RUN addgroup clickhouse \ && chown root:clickhouse /var/log/clickhouse-server \ && chmod 775 /var/log/clickhouse-server \ && chmod +x /entrypoint.sh \ - && apk add --no-cache su-exec + && apk add --no-cache su-exec bash EXPOSE 9000 8123 9009 diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh index c9ba03f7f35..0142149b5bd 100755 --- a/docker/server/alpine-build.sh +++ b/docker/server/alpine-build.sh @@ -4,6 +4,7 @@ set -x REPO_CHANNEL="${REPO_CHANNEL:-stable}" # lts / testing / prestable / etc REPO_URL="${REPO_URL:-"https://repo.yandex.ru/clickhouse/tgz/${REPO_CHANNEL}"}" VERSION="${VERSION:-20.9.3.45}" +DOCKER_IMAGE="${DOCKER_IMAGE:-yandex/clickhouse-server}" # where original files live DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}" @@ -11,12 +12,12 @@ DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}" # we will create root for our image here CONTAINER_ROOT_FOLDER="${DOCKER_BUILD_FOLDER}/alpine-root" -# where to put downloaded tgz -TGZ_PACKAGES_FOLDER="${CONTAINER_ROOT_FOLDER}/tgz-packages" - -# clean up the root from old runs +# clean up the root from old runs, it's reconstructed each time rm -rf "$CONTAINER_ROOT_FOLDER" +mkdir -p "$CONTAINER_ROOT_FOLDER" +# where to put downloaded tgz +TGZ_PACKAGES_FOLDER="${DOCKER_BUILD_FOLDER}/tgz-packages" mkdir -p "$TGZ_PACKAGES_FOLDER" PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" ) @@ -24,7 +25,7 @@ PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" ) # download tars from the repo for package in "${PACKAGES[@]}" do - wget -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" + wget -c -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" done # unpack tars @@ -42,7 +43,7 @@ mkdir -p "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/users.d" \ "${CONTAINER_ROOT_FOLDER}/lib64" cp "${DOCKER_BUILD_FOLDER}/docker_related_config.xml" "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d/" -cp "${DOCKER_BUILD_FOLDER}/entrypoint.alpine.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh" +cp "${DOCKER_BUILD_FOLDER}/entrypoint.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh" ## get glibc components from ubuntu 20.04 and put them to expected place docker pull ubuntu:20.04 @@ -56,4 +57,5 @@ docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAIN docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" docker cp -L "${ubuntu20image}":/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" -docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull +docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "${DOCKER_IMAGE}:${VERSION}-alpine" --pull +rm -rf "$CONTAINER_ROOT_FOLDER" diff --git a/docker/server/entrypoint.alpine.sh b/docker/server/entrypoint.alpine.sh deleted file mode 100755 index f0cc62d276d..00000000000 --- a/docker/server/entrypoint.alpine.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/sh -#set -x - -DO_CHOWN=1 -if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then - DO_CHOWN=0 -fi - -CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" -CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" - -# support --user -if [ "$(id -u)" = "0" ]; then - USER=$CLICKHOUSE_UID - GROUP=$CLICKHOUSE_GID - # busybox has setuidgid & chpst buildin - gosu="su-exec $USER:$GROUP" -else - USER="$(id -u)" - GROUP="$(id -g)" - gosu="" - DO_CHOWN=0 -fi - -# set some vars -CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" - -# port is needed to check if clickhouse-server is ready for connections -HTTP_PORT="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=http_port)" - -# get CH directories locations -DATA_DIR="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=path || true)" -TMP_DIR="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=tmp_path || true)" -USER_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=user_files_path || true)" -LOG_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=logger.log || true)" -LOG_DIR="$(dirname "${LOG_PATH}" || true)" -ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=logger.errorlog || true)" -ERROR_LOG_DIR="$(dirname "${ERROR_LOG_PATH}" || true)" -FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "${CLICKHOUSE_CONFIG}" --key=format_schema_path || true)" - -CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" -CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" -CLICKHOUSE_DB="${CLICKHOUSE_DB:-}" - -for dir in "$DATA_DIR" \ - "$ERROR_LOG_DIR" \ - "$LOG_DIR" \ - "$TMP_DIR" \ - "$USER_PATH" \ - "$FORMAT_SCHEMA_PATH" -do - # check if variable not empty - [ -z "$dir" ] && continue - # ensure directories exist - if ! mkdir -p "$dir"; then - echo "Couldn't create necessary directory: $dir" - exit 1 - fi - - if [ "$DO_CHOWN" = "1" ]; then - # ensure proper directories permissions - chown -R "$USER:$GROUP" "$dir" - elif [ "$(stat -c %u "$dir")" != "$USER" ]; then - echo "Necessary directory '$dir' isn't owned by user with id '$USER'" - exit 1 - fi -done - -# if clickhouse user is defined - create it (user "default" already exists out of box) -if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ]; then - echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'" - cat < /etc/clickhouse-server/users.d/default-user.xml - - - - - - - - <${CLICKHOUSE_USER}> - default - - ::/0 - - ${CLICKHOUSE_PASSWORD} - default - - - -EOT -fi - -if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then - # Listen only on localhost until the initialization is done - $gosu /usr/bin/clickhouse-server --config-file="${CLICKHOUSE_CONFIG}" -- --listen_host=127.0.0.1 & - pid="$!" - - # check if clickhouse is ready to accept connections - # will try to send ping clickhouse via http_port (max 6 retries, with 1 sec timeout and 1 sec delay between retries) - tries=6 - while ! wget --spider -T 1 -q "http://localhost:$HTTP_PORT/ping" 2>/dev/null; do - if [ "$tries" -le "0" ]; then - echo >&2 'ClickHouse init process failed.' - exit 1 - fi - tries=$(( tries-1 )) - sleep 1 - done - - if [ -n "$CLICKHOUSE_PASSWORD" ]; then - printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" - fi - - clickhouseclient="clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD " - - # create default database, if defined - if [ -n "$CLICKHOUSE_DB" ]; then - echo "$0: create database '$CLICKHOUSE_DB'" - "$clickhouseclient" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB"; - fi - - for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sh) - if [ -x "$f" ]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - . "$f" - fi - ;; - *.sql) echo "$0: running $f"; "$clickhouseclient" < "$f" ; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "$clickhouseclient"; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo - done - - if ! kill -s TERM "$pid" || ! wait "$pid"; then - echo >&2 'Finishing of ClickHouse init process failed.' - exit 1 - fi -fi - -# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments -if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - exec $gosu /usr/bin/clickhouse-server --config-file="${CLICKHOUSE_CONFIG}" "$@" -fi - -# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image -exec "$@" diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh old mode 100644 new mode 100755 index 6048fdffe38..8a4d02a6014 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -1,7 +1,10 @@ #!/bin/bash +set -eo pipefail +shopt -s nullglob + DO_CHOWN=1 -if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then +if [ "${CLICKHOUSE_DO_NOT_CHOWN:-0}" = "1" ]; then DO_CHOWN=0 fi @@ -9,10 +12,17 @@ CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" # support --user -if [ x"$UID" == x0 ]; then +if [ "$(id -u)" = "0" ]; then USER=$CLICKHOUSE_UID GROUP=$CLICKHOUSE_GID - gosu="gosu $USER:$GROUP" + if command -v gosu &> /dev/null; then + gosu="gosu $USER:$GROUP" + elif command -v su-exec &> /dev/null; then + gosu="su-exec $USER:$GROUP" + else + echo "No gosu/su-exec detected!" + exit 1 + fi else USER="$(id -u)" GROUP="$(id -g)" @@ -23,18 +33,23 @@ fi # set some vars CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" +if ! $gosu test -f "$CLICKHOUSE_CONFIG" -a -r "$CLICKHOUSE_CONFIG"; then + echo "Configuration file '$dir' isn't readable by user with id '$USER'" + exit 1 +fi + # port is needed to check if clickhouse-server is ready for connections -HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" +HTTP_PORT="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=http_port)" # get CH directories locations -DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" -TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" -USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" -LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" -LOG_DIR="$(dirname $LOG_PATH || true)" -ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" -ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" -FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" +DATA_DIR="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=logger.log || true)" +LOG_DIR="$(dirname "$LOG_PATH" || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname "$ERROR_LOG_PATH" || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key=format_schema_path || true)" CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" @@ -58,8 +73,8 @@ do if [ "$DO_CHOWN" = "1" ]; then # ensure proper directories permissions chown -R "$USER:$GROUP" "$dir" - elif [ "$(stat -c %u "$dir")" != "$USER" ]; then - echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + elif ! $gosu test -d "$dir" -a -w "$dir" -a -r "$dir"; then + echo "Necessary directory '$dir' isn't accessible by user with id '$USER'" exit 1 fi done @@ -90,21 +105,22 @@ fi if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then # Listen only on localhost until the initialization is done - $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & + $gosu /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" -- --listen_host=127.0.0.1 & pid="$!" # check if clickhouse is ready to accept connections - # will try to send ping clickhouse via http_port (max 12 retries by default, with 1 sec delay) - if ! wget --spider --quiet --prefer-family=IPv6 --tries="${CLICKHOUSE_INIT_TIMEOUT:-12}" --waitretry=1 --retry-connrefused "http://localhost:$HTTP_PORT/ping" ; then - echo >&2 'ClickHouse init process failed.' - exit 1 - fi + # will try to send ping clickhouse via http_port (max 12 retries by default, with 1 sec timeout and 1 sec delay between retries) + tries=${CLICKHOUSE_INIT_TIMEOUT:-12} + while ! wget --spider -T 1 -q "http://127.0.0.1:$HTTP_PORT/ping" 2>/dev/null; do + if [ "$tries" -le "0" ]; then + echo >&2 'ClickHouse init process failed.' + exit 1 + fi + tries=$(( tries-1 )) + sleep 1 + done - if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then - printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" - fi - - clickhouseclient=( clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD ) + clickhouseclient=( clickhouse-client --multiquery -u "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" ) echo @@ -122,10 +138,11 @@ if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then "$f" else echo "$0: sourcing $f" + # shellcheck source=/dev/null . "$f" fi ;; - *.sql) echo "$0: running $f"; cat "$f" | "${clickhouseclient[@]}" ; echo ;; + *.sql) echo "$0: running $f"; "${clickhouseclient[@]}" < "$f" ; echo ;; *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${clickhouseclient[@]}"; echo ;; *) echo "$0: ignoring $f" ;; esac @@ -140,7 +157,7 @@ fi # if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" + exec $gosu /usr/bin/clickhouse-server --config-file="$CLICKHOUSE_CONFIG" "$@" fi # Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 8e3b5193874..2e17151b31f 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=20.13.1.* +ARG version=21.1.0 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index c782ac49d27..dd98390cdcc 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -329,6 +329,7 @@ function run_tests # nc - command not found 01601_proxy_protocol + 01622_defaults_for_url_engine ) time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 99095de60fb..4963ff0094d 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -30,3 +30,4 @@ RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-od ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile new file mode 100644 index 00000000000..38a773e65ad --- /dev/null +++ b/docker/test/sqlancer/Dockerfile @@ -0,0 +1,13 @@ +# docker build -t yandex/clickhouse-sqlancer-test . +FROM ubuntu:20.04 + +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven --yes --no-install-recommends + +RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip +RUN mkdir /sqlancer && \ + cd /sqlancer && \ + unzip /sqlancer.zip +RUN cd /sqlancer/sqlancer-master && mvn package -DskipTests + +COPY run.sh / +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/sqlancer/run.sh b/docker/test/sqlancer/run.sh new file mode 100755 index 00000000000..9e1c0c547d0 --- /dev/null +++ b/docker/test/sqlancer/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e -x + +dpkg -i package_folder/clickhouse-common-static_*.deb +dpkg -i package_folder/clickhouse-common-static-dbg_*.deb +dpkg -i package_folder/clickhouse-server_*.deb +dpkg -i package_folder/clickhouse-client_*.deb + +service clickhouse-server start && sleep 5 + +cd /sqlancer/sqlancer-master +CLICKHOUSE_AVAILABLE=true mvn -Dtest=TestClickHouse test + +cp /sqlancer/sqlancer-master/target/surefire-reports/TEST-sqlancer.dbms.TestClickHouse.xml /test_output/result.xml \ No newline at end of file diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 309328bc8e2..d9a03f84726 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -66,3 +66,6 @@ function run_tests() export -f run_tests timeout "$MAX_RUN_TIME" bash -c run_tests ||: + +tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||: +tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||: diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index d212290d553..9efe08dbf23 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -86,3 +86,4 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone COPY run.sh / CMD ["/bin/bash", "/run.sh"] + diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index 0f65649fb76..f01ed613918 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -7,3 +7,4 @@ RUN apt-get install gdb CMD service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; \ gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt + diff --git a/docs/en/development/build-cross-osx.md b/docs/en/development/build-cross-osx.md index 3e8c5419158..6f3b3a717d0 100644 --- a/docs/en/development/build-cross-osx.md +++ b/docs/en/development/build-cross-osx.md @@ -42,9 +42,9 @@ Also, we need to download macOS X SDK into the working tree. ``` bash cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz' mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +tar xJf MacOSX10.15.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 ``` ## Build ClickHouse {#build-clickhouse} diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 4f81a679b8e..80769fe9954 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -98,7 +98,9 @@ For a description of parameters, see the [CREATE query description](../../../sql - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - `storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). - `min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage). - - `max_parts_in_total` — Maximum number of parts in all partitions. + - `max_parts_in_total` — Maximum number of parts in all partitions. + - `max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. + - `min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. **Example of Sections Setting** diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index 9f96ca3fe8c..7fffa962480 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -25,10 +25,27 @@ The Distributed engine accepts parameters: - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples +Also it accept the following settings: + +- `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. + +- `fsync_directories` - do the `fsync` for directories. Guarantees that the OS refreshed directory metadata after operations related to asynchronous inserts on Distributed table (after insert, after sending the data to shard, etc). + +!!! note "Note" + + **Durability settings** (`fsync_...`): + + - Affect only asynchronous INSERTs (i.e. `insert_distributed_sync=false`) when data first stored on the initiator node disk and later asynchronously send to shards. + - May significantly decrease the inserts' performance + - Affect writing the data stored inside Distributed table folder into the **node which accepted your insert**. If you need to have guarantees of writing data to underlying MergeTree tables - see durability settings (`...fsync...`) in `system.merge_tree_settings` + Example: ``` sql Distributed(logs, default, hits[, sharding_key[, policy_name]]) +SETTINGS + fsync_after_insert=0, + fsync_directories=0; ``` Data will be read from all servers in the `logs` cluster, from the default.hits table located on every server in the cluster. diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index f3a6381aeca..c08eec61b1c 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e5f836e3b5a..d3a4d50d21c 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -844,23 +844,27 @@ Higher values will lead to higher memory usage. ## max_compress_block_size {#max-compress-block-size} -The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). If the size is reduced, the compression rate is significantly reduced, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. There usually isn’t any reason to change this setting. +The maximum size of blocks of uncompressed data before compressing for writing to a table. By default, 1,048,576 (1 MiB). Specifying smaller block size generally leads to slightly reduced compression ratio, the compression and decompression speed increases slightly due to cache locality, and memory consumption is reduced. + +!!! note "Warning" + This is an expert-level setting, and you shouldn't change it if you're just getting started with Clickhouse. Don’t confuse blocks for compression (a chunk of memory consisting of bytes) with blocks for query processing (a set of rows from a table). ## min_compress_block_size {#min-compress-block-size} -For [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md)" tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least ‘min_compress_block_size’. By default, 65,536. +For [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) tables. In order to reduce latency when processing queries, a block is compressed when writing the next mark if its size is at least `min_compress_block_size`. By default, 65,536. -The actual size of the block, if the uncompressed data is less than ‘max_compress_block_size’, is no less than this value and no less than the volume of data for one mark. +The actual size of the block, if the uncompressed data is less than `max_compress_block_size`, is no less than this value and no less than the volume of data for one mark. -Let’s look at an example. Assume that ‘index_granularity’ was set to 8192 during table creation. +Let’s look at an example. Assume that `index_granularity` was set to 8192 during table creation. We are writing a UInt32-type column (4 bytes per value). When writing 8192 rows, the total will be 32 KB of data. Since min_compress_block_size = 65,536, a compressed block will be formed for every two marks. We are writing a URL column with the String type (average size of 60 bytes per value). When writing 8192 rows, the average will be slightly less than 500 KB of data. Since this is more than 65,536, a compressed block will be formed for each mark. In this case, when reading data from the disk in the range of a single mark, extra data won’t be decompressed. -There usually isn’t any reason to change this setting. +!!! note "Warning" + This is an expert-level setting, and you shouldn't change it if you're just getting started with Clickhouse. ## max_query_size {#settings-max_query_size} @@ -2470,6 +2474,45 @@ Possible values: Default value: `0`. + +## aggregate_functions_null_for_empty {#aggregate_functions_null_for_empty} + +Enables or disables rewriting all aggregate functions in a query, adding [-OrNull](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-ornull) suffix to them. Enable it for SQL standard compatibility. +It is implemented via query rewrite (similar to [count_distinct_implementation](#settings-count_distinct_implementation) setting) to get consistent results for distributed queries. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. + +**Example** + +Consider the following query with aggregate functions: +```sql +SELECT + SUM(-1), + MAX(0) +FROM system.one +WHERE 0 +``` + +With `aggregate_functions_null_for_empty = 0` it would produce: +```text +┌─SUM(-1)─┬─MAX(0)─┐ +│ 0 │ 0 │ +└─────────┴────────┘ +``` + +With `aggregate_functions_null_for_empty = 1` the result would be: +```text +┌─SUMOrNull(-1)─┬─MAXOrNull(0)─┐ +│ NULL │ NULL │ +└───────────────┴──────────────┘ +``` + + ## union_default_mode {#union-default-mode} Sets a mode for combining `SELECT` query results. The setting is only used when shared with [UNION](../../sql-reference/statements/select/union.md) without explicitly specifying the `UNION ALL` or `UNION DISTINCT`. @@ -2484,6 +2527,7 @@ Default value: `''`. See examples in [UNION](../../sql-reference/statements/select/union.md). + ## data_type_default_nullable {#data_type_default_nullable} Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). @@ -2495,6 +2539,7 @@ Possible values: Default value: `0`. + ## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold} Enables special logic to perform merges on replicas. diff --git a/docs/en/operations/system-tables/distributed_ddl_queue.md b/docs/en/operations/system-tables/distributed_ddl_queue.md new file mode 100644 index 00000000000..643bdee6def --- /dev/null +++ b/docs/en/operations/system-tables/distributed_ddl_queue.md @@ -0,0 +1,67 @@ +# system.distributed_ddl_queue {#system_tables-distributed_ddl_queue} + +Contains information about distributed ddl queries (ON CLUSTER queries) that were executed on a cluster. + +Columns: + +- `entry` ([String](../../sql-reference/data-types/string.md)) - Query id. +- `host_name` ([String](../../sql-reference/data-types/string.md)) - Hostname. +- `host_address` ([String](../../sql-reference/data-types/string.md)) - IP address that the Hostname resolves to. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) - Host Port. +- `status` ([Enum](../../sql-reference/data-types/enum.md)) - Stats of the query. +- `cluster` ([String](../../sql-reference/data-types/string.md)) - Cluster name. +- `query` ([String](../../sql-reference/data-types/string.md)) - Query executed. +- `initiator` ([String](../../sql-reference/data-types/string.md)) - Nod that executed the query. +- `query_start_time` ([Date](../../sql-reference/data-types/date.md)) — Query start time. +- `query_finish_time` ([Date](../../sql-reference/data-types/date.md)) — Query finish time. +- `query_duration_ms` ([UInt64](../../sql-reference/data-types/datetime64.md)) — Duration of query execution in milliseconds. +- `exception_code` ([Enum](../../sql-reference/data-types/enum.md)) - Exception code from ZooKeeper. + + +**Example** + +``` sql +SELECT * +FROM system.distributed_ddl_queue +WHERE cluster = 'test_cluster' +LIMIT 2 +FORMAT Vertical + +Query id: f544e72a-6641-43f1-836b-24baa1c9632a + +Row 1: +────── +entry: query-0000000000 +host_name: clickhouse01 +host_address: 172.23.0.11 +port: 9000 +status: Finished +cluster: test_cluster +query: CREATE DATABASE test_db UUID '4a82697e-c85e-4e5b-a01e-a36f2a758456' ON CLUSTER test_cluster +initiator: clickhouse01:9000 +query_start_time: 2020-12-30 13:07:51 +query_finish_time: 2020-12-30 13:07:51 +query_duration_ms: 6 +exception_code: ZOK + +Row 2: +────── +entry: query-0000000000 +host_name: clickhouse02 +host_address: 172.23.0.12 +port: 9000 +status: Finished +cluster: test_cluster +query: CREATE DATABASE test_db UUID '4a82697e-c85e-4e5b-a01e-a36f2a758456' ON CLUSTER test_cluster +initiator: clickhouse01:9000 +query_start_time: 2020-12-30 13:07:51 +query_finish_time: 2020-12-30 13:07:51 +query_duration_ms: 6 +exception_code: ZOK + +2 rows in set. Elapsed: 0.025 sec. +``` + + +[Original article](https://clickhouse.tech/docs/en/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) + \ No newline at end of file diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 7a9e386d419..5dc23aee686 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,33 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. + +System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: + +- `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`. +- `table`: table to insert data. +- `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression. +- `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md) expression. +- `flush_interval_milliseconds`: interval of flushing data to disk. +- `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit. + +An example: + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables. diff --git a/docs/en/operations/system-tables/table_engines.md b/docs/en/operations/system-tables/table_engines.md index 4ca1fc657ee..30122cb133e 100644 --- a/docs/en/operations/system-tables/table_engines.md +++ b/docs/en/operations/system-tables/table_engines.md @@ -11,6 +11,7 @@ This table contains the following columns (the column type is shown in brackets) - `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`. - `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](../../engines/table-engines/mergetree-family/replication.md). - `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. +- `supports_parallel_insert` (UInt8) — Flag that indicates if table engine supports parallel insert (see [`max_insert_threads`](../../operations/settings/settings.md#settings-max-insert-threads) setting). Example: @@ -21,11 +22,11 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') ``` ``` text -┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ -│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ -└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ ``` **See also** diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index c1dd11b1cc6..20486ebbcc8 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -55,10 +55,10 @@ In this case, ClickHouse can reload the dictionary earlier if the dictionary con When upgrading the dictionaries, the ClickHouse server applies different logic depending on the type of [source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md): - For a text file, it checks the time of modification. If the time differs from the previously recorded time, the dictionary is updated. -- For MyISAM tables, the time of modification is checked using a `SHOW TABLE STATUS` query. +- For MySQL source, the time of modification is checked using a `SHOW TABLE STATUS` query (in case of MySQL 8 you need to disable meta-information caching in MySQL by `set global information_schema_stats_expiry=0`. - Dictionaries from other sources are updated every time by default. -For MySQL (InnoDB), ODBC and ClickHouse sources, you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: +For other sources (ODBC, ClickHouse, etc), you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: - The dictionary table must have a field that always changes when the source data is updated. - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `` field in the settings for the [source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md). diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index e86ac7fe105..b86f0d6bc4e 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -583,7 +583,7 @@ Example of settings: or ``` sql -SOURCE(MONGO( +SOURCE(MONGODB( host 'localhost' port 27017 user '' diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index ee4449d46fa..dc7727bdfd8 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1290,22 +1290,65 @@ Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference ## arrayMin(\[func,\] arr1, …) {#array-min} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the min of the `func` values. If the function is omitted, it just returns the min of the array elements. Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMin([1, 2, 4]) AS res +┌─res─┐ +│ 1 │ +└─────┘ + + +SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -4 │ +└─────┘ +``` + ## arrayMax(\[func,\] arr1, …) {#array-max} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the max of the `func` values. If the function is omitted, it just returns the max of the array elements. Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMax([1, 2, 4]) AS res +┌─res─┐ +│ 4 │ +└─────┘ + + +SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -1 │ +└─────┘ +``` + ## arraySum(\[func,\] arr1, …) {#array-sum} Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements. Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arraySum([2,3]) AS res +┌─res─┐ +│ 5 │ +└─────┘ + + +SELECT arraySum(x -> x*x, [2, 3]) AS res +┌─res─┐ +│ 13 │ +└─────┘ +``` + + ## arrayAvg(\[func,\] arr1, …) {#array-avg} Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements. diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 825399422d1..0ea4d4b3dc5 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -23,6 +23,7 @@ The following actions are supported: - [CLEAR COLUMN](#alter_clear-column) — Resets column values. - [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. - [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. +- [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties. These actions are described in detail below. @@ -145,6 +146,26 @@ The `ALTER` query is atomic. For MergeTree tables it is also lock-free. The `ALTER` query for changing columns is replicated. The instructions are saved in ZooKeeper, then each replica applies them. All `ALTER` queries are run in the same order. The query waits for the appropriate actions to be completed on the other replicas. However, a query to change columns in a replicated table can be interrupted, and all actions will be performed asynchronously. +## MODIFY COLUMN REMOVE {#modify-remove} + +Removes one of the column properties: `DEFAULT`, `ALIAS`, `MATERIALIZED`, `CODEC`, `COMMENT`, `TTL`. + +Syntax: + +```sql +ALTER TABLE table_name MODIFY column_name REMOVE property; +``` + +**Example** + +```sql +ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; +``` + +## See Also + +- [REMOVE TTL](ttl.md). + ## Limitations {#alter-query-limitations} The `ALTER` query lets you create and delete separate elements (columns) in nested data structures, but not whole nested data structures. To add a nested data structure, you can add columns with a name like `name.nested_name` and the type `Array(T)`. A nested data structure is equivalent to multiple array columns with a name that has the same prefix before the dot. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 2d46ee609f1..42396223b86 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -286,7 +286,7 @@ ALTER TABLE mt DELETE IN PARTITION 2 WHERE p = 2; You can specify the partition expression in `ALTER ... PARTITION` queries in different ways: - As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`. -- As the expression from the table column. Constants and constant expressions are supported. For example, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. +- As a tuple of expressions or constants that matches (in types) the table partitioning keys tuple. In the case of a single element partitioning key, the expression should be wrapped in the `tuple (...)` function. For example, `ALTER TABLE visits DETACH PARTITION tuple(toYYYYMM(toDate('2019-01-25')))`. - Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`. - In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](../../../operations/system-tables/detached_parts.md#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md index 05040e38ccf..5331afdb2f8 100644 --- a/docs/en/sql-reference/statements/alter/ttl.md +++ b/docs/en/sql-reference/statements/alter/ttl.md @@ -3,10 +3,83 @@ toc_priority: 44 toc_title: TTL --- -### Manipulations with Table TTL {#manipulations-with-table-ttl} +# Manipulations with Table TTL {#manipulations-with-table-ttl} + +## MODIFY TTL {#modify-ttl} You can change [table TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) with a request of the following form: ``` sql -ALTER TABLE table-name MODIFY TTL ttl-expression +ALTER TABLE table_name MODIFY TTL ttl_expression; ``` + +## REMOVE TTL {#remove-ttl} + +TTL-property can be removed from table with the following query: + +```sql +ALTER TABLE table_name REMOVE TTL +``` + +**Example** + +Consider the table with table `TTL`: + +```sql +CREATE TABLE table_with_ttl +( + event_time DateTime, + UserID UInt64, + Comment String +) +ENGINE MergeTree() +ORDER BY tuple() +TTL event_time + INTERVAL 3 MONTH; +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO table_with_ttl VALUES (now(), 1, 'username1'); + +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +``` + +Run `OPTIMIZE` to force `TTL` cleanup: + +```sql +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl FORMAT PrettyCompact; +``` +Second row was deleted from table. + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +Now remove table `TTL` with the following query: + +```sql +ALTER TABLE table_with_ttl REMOVE TTL; +``` + +Re-insert the deleted row and force the `TTL` cleanup again with `OPTIMIZE`: + +```sql +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl FORMAT PrettyCompact; +``` + +The `TTL` is no longer there, so the second row is not deleted: + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +│ 2020-08-11 12:44:57 │ 2 │ username2 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +### See Also + +- More about the [TTL-expression](../../../sql-reference/statements/create/table#ttl-expression). +- Modify column [with TTL](../../../sql-reference/statements/alter/column#alter_modify-column). diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index ae5e074fd15..7acf4018812 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -13,9 +13,7 @@ Basic query format: INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... ``` -You can specify a list of columns to insert using the `(c1, c2, c3)` or `COLUMNS(c1,c2,c3)` syntax. - -Instead of listing all the required columns you can use the `(* EXCEPT(column_list))` syntax. +You can specify a list of columns to insert using the `(c1, c2, c3)`. You can also use an expression with column [matcher](../../sql-reference/statements/select/index.md#asterisk) such as `*` and/or [modifiers](../../sql-reference/statements/select/index.md#select-modifiers) such as [APPLY](../../sql-reference/statements/select/index.md#apply-modifier), [EXCEPT](../../sql-reference/statements/select/index.md#apply-modifier), [REPLACE](../../sql-reference/statements/select/index.md#replace-modifier). For example, consider the table: @@ -23,9 +21,8 @@ For example, consider the table: SHOW CREATE insert_select_testtable; ``` -``` -┌─statement────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ CREATE TABLE insert_select_testtable +```text +CREATE TABLE insert_select_testtable ( `a` Int8, `b` String, @@ -33,8 +30,7 @@ SHOW CREATE insert_select_testtable; ) ENGINE = MergeTree() ORDER BY a -SETTINGS index_granularity = 8192 │ -└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +SETTINGS index_granularity = 8192 ``` ``` sql diff --git a/docs/en/sql-reference/statements/select/all.md b/docs/en/sql-reference/statements/select/all.md new file mode 100644 index 00000000000..5e0de4c142b --- /dev/null +++ b/docs/en/sql-reference/statements/select/all.md @@ -0,0 +1,21 @@ +--- +toc_title: ALL +--- + +# ALL Clause {#select-all} + +`SELECT ALL` is identical to `SELECT` without `DISTINCT`. + +- If `ALL` specified, ignore it. +- If both `ALL` and `DISTINCT` specified, exception will be thrown. + +`ALL` can also be specified inside aggregate function with the same effect(noop), for instance: + +```sql +SELECT sum(ALL number) FROM numbers(10); +``` +equals to + +```sql +SELECT sum(number) FROM numbers(10); +``` diff --git a/docs/en/sql-reference/statements/select/distinct.md b/docs/en/sql-reference/statements/select/distinct.md index 71365b18855..87154cba05a 100644 --- a/docs/en/sql-reference/statements/select/distinct.md +++ b/docs/en/sql-reference/statements/select/distinct.md @@ -18,10 +18,6 @@ It is possible to obtain the same result by applying [GROUP BY](../../../sql-ref - When [ORDER BY](../../../sql-reference/statements/select/order-by.md) is omitted and [LIMIT](../../../sql-reference/statements/select/limit.md) is defined, the query stops running immediately after the required number of different rows has been read. - Data blocks are output as they are processed, without waiting for the entire query to finish running. -## Limitations {#limitations} - -`DISTINCT` is not supported if `SELECT` has at least one array column. - ## Examples {#examples} ClickHouse supports using the `DISTINCT` and `ORDER BY` clauses for different columns in one query. The `DISTINCT` clause is executed before the `ORDER BY` clause. diff --git a/docs/es/development/build-cross-osx.md b/docs/es/development/build-cross-osx.md index 35cf01a8ce3..d00e57c5d31 100644 --- a/docs/es/development/build-cross-osx.md +++ b/docs/es/development/build-cross-osx.md @@ -44,9 +44,9 @@ Además, necesitamos descargar macOS X SDK en el árbol de trabajo. ``` bash cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz' mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +tar xJf MacOSX10.15.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 ``` # Construir ClickHouse {#build-clickhouse} diff --git a/docs/es/interfaces/third-party/client-libraries.md b/docs/es/interfaces/third-party/client-libraries.md index 818bdbbc6f0..b61ab1a5d9c 100644 --- a/docs/es/interfaces/third-party/client-libraries.md +++ b/docs/es/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/fr/development/build-cross-osx.md b/docs/fr/development/build-cross-osx.md index 4fa97a65c7b..a15d4650edb 100644 --- a/docs/fr/development/build-cross-osx.md +++ b/docs/fr/development/build-cross-osx.md @@ -44,9 +44,9 @@ En outre, nous devons télécharger macOS X SDK dans l'arbre de travail. ``` bash cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz' mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +tar xJf MacOSX10.15.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 ``` # Construire ClickHouse {#build-clickhouse} diff --git a/docs/fr/interfaces/third-party/client-libraries.md b/docs/fr/interfaces/third-party/client-libraries.md index 5a86d12a09c..7949aa1d7cf 100644 --- a/docs/fr/interfaces/third-party/client-libraries.md +++ b/docs/fr/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [clickhouse-chauffeur](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2 / phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov / clickhouse-PHP-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/ja/development/build-cross-osx.md b/docs/ja/development/build-cross-osx.md index 5d7dee09827..7444a384e05 100644 --- a/docs/ja/development/build-cross-osx.md +++ b/docs/ja/development/build-cross-osx.md @@ -45,9 +45,9 @@ make install ``` bash cd ClickHouse -wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz' +wget 'https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz' mkdir -p build-darwin/cmake/toolchain/darwin-x86_64 -tar xJf MacOSX10.14.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 +tar xJf MacOSX10.15.sdk.tar.xz -C build-darwin/cmake/toolchain/darwin-x86_64 --strip-components=1 ``` # ビルドClickHouse {#build-clickhouse} diff --git a/docs/ja/interfaces/third-party/client-libraries.md b/docs/ja/interfaces/third-party/client-libraries.md index ffe7b641c38..c7bd368bc4c 100644 --- a/docs/ja/interfaces/third-party/client-libraries.md +++ b/docs/ja/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" - [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-クライアント](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index 53c007e078f..de8fba1bc4b 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -133,7 +133,7 @@ ClickHouse имеет сильную типизацию, поэтому нет ## Агрегатные функции {#aggregate-functions} -Агрегатные функции - это функции с состоянием (stateful). Они накапливают переданные значения в некотором состоянии и позволяют получать результаты из этого состояния. Работа с ними осуществляется с помощью интерфейса `IAggregateFunction`. Состояния могут быть как простыми (состояние для `AggregateFunctionCount` это всего лишь один человек `UInt64` значение) так и довольно сложными (состояние `AggregateFunctionUniqCombined` представляет собой комбинацию линейного массива, хэш-таблицы и вероятностной структуры данных `HyperLogLog`). +Агрегатные функции - это функции с состоянием (stateful). Они накапливают переданные значения в некотором состоянии и позволяют получать результаты из этого состояния. Работа с ними осуществляется с помощью интерфейса `IAggregateFunction`. Состояния могут быть как простыми (состояние для `AggregateFunctionCount` это всего лишь одна переменная типа `UInt64`) так и довольно сложными (состояние `AggregateFunctionUniqCombined` представляет собой комбинацию линейного массива, хэш-таблицы и вероятностной структуры данных `HyperLogLog`). Состояния распределяются в `Arena` (пул памяти) для работы с несколькими состояниями при выполнении запроса `GROUP BY` высокой кардинальности (большим числом уникальных данных). Состояния могут иметь нетривиальный конструктор и деструктор: например, сложные агрегатные состояния могут сами аллоцировать дополнительную память. Потому к созданию и уничтожению состояний, правильной передаче владения и порядку уничтожения следует уделять больше внимание. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 501f7732745..c7bd64c4ab1 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -77,17 +77,19 @@ ORDER BY expr - `SETTINGS` — дополнительные параметры, регулирующие поведение `MergeTree` (необязательные): - - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). - - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). + - `index_granularity` — максимальное количество строк данных между засечками индекса. По умолчанию — 8192. Смотрите [Хранение данных](#mergetree-data-storage). + - `index_granularity_bytes` — максимальный размер гранул данных в байтах. По умолчанию — 10Mb. Чтобы ограничить размер гранул только количеством строк, установите значение 0 (не рекомендовано). Смотрите [Хранение данных](#mergetree-data-storage). - `min_index_granularity_bytes` — минимально допустимый размер гранул данных в байтах. Значение по умолчанию — 1024b. Для обеспечения защиты от случайного создания таблиц с очень низким значением `index_granularity_bytes`. Смотрите [Хранение данных](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. - - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". - - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. - - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). - - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных, указывающей за последний байт. По умолчанию — 1. Не отключайте её. - - `merge_max_block_size` — максимальное количество строк в блоке для операций слияния. Значение по умолчанию: 8192. - - `storage_policy` — политика хранения данных. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](#table_engine-mergetree-multiple-volumes). + - `enable_mixed_granularity_parts` — включает или выключает переход к ограничению размера гранул с помощью настройки `index_granularity_bytes`. Настройка `index_granularity_bytes` улучшает производительность ClickHouse при выборке данных из таблиц с большими (десятки и сотни мегабайтов) строками. Если у вас есть таблицы с большими строками, можно включить эту настройку, чтобы повысить эффективность запросов `SELECT`. + - `use_minimalistic_part_header_in_zookeeper` — Способ хранения заголовков кусков данных в ZooKeeper. Если `use_minimalistic_part_header_in_zookeeper = 1`, то ZooKeeper хранит меньше данных. Подробнее читайте в [описании настройки](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) в разделе "Конфигурационные параметры сервера". + - `min_merge_bytes_to_use_direct_io` — минимальный объём данных при слиянии, необходимый для прямого (небуферизованного) чтения/записи (direct I/O) на диск. При слиянии частей данных ClickHouse вычисляет общий объём хранения всех данных, подлежащих слиянию. Если общий объём хранения всех данных для чтения превышает `min_bytes_to_use_direct_io` байт, тогда ClickHouse использует флаг `O_DIRECT` при чтении данных с диска. Если `min_merge_bytes_to_use_direct_io = 0`, тогда прямой ввод-вывод отключен. Значение по умолчанию: `10 * 1024 * 1024 * 1024` байтов. + - `merge_with_ttl_timeout` — минимальное время в секундах перед повторным слиянием с TTL. По умолчанию — 86400 (1 день). + - `write_final_mark` — включает или отключает запись последней засечки индекса в конце куска данных, указывающей за последний байт. По умолчанию — 1. Не отключайте её. + - `merge_max_block_size` — максимальное количество строк в блоке для операций слияния. Значение по умолчанию: 8192. + - `storage_policy` — политика хранения данных. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](#table_engine-mergetree-multiple-volumes). - `min_bytes_for_wide_part`, `min_rows_for_wide_part` — минимальное количество байт/строк в куске данных для хранения в формате `Wide`. Можно задать одну или обе настройки или не задавать ни одной. Подробнее см. в разделе [Хранение данных](#mergetree-data-storage). + - `max_compress_block_size` — максимальный размер блоков несжатых данных перед сжатием для записи в таблицу. Вы также можете задать этот параметр в глобальных настройках (смотрите [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size)). Настройка, которая задается при создании таблицы, имеет более высокий приоритет, чем глобальная. + - `min_compress_block_size` — минимальный размер блоков несжатых данных, необходимых для сжатия при записи следующей засечки. Вы также можете задать этот параметр в глобальных настройках (смотрите [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size)). Настройка, которая задается при создании таблицы, имеет более высокий приоритет, чем глобальная. **Пример задания секций** diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index c07aab5826c..26e05b02509 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index c4f5cdaf2ca..aa549fc5776 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -811,23 +811,27 @@ log_query_threads=1 ## max_compress_block_size {#max-compress-block-size} -Максимальный размер блоков не сжатых данных перед сжатием при записи в таблицу. По умолчанию - 1 048 576 (1 MiB). При уменьшении размера, незначительно уменьшается коэффициент сжатия, незначительно возрастает скорость сжатия и разжатия за счёт кэш-локальности, и уменьшается потребление оперативки. Как правило, не имеет смысла менять эту настройку. +Максимальный размер блоков несжатых данных перед сжатием при записи в таблицу. По умолчанию - 1 048 576 (1 MiB). При уменьшении размера, незначительно уменьшается коэффициент сжатия, незначительно возрастает скорость сжатия и разжатия за счёт кэш-локальности, и уменьшается потребление оперативной памяти. + +!!! note "Предупреждение" + Эта настройка экспертного уровня, не используйте ее, если вы только начинаете работать с Clickhouse. Не путайте блоки для сжатия (кусок памяти, состоящий из байт) и блоки для обработки запроса (пачка строк из таблицы). ## min_compress_block_size {#min-compress-block-size} -Для таблиц типа [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). В целях уменьшения задержек при обработке запросов, блок сжимается при записи следующей засечки, если его размер не меньше min_compress_block_size. По умолчанию - 65 536. +Для таблиц типа [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). В целях уменьшения задержек при обработке запросов, блок сжимается при записи следующей засечки, если его размер не меньше `min_compress_block_size`. По умолчанию - 65 536. -Реальный размер блока, если несжатых данных меньше max_compress_block_size, будет не меньше этого значения и не меньше объёма данных на одну засечку. +Реальный размер блока, если несжатых данных меньше `max_compress_block_size`, будет не меньше этого значения и не меньше объёма данных на одну засечку. -Рассмотрим пример. Пусть index_granularity, указанная при создании таблицы - 8192. +Рассмотрим пример. Пусть `index_granularity`, указанная при создании таблицы - 8192. -Пусть мы записываем столбец типа UInt32 (4 байта на значение). При записи 8192 строк, будет всего 32 КБ данных. Так как min_compress_block_size = 65 536, сжатый блок будет сформирован на каждые две засечки. +Пусть мы записываем столбец типа UInt32 (4 байта на значение). При записи 8192 строк, будет всего 32 КБ данных. Так как `min_compress_block_size` = 65 536, сжатый блок будет сформирован на каждые две засечки. Пусть мы записываем столбец URL типа String (средний размер - 60 байт на значение). При записи 8192 строк, будет, в среднем, чуть меньше 500 КБ данных. Так как это больше 65 536 строк, то сжатый блок будет сформирован на каждую засечку. В этом случае, при чтении с диска данных из диапазона в одну засечку, не будет разжато лишних данных. -Как правило, не имеет смысла менять эту настройку. +!!! note "Предупреждение" + Эта настройка экспертного уровня, не используйте ее, если вы только начинаете работать с Clickhouse. ## max_query_size {#settings-max_query_size} @@ -2339,6 +2343,45 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Значение по умолчанию: `0`. + +## aggregate_functions_null_for_empty {#aggregate_functions_null_for_empty} + +Включает или отключает перезапись всех агрегатных функций в запросе, с добавлением к ним суффикса [-OrNull](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-ornull). Включите для совместимости со стандартом SQL. +Реализуется с помощью перезаписи запросов (аналогично настройке [count_distinct_implementation](#settings-count_distinct_implementation)), чтобы получить согласованные результаты для распределенных запросов. + +Возможные значения: + +- 0 — выключена. +- 1 — включена. + +Значение по умолчанию: 0. + +**Пример** + +Рассмотрим запрос с агрегирующими функциями: +```sql +SELECT + SUM(-1), + MAX(0) +FROM system.one +WHERE 0 +``` + +Результат запроса с настройкой `aggregate_functions_null_for_empty = 0`: +```text +┌─SUM(-1)─┬─MAX(0)─┐ +│ 0 │ 0 │ +└─────────┴────────┘ +``` + +Результат запроса с настройкой `aggregate_functions_null_for_empty = 1`: +```text +┌─SUMOrNull(-1)─┬─MAXOrNull(0)─┐ +│ NULL │ NULL │ +└───────────────┴──────────────┘ +``` + + ## union_default_mode {#union-default-mode} Устанавливает режим объединения результатов `SELECT` запросов. Настройка используется только при совместном использовании с [UNION](../../sql-reference/statements/select/union.md) без явного указания `UNION ALL` или `UNION DISTINCT`. @@ -2353,6 +2396,7 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Смотрите примеры в разделе [UNION](../../sql-reference/statements/select/union.md). + ## execute_merges_on_single_replica_time_threshold {#execute-merges-on-single-replica-time-threshold} Включает особую логику выполнения слияний на репликах. diff --git a/docs/ru/operations/system-tables/metrics.md b/docs/ru/operations/system-tables/metrics.md index 39ebab58624..db4016687d6 100644 --- a/docs/ru/operations/system-tables/metrics.md +++ b/docs/ru/operations/system-tables/metrics.md @@ -8,7 +8,7 @@ - `value` ([Int64](../../sql-reference/data-types/int-uint.md)) — значение метрики. - `description` ([String](../../sql-reference/data-types/string.md)) — описание метрики. -Список поддержанных метрик смотрите в файле [src/Common/CurrentMetrics.cpp](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp). +Список поддерживаемых метрик смотрите в файле [src/Common/CurrentMetrics.cpp](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/CurrentMetrics.cpp). **Пример** diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 4dbf4be9f96..ec0fb8e0ee5 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -54,10 +54,10 @@ LIFETIME(MIN 300 MAX 360) При обновлении словарей сервер ClickHouse применяет различную логику в зависимости от типа [источника](external-dicts-dict-sources.md): > - У текстового файла проверяется время модификации. Если время изменилось по отношению к запомненному ранее, то словарь обновляется. -> - Для таблиц типа MyISAM, время модификации проверяется запросом `SHOW TABLE STATUS`. +> - Для MySQL источника, время модификации проверяется запросом `SHOW TABLE STATUS` (для MySQL 8 необходимо отключить кеширование мета-информации в MySQL `set global information_schema_stats_expiry=0`. > - Словари из других источников по умолчанию обновляются каждый раз. -Для источников MySQL (InnoDB), ODBC и ClickHouse можно настроить запрос, который позволит обновлять словари только в случае их фактического изменения, а не каждый раз. Чтобы это сделать необходимо выполнить следующие условия/действия: +Для других источников (ODBC, ClickHouse и т.д.) можно настроить запрос, который позволит обновлять словари только в случае их фактического изменения, а не каждый раз. Чтобы это сделать необходимо выполнить следующие условия/действия: > - В таблице словаря должно быть поле, которое гарантированно изменяется при обновлении данных в источнике. > - В настройках источника указывается запрос, который получает изменяющееся поле. Результат запроса сервер ClickHouse интерпретирует как строку и если эта строка изменилась по отношению к предыдущему состоянию, то словарь обновляется. Запрос следует указывать в поле `` настроек [источника](external-dicts-dict-sources.md). diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 7fe1b4e4e78..7a394e2f684 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -12,6 +12,7 @@ toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u - [CLEAR COLUMN](#alter_clear-column) — сбрасывает все значения в столбце для заданной партиции; - [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; - [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL. +- [MODIFY COLUMN REMOVE](#modify-remove) — удаляет какое-либо из свойств столбца. Подробное описание для каждого действия приведено ниже. @@ -135,6 +136,28 @@ ALTER TABLE visits MODIFY COLUMN browser Array(String) Запрос `ALTER` на изменение столбцов реплицируется. Соответствующие инструкции сохраняются в ZooKeeper, и затем каждая реплика их применяет. Все запросы `ALTER` выполняются в одном и том же порядке. Запрос ждёт выполнения соответствующих действий на всех репликах. Но при этом, запрос на изменение столбцов в реплицируемой таблице можно прервать, и все действия будут осуществлены асинхронно. +## MODIFY COLUMN REMOVE {#modify-remove} + +Удаляет какое-либо из свойств столбца: `DEFAULT`, `ALIAS`, `MATERIALIZED`, `CODEC`, `COMMENT`, `TTL`. + +Синтаксис: + +```sql +ALTER TABLE table_name MODIFY column_name REMOVE property; +``` + +**Пример** + +Удаление свойства TTL: + +```sql +ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; +``` + +## Смотрите также + +- [REMOVE TTL](ttl.md). + ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки. diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md index b43340467fc..8776c70c89e 100644 --- a/docs/ru/sql-reference/statements/alter/partition.md +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -288,7 +288,7 @@ ALTER TABLE mt DELETE IN PARTITION 2 WHERE p = 2; Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: - Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../../../operations/system-tables/parts.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. -- Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. +- Кортеж из выражений или констант, совпадающий (в типах) с кортежем партиционирования. В случае ключа партиционирования из одного элемента, выражение следует обернуть в функцию `tuple(...)`. Например, `ALTER TABLE visits DETACH PARTITION tuple(toYYYYMM(toDate('2019-01-25')))`. - Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. - Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../../../operations/system-tables/detached_parts.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. @@ -306,4 +306,4 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/partition/) \ No newline at end of file +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/partition/) diff --git a/docs/ru/sql-reference/statements/alter/ttl.md b/docs/ru/sql-reference/statements/alter/ttl.md index 5e5f47c22e3..5721ec9cf27 100644 --- a/docs/ru/sql-reference/statements/alter/ttl.md +++ b/docs/ru/sql-reference/statements/alter/ttl.md @@ -5,10 +5,82 @@ toc_title: TTL # Манипуляции с TTL таблицы {#manipuliatsii-s-ttl-tablitsy} +## MODIFY TTL {#modify-ttl} + Вы можете изменить [TTL для таблицы](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl) запросом следующего вида: ``` sql ALTER TABLE table-name MODIFY TTL ttl-expression ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/ttl/) \ No newline at end of file +## REMOVE TTL {#remove-ttl} + +Удалить табличный TTL можно запросом следующего вида: + +```sql +ALTER TABLE table_name REMOVE TTL +``` + +**Пример** + +Создадим таблицу с табличным `TTL` и заполним её данными: + +```sql +CREATE TABLE table_with_ttl +( + event_time DateTime, + UserID UInt64, + Comment String +) +ENGINE MergeTree() +ORDER BY tuple() +TTL event_time + INTERVAL 3 MONTH; +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO table_with_ttl VALUES (now(), 1, 'username1'); + +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +``` + +Выполним `OPTIMIZE` для принудительной очистки по `TTL`: + +```sql +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl; +``` +В результате видно, что вторая строка удалена. + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +Удаляем табличный `TTL`: + +```sql +ALTER TABLE table_with_ttl REMOVE TTL; +``` + +Заново вставляем удаленную строку и снова принудительно запускаем очистку по `TTL` с помощью `OPTIMIZE`: + +```sql +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl; +``` + +`TTL` больше нет, поэтому данные не удаляются: + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +│ 2020-08-11 12:44:57 │ 2 │ username2 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +### Смотрите также + +- Подробнее о [свойстве TTL](../../../engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl). + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/ttl/) diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 0d38be81ac6..e3cea4aecc5 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -13,9 +13,7 @@ toc_title: INSERT INTO INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... ``` -Вы можете указать список столбцов для вставки, используя следующий синтаксис: `(c1, c2, c3)` или `COLUMNS(c1,c2,c3)`. - -Можно не перечислять все необходимые столбцы, а использовать синтаксис `(* EXCEPT(column_list))`. +Вы можете указать список столбцов для вставки, используя синтаксис `(c1, c2, c3)`. Также можно использовать выражение cо [звездочкой](../../sql-reference/statements/select/index.md#asterisk) и/или модификаторами, такими как `APPLY`, `EXCEPT`, `REPLACE`. В качестве примера рассмотрим таблицу: diff --git a/docs/ru/sql-reference/statements/select/distinct.md b/docs/ru/sql-reference/statements/select/distinct.md index 9d620079f6b..6616f421486 100644 --- a/docs/ru/sql-reference/statements/select/distinct.md +++ b/docs/ru/sql-reference/statements/select/distinct.md @@ -18,10 +18,6 @@ toc_title: DISTINCT - Когда секция [ORDER BY](order-by.md) опущена, а секция [LIMIT](limit.md) присутствует, запрос прекращает выполнение сразу после считывания необходимого количества различных строк. - Блоки данных выводятся по мере их обработки, не дожидаясь завершения выполнения всего запроса. -## Ограничения {#limitations} - -`DISTINCT` не поддерживается, если `SELECT` имеет по крайней мере один столбец-массив. - ## Примеры {#examples} ClickHouse поддерживает использование секций `DISTINCT` и `ORDER BY` для разных столбцов в одном запросе. Секция `DISTINCT` выполняется до секции `ORDER BY`. diff --git a/docs/zh/development/build-cross-osx.md b/docs/zh/development/build-cross-osx.md index 6a941f4e834..ee3335553d3 100644 --- a/docs/zh/development/build-cross-osx.md +++ b/docs/zh/development/build-cross-osx.md @@ -33,8 +33,8 @@ cd cctools-port/cctools make install cd ${CCTOOLS} -wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz -tar xJf MacOSX10.14.sdk.tar.xz +wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz +tar xJf MacOSX10.15.sdk.tar.xz ``` # 编译 ClickHouse {#bian-yi-clickhouse} @@ -46,7 +46,7 @@ CC=clang-8 CXX=clang++-8 cmake . -Bbuild-osx -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar \ -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib \ -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld \ - -DSDK_PATH=${CCTOOLS}/MacOSX10.14.sdk + -DSDK_PATH=${CCTOOLS}/MacOSX10.15.sdk ninja -C build-osx ``` diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index e94eb8bcfc0..e2412f2b8de 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试 - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index fcf6741761b..56067bc5057 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -22,9 +22,35 @@ toc_title: "\u7CFB\u7EDF\u8868" 大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。 -与其他系统表不同，系统表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) 由 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 +与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 -默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 +用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下: + +- `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。 +- `table`: 系统日志表名。 +- `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。 +- `ttl`: 指定系统日志表TTL选项。 +- `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。 +- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。 + +一个配置定义的例子如下： + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` + +默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 ## 系统指标的来源 {#system-tables-sources-of-system-metrics} diff --git a/docs/zh/sql-reference/data-types/float.md b/docs/zh/sql-reference/data-types/float.md index e0924eb0178..0fe6d2cf000 100644 --- a/docs/zh/sql-reference/data-types/float.md +++ b/docs/zh/sql-reference/data-types/float.md @@ -29,7 +29,7 @@ SELECT 1 - 0.9 - 当一行行阅读浮点数的时候，浮点数的结果可能不是机器最近显示的数值。 -## 南和Inf {#data_type-float-nan-inf} +## NaN和Inf {#data_type-float-nan-inf} 与标准SQL相比，ClickHouse 支持以下类别的浮点数： diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a1b5467f234..9adca58b55a 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -318,6 +318,10 @@ else () if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) endif() + + if (USE_BINARY_HASH) + add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM) + endif() endif () if (ENABLE_TESTS AND USE_GTEST) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 80ad8da837c..ca2a3db193f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -801,7 +801,8 @@ private: connection->setDefaultDatabase(connection_parameters.default_database); ReadBufferFromFile in(queries_file); readStringUntilEOF(text, in); - processMultiQuery(text); + if (!processMultiQuery(text)) + break; } return; } @@ -984,7 +985,8 @@ private: if (query_fuzzer_runs) { - processWithFuzzing(full_query); + if (!processWithFuzzing(full_query)) + return false; } else { @@ -1034,7 +1036,8 @@ private: } - void processWithFuzzing(const String & text) + /// Returns false when server is not available. + bool processWithFuzzing(const String & text) { ASTPtr orig_ast; @@ -1052,7 +1055,7 @@ private: if (!orig_ast) { // Can't continue after a parsing error - return; + return true; } // Don't repeat inserts, the tables grow too big. Also don't repeat @@ -1147,7 +1150,7 @@ private: // Probably the server is dead because we found an assertion // failure. Fail fast. fmt::print(stderr, "Lost connection to the server\n"); - return; + return false; } // The server is still alive so we're going to continue fuzzing. @@ -1173,6 +1176,8 @@ private: fuzz_base = ast_to_process; } } + + return true; } void processTextAsSingleQuery(const String & text_) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index dbf153eeb81..5a8d35e204d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -273,11 +273,12 @@ try global_context->setCurrentDatabase(default_database); applyCmdOptions(*global_context); - String path = global_context->getPath(); - if (!path.empty()) + if (config().has("path")) { + String path = global_context->getPath(); + /// Lock path directory before read - status.emplace(global_context->getPath() + "status", StatusFile::write_full_info); + status.emplace(path + "status", StatusFile::write_full_info); LOG_DEBUG(log, "Loading metadata from {}", path); Poco::File(path + "data/").createDirectories(); @@ -288,7 +289,7 @@ try DatabaseCatalog::instance().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } - else + else if (!config().has("no-system-tables")) { attachSystemTables(*global_context); } @@ -540,6 +541,7 @@ void LocalServer::init(int argc, char ** argv) ("logger.log", po::value(), "Log file name") ("logger.level", po::value(), "Log level") ("ignore-error", "do not stop processing if a query failed") + ("no-system-tables", "do not attach system tables (better startup time)") ("version,V", "print version information and exit") ; @@ -602,6 +604,8 @@ void LocalServer::init(int argc, char ** argv) config().setString("logger.level", options["logger.level"].as()); if (options.count("ignore-error")) config().setBool("ignore-error", true); + if (options.count("no-system-tables")) + config().setBool("no-system-tables", true); std::vector arguments; for (int arg_num = 1; arg_num < argc; ++arg_num) diff --git a/programs/main.cpp b/programs/main.cpp index dee02c55832..cbb22b7a87b 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -18,6 +18,7 @@ #endif #include +#include #include #include @@ -62,6 +63,14 @@ int mainEntryClickHouseStatus(int argc, char ** argv); int mainEntryClickHouseRestart(int argc, char ** argv); #endif +int mainEntryClickHouseHashBinary(int, char **) +{ + /// Intentionally without newline. So you can run: + /// objcopy --add-section .note.ClickHouse.hash=<(./clickhouse hash-binary) clickhouse + std::cout << getHashOfLoadedBinaryHex(); + return 0; +} + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) namespace @@ -110,6 +119,7 @@ std::pair clickhouse_applications[] = {"status", mainEntryClickHouseStatus}, {"restart", mainEntryClickHouseRestart}, #endif + {"hash-binary", mainEntryClickHouseHashBinary}, }; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 76765c0374c..2f8029fc39c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -65,6 +65,8 @@ #include #include #include +#include +#include #include #include #include @@ -184,6 +186,7 @@ namespace ErrorCodes extern const int FAILED_TO_GETPWUID; extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; extern const int NETWORK_ERROR; + extern const int CORRUPTED_DATA; } @@ -436,7 +439,44 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) std::string executable_path = getExecutablePath(); - if (executable_path.empty()) + + if (!executable_path.empty()) + { + /// Integrity check based on checksum of the executable code. + /// Note: it is not intended to protect from malicious party, + /// because the reference checksum can be easily modified as well. + /// And we don't involve asymmetric encryption with PKI yet. + /// It's only intended to protect from faulty hardware. + /// Note: it is only based on machine code. + /// But there are other sections of the binary (e.g. exception handling tables) + /// that are interpreted (not executed) but can alter the behaviour of the program as well. + + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + + if (stored_binary_hash.empty()) + { + LOG_WARNING(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == stored_binary_hash) + { + LOG_INFO(log, "Calculated checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file {2} was changed just after startup;" + " - the file {2} is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file {2} was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, stored_binary_hash, executable_path); + } + } + else executable_path = "/usr/bin/clickhouse"; /// It is used for information messages. /// After full config loaded diff --git a/programs/server/config.xml b/programs/server/config.xml index a4c13be493e..372315c7922 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -676,7 +676,7 @@ system query_log
- : - : - DEFAULT_AIO_FILE_BLOCK_SIZE - - */ - - /* - Representation of data on a disk - - XXX : the data you want to write - ZZZ : data that is already on disk or zeros, if there is no data - - region_aligned_begin region_aligned_end - : region_begin region_end : - : : : : - : : : : - +---:-----------+---------------+---------------+---------------+--:------------+ - | : | | | | : | - | +-----------+---------------+---------------+---------------+--+ | - |ZZZ|XXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XX|ZZZZZZZZZZZZ| - |ZZZ|XXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XX|ZZZZZZZZZZZZ| - | +-----------+---------------+---------------+---------------+--+ | - | | | | | | - +---------------+---------------+---------------+---------------+---------------+ - - <--><--------------------------------------------------------------><-----------> - : : : - : : : - region_left_padding region_size region_right_padding - - <-------------------------------------------------------------------------------> - : - : - region_aligned_size - */ - - /// Region of the disk in which we want to write data. - const off_t region_begin = pos_in_file; - - if ((flush_buffer.offset() > static_cast(std::numeric_limits::max())) || - (pos_in_file > (std::numeric_limits::max() - static_cast(flush_buffer.offset())))) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - - const off_t region_end = pos_in_file + flush_buffer.offset(); - const size_t region_size = region_end - region_begin; - - /// The aligned region of the disk into which we want to write the data. - const size_t region_left_padding = region_begin % DEFAULT_AIO_FILE_BLOCK_SIZE; - const size_t region_right_padding = (DEFAULT_AIO_FILE_BLOCK_SIZE - (region_end % DEFAULT_AIO_FILE_BLOCK_SIZE)) % DEFAULT_AIO_FILE_BLOCK_SIZE; - - region_aligned_begin = region_begin - region_left_padding; - - if (region_end > (std::numeric_limits::max() - static_cast(region_right_padding))) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - - const off_t region_aligned_end = region_end + region_right_padding; - region_aligned_size = region_aligned_end - region_aligned_begin; - - bytes_to_write = region_aligned_size; - - /* - Representing data in the buffer before processing - - XXX : the data you want to write - - buffer_begin buffer_end - : : - : : - +---------------+---------------+---------------+-------------:-+ - | | | | : | - +---------------+---------------+---------------+-------------+ | - |XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXX| | - |XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXX| | - +---------------+---------------+---------------+-------------+ | - | | | | | - +---------------+---------------+---------------+---------------+ - - <-------------------------------------------------------------> - : - : - buffer_size - */ - - /// The buffer of data that we want to write to the disk. - buffer_begin = flush_buffer.buffer().begin(); - Position buffer_end = buffer_begin + region_size; - size_t buffer_size = buffer_end - buffer_begin; - - /// Process the buffer so that it reflects the structure of the disk region. - - /* - Representation of data in the buffer after processing - - XXX : the data you want to write - ZZZ : data from disk or zeros, if there is no data - - `buffer_begin` `buffer_end` extra page - : : : - : : : - +---:-----------+---------------+---------------+---------------+--:------------+ - | | | | | : | - | +-----------+---------------+---------------+---------------+--+ | - |ZZZ|XXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XX|ZZZZZZZZZZZZ| - |ZZZ|XXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XXXXXXXXXXXXXXX|XX|ZZZZZZZZZZZZ| - | +-----------+---------------+---------------+---------------+--+ | - | | | | | | - +---------------+---------------+---------------+---------------+---------------+ - - <--><--------------------------------------------------------------><-----------> - : : : - : : : - region_left_padding region_size region_right_padding - - <-------------------------------------------------------------------------------> - : - : - region_aligned_size - */ - - if ((region_left_padding > 0) || (region_right_padding > 0)) - { - char memory_page[DEFAULT_AIO_FILE_BLOCK_SIZE] __attribute__ ((aligned (DEFAULT_AIO_FILE_BLOCK_SIZE))); - - if (region_left_padding > 0) - { - /// Move the buffer data to the right. Complete the beginning of the buffer with data from the disk. - buffer_size += region_left_padding; - buffer_end = buffer_begin + buffer_size; - - ::memmove(buffer_begin + region_left_padding, buffer_begin, (buffer_size - region_left_padding) * sizeof(*buffer_begin)); - - ssize_t read_count = ::pread(fd, memory_page, DEFAULT_AIO_FILE_BLOCK_SIZE, region_aligned_begin); - if (read_count < 0) - throw Exception("Read error", ErrorCodes::AIO_READ_ERROR); - - size_t to_copy = std::min(static_cast(read_count), region_left_padding); - ::memcpy(buffer_begin, memory_page, to_copy * sizeof(*buffer_begin)); - ::memset(buffer_begin + to_copy, 0, (region_left_padding - to_copy) * sizeof(*buffer_begin)); - } - - if (region_right_padding > 0) - { - /// Add the end of the buffer with data from the disk. - ssize_t read_count = ::pread(fd, memory_page, DEFAULT_AIO_FILE_BLOCK_SIZE, region_aligned_end - DEFAULT_AIO_FILE_BLOCK_SIZE); - if (read_count < 0) - throw Exception("Read error", ErrorCodes::AIO_READ_ERROR); - - Position truncation_begin; - off_t offset = DEFAULT_AIO_FILE_BLOCK_SIZE - region_right_padding; - if (read_count > offset) - { - ::memcpy(buffer_end, memory_page + offset, (read_count - offset) * sizeof(*buffer_end)); - truncation_begin = buffer_end + (read_count - offset); - truncation_count = DEFAULT_AIO_FILE_BLOCK_SIZE - read_count; - } - else - { - truncation_begin = buffer_end; - truncation_count = region_right_padding; - } - - ::memset(truncation_begin, 0, truncation_count * sizeof(*truncation_begin)); - } - } -} - -void WriteBufferAIO::finalize() -{ - if (bytes_written < bytes_to_write) - throw Exception("Asynchronous write error on file " + filename, ErrorCodes::AIO_WRITE_ERROR); - - bytes_written -= truncation_count; - -#if defined(__FreeBSD__) - off_t aio_offset = request.aio.aio_offset; -#else - off_t aio_offset = request.aio_offset; -#endif - off_t pos_offset = bytes_written - (pos_in_file - aio_offset); - - if (pos_in_file > (std::numeric_limits::max() - pos_offset)) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - pos_in_file += pos_offset; - - if (pos_in_file > max_pos_in_file) - max_pos_in_file = pos_in_file; - - if (truncation_count > 0) - { - /// Truncate the file to remove unnecessary zeros from it. - int res = ::ftruncate(fd, max_pos_in_file); - if (res == -1) - throwFromErrnoWithPath("Cannot truncate file " + filename, filename, ErrorCodes::CANNOT_TRUNCATE_FILE); - } -} - -} - -#endif diff --git a/src/IO/WriteBufferAIO.h b/src/IO/WriteBufferAIO.h deleted file mode 100644 index f514acab359..00000000000 --- a/src/IO/WriteBufferAIO.h +++ /dev/null @@ -1,102 +0,0 @@ -#pragma once - -#if defined(OS_LINUX) || defined(__FreeBSD__) - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - - -namespace CurrentMetrics -{ - extern const Metric OpenFileForWrite; -} - -namespace DB -{ - -/** Class for asynchronous data writing. - */ -class WriteBufferAIO final : public WriteBufferFromFileBase -{ -public: - WriteBufferAIO(const std::string & filename_, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, int flags_ = -1, mode_t mode_ = 0666, - char * existing_memory_ = nullptr); - ~WriteBufferAIO() override; - - WriteBufferAIO(const WriteBufferAIO &) = delete; - WriteBufferAIO & operator=(const WriteBufferAIO &) = delete; - - off_t getPositionInFile(); - off_t seek(off_t off, int whence); - void truncate(off_t length); - void sync() override; - std::string getFileName() const override { return filename; } - int getFD() const { return fd; } - -private: - void nextImpl() override; - - /// If there's still data in the buffer, we'll write them. - void flush(); - /// Wait for the end of the current asynchronous task. - bool waitForAIOCompletion(); - /// Prepare an asynchronous request. - void prepare(); - /// - void finalize() override; - -private: - /// Buffer for asynchronous data writes. - BufferWithOwnMemory flush_buffer; - - /// Description of the asynchronous write request. - iocb request{}; - iocb * request_ptr{&request}; - - AIOContext aio_context{1}; - - const std::string filename; - - /// The number of bytes to be written to the disk. - off_t bytes_to_write = 0; - /// Number of bytes written with the last request. - off_t bytes_written = 0; - /// The number of zero bytes to be cut from the end of the file - /// after the data write operation completes. - off_t truncation_count = 0; - - /// The current position in the file. - off_t pos_in_file = 0; - /// The maximum position reached in the file. - off_t max_pos_in_file = 0; - - /// The starting position of the aligned region of the disk to which the data is written. - off_t region_aligned_begin = 0; - /// The size of the aligned region of the disk. - size_t region_aligned_size = 0; - - /// The file descriptor for writing. - int fd = -1; - - /// The data buffer that we want to write to the disk. - Position buffer_begin = nullptr; - - /// Is the asynchronous write operation still in progress? - bool is_pending_write = false; - /// Did the asynchronous operation fail? - bool aio_failed = false; - - CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForWrite}; -}; - -} - -#endif diff --git a/src/IO/WriteBufferFromFile.h b/src/IO/WriteBufferFromFile.h index 77530c323d2..b7d58638113 100644 --- a/src/IO/WriteBufferFromFile.h +++ b/src/IO/WriteBufferFromFile.h @@ -50,11 +50,6 @@ public: /// Close file before destruction of object. void close(); - void finalize() override - { - close(); - } - std::string getFileName() const override { return file_name; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 624a6c3496a..e6acd0b8880 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -483,6 +483,10 @@ inline void writeEscapedString(const StringRef & ref, WriteBuffer & buf) writeEscapedString(ref.data, ref.size, buf); } +inline void writeEscapedString(const std::string_view & ref, WriteBuffer & buf) +{ + writeEscapedString(ref.data(), ref.size(), buf); +} template void writeAnyQuotedString(const char * begin, const char * end, WriteBuffer & buf) @@ -512,17 +516,31 @@ inline void writeQuotedString(const String & s, WriteBuffer & buf) writeAnyQuotedString<'\''>(s, buf); } - inline void writeQuotedString(const StringRef & ref, WriteBuffer & buf) { writeAnyQuotedString<'\''>(ref, buf); } +inline void writeQuotedString(const std::string_view & ref, WriteBuffer & buf) +{ + writeAnyQuotedString<'\''>(ref.data(), ref.data() + ref.size(), buf); +} + +inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf) +{ + writeAnyQuotedString<'"'>(s, buf); +} + inline void writeDoubleQuotedString(const StringRef & s, WriteBuffer & buf) { writeAnyQuotedString<'"'>(s, buf); } +inline void writeDoubleQuotedString(const std::string_view & s, WriteBuffer & buf) +{ + writeAnyQuotedString<'"'>(s.data(), s.data() + s.size(), buf); +} + /// Outputs a string in backquotes. inline void writeBackQuotedString(const StringRef & s, WriteBuffer & buf) { @@ -901,6 +919,7 @@ writeBinary(const T & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary(x, buf); } inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); } +inline void writeBinary(const std::string_view & x, WriteBuffer & buf) { writeStringBinary(x, buf); } inline void writeBinary(const Int128 & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const UInt128 & x, WriteBuffer & buf) { writePODBinary(x, buf); } inline void writeBinary(const DummyUInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); } @@ -1001,6 +1020,10 @@ writeQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeQuoted(const String & x, WriteBuffer & buf) { writeQuotedString(x, buf); } +inline void writeQuoted(const std::string_view & x, WriteBuffer & buf) { writeQuotedString(x, buf); } + +inline void writeQuoted(const StringRef & x, WriteBuffer & buf) { writeQuotedString(x, buf); } + inline void writeQuoted(const LocalDate & x, WriteBuffer & buf) { writeChar('\'', buf); @@ -1043,6 +1066,10 @@ writeDoubleQuoted(const T & x, WriteBuffer & buf) { writeText(x, buf); } inline void writeDoubleQuoted(const String & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } +inline void writeDoubleQuoted(const std::string_view & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } + +inline void writeDoubleQuoted(const StringRef & x, WriteBuffer & buf) { writeDoubleQuotedString(x, buf); } + inline void writeDoubleQuoted(const LocalDate & x, WriteBuffer & buf) { writeChar('"', buf); diff --git a/src/IO/createWriteBufferFromFileBase.cpp b/src/IO/createWriteBufferFromFileBase.cpp deleted file mode 100644 index 6022457f32e..00000000000 --- a/src/IO/createWriteBufferFromFileBase.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#if defined(OS_LINUX) || defined(__FreeBSD__) -#include -#endif -#include - - -namespace ProfileEvents -{ - extern const Event CreatedWriteBufferOrdinary; - extern const Event CreatedWriteBufferAIO; - extern const Event CreatedWriteBufferAIOFailed; -} - -namespace DB -{ - -std::unique_ptr createWriteBufferFromFileBase(const std::string & filename_, size_t estimated_size, - size_t aio_threshold, size_t buffer_size_, int flags_, mode_t mode, char * existing_memory_, - size_t alignment) -{ -#if defined(OS_LINUX) || defined(__FreeBSD__) - if (aio_threshold && estimated_size >= aio_threshold) - { - /// Attempt to open a file with O_DIRECT - try - { - auto res = std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_); - ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIO); - return res; - } - catch (const ErrnoException &) - { - /// Fallback to cached IO if O_DIRECT is not supported. - ProfileEvents::increment(ProfileEvents::CreatedWriteBufferAIOFailed); - } - } -#else - (void)aio_threshold; - (void)estimated_size; -#endif - - ProfileEvents::increment(ProfileEvents::CreatedWriteBufferOrdinary); - return std::make_unique(filename_, buffer_size_, flags_, mode, existing_memory_, alignment); -} - -} diff --git a/src/IO/createWriteBufferFromFileBase.h b/src/IO/createWriteBufferFromFileBase.h deleted file mode 100644 index 42cad88303b..00000000000 --- a/src/IO/createWriteBufferFromFileBase.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -/** Create an object to write data to a file. - * estimated_size - number of bytes to write - * aio_threshold - the minimum number of bytes for asynchronous writes - * - * If aio_threshold = 0 or estimated_size < aio_threshold, the write operations are executed synchronously. - * Otherwise, write operations are performed asynchronously. - */ -std::unique_ptr createWriteBufferFromFileBase( - const std::string & filename_, - size_t estimated_size, - size_t aio_threshold, - size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, - int flags_ = -1, - mode_t mode = 0666, - char * existing_memory_ = nullptr, - size_t alignment = 0); - -} diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/tests/CMakeLists.txt index da4d330f0a9..fcd59d94cb0 100644 --- a/src/IO/tests/CMakeLists.txt +++ b/src/IO/tests/CMakeLists.txt @@ -55,9 +55,6 @@ add_executable (write_int write_int.cpp) target_link_libraries (write_int PRIVATE clickhouse_common_io) if (OS_LINUX OR OS_FREEBSD) - add_executable(write_buffer_aio write_buffer_aio.cpp) - target_link_libraries (write_buffer_aio PRIVATE clickhouse_common_io) - add_executable(read_buffer_aio read_buffer_aio.cpp) target_link_libraries (read_buffer_aio PRIVATE clickhouse_common_io) endif () diff --git a/src/IO/tests/gtest_manip.cpp b/src/IO/tests/gtest_manip.cpp new file mode 100644 index 00000000000..c3989ca0d7a --- /dev/null +++ b/src/IO/tests/gtest_manip.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +template +void checkString(const T & str, U manip, const std::string & expected) +{ + WriteBufferFromOwnString buf; + + buf << manip << str; + EXPECT_EQ(expected, buf.str()) << "str type:" << typeid(str).name(); +} + +TEST(OperatorsManipTest, EscapingTest) +{ + checkString("Hello 'world'", escape, "Hello \\'world\\'"); + checkString("Hello \\world\\", escape, "Hello \\\\world\\\\"); // NOLINT + + std::string s1 = "Hello 'world'"; + checkString(s1, escape, "Hello \\'world\\'"); + std::string s2 = "Hello \\world\\"; + checkString(s2, escape, "Hello \\\\world\\\\"); // NOLINT + + std::string_view sv1 = s1; + checkString(sv1, escape, "Hello \\'world\\'"); + std::string_view sv2 = s2; + checkString(sv2, escape, "Hello \\\\world\\\\"); // NOLINT + + StringRef sr1 = s1; + checkString(sr1, escape, "Hello \\'world\\'"); + StringRef sr2 = s2; + checkString(sr2, escape, "Hello \\\\world\\\\"); // NOLINT +} + +TEST(OperatorsManipTest, QuouteTest) +{ + checkString("Hello 'world'", quote, "'Hello \\'world\\''"); + + std::string s1 = "Hello 'world'"; + checkString(s1, quote, "'Hello \\'world\\''"); + + std::string_view sv1 = s1; + checkString(sv1, quote, "'Hello \\'world\\''"); + + StringRef sr1 = s1; + checkString(sr1, quote, "'Hello \\'world\\''"); +} + +TEST(OperatorsManipTest, DoubleQuouteTest) +{ + checkString("Hello 'world'", double_quote, "\"Hello 'world'\""); + + std::string s1 = "Hello 'world'"; + checkString(s1, double_quote, "\"Hello 'world'\""); + + std::string_view sv1 = s1; + checkString(sv1, double_quote, "\"Hello 'world'\""); + + StringRef sr1 = s1; + checkString(sr1, double_quote, "\"Hello 'world'\""); +} + +TEST(OperatorsManipTest, binary) +{ + checkString("Hello", binary, "\x5Hello"); + + std::string s1 = "Hello"; + checkString(s1, binary, "\x5Hello"); + + std::string_view sv1 = s1; + checkString(sv1, binary, "\x5Hello"); + + StringRef sr1 = s1; + checkString(sr1, binary, "\x5Hello"); +} diff --git a/src/IO/tests/gtest_peekable_read_buffer.cpp b/src/IO/tests/gtest_peekable_read_buffer.cpp index fb4b0b799b4..8c491338bd3 100644 --- a/src/IO/tests/gtest_peekable_read_buffer.cpp +++ b/src/IO/tests/gtest_peekable_read_buffer.cpp @@ -9,7 +9,6 @@ namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; - extern const int MEMORY_LIMIT_EXCEEDED; } static void readAndAssert(DB::ReadBuffer & buf, const char * str) @@ -40,7 +39,7 @@ try DB::ReadBufferFromString b4(s4); DB::ConcatReadBuffer concat({&b1, &b2, &b3, &b4}); - DB::PeekableReadBuffer peekable(concat, 0, 16); + DB::PeekableReadBuffer peekable(concat, 0); ASSERT_TRUE(!peekable.eof()); assertAvailable(peekable, "0123456789"); @@ -48,6 +47,8 @@ try DB::PeekableReadBufferCheckpoint checkpoint{peekable}; readAndAssert(peekable, "01234"); } + +#ifndef ABORT_ON_LOGICAL_ERROR bool exception = false; try { @@ -60,6 +61,7 @@ try exception = true; } ASSERT_TRUE(exception); +#endif assertAvailable(peekable, "56789"); readAndAssert(peekable, "56"); @@ -70,19 +72,10 @@ try peekable.dropCheckpoint(); assertAvailable(peekable, "789"); - exception = false; - try { DB::PeekableReadBufferCheckpoint checkpoint{peekable, true}; - peekable.ignore(30); + peekable.ignore(20); } - catch (DB::Exception & e) - { - if (e.code() != DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED) - throw; - exception = true; - } - ASSERT_TRUE(exception); assertAvailable(peekable, "789qwertyuiop"); readAndAssert(peekable, "789qwertyu"); diff --git a/src/IO/tests/write_buffer_aio.cpp b/src/IO/tests/write_buffer_aio.cpp deleted file mode 100644 index 9274e5abee5..00000000000 --- a/src/IO/tests/write_buffer_aio.cpp +++ /dev/null @@ -1,498 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace -{ - -namespace fs = std::filesystem; - -void run(); -[[noreturn]] void die(const std::string & msg); -void runTest(unsigned int num, const std::function & func); -std::string createTmpFile(); -std::string generateString(size_t n); - -bool test1(); -bool test2(); -bool test3(); -bool test4(); -bool test5(); -bool test6(); -bool test7(); -bool test8(); -bool test9(); -bool test10(); - -void run() -{ - const std::vector> tests = - { - test1, - test2, - test3, - test4, - test5, - test6, - test7, - test8, - test9, - test10 - }; - - unsigned int num = 0; - for (const auto & test : tests) - { - ++num; - runTest(num, test); - } -} - -void die(const std::string & msg) -{ - std::cout << msg; - ::exit(EXIT_FAILURE); -} - -void runTest(unsigned int num, const std::function & func) -{ - bool ok; - - try - { - ok = func(); - } - catch (const DB::Exception & ex) - { - ok = false; - std::cout << "Caught exception " << ex.displayText() << "\n"; - } - catch (const std::exception & ex) - { - ok = false; - std::cout << "Caught exception " << ex.what() << "\n"; - } - - if (ok) - std::cout << "Test " << num << " passed\n"; - else - std::cout << "Test " << num << " failed\n"; -} - -std::string createTmpFile() -{ - char pattern[] = "/tmp/fileXXXXXX"; - char * dir = ::mkdtemp(pattern); - if (dir == nullptr) - die("Could not create directory"); - - return std::string(dir) + "/foo"; -} - -std::string generateString(size_t n) -{ - static const std::string symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - std::string buf; - buf.reserve(n); - - for (size_t i = 0; i < n; ++i) - buf += symbols[i % symbols.length()]; - - return buf; -} - -bool test1() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.write(buf.data(), buf.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return (received == buf); -} - -bool test2() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.write(buf.data(), buf.length() / 2); - out.seek(DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_CUR); - out.write(&buf[buf.length() / 2], buf.length() / 2); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - if (received.substr(0, buf.length() / 2) != buf.substr(0, buf.length() / 2)) - return false; - if (received.substr(buf.length() / 2, DEFAULT_AIO_FILE_BLOCK_SIZE) != std::string(DEFAULT_AIO_FILE_BLOCK_SIZE, '\0')) - return false; - if (received.substr(buf.length() / 2 + DEFAULT_AIO_FILE_BLOCK_SIZE) != buf.substr(buf.length() / 2)) - return false; - - return true; -} - -bool test3() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.write(buf.data(), buf.length()); - - off_t pos1 = out.getPositionInFile(); - - out.truncate(buf.length() / 2); - - off_t pos2 = out.getPositionInFile(); - - if (pos1 != pos2) - return false; - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return (received == buf.substr(0, buf.length() / 2)); -} - -bool test4() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.write(buf.data(), buf.length()); - - off_t pos1 = out.getPositionInFile(); - - out.truncate(3 * buf.length() / 2); - - off_t pos2 = out.getPositionInFile(); - - if (pos1 != pos2) - return false; - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - if (received.substr(0, buf.length()) != buf) - return false; - - if (received.substr(buf.length()) != std::string(buf.length() / 2, '\0')) - return false; - - return true; -} - -bool test5() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.seek(1, SEEK_SET); - out.write(buf.data(), buf.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return received.substr(1) == buf; -} - -bool test6() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - std::string buf2 = "1111111111"; - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.seek(3, SEEK_SET); - out.write(buf.data(), buf.length()); - out.seek(-2 * DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_CUR); - out.write(buf2.data(), buf2.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - if (received.substr(3, 8 * DEFAULT_AIO_FILE_BLOCK_SIZE) != buf.substr(0, 8 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - - if (received.substr(3 + 8 * DEFAULT_AIO_FILE_BLOCK_SIZE, 10) != buf2) - return false; - - if (received.substr(13 + 8 * DEFAULT_AIO_FILE_BLOCK_SIZE) != buf.substr(10 + 8 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - - return true; -} - -bool test7() -{ - std::string filename = createTmpFile(); - - std::string buf2 = "11111111112222222222"; - - { - DB::WriteBufferAIO out(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.seek(DEFAULT_AIO_FILE_BLOCK_SIZE - (buf2.length() / 2), SEEK_SET); - out.write(buf2.data(), buf2.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - if (received.length() != 4106) - return false; - if (received.substr(0, 4086) != std::string(4086, '\0')) - return false; - if (received.substr(4086, 20) != buf2) - return false; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return true; -} - -bool test8() -{ - std::string filename = createTmpFile(); - - std::string buf2 = "11111111112222222222"; - - { - DB::WriteBufferAIO out(filename, 2 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.seek(2 * DEFAULT_AIO_FILE_BLOCK_SIZE - (buf2.length() / 2), SEEK_SET); - out.write(buf2.data(), buf2.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - if (received.length() != 8202) - return false; - if (received.substr(0, 8182) != std::string(8182, '\0')) - return false; - if (received.substr(8182, 20) != buf2) - return false; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return true; -} - -bool test9() -{ - std::string filename = createTmpFile(); - - size_t n = 3 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - std::string buf = generateString(n); - - std::string buf2(DEFAULT_AIO_FILE_BLOCK_SIZE + 10, '1'); - - { - DB::WriteBufferAIO out(filename, 2 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.seek(3, SEEK_SET); - out.write(buf.data(), buf.length()); - out.seek(-DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_CUR); - out.write(buf2.data(), buf2.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - if (received.substr(3, 2 * DEFAULT_AIO_FILE_BLOCK_SIZE) != buf.substr(0, 2 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - - if (received.substr(3 + 2 * DEFAULT_AIO_FILE_BLOCK_SIZE, DEFAULT_AIO_FILE_BLOCK_SIZE + 10) != buf2) - return false; - - return true; -} - -bool test10() -{ - std::string filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE + 3; - - std::string buf = generateString(n); - - { - DB::WriteBufferAIO out(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (out.getFileName() != filename) - return false; - if (out.getFD() == -1) - return false; - - out.write(buf.data(), buf.length()); - } - - std::ifstream in(filename.c_str()); - if (!in.is_open()) - die("Could not open file"); - - std::string received{ std::istreambuf_iterator(in), std::istreambuf_iterator() }; - - in.close(); - fs::remove_all(fs::path(filename).parent_path().string()); - - return (received == buf); -} - -} - -int main() -{ - run(); - return 0; -} diff --git a/src/IO/ya.make b/src/IO/ya.make index 4dc3afb2f11..2ef8bd0a986 100644 --- a/src/IO/ya.make +++ b/src/IO/ya.make @@ -51,7 +51,6 @@ SRCS( ReadHelpers.cpp SeekAvoidingReadBuffer.cpp UseSSL.cpp - WriteBufferAIO.cpp WriteBufferFromFile.cpp WriteBufferFromFileBase.cpp WriteBufferFromFileDescriptor.cpp @@ -69,7 +68,6 @@ SRCS( ZstdInflatingReadBuffer.cpp copyData.cpp createReadBufferFromFileBase.cpp - createWriteBufferFromFileBase.cpp parseDateTimeBestEffort.cpp readFloatText.cpp diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 2fc78261f17..fe0bba2bf3e 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -436,12 +436,6 @@ void ActionsDAG::project(const NamesWithAliases & projection) settings.projected_output = true; } -void ActionsDAG::removeColumn(const std::string & column_name) -{ - auto & node = getNode(column_name); - index.remove(&node); -} - bool ActionsDAG::tryRestoreColumn(const std::string & column_name) { if (index.contains(column_name)) @@ -550,6 +544,11 @@ std::string ActionsDAG::dumpDAG() const out << "\n"; } + out << "Index:"; + for (const auto * node : index) + out << ' ' << map[node]; + out << '\n'; + return out.str(); } @@ -698,7 +697,8 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) /// Will store merged result in `first`. /// This map contains nodes which should be removed from `first` index, cause they are used as inputs for `second`. - std::unordered_set removed_first_result; + /// The second element is the number of removes (cause one node may be repeated several times in result). + std::unordered_map removed_first_result; /// Map inputs of `second` to nodes of `first`. std::unordered_map inputs_map; @@ -723,7 +723,7 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) else { inputs_map[node] = it->second.front(); - removed_first_result.emplace(it->second.front()); + removed_first_result[it->second.front()] += 1; it->second.pop_front(); } } @@ -767,8 +767,12 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second) auto cur = it; ++it; - if (removed_first_result.count(*cur)) + auto jt = removed_first_result.find(*cur); + if (jt != removed_first_result.end() && jt->second > 0) + { first.index.remove(cur); + --jt->second; + } } for (auto * node : second.index) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index a0ac210e752..6a26927374e 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -80,7 +80,7 @@ public: }; /// Index is used to: - /// * find Node buy it's result_name + /// * find Node by it's result_name /// * specify order of columns in result /// It represents a set of available columns. /// Removing of column from index is equivalent to removing of column from final result. @@ -133,16 +133,6 @@ public: insert(node); } - void remove(Node * node) - { - auto it = map.find(node->result_name); - if (it != map.end()) - return; - - list.erase(it->second); - map.erase(it); - } - void remove(std::list::iterator it) { auto map_it = map.find((*it)->result_name); @@ -219,8 +209,6 @@ public: /// Add alias actions and remove unused columns from index. Also specify result columns order in index. void project(const NamesWithAliases & projection); - /// Removes column from index. - void removeColumn(const std::string & column_name); /// If column is not in index, try to find it in nodes and insert back into index. bool tryRestoreColumn(const std::string & column_name); diff --git a/src/Interpreters/ColumnAliasesVisitor.cpp b/src/Interpreters/ColumnAliasesVisitor.cpp new file mode 100644 index 00000000000..dcc4c3d75d4 --- /dev/null +++ b/src/Interpreters/ColumnAliasesVisitor.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool ColumnAliasesMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) +{ + if (const auto * f = node->as()) + { + /// "lambda" visits children itself. + if (f->name == "lambda") + return false; + } + + return !(node->as() + || node->as() + || node->as() + || node->as() + || node->as()); +} + +void ColumnAliasesMatcher::visit(ASTPtr & ast, Data & data) +{ + // If it's select query, only replace filters. + if (auto * query = ast->as()) + { + if (query->where()) + Visitor(data).visit(query->refWhere()); + if (query->prewhere()) + Visitor(data).visit(query->refPrewhere()); + + return; + } + + if (auto * node = ast->as()) + { + visit(*node, ast, data); + return; + } + + if (auto * node = ast->as()) + { + visit(*node, ast, data); + return; + } +} + +void ColumnAliasesMatcher::visit(ASTFunction & node, ASTPtr & /*ast*/, Data & data) +{ + /// Do not add formal parameters of the lambda expression + if (node.name == "lambda") + { + Names local_aliases; + auto names_from_lambda = RequiredSourceColumnsMatcher::extractNamesFromLambda(node); + for (const auto & name : names_from_lambda) + { + if (data.private_aliases.insert(name).second) + { + local_aliases.push_back(name); + } + } + /// visit child with masked local aliases + Visitor(data).visit(node.arguments->children[1]); + for (const auto & name : local_aliases) + data.private_aliases.erase(name); + } +} + +void ColumnAliasesMatcher::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) +{ + if (auto column_name = IdentifierSemantic::getColumnName(node)) + { + if (data.forbidden_columns.count(*column_name) || data.private_aliases.count(*column_name) || !data.columns.has(*column_name)) + return; + + const auto & col = data.columns.get(*column_name); + if (col.default_desc.kind == ColumnDefaultKind::Alias) + { + ast = addTypeConversionToAST(col.default_desc.expression->clone(), col.type->getName(), data.columns.getAll(), data.context); + auto str = queryToString(ast); + // revisit ast to track recursive alias columns + Visitor(data).visit(ast); + } + } +} + + +} diff --git a/src/Interpreters/ColumnAliasesVisitor.h b/src/Interpreters/ColumnAliasesVisitor.h new file mode 100644 index 00000000000..2cd9ad796cd --- /dev/null +++ b/src/Interpreters/ColumnAliasesVisitor.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class IAST; +using ASTPtr = std::shared_ptr; +class IDataType; +class ASTFunction; +class ASTIdentifier; +using DataTypePtr = std::shared_ptr; + +/// Visits AST node to rewrite alias columns in query +/// Currently works only 3 kind ways below + +/// For example: +// CREATE TABLE test_table +// ( +// `timestamp` DateTime, +// `value` UInt64, +// `day` Date ALIAS toDate(timestamp), +// `day1` Date ALIAS day + 1, +// `day2` Date ALIAS day1 + 1, +// `time` DateTime ALIAS timestamp +// )ENGINE = MergeTree +// PARTITION BY toYYYYMMDD(timestamp) +// ORDER BY timestamp SETTINGS index_granularity = 1; + +/// 1. Rewrite the filters in query when enable optimize_respect_aliases +/// this could help with `optimize_trivial_count`, Partition Prune in `KeyCondition` and secondary indexes. +/// eg: select max(value) from test_table where day2 = today(), filters will be: ((toDate(timestamp) + 1) + 1) = today() . + +/// 2. Alias on alias for `required_columns` extracted in `InterpreterSelectQuery.cpp`, it could help get all dependent physical columns for query. +/// eg: select day2 from test_table. `required_columns` can got require columns from the temporary rewritten AST `((toDate(timestamp) + 1) + 1)`. + +/// 3. Help with `optimize_aggregation_in_order` and `optimize_read_in_order` in `ReadInOrderOptimizer.cpp`: +/// For queries with alias columns in `orderBy` and `groupBy`, these ASTs will not change. +/// But we generate temporary asts and generate temporary Actions to get the `InputOrderInfo` +/// eg: select day1 from test_table order by day1; + + +class ColumnAliasesMatcher +{ +public: + using Visitor = InDepthNodeVisitor; + + struct Data + { + const ColumnsDescription & columns; + + /// forbidden_columns are from array join, we can't rewrite alias columns involved in array join. + /// Do not analyze joined columns. + /// They may have aliases and come to description as is. + const NameSet & forbidden_columns; + const Context & context; + + /// private_aliases are from lambda, so these are local names. + NameSet private_aliases; + + Data(const ColumnsDescription & columns_, const NameSet & forbidden_columns_, const Context & context_) + : columns(columns_) + , forbidden_columns(forbidden_columns_) + , context(context_) + {} + }; + + static void visit(ASTPtr & ast, Data & data); + static bool needChildVisit(const ASTPtr & node, const ASTPtr & child); + +private: + static void visit(ASTIdentifier & node, ASTPtr & ast, Data & data); + static void visit(ASTFunction & node, ASTPtr & ast, Data & data); +}; + +using ColumnAliasesVisitor = ColumnAliasesMatcher::Visitor; + +} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 9fb74a4f800..2a8fdce869b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -429,7 +428,6 @@ struct ContextShared if (system_logs) system_logs->shutdown(); - TemporaryLiveViewCleaner::shutdown(); DatabaseCatalog::shutdown(); /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). @@ -493,7 +491,6 @@ Context Context::createGlobal(ContextShared * shared) void Context::initGlobal() { DatabaseCatalog::init(*this); - TemporaryLiveViewCleaner::init(*this); } SharedContextHolder Context::createShared() diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index cce62b1a6c4..6dab60f57d1 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -21,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +32,6 @@ #include #include #include -#include #include #include #include @@ -62,107 +59,46 @@ namespace ErrorCodes } -namespace +String DDLLogEntry::toString() { + WriteBufferFromOwnString wb; -struct HostID -{ - String host_name; - UInt16 port; + Strings host_id_strings(hosts.size()); + std::transform(hosts.begin(), hosts.end(), host_id_strings.begin(), HostID::applyToString); - HostID() = default; - - explicit HostID(const Cluster::Address & address) - : host_name(address.host_name), port(address.port) {} - - static HostID fromString(const String & host_port_str) - { - HostID res; - std::tie(res.host_name, res.port) = Cluster::Address::fromString(host_port_str); - return res; - } - - String toString() const - { - return Cluster::Address::toString(host_name, port); - } - - String readableString() const - { - return host_name + ":" + DB::toString(port); - } - - bool isLocalAddress(UInt16 clickhouse_port) const - { - try - { - return DB::isLocalAddress(DNSResolver::instance().resolveAddress(host_name, port), clickhouse_port); - } - catch (const Poco::Net::NetException &) - { - /// Avoid "Host not found" exceptions - return false; - } - } - - static String applyToString(const HostID & host_id) - { - return host_id.toString(); - } -}; + auto version = CURRENT_VERSION; + wb << "version: " << version << "\n"; + wb << "query: " << escape << query << "\n"; + wb << "hosts: " << host_id_strings << "\n"; + wb << "initiator: " << initiator << "\n"; + return wb.str(); } - -struct DDLLogEntry +void DDLLogEntry::parse(const String & data) { - String query; - std::vector hosts; - String initiator; // optional + ReadBufferFromString rb(data); - static constexpr int CURRENT_VERSION = 1; + int version; + rb >> "version: " >> version >> "\n"; - String toString() - { - WriteBufferFromOwnString wb; + if (version != CURRENT_VERSION) + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}", version); - Strings host_id_strings(hosts.size()); - std::transform(hosts.begin(), hosts.end(), host_id_strings.begin(), HostID::applyToString); + Strings host_id_strings; + rb >> "query: " >> escape >> query >> "\n"; + rb >> "hosts: " >> host_id_strings >> "\n"; - auto version = CURRENT_VERSION; - wb << "version: " << version << "\n"; - wb << "query: " << escape << query << "\n"; - wb << "hosts: " << host_id_strings << "\n"; - wb << "initiator: " << initiator << "\n"; + if (!rb.eof()) + rb >> "initiator: " >> initiator >> "\n"; + else + initiator.clear(); - return wb.str(); - } + assertEOF(rb); - void parse(const String & data) - { - ReadBufferFromString rb(data); - - int version; - rb >> "version: " >> version >> "\n"; - - if (version != CURRENT_VERSION) - throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown DDLLogEntry format version: {}", version); - - Strings host_id_strings; - rb >> "query: " >> escape >> query >> "\n"; - rb >> "hosts: " >> host_id_strings >> "\n"; - - if (!rb.eof()) - rb >> "initiator: " >> initiator >> "\n"; - else - initiator.clear(); - - assertEOF(rb); - - hosts.resize(host_id_strings.size()); - std::transform(host_id_strings.begin(), host_id_strings.end(), hosts.begin(), HostID::fromString); - } -}; + hosts.resize(host_id_strings.size()); + std::transform(host_id_strings.begin(), host_id_strings.end(), hosts.begin(), HostID::fromString); +} struct DDLTask @@ -315,7 +251,7 @@ DDLWorker::DDLWorker(int pool_size_, const std::string & zk_root_dir, Context & : context(context_) , log(&Poco::Logger::get("DDLWorker")) , pool_size(pool_size_) - , worker_pool(pool_size_) + , worker_pool(std::make_unique(pool_size)) { CurrentMetrics::set(CurrentMetrics::MaxDDLEntryID, 0); last_tasks.reserve(pool_size); @@ -352,7 +288,7 @@ DDLWorker::~DDLWorker() stop_flag = true; queue_updated_event->set(); cleanup_event->set(); - worker_pool.wait(); + worker_pool.reset(); main_thread.join(); cleanup_thread.join(); } @@ -517,7 +453,7 @@ void DDLWorker::scheduleTasks() if (!already_processed) { - worker_pool.scheduleOrThrowOnError([this, task_ptr = task.release()]() + worker_pool->scheduleOrThrowOnError([this, task_ptr = task.release()]() { setThreadName("DDLWorkerExec"); enqueueTask(DDLTaskPtr(task_ptr)); @@ -847,6 +783,13 @@ bool DDLWorker::taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, const Storage if (!ast_ddl->as() && !ast_ddl->as() && !ast_ddl->as()) return false; + if (auto * alter = ast_ddl->as()) + { + // Setting alters should be executed on all replicas + if (alter->isSettingsAlter()) + return false; + } + return storage->supportsReplication(); } @@ -1131,6 +1074,17 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry) void DDLWorker::runMainThread() { + auto reset_state = [&](bool reset_pool = true) + { + /// It will wait for all threads in pool to finish and will not rethrow exceptions (if any). + /// We create new thread pool to forget previous exceptions. + if (reset_pool) + worker_pool = std::make_unique(pool_size); + /// Clear other in-memory state, like server just started. + last_tasks.clear(); + max_id = 0; + }; + setThreadName("DDLWorker"); LOG_DEBUG(log, "Started DDLWorker thread"); @@ -1146,7 +1100,12 @@ void DDLWorker::runMainThread() catch (const Coordination::Exception & e) { if (!Coordination::isHardwareError(e.code)) - throw; /// A logical error. + { + /// A logical error. + LOG_ERROR(log, "ZooKeeper error: {}. Failed to start DDLWorker.",getCurrentExceptionMessage(true)); + reset_state(false); + assert(false); /// Catch such failures in tests with debug build + } tryLogCurrentException(__PRETTY_FUNCTION__); @@ -1155,8 +1114,8 @@ void DDLWorker::runMainThread() } catch (...) { - tryLogCurrentException(log, "Terminating. Cannot initialize DDL queue."); - return; + tryLogCurrentException(log, "Cannot initialize DDL queue."); + reset_state(false); } } while (!initialized && !stop_flag); @@ -1185,14 +1144,14 @@ void DDLWorker::runMainThread() } else { - LOG_ERROR(log, "Unexpected ZooKeeper error: {}. Terminating.", getCurrentExceptionMessage(true)); - return; + LOG_ERROR(log, "Unexpected ZooKeeper error: {}", getCurrentExceptionMessage(true)); + reset_state(); } } catch (...) { - tryLogCurrentException(log, "Unexpected error, will terminate:"); - return; + tryLogCurrentException(log, "Unexpected error:"); + reset_state(); } } } diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 1c365a0d9ba..9a4e55dcfc4 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -1,12 +1,15 @@ #pragma once +#include #include #include -#include +#include +#include #include +#include #include +#include #include -#include #include #include @@ -16,24 +19,80 @@ namespace zkutil { - class ZooKeeper; +class ZooKeeper; } namespace DB { - class Context; class ASTAlterQuery; class AccessRightsElements; -struct DDLLogEntry; + +struct HostID +{ + String host_name; + UInt16 port; + + HostID() = default; + + explicit HostID(const Cluster::Address & address) : host_name(address.host_name), port(address.port) { } + + static HostID fromString(const String & host_port_str) + { + HostID res; + std::tie(res.host_name, res.port) = Cluster::Address::fromString(host_port_str); + return res; + } + + String toString() const { return Cluster::Address::toString(host_name, port); } + + String readableString() const { return host_name + ":" + DB::toString(port); } + + bool isLocalAddress(UInt16 clickhouse_port) const + { + try + { + return DB::isLocalAddress(DNSResolver::instance().resolveAddress(host_name, port), clickhouse_port); + } + catch (const Poco::Net::NetException &) + { + /// Avoid "Host not found" exceptions + return false; + } + } + + static String applyToString(const HostID & host_id) { return host_id.toString(); } +}; + +struct DDLLogEntry +{ + String query; + std::vector hosts; + String initiator; // optional + + static constexpr int CURRENT_VERSION = 1; + +public: + String toString(); + void parse(const String & data); +}; + struct DDLTask; using DDLTaskPtr = std::unique_ptr; /// Pushes distributed DDL query to the queue BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context); -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option = false); -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option = false); +BlockIO executeDDLQueryOnCluster( + const ASTPtr & query_ptr, + const Context & context, + const AccessRightsElements & query_requires_access, + bool query_requires_grant_option = false); +BlockIO executeDDLQueryOnCluster( + const ASTPtr & query_ptr, + const Context & context, + AccessRightsElements && query_requires_access, + bool query_requires_grant_option = false); class DDLWorker @@ -127,7 +186,7 @@ private: /// Size of the pool for query execution. size_t pool_size = 1; - ThreadPool worker_pool; + std::unique_ptr worker_pool; /// Cleaning starts after new node event is received if the last cleaning wasn't made sooner than N seconds ago Int64 cleanup_delay_period = 60; // minute (in seconds) diff --git a/src/Interpreters/DNSCacheUpdater.cpp b/src/Interpreters/DNSCacheUpdater.cpp index 248c0ffa4dd..fb0298f480f 100644 --- a/src/Interpreters/DNSCacheUpdater.cpp +++ b/src/Interpreters/DNSCacheUpdater.cpp @@ -1,7 +1,7 @@ #include "DNSCacheUpdater.h" #include #include -#include + namespace DB { diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index 07a41c12983..b889509c264 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -49,7 +49,9 @@ struct TableWithColumnNamesAndTypes { DatabaseAndTableWithAlias table; NamesAndTypesList columns; - NamesAndTypesList hidden_columns; /// Not general columns like MATERIALIZED and ALIAS. They are omitted in * and t.* results. + NamesAndTypesList hidden_columns; /// Not general columns like MATERIALIZED, ALIAS, VIRTUAL. They are omitted in * and t.* results by default. + NamesAndTypesList alias_columns; + NamesAndTypesList materialized_columns; TableWithColumnNamesAndTypes(const DatabaseAndTableWithAlias & table_, const NamesAndTypesList & columns_) : table(table_) @@ -63,11 +65,28 @@ struct TableWithColumnNamesAndTypes void addHiddenColumns(const NamesAndTypesList & addition) { - hidden_columns.insert(hidden_columns.end(), addition.begin(), addition.end()); + addAdditionalColumns(hidden_columns, addition); + } + + void addAliasColumns(const NamesAndTypesList & addition) + { + addAdditionalColumns(alias_columns, addition); + } + + void addMaterializedColumns(const NamesAndTypesList & addition) + { + addAdditionalColumns(alias_columns, addition); + } + +private: + void addAdditionalColumns(NamesAndTypesList & target, const NamesAndTypesList & addition) + { + target.insert(target.end(), addition.begin(), addition.end()); for (auto & col : addition) names.insert(col.name); } + private: NameSet names; }; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index e9caaab4ef9..18cf69675ba 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -148,10 +149,16 @@ void DatabaseCatalog::loadDatabases() std::lock_guard lock{tables_marked_dropped_mutex}; if (!tables_marked_dropped.empty()) (*drop_task)->schedule(); + + /// Another background thread which drops temporary LiveViews. + /// We should start it after loadMarkedAsDroppedTables() to avoid race condition. + TemporaryLiveViewCleaner::instance().startupIfNecessary(); } void DatabaseCatalog::shutdownImpl() { + TemporaryLiveViewCleaner::shutdown(); + if (drop_task) (*drop_task)->deactivate(); @@ -524,6 +531,7 @@ std::unique_ptr DatabaseCatalog::database_catalog; DatabaseCatalog::DatabaseCatalog(Context & global_context_) : global_context(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) { + TemporaryLiveViewCleaner::init(global_context); } DatabaseCatalog & DatabaseCatalog::init(Context & global_context_) diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 5104f1e8e72..1b5c48fd43e 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -62,7 +62,7 @@ public: using Actions = std::vector; - /// This map helps to find input position bu it's name. + /// This map helps to find input position by it's name. /// Key is a view to input::result_name. /// Result is a list because it is allowed for inputs to have same names. using NameToInputMap = std::unordered_map>; @@ -87,6 +87,7 @@ public: const Actions & getActions() const { return actions; } const std::list & getNodes() const { return actions_dag->getNodes(); } const ActionsDAG & getActionsDAG() const { return *actions_dag; } + const ColumnNumbers & getResultPositions() const { return result_positions; } /// Get a list of input columns. Names getRequiredColumns() const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index a80b7799a98..ae1aa1a0bb1 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1489,23 +1489,6 @@ void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, si columns_to_remove.insert(step.required_output[i]); } - if (!columns_to_remove.empty()) - { - auto columns = prewhere_info->prewhere_actions->getResultColumns(); - - auto remove_actions = std::make_shared(); - for (const auto & column : columns) - { - if (columns_to_remove.count(column.name)) - { - remove_actions->addInput(column); - remove_actions->removeColumn(column.name); - } - } - - prewhere_info->remove_columns_actions = std::move(remove_actions); - } - columns_to_remove_after_prewhere = std::move(columns_to_remove); } else if (hasFilter()) diff --git a/src/Interpreters/InterpreterExistsQuery.cpp b/src/Interpreters/InterpreterExistsQuery.cpp index 94a31db2e99..b94f29f9194 100644 --- a/src/Interpreters/InterpreterExistsQuery.cpp +++ b/src/Interpreters/InterpreterExistsQuery.cpp @@ -53,6 +53,13 @@ BlockInputStreamPtr InterpreterExistsQuery::executeImpl() result = DatabaseCatalog::instance().isTableExist({database, exists_query->table}, context); } } + else if ((exists_query = query_ptr->as())) + { + String database = context.resolveDatabase(exists_query->database); + context.checkAccess(AccessType::SHOW_TABLES, database, exists_query->table); + auto tbl = DatabaseCatalog::instance().tryGetTable({database, exists_query->table}, context); + result = tbl != nullptr && tbl->isView(); + } else if ((exists_query = query_ptr->as())) { String database = context.resolveDatabase(exists_query->database); diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 3fcbd1c3219..15e4c52f040 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -156,6 +156,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } + else if (query->as()) + { + return std::make_unique(query, context); + } else if (query->as()) { return std::make_unique(query, context); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 742c9f6736f..ab5fe4eae9f 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -106,7 +106,7 @@ Block InterpreterInsertQuery::getSampleBlock( /// The table does not have a column with that name if (!table_sample.has(current_name)) - throw Exception("No such column " + current_name + " in table " + query.table_id.getNameForLogs(), + throw Exception("No such column " + current_name + " in table " + table->getStorageID().getNameForLogs(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); if (!allow_materialized && !table_sample_non_materialized.has(current_name)) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f7adfde5b01..b5313473252 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -1223,6 +1224,7 @@ void InterpreterSelectQuery::executeFetchColumns( temp_query_info.query = query_ptr; temp_query_info.syntax_analyzer_result = syntax_analyzer_result; temp_query_info.sets = query_analyzer->getPreparedSets(); + num_rows = storage->totalRowsByPartitionPredicate(temp_query_info, *context); } if (num_rows) @@ -1329,9 +1331,12 @@ void InterpreterSelectQuery::executeFetchColumns( if (is_alias) { auto column_decl = storage_columns.get(column); - /// TODO: can make CAST only if the type is different (but requires SyntaxAnalyzer). - auto cast_column_default = addTypeConversionToAST(column_default->expression->clone(), column_decl.type->getName()); - column_expr = setAlias(cast_column_default->clone(), column); + column_expr = column_default->expression->clone(); + // recursive visit for alias to alias + replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), syntax_analyzer_result->getArrayJoinSourceNameSet(), *context); + + column_expr = addTypeConversionToAST(std::move(column_expr), column_decl.type->getName(), metadata_snapshot->getColumns().getAll(), *context); + column_expr = setAlias(column_expr, column); } else column_expr = std::make_shared(column); @@ -1543,7 +1548,7 @@ void InterpreterSelectQuery::executeFetchColumns( getSortDescriptionFromGroupBy(query), query_info.syntax_analyzer_result); - query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot); + query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot, *context); } StreamLocalLimits limits; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index fd36f3a6fd6..86706701141 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -221,21 +222,43 @@ BlockIO InterpreterSystemQuery::execute() switch (query.type) { case Type::SHUTDOWN: + { context.checkAccess(AccessType::SYSTEM_SHUTDOWN); if (kill(0, SIGTERM)) throwFromErrno("System call kill(0, SIGTERM) failed", ErrorCodes::CANNOT_KILL); break; + } case Type::KILL: + { context.checkAccess(AccessType::SYSTEM_SHUTDOWN); - if (kill(0, SIGKILL)) - throwFromErrno("System call kill(0, SIGKILL) failed", ErrorCodes::CANNOT_KILL); + /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL. + /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker). + LOG_INFO(log, "Exit immediately as the SYSTEM KILL command has been issued."); + _exit(128 + SIGKILL); + // break; /// unreachable + } + case Type::SUSPEND: + { + auto command = fmt::format("kill -STOP {0} && sleep {1} && kill -CONT {0}", getpid(), query.seconds); + LOG_DEBUG(log, "Will run {}", command); + auto res = ShellCommand::execute(command); + res->in.close(); + WriteBufferFromOwnString out; + copyData(res->out, out); + copyData(res->err, out); + if (!out.str().empty()) + LOG_DEBUG(log, "The command returned output: {}", command, out.str()); + res->wait(); break; + } case Type::DROP_DNS_CACHE: + { context.checkAccess(AccessType::SYSTEM_DROP_DNS_CACHE); DNSResolver::instance().dropCache(); /// Reinitialize clusters to update their resolved_addresses system_context.reloadClusterConfig(); break; + } case Type::DROP_MARK_CACHE: context.checkAccess(AccessType::SYSTEM_DROP_MARK_CACHE); system_context.dropMarkCache(); @@ -251,12 +274,15 @@ BlockIO InterpreterSystemQuery::execute() break; #endif case Type::RELOAD_DICTIONARY: + { context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY); system_context.getExternalDictionariesLoader().loadOrReload( DatabaseCatalog::instance().resolveDictionaryName(query.target_dictionary)); ExternalDictionariesLoader::resetAll(); break; + } case Type::RELOAD_DICTIONARIES: + { context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY); executeCommandsAndThrowIfError( [&] () { system_context.getExternalDictionariesLoader().reloadAllTriedToLoad(); }, @@ -264,6 +290,7 @@ BlockIO InterpreterSystemQuery::execute() ); ExternalDictionariesLoader::resetAll(); break; + } case Type::RELOAD_EMBEDDED_DICTIONARIES: context.checkAccess(AccessType::SYSTEM_RELOAD_EMBEDDED_DICTIONARIES); system_context.getEmbeddedDictionaries().reload(); @@ -273,6 +300,7 @@ BlockIO InterpreterSystemQuery::execute() system_context.reloadConfig(); break; case Type::RELOAD_SYMBOLS: + { #if defined(__ELF__) && !defined(__FreeBSD__) context.checkAccess(AccessType::SYSTEM_RELOAD_SYMBOLS); (void)SymbolIndex::instance(true); @@ -280,6 +308,7 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception("SYSTEM RELOAD SYMBOLS is not supported on current platform", ErrorCodes::NOT_IMPLEMENTED); #endif + } case Type::STOP_MERGES: startStopAction(ActionLocks::PartsMerge, false); break; @@ -340,6 +369,7 @@ BlockIO InterpreterSystemQuery::execute() ErrorCodes::BAD_ARGUMENTS); break; case Type::FLUSH_LOGS: + { context.checkAccess(AccessType::SYSTEM_FLUSH_LOGS); executeCommandsAndThrowIfError( [&] () { if (auto query_log = context.getQueryLog()) query_log->flush(true); }, @@ -352,6 +382,7 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); } ); break; + } case Type::STOP_LISTEN_QUERIES: case Type::START_LISTEN_QUERIES: throw Exception(String(ASTSystemQuery::typeToString(query.type)) + " is not supported yet", ErrorCodes::NOT_IMPLEMENTED); @@ -436,8 +467,11 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) guard.second = catalog.getDDLGuard(guard.first.database_name, guard.first.table_name); ThreadPool pool(std::min(size_t(getNumberOfPhysicalCPUCores()), replica_names.size())); - for (auto & table : replica_names) - pool.scheduleOrThrowOnError([&]() { tryRestartReplica(table, system_context, false); }); + for (auto & replica : replica_names) + { + LOG_TRACE(log, "Restarting replica on {}", replica.getNameForLogs()); + pool.scheduleOrThrowOnError([&]() { tryRestartReplica(replica, system_context, false); }); + } pool.wait(); } @@ -586,7 +620,8 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() switch (query.type) { case Type::SHUTDOWN: [[fallthrough]]; - case Type::KILL: + case Type::KILL: [[fallthrough]]; + case Type::SUSPEND: { required_access.emplace_back(AccessType::SYSTEM_SHUTDOWN); break; diff --git a/src/Interpreters/RewriteAnyFunctionVisitor.cpp b/src/Interpreters/RewriteAnyFunctionVisitor.cpp index e8f05962862..f7c128a5749 100644 --- a/src/Interpreters/RewriteAnyFunctionVisitor.cpp +++ b/src/Interpreters/RewriteAnyFunctionVisitor.cpp @@ -78,6 +78,9 @@ void RewriteAnyFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, Da auto & func_arguments = func.arguments->children; + if (func_arguments.size() != 1) + return; + const auto * first_arg_func = func_arguments[0]->as(); if (!first_arg_func || first_arg_func->arguments->children.empty()) return; diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 5db914bc457..2d3bffa8234 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -230,16 +230,8 @@ void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column) void TableJoin::addJoinedColumnsAndCorrectNullability(ColumnsWithTypeAndName & columns) const { for (auto & col : columns) - { - /// Materialize column. - /// Column is not empty if it is constant, but after Join all constants will be materialized. - /// So, we need remove constants from header. - if (col.column) - col.column = nullptr; - if (leftBecomeNullable(col.type)) col.type = makeNullable(col.type); - } for (const auto & col : columns_added_by_join) { diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 0129e6bdce9..bf3bbf22b8c 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -224,11 +224,14 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt bool first_table = true; for (const auto & table : tables_with_columns) { - for (const auto & column : table.columns) + for (const auto * cols : {&table.columns, &table.alias_columns, &table.materialized_columns}) { - if (first_table || !data.join_using_columns.count(column.name)) + for (const auto & column : *cols) { - addIdentifier(columns, table.table, column.name); + if (first_table || !data.join_using_columns.count(column.name)) + { + addIdentifier(columns, table.table, column.name); + } } } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 2b801500958..554fedeed64 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -18,6 +18,7 @@ #include /// getSmallestColumn() #include #include +#include #include #include @@ -427,6 +428,7 @@ void collectJoinedColumns(TableJoin & analyzed_join, const ASTSelectQuery & sele } } + std::vector getAggregates(ASTPtr & query, const ASTSelectQuery & select_query) { /// There can not be aggregate functions inside the WHERE and PREWHERE. @@ -730,6 +732,13 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select required_source_columns.swap(source_columns); } +NameSet TreeRewriterResult::getArrayJoinSourceNameSet() const +{ + NameSet forbidden_columns; + for (const auto & elem : array_join_result_to_source) + forbidden_columns.insert(elem.first); + return forbidden_columns; +} TreeRewriterResultPtr TreeRewriter::analyzeSelect( ASTPtr & query, @@ -793,6 +802,12 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( result.analyzed_join->table_join); collectJoinedColumns(*result.analyzed_join, *select_query, tables_with_columns, result.aliases); + /// rewrite filters for select query, must go after getArrayJoinedColumns + if (settings.optimize_respect_aliases && result.metadata_snapshot) + { + replaceAliasColumnsInQuery(query, result.metadata_snapshot->getColumns(), result.getArrayJoinSourceNameSet(), context); + } + result.aggregates = getAggregates(query, *select_query); result.window_function_asts = getWindowFunctions(query, *select_query); result.collectUsedColumns(query, true); diff --git a/src/Interpreters/TreeRewriter.h b/src/Interpreters/TreeRewriter.h index d9f98ee40bd..3a5f9d6fdb2 100644 --- a/src/Interpreters/TreeRewriter.h +++ b/src/Interpreters/TreeRewriter.h @@ -70,6 +70,7 @@ struct TreeRewriterResult void collectSourceColumns(bool add_special); void collectUsedColumns(const ASTPtr & query, bool is_select); Names requiredSourceColumns() const { return required_source_columns.getNames(); } + NameSet getArrayJoinSourceNameSet() const; const Scalars & getScalars() const { return scalars; } }; diff --git a/src/Interpreters/addTypeConversionToAST.cpp b/src/Interpreters/addTypeConversionToAST.cpp index 07d0d43e0f3..bb42ad79daa 100644 --- a/src/Interpreters/addTypeConversionToAST.cpp +++ b/src/Interpreters/addTypeConversionToAST.cpp @@ -4,11 +4,20 @@ #include #include #include - +#include +#include +#include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int THERE_IS_NO_DEFAULT_VALUE; +} + ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name) { auto func = makeASTFunction("cast", ast, std::make_shared(type_name)); @@ -23,4 +32,23 @@ ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name) return func; } +ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, const Context & context) +{ + auto syntax_analyzer_result = TreeRewriter(context).analyze(ast, all_columns); + const auto actions = ExpressionAnalyzer(ast, syntax_analyzer_result, context).getActions(true); + + for (const auto & action : actions->getActions()) + if (action.node->type == ActionsDAG::ActionType::ARRAY_JOIN) + throw Exception("Unsupported default value that requires ARRAY JOIN action", ErrorCodes::THERE_IS_NO_DEFAULT_VALUE); + + auto block = actions->getSampleBlock(); + + auto desc_type = block.getByName(ast->getColumnName()).type; + if (desc_type->getName() != type_name) + return addTypeConversionToAST(std::move(ast), type_name); + + return std::move(ast); +} + + } diff --git a/src/Interpreters/addTypeConversionToAST.h b/src/Interpreters/addTypeConversionToAST.h index 1951eebc3f5..16fa98f6e0c 100644 --- a/src/Interpreters/addTypeConversionToAST.h +++ b/src/Interpreters/addTypeConversionToAST.h @@ -6,8 +6,12 @@ namespace DB { - +class Context; +class NamesAndTypesList; /// It will produce an expression with CAST to get an AST with the required type. ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name); +// If same type, then ignore the wrapper of CAST function +ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name, const NamesAndTypesList & all_columns, const Context & context); + } diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 56ca614dc2d..9234aaa831a 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -124,6 +124,8 @@ TablesWithColumns getDatabaseAndTablesWithColumns(const std::vector +#include +#include +#include + +namespace DB +{ + +void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, const Context & context) +{ + ColumnAliasesVisitor::Data aliase_column_data(columns, forbidden_columns, context); + ColumnAliasesVisitor aliase_column_visitor(aliase_column_data); + aliase_column_visitor.visit(ast); +} + +} diff --git a/src/Interpreters/replaceAliasColumnsInQuery.h b/src/Interpreters/replaceAliasColumnsInQuery.h new file mode 100644 index 00000000000..bf7143ba099 --- /dev/null +++ b/src/Interpreters/replaceAliasColumnsInQuery.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +class ColumnsDescription; +class Context; +void replaceAliasColumnsInQuery(ASTPtr & ast, const ColumnsDescription & columns, const NameSet & forbidden_columns, const Context & context); + +} diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 92da029d681..d6671b0973c 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -37,6 +37,7 @@ SRCS( ClusterProxy/SelectStreamFactory.cpp ClusterProxy/executeQuery.cpp CollectJoinOnKeysVisitor.cpp + ColumnAliasesVisitor.cpp Context.cpp CrashLog.cpp CrossToInnerJoinVisitor.cpp @@ -157,6 +158,7 @@ SRCS( interpretSubquery.cpp join_common.cpp loadMetadata.cpp + replaceAliasColumnsInQuery.cpp processColumnTransformers.cpp sortBlock.cpp diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index b997164684b..8a44dcc7c3b 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -344,6 +344,22 @@ void ASTAlterCommand::formatImpl( throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE); } +bool ASTAlterQuery::isSettingsAlter() const +{ + if (command_list) + { + if (command_list->children.empty()) + return false; + for (const auto & child : command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::MODIFY_SETTING) + return false; + } + return true; + } + return false; +} /** Get the text that identifies this element. */ String ASTAlterQuery::getID(char delim) const diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 91c80867738..f53a987905e 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -187,6 +187,8 @@ public: ASTExpressionList * command_list = nullptr; + bool isSettingsAlter() const; + String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 0d6e15a3d8c..f3a43d7f3fd 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -22,6 +22,8 @@ const char * ASTSystemQuery::typeToString(Type type) return "SHUTDOWN"; case Type::KILL: return "KILL"; + case Type::SUSPEND: + return "SUSPEND"; case Type::DROP_DNS_CACHE: return "DROP DNS CACHE"; case Type::DROP_MARK_CACHE: @@ -146,7 +148,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, auto print_on_volume = [&] { - settings.ostr << " ON VOLUME " + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON VOLUME " << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(storage_policy) << (settings.hilite ? hilite_none : "") << "." @@ -182,9 +184,20 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, print_database_table(); } else if (type == Type::RELOAD_DICTIONARY) + { print_database_dictionary(); + } else if (type == Type::DROP_REPLICA) + { print_drop_replica(); + } + else if (type == Type::SUSPEND) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR " + << (settings.hilite ? hilite_none : "") << seconds + << (settings.hilite ? hilite_keyword : "") << " SECOND" + << (settings.hilite ? hilite_none : ""); + } } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 756b5b52600..ad7eb664659 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -20,6 +20,7 @@ public: UNKNOWN, SHUTDOWN, KILL, + SUSPEND, DROP_DNS_CACHE, DROP_MARK_CACHE, DROP_UNCOMPRESSED_CACHE, @@ -65,9 +66,10 @@ public: String table; String replica; String replica_zk_path; - bool is_drop_whole_replica; + bool is_drop_whole_replica{}; String storage_policy; String volume; + UInt64 seconds{}; String getID(char) const override { return "SYSTEM query"; } diff --git a/src/Parsers/ASTWithAlias.cpp b/src/Parsers/ASTWithAlias.cpp index 20f647fb575..88f6568a719 100644 --- a/src/Parsers/ASTWithAlias.cpp +++ b/src/Parsers/ASTWithAlias.cpp @@ -48,4 +48,9 @@ void ASTWithAlias::appendColumnName(WriteBuffer & ostr) const appendColumnNameImpl(ostr); } +void ASTWithAlias::appendColumnNameWithoutAlias(WriteBuffer & ostr) const +{ + appendColumnNameImpl(ostr); +} + } diff --git a/src/Parsers/ASTWithAlias.h b/src/Parsers/ASTWithAlias.h index 7a272a157e2..ea4419402b0 100644 --- a/src/Parsers/ASTWithAlias.h +++ b/src/Parsers/ASTWithAlias.h @@ -21,6 +21,7 @@ public: using IAST::IAST; void appendColumnName(WriteBuffer & ostr) const final; + void appendColumnNameWithoutAlias(WriteBuffer & ostr) const final; String getAliasOrColumnName() const override { return alias.empty() ? getColumnName() : alias; } String tryGetAlias() const override { return alias; } void setAlias(const String & to) override { alias = to; } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 649be7e8fa7..501d3329593 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -261,11 +261,13 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserIdentifier id_parser; ParserKeyword distinct("DISTINCT"); + ParserKeyword all("ALL"); ParserExpressionList contents(false); ParserSelectWithUnionQuery select; ParserKeyword over("OVER"); - bool has_distinct_modifier = false; + bool has_all = false; + bool has_distinct = false; ASTPtr identifier; ASTPtr query; @@ -279,10 +281,34 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; ++pos; + auto pos_after_bracket = pos; + auto old_expected = expected; + + if (all.ignore(pos, expected)) + has_all = true; if (distinct.ignore(pos, expected)) - has_distinct_modifier = true; - else + has_distinct = true; + + if (!has_all && all.ignore(pos, expected)) + has_all = true; + + if (has_all && has_distinct) + return false; + + if (has_all || has_distinct) + { + /// case f(ALL), f(ALL, x), f(DISTINCT), f(DISTINCT, x), ALL and DISTINCT should be treat as identifier + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingRoundBracket) + { + pos = pos_after_bracket; + expected = old_expected; + has_all = false; + has_distinct = false; + } + } + + if (!has_distinct && !has_all) { auto old_pos = pos; auto maybe_an_subquery = pos->type == TokenType::OpeningRoundBracket; @@ -370,14 +396,37 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ++pos; /// Parametric aggregate functions cannot have DISTINCT in parameters list. - if (has_distinct_modifier) + if (has_distinct) return false; expr_list_params = expr_list_args; expr_list_args = nullptr; + pos_after_bracket = pos; + old_expected = expected; + + if (all.ignore(pos, expected)) + has_all = true; + if (distinct.ignore(pos, expected)) - has_distinct_modifier = true; + has_distinct = true; + + if (!has_all && all.ignore(pos, expected)) + has_all = true; + + if (has_all && has_distinct) + return false; + + if (has_all || has_distinct) + { + /// case f(ALL), f(ALL, x), f(DISTINCT), f(DISTINCT, x), ALL and DISTINCT should be treat as identifier + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingRoundBracket) + { + pos = pos_after_bracket; + expected = old_expected; + has_distinct = false; + } + } if (!contents.parse(pos, expr_list_args, expected)) return false; @@ -391,7 +440,7 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) tryGetIdentifierNameInto(identifier, function_node->name); /// func(DISTINCT ...) is equivalent to funcDistinct(...) - if (has_distinct_modifier) + if (has_distinct) function_node->name += "Distinct"; function_node->arguments = expr_list_args; diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index e223235b8e4..6451a9d0c0d 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -109,6 +109,14 @@ String IAST::getColumnName() const } +String IAST::getColumnNameWithoutAlias() const +{ + WriteBufferFromOwnString write_buffer; + appendColumnNameWithoutAlias(write_buffer); + return write_buffer.str(); +} + + void IAST::FormatSettings::writeIdentifier(const String & name) const { switch (identifier_quoting_style) diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index bed6c5bcdf9..54e08b2700e 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -41,11 +41,18 @@ public: /** Get the canonical name of the column if the element is a column */ String getColumnName() const; + /** Same as the above but ensure no alias names are used. This is for index analysis */ + String getColumnNameWithoutAlias() const; virtual void appendColumnName(WriteBuffer &) const { throw Exception("Trying to get name of not a column: " + getID(), ErrorCodes::LOGICAL_ERROR); } + virtual void appendColumnNameWithoutAlias(WriteBuffer &) const + { + throw Exception("Trying to get name of not a column: " + getID(), ErrorCodes::LOGICAL_ERROR); + } + /** Get the alias, if any, or the canonical name of the column, if it is not. */ virtual String getAliasOrColumnName() const { return getColumnName(); } diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 91c48fc362d..f515901edd4 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -30,6 +30,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) node = select_query; ParserKeyword s_select("SELECT"); + ParserKeyword s_all("ALL"); ParserKeyword s_distinct("DISTINCT"); ParserKeyword s_from("FROM"); ParserKeyword s_prewhere("PREWHERE"); @@ -91,14 +92,24 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } - /// SELECT [DISTINCT] [TOP N [WITH TIES]] expr list + /// SELECT [ALL/DISTINCT] [TOP N [WITH TIES]] expr list { + bool has_all = false; if (!s_select.ignore(pos, expected)) return false; + if (s_all.ignore(pos, expected)) + has_all = true; + if (s_distinct.ignore(pos, expected)) select_query->distinct = true; + if (!has_all && s_all.ignore(pos, expected)) + has_all = true; + + if (has_all && select_query->distinct) + return false; + if (s_top.ignore(pos, expected)) { ParserNumber num; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index b6a90b348a0..491037da9a9 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -169,6 +169,20 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & parseDatabaseAndTableName(pos, expected, res->database, res->table); break; + case Type::SUSPEND: + { + ASTPtr seconds; + if (!(ParserKeyword{"FOR"}.ignore(pos, expected) + && ParserUnsignedInteger().parse(pos, seconds, expected) + && ParserKeyword{"SECOND"}.ignore(pos, expected))) /// SECOND, not SECONDS to be consistent with INTERVAL parsing in SQL + { + return false; + } + + res->seconds = seconds->as()->value.get(); + break; + } + default: /// There are no [db.table] after COMMAND NAME break; diff --git a/src/Parsers/ParserTablePropertiesQuery.cpp b/src/Parsers/ParserTablePropertiesQuery.cpp index f42414045e3..30be37bc4a1 100644 --- a/src/Parsers/ParserTablePropertiesQuery.cpp +++ b/src/Parsers/ParserTablePropertiesQuery.cpp @@ -32,6 +32,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & bool parse_only_database_name = false; bool parse_show_create_view = false; + bool exists_view = false; bool temporary = false; if (s_exists.ignore(pos, expected)) @@ -41,6 +42,11 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query = std::make_shared(); parse_only_database_name = true; } + else if (s_view.ignore(pos, expected)) + { + query = std::make_shared(); + exists_view = true; + } else { if (s_temporary.ignore(pos, expected)) @@ -86,7 +92,7 @@ bool ParserTablePropertiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & } else { - if (!parse_show_create_view) + if (!(exists_view || parse_show_create_view)) { if (temporary || s_temporary.ignore(pos, expected)) query->temporary = true; diff --git a/src/Parsers/TablePropertiesQueriesASTs.h b/src/Parsers/TablePropertiesQueriesASTs.h index c9fd453604f..bfd3135aec4 100644 --- a/src/Parsers/TablePropertiesQueriesASTs.h +++ b/src/Parsers/TablePropertiesQueriesASTs.h @@ -22,6 +22,15 @@ struct ASTExistsTableQueryIDAndQueryNames static constexpr auto QueryTemporary = "EXISTS TEMPORARY TABLE"; }; +struct ASTExistsViewQueryIDAndQueryNames +{ + static constexpr auto ID = "ExistsViewQuery"; + static constexpr auto Query = "EXISTS VIEW"; + /// No temporary view are supported, just for parsing + static constexpr auto QueryTemporary = ""; +}; + + struct ASTExistsDictionaryQueryIDAndQueryNames { static constexpr auto ID = "ExistsDictionaryQuery"; @@ -69,6 +78,7 @@ struct ASTDescribeQueryExistsQueryIDAndQueryNames using ASTExistsDatabaseQuery = ASTQueryWithTableAndOutputImpl; using ASTExistsTableQuery = ASTQueryWithTableAndOutputImpl; +using ASTExistsViewQuery = ASTQueryWithTableAndOutputImpl; using ASTExistsDictionaryQuery = ASTQueryWithTableAndOutputImpl; using ASTShowCreateTableQuery = ASTQueryWithTableAndOutputImpl; using ASTShowCreateViewQuery = ASTQueryWithTableAndOutputImpl; diff --git a/src/Processors/Executors/PollingQueue.cpp b/src/Processors/Executors/PollingQueue.cpp index 7383824a592..93edfe53987 100644 --- a/src/Processors/Executors/PollingQueue.cpp +++ b/src/Processors/Executors/PollingQueue.cpp @@ -7,6 +7,9 @@ #include #include +#include +#include + namespace DB { @@ -46,7 +49,7 @@ void PollingQueue::addTask(size_t thread_number, void * data, int fd) { std::uintptr_t key = reinterpret_cast(data); if (tasks.count(key)) - throw Exception("Task was already added to task queue", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Task {} was already added to task queue", key); tasks[key] = TaskData{thread_number, data, fd}; @@ -58,6 +61,22 @@ void PollingQueue::addTask(size_t thread_number, void * data, int fd) throwFromErrno("Cannot add socket descriptor to epoll", ErrorCodes::CANNOT_OPEN_FILE); } +static std::string dumpTasks(const std::unordered_map & tasks) +{ + WriteBufferFromOwnString res; + res << "Tasks = ["; + + for (const auto & task : tasks) + { + res << "(id " << task.first << " thread " << task.second.thread_num << " ptr "; + writePointerHex(task.second.data, res); + res << " fd " << task.second.fd << ")"; + } + + res << "]"; + return res.str(); +} + PollingQueue::TaskData PollingQueue::wait(std::unique_lock & lock) { if (is_finished) @@ -81,10 +100,14 @@ PollingQueue::TaskData PollingQueue::wait(std::unique_lock & lock) if (event.data.ptr == pipe_fd) return {}; - std::uintptr_t key = reinterpret_cast(event.data.ptr); + void * ptr = event.data.ptr; + std::uintptr_t key = reinterpret_cast(ptr); auto it = tasks.find(key); if (it == tasks.end()) - throw Exception("Task was not found in task queue", ErrorCodes::LOGICAL_ERROR); + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Task {} ({}) was not found in task queue: {}", + key, ptr, dumpTasks(tasks)); + } auto res = it->second; tasks.erase(it); @@ -98,7 +121,6 @@ PollingQueue::TaskData PollingQueue::wait(std::unique_lock & lock) void PollingQueue::finish() { is_finished = true; - tasks.clear(); uint64_t buf = 0; while (-1 == write(pipe_fd[1], &buf, sizeof(buf))) diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index f7ca0a972dc..e23954cc063 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -100,6 +100,11 @@ void ExpressionStep::describeActions(FormatSettings & settings) const first = false; settings.out << action.toString() << '\n'; } + + settings.out << prefix << "Positions:"; + for (const auto & pos : expression->getResultPositions()) + settings.out << ' ' << pos; + settings.out << '\n'; } JoinStep::JoinStep(const DataStream & input_stream_, JoinPtr join_) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 5d089eb9f80..ade75506a38 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -6,8 +6,9 @@ #include #include #include -#include #include +#include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include @@ -80,18 +82,26 @@ namespace StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool) + StorageDistributed & storage_, + const DiskPtr & disk_, + const std::string & relative_path_, + ConnectionPoolPtr pool_, + ActionBlocker & monitor_blocker_, + BackgroundSchedulePool & bg_pool) : storage(storage_) , pool(std::move(pool_)) - , path{path_ + '/'} + , disk(disk_) + , relative_path(relative_path_) + , path(disk->getPath() + relative_path + '/') , should_batch_inserts(storage.global_context.getSettingsRef().distributed_directory_monitor_batch_inserts) + , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) , min_batched_block_size_rows(storage.global_context.getSettingsRef().min_insert_block_size_rows) , min_batched_block_size_bytes(storage.global_context.getSettingsRef().min_insert_block_size_bytes) - , current_batch_file_path{path + "current_batch.txt"} - , default_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()} - , sleep_time{default_sleep_time} - , max_sleep_time{storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()} - , log{&Poco::Logger::get(getLoggerName())} + , current_batch_file_path(path + "current_batch.txt") + , default_sleep_time(storage.global_context.getSettingsRef().distributed_directory_monitor_sleep_time_ms.totalMilliseconds()) + , sleep_time(default_sleep_time) + , max_sleep_time(storage.global_context.getSettingsRef().distributed_directory_monitor_max_sleep_time_ms.totalMilliseconds()) + , log(&Poco::Logger::get(getLoggerName())) , monitor_blocker(monitor_blocker_) , metric_pending_files(CurrentMetrics::DistributedFilesToInsert, 0) { @@ -134,6 +144,10 @@ void StorageDistributedDirectoryMonitor::shutdownAndDropAllData() task_handle->deactivate(); } + std::optional dir_sync_guard; + if (dir_fsync) + dir_sync_guard.emplace(disk, relative_path); + Poco::File(path).remove(true); } @@ -337,6 +351,10 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa throw; } + std::optional dir_sync_guard; + if (dir_fsync) + dir_sync_guard.emplace(disk, relative_path); + Poco::File{file_path}.remove(); metric_pending_files.sub(); @@ -444,10 +462,16 @@ struct StorageDistributedDirectoryMonitor::Batch StorageDistributedDirectoryMonitor & parent; const std::map & file_index_to_path; + bool fsync = false; + bool dir_fsync = false; + Batch( StorageDistributedDirectoryMonitor & parent_, const std::map & file_index_to_path_) - : parent(parent_), file_index_to_path(file_index_to_path_) + : parent(parent_) + , file_index_to_path(file_index_to_path_) + , fsync(parent.storage.getDistributedSettingsRef().fsync_after_insert) + , dir_fsync(parent.dir_fsync) {} bool isEnoughSize() const @@ -474,12 +498,20 @@ struct StorageDistributedDirectoryMonitor::Batch /// Temporary file is required for atomicity. String tmp_file{parent.current_batch_file_path + ".tmp"}; + std::optional dir_sync_guard; + if (dir_fsync) + dir_sync_guard.emplace(parent.disk, parent.relative_path); + if (Poco::File{tmp_file}.exists()) LOG_ERROR(parent.log, "Temporary file {} exists. Unclean shutdown?", backQuote(tmp_file)); { WriteBufferFromFile out{tmp_file, O_WRONLY | O_TRUNC | O_CREAT}; writeText(out); + + out.finalize(); + if (fsync) + out.sync(); } Poco::File{tmp_file}.renameTo(parent.current_batch_file_path); @@ -537,6 +569,10 @@ struct StorageDistributedDirectoryMonitor::Batch { LOG_TRACE(parent.log, "Sent a batch of {} files.", file_indices.size()); + std::optional dir_sync_guard; + if (dir_fsync) + dir_sync_guard.emplace(parent.disk, parent.relative_path); + for (UInt64 file_index : file_indices) Poco::File{file_index_to_path.at(file_index)}.remove(); } @@ -734,10 +770,16 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map metric_pending_files.sub(batch.file_indices.size()); } - /// current_batch.txt will not exist if there was no send - /// (this is the case when all batches that was pending has been marked as pending) - if (Poco::File{current_batch_file_path}.exists()) - Poco::File{current_batch_file_path}.remove(); + { + std::optional dir_sync_guard; + if (dir_fsync) + dir_sync_guard.emplace(disk, relative_path); + + /// current_batch.txt will not exist if there was no send + /// (this is the case when all batches that was pending has been marked as pending) + if (Poco::File{current_batch_file_path}.exists()) + Poco::File{current_batch_file_path}.remove(); + } } bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code) @@ -759,6 +801,15 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p const auto & broken_file_path = broken_path + file_name; Poco::File{broken_path}.createDirectory(); + + std::optional dir_sync_guard; + std::optional broken_dir_sync_guard; + if (dir_fsync) + { + broken_dir_sync_guard.emplace(disk, relative_path + "/broken/"); + dir_sync_guard.emplace(disk, relative_path); + } + Poco::File{file_path}.renameTo(broken_file_path); LOG_ERROR(log, "Renamed `{}` to `{}`", file_path, broken_file_path); @@ -781,14 +832,15 @@ std::string StorageDistributedDirectoryMonitor::getLoggerName() const return storage.getStorageID().getFullTableName() + ".DirectoryMonitor"; } -void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path) +void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_relative_path) { task_handle->deactivate(); std::lock_guard lock{mutex}; { std::unique_lock metrics_lock(metrics_mutex); - path = new_path; + relative_path = new_relative_path; + path = disk->getPath() + relative_path + '/'; } current_batch_file_path = path + "current_batch.txt"; diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 1d34357b3b1..a6175b44d7b 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -14,6 +14,9 @@ namespace CurrentMetrics { class Increment; } namespace DB { +class IDisk; +using DiskPtr = std::shared_ptr; + class StorageDistributed; class ActionBlocker; class BackgroundSchedulePool; @@ -25,13 +28,18 @@ class StorageDistributedDirectoryMonitor { public: StorageDistributedDirectoryMonitor( - StorageDistributed & storage_, std::string path_, ConnectionPoolPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool); + StorageDistributed & storage_, + const DiskPtr & disk_, + const std::string & relative_path_, + ConnectionPoolPtr pool_, + ActionBlocker & monitor_blocker_, + BackgroundSchedulePool & bg_pool); ~StorageDistributedDirectoryMonitor(); static ConnectionPoolPtr createPool(const std::string & name, const StorageDistributed & storage); - void updatePath(const std::string & new_path); + void updatePath(const std::string & new_relative_path); void flushAllData(); @@ -70,9 +78,13 @@ private: StorageDistributed & storage; const ConnectionPoolPtr pool; + + DiskPtr disk; + std::string relative_path; std::string path; const bool should_batch_inserts = false; + const bool dir_fsync = false; const size_t min_batched_block_size_rows = 0; const size_t min_batched_block_size_bytes = 0; String current_batch_file_path; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 8d901028057..7299953c88e 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -588,6 +589,10 @@ void DistributedBlockOutputStream::writeToLocal(const Block & block, const size_ void DistributedBlockOutputStream::writeToShard(const Block & block, const std::vector & dir_names) { const auto & settings = context.getSettingsRef(); + const auto & distributed_settings = storage.getDistributedSettingsRef(); + + bool fsync = distributed_settings.fsync_after_insert; + bool dir_fsync = distributed_settings.fsync_directories; std::string compression_method = Poco::toUpper(settings.network_compression_method.toString()); std::optional compression_level; @@ -603,14 +608,26 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: std::string first_file_tmp_path{}; auto reservation = storage.getStoragePolicy()->reserveAndCheck(block.bytes()); - auto disk = reservation->getDisk()->getPath(); + const auto disk = reservation->getDisk(); + auto disk_path = disk->getPath(); auto data_path = storage.getRelativeDataPath(); + auto make_directory_sync_guard = [&](const std::string & current_path) + { + std::unique_ptr guard; + if (dir_fsync) + { + const std::string relative_path(data_path + current_path); + guard = std::make_unique(disk, relative_path); + } + return guard; + }; + auto it = dir_names.begin(); /// on first iteration write block to a temporary directory for subsequent /// hardlinking to ensure the inode is not freed until we're done { - const std::string path(disk + data_path + *it); + const std::string path(disk_path + data_path + *it); Poco::File(path).createDirectory(); const std::string tmp_path(path + "/tmp/"); @@ -622,6 +639,8 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: /// Write batch to temporary location { + auto tmp_dir_sync_guard = make_directory_sync_guard(*it + "/tmp/"); + WriteBufferFromFile out{first_file_tmp_path}; CompressedWriteBuffer compress{out, compression_codec}; NativeBlockOutputStream stream{compress, DBMS_TCP_PROTOCOL_VERSION, block.cloneEmpty()}; @@ -647,22 +666,28 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: stream.writePrefix(); stream.write(block); stream.writeSuffix(); + + out.finalize(); + if (fsync) + out.sync(); } // Create hardlink here to reuse increment number const std::string block_file_path(path + '/' + file_name); createHardLink(first_file_tmp_path, block_file_path); + auto dir_sync_guard = make_directory_sync_guard(*it); } ++it; /// Make hardlinks for (; it != dir_names.end(); ++it) { - const std::string path(disk + data_path + *it); + const std::string path(disk_path + data_path + *it); Poco::File(path).createDirectory(); - const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin"); + const std::string block_file_path(path + '/' + toString(storage.file_names_increment.get()) + ".bin"); createHardLink(first_file_tmp_path, block_file_path); + auto dir_sync_guard = make_directory_sync_guard(*it); } /// remove the temporary file, enabling the OS to reclaim inode after all threads diff --git a/src/Storages/Distributed/DistributedSettings.cpp b/src/Storages/Distributed/DistributedSettings.cpp new file mode 100644 index 00000000000..555aeba7c58 --- /dev/null +++ b/src/Storages/Distributed/DistributedSettings.cpp @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_SETTING; +} + +IMPLEMENT_SETTINGS_TRAITS(DistributedSettingsTraits, LIST_OF_DISTRIBUTED_SETTINGS) + +void DistributedSettings::loadFromQuery(ASTStorage & storage_def) +{ + if (storage_def.settings) + { + try + { + applyChanges(storage_def.settings->changes); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::UNKNOWN_SETTING) + e.addMessage("for storage " + storage_def.engine->name); + throw; + } + } + else + { + auto settings_ast = std::make_shared(); + settings_ast->is_standalone = false; + storage_def.set(storage_def.settings, settings_ast); + } +} + +} + diff --git a/src/Storages/Distributed/DistributedSettings.h b/src/Storages/Distributed/DistributedSettings.h new file mode 100644 index 00000000000..9df787428df --- /dev/null +++ b/src/Storages/Distributed/DistributedSettings.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include + + +namespace Poco::Util +{ + class AbstractConfiguration; +} + + +namespace DB +{ +class ASTStorage; + +#define LIST_OF_DISTRIBUTED_SETTINGS(M) \ + M(Bool, fsync_after_insert, false, "Do fsync for every inserted. Will decreases performance of inserts (only for async INSERT, i.e. insert_distributed_sync=false)", 0) \ + M(Bool, fsync_directories, false, "Do fsync for temporary directory (that is used for async INSERT only) after all part operations (writes, renames, etc.).", 0) \ + +DECLARE_SETTINGS_TRAITS(DistributedSettingsTraits, LIST_OF_DISTRIBUTED_SETTINGS) + + +/** Settings for the Distributed family of engines. + */ +struct DistributedSettings : public BaseSettings +{ + void loadFromQuery(ASTStorage & storage_def); +}; + +} diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 42ece547e1c..cfb4c4e9646 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -120,9 +120,6 @@ public: /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } - /// Returns true if the storage supports settings. - virtual bool supportsSettings() const { return false; } - /// Returns true if the blocks shouldn't be pushed to associated views on insert. virtual bool noPushingToViews() const { return false; } diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index 8ec8e718011..53871990810 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -36,7 +36,6 @@ class StorageKafka final : public ext::shared_ptr_helper, public I public: std::string getName() const override { return "Kafka"; } - bool supportsSettings() const override { return true; } bool noPushingToViews() const override { return true; } void startup() override; diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp index 052b22fd666..143e7460cc3 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.cpp @@ -48,6 +48,39 @@ void TemporaryLiveViewCleaner::init(Context & global_context_) the_instance.reset(new TemporaryLiveViewCleaner(global_context_)); } +void TemporaryLiveViewCleaner::startupIfNecessary() +{ + std::lock_guard lock{mutex}; + if (background_thread_should_exit) + return; + if (!views.empty()) + startupIfNecessaryImpl(lock); + else + can_start_background_thread = true; +} + +void TemporaryLiveViewCleaner::startupIfNecessaryImpl(const std::lock_guard &) +{ + /// If views.empty() the background thread isn't running or it's going to stop right now. + /// If can_start_background_thread is false, then the thread has not been started previously. + bool background_thread_is_running; + if (can_start_background_thread) + { + background_thread_is_running = !views.empty(); + } + else + { + can_start_background_thread = true; + background_thread_is_running = false; + } + + if (!background_thread_is_running) + { + if (background_thread.joinable()) + background_thread.join(); + background_thread = ThreadFromGlobalPool{&TemporaryLiveViewCleaner::backgroundThreadFunc, this}; + } +} void TemporaryLiveViewCleaner::shutdown() { @@ -79,20 +112,13 @@ void TemporaryLiveViewCleaner::addView(const std::shared_ptr & if (background_thread_should_exit) return; - /// If views.empty() the background thread isn't running or it's going to stop right now. - bool background_thread_is_running = !views.empty(); + if (can_start_background_thread) + startupIfNecessaryImpl(lock); /// Keep the vector `views` sorted by time of next check. StorageAndTimeOfCheck storage_and_time_of_check{view, time_of_next_check}; views.insert(std::upper_bound(views.begin(), views.end(), storage_and_time_of_check), storage_and_time_of_check); - if (!background_thread_is_running) - { - if (background_thread.joinable()) - background_thread.join(); - background_thread = ThreadFromGlobalPool{&TemporaryLiveViewCleaner::backgroundThreadFunc, this}; - } - background_thread_wake_up.notify_one(); } diff --git a/src/Storages/LiveView/TemporaryLiveViewCleaner.h b/src/Storages/LiveView/TemporaryLiveViewCleaner.h index 57c12bd1c07..8d57aa9fbfa 100644 --- a/src/Storages/LiveView/TemporaryLiveViewCleaner.h +++ b/src/Storages/LiveView/TemporaryLiveViewCleaner.h @@ -23,6 +23,9 @@ public: static void init(Context & global_context_); static void shutdown(); + void startupIfNecessary(); + void startupIfNecessaryImpl(const std::lock_guard &); + private: friend std::unique_ptr::deleter_type; @@ -44,6 +47,7 @@ private: std::mutex mutex; std::vector views; ThreadFromGlobalPool background_thread; + bool can_start_background_thread = false; std::atomic background_thread_should_exit = false; std::condition_variable background_thread_wake_up; }; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f85f9501fcf..5d50f29756c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -398,7 +398,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( disk->createDirectories(part_download_path); - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, part_download_path); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0b4297e37b3..5d0f79f4679 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -835,10 +835,6 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ String from = getFullRelativePath(); String to = storage.relative_data_path + new_relative_path + "/"; - std::optional sync_guard; - if (storage.getSettings()->fsync_part_directory) - sync_guard.emplace(volume->getDisk(), to); - if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -862,6 +858,10 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); volume->getDisk()->moveFile(from, to); relative_path = new_relative_path; + + std::optional sync_guard; + if (storage.getSettings()->fsync_part_directory) + sync_guard.emplace(volume->getDisk(), to); } diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 2a52d7d53f0..dd293bf2502 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index ead93a6e1d9..eb58b2911c9 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -444,7 +444,7 @@ bool KeyCondition::addCondition(const String & column, const Range & range) */ bool KeyCondition::getConstant(const ASTPtr & expr, Block & block_with_constants, Field & out_value, DataTypePtr & out_type) { - String column_name = expr->getColumnName(); + String column_name = expr->getColumnNameWithoutAlias(); if (const auto * lit = expr->as()) { @@ -607,7 +607,7 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( if (strict) return false; - String expr_name = node->getColumnName(); + String expr_name = node->getColumnNameWithoutAlias(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) return false; @@ -675,7 +675,7 @@ bool KeyCondition::canConstantBeWrappedByFunctions( if (strict) return false; - String expr_name = ast->getColumnName(); + String expr_name = ast->getColumnNameWithoutAlias(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) return false; @@ -934,7 +934,7 @@ bool KeyCondition::isKeyPossiblyWrappedByMonotonicFunctionsImpl( * Therefore, use the full name of the expression for search. */ const auto & sample_block = key_expr->getSampleBlock(); - String name = node->getColumnName(); + String name = node->getColumnNameWithoutAlias(); auto it = key_columns.find(name); if (key_columns.end() != it) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 897aea4363a..f84de847acd 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1506,6 +1506,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S "ALTER MODIFY ORDER BY is not supported for default-partitioned tables created with the old syntax", ErrorCodes::BAD_ARGUMENTS); } + if (command.type == AlterCommand::MODIFY_TTL && !is_custom_partitioned) + { + throw Exception( + "ALTER MODIFY TTL is not supported for default-partitioned tables created with the old syntax", + ErrorCodes::BAD_ARGUMENTS); + } if (command.type == AlterCommand::MODIFY_SAMPLE_BY) { if (!is_custom_partitioned) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index e5ffe8c025b..52c0b61b977 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -357,7 +357,6 @@ public: || merging_params.mode == MergingParams::VersionedCollapsing; } - bool supportsSettings() const override { return true; } NamesAndTypesList getVirtuals() const override; bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context &, const StorageMetadataPtr & metadata_snapshot) const override; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 64a0e7329ee..a735b939cd5 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include @@ -780,7 +780,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor gathering_column_names.clear(); } - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, new_part_tmp_path); @@ -910,7 +910,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor if (metadata_snapshot->hasSecondaryIndices()) { const auto & indices = metadata_snapshot->getSecondaryIndices(); - merged_stream = std::make_shared(merged_stream, indices.getSingleExpressionForIndices(metadata_snapshot->getColumns(), data.global_context)); + merged_stream = std::make_shared( + merged_stream, indices.getSingleExpressionForIndices(metadata_snapshot->getColumns(), data.global_context)); merged_stream = std::make_shared(merged_stream); } @@ -921,7 +922,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merging_columns, index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec, - data_settings->min_merge_bytes_to_use_direct_io, blocks_are_granules_size}; merged_stream->readPrefix(); @@ -1182,7 +1182,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor disk->createDirectories(new_part_tmp_path); - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, new_part_tmp_path); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index d5e7009efd6..ef3b5eb7d24 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -24,9 +24,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( , plain_file(data_part->volume->getDisk()->writeFile( part_path + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION, settings.max_compress_block_size, - WriteMode::Rewrite, - settings.estimated_size, - settings.aio_threshold)) + WriteMode::Rewrite)) , plain_hashing(*plain_file) , marks_file(data_part->volume->getDisk()->writeFile( part_path + MergeTreeDataPartCompact::DATA_FILE_NAME + marks_file_extension_, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 8e6ffe9ee68..697f8c809b2 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -39,14 +39,14 @@ MergeTreeDataPartWriterOnDisk::Stream::Stream( const std::string & marks_path_, const std::string & marks_file_extension_, const CompressionCodecPtr & compression_codec_, - size_t max_compress_block_size_, - size_t estimated_size_, - size_t aio_threshold_) : + size_t max_compress_block_size_) : escaped_column_name(escaped_column_name_), data_file_extension{data_file_extension_}, marks_file_extension{marks_file_extension_}, - plain_file(disk_->writeFile(data_path_ + data_file_extension, max_compress_block_size_, WriteMode::Rewrite, estimated_size_, aio_threshold_)), - plain_hashing(*plain_file), compressed_buf(plain_hashing, compression_codec_), compressed(compressed_buf), + plain_file(disk_->writeFile(data_path_ + data_file_extension, max_compress_block_size_, WriteMode::Rewrite)), + plain_hashing(*plain_file), + compressed_buf(plain_hashing, compression_codec_, max_compress_block_size_), + compressed(compressed_buf), marks_file(disk_->writeFile(marks_path_ + marks_file_extension, 4096, WriteMode::Rewrite)), marks(*marks_file) { } @@ -164,8 +164,7 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices() data_part->volume->getDisk(), part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, - default_codec, settings.max_compress_block_size, - 0, settings.aio_threshold)); + default_codec, settings.max_compress_block_size)); skip_indices_aggregators.push_back(index_helper->createIndexAggregator()); skip_index_accumulated_marks.push_back(0); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index a7b84c95e0a..704b38ba6d5 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -56,9 +56,7 @@ public: const std::string & marks_path_, const std::string & marks_file_extension_, const CompressionCodecPtr & compression_codec_, - size_t max_compress_block_size_, - size_t estimated_size_, - size_t aio_threshold_); + size_t max_compress_block_size_); String escaped_column_name; std::string data_file_extension; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index be735104e99..ef64ec28e79 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -80,14 +80,13 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( { const auto & columns = metadata_snapshot->getColumns(); for (const auto & it : columns_list) - addStreams(it.name, *it.type, columns.getCodecDescOrDefault(it.name, default_codec), settings.estimated_size); + addStreams(it.name, *it.type, columns.getCodecDescOrDefault(it.name, default_codec)); } void MergeTreeDataPartWriterWide::addStreams( const String & name, const IDataType & type, - const ASTPtr & effective_codec_desc, - size_t estimated_size) + const ASTPtr & effective_codec_desc) { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path, const IDataType & substream_type) { @@ -109,9 +108,7 @@ void MergeTreeDataPartWriterWide::addStreams( part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, compression_codec, - settings.max_compress_block_size, - estimated_size, - settings.aio_threshold); + settings.max_compress_block_size); }; IDataType::SubstreamPath stream_path; @@ -143,6 +140,12 @@ void MergeTreeDataPartWriterWide::shiftCurrentMark(const Granules & granules_wri /// If we didn't finished last granule than we will continue to write it from new block if (!last_granule.is_complete) { + if (settings.blocks_are_granules_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incomplete granules are not allowed while blocks are granules size. " + "Mark number {} (rows {}), rows written in last mark {}, rows to write in last mark from block {} (from row {}), total marks currently {}", + last_granule.mark_number, index_granularity.getMarkRows(last_granule.mark_number), rows_written_in_last_mark, + last_granule.rows_to_write, last_granule.start_row, index_granularity.getMarksCount()); + /// Shift forward except last granule setCurrentMark(getCurrentMark() + granules_written.size() - 1); bool still_in_the_same_granule = granules_written.size() == 1; @@ -164,7 +167,7 @@ void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Perm { /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, - /// but not in case of vertical merge) + /// but not in case of vertical part of vertical merge) if (compute_granularity) { size_t index_granularity_for_block = computeIndexGranularity(block); @@ -454,12 +457,25 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const String & name, /// Now they must be equal if (column->size() != index_granularity_rows) { - if (must_be_last && !settings.can_use_adaptive_granularity) - break; + + if (must_be_last) + { + /// The only possible mark after bin.eof() is final mark. When we + /// cannot use adaptive granularity we cannot have last mark. + /// So finish validation. + if (!settings.can_use_adaptive_granularity) + break; + + /// If we don't compute granularity then we are not responsible + /// for last mark (for example we mutating some column from part + /// with fixed granularity where last mark is not adjusted) + if (!compute_granularity) + continue; + } throw Exception( - ErrorCodes::LOGICAL_ERROR, "Incorrect mark rows for mark #{} (compressed offset {}, decompressed offset {}), actually in bin file {}, in mrk file {}", - mark_num, offset_in_compressed_file, offset_in_decompressed_block, column->size(), index_granularity.getMarkRows(mark_num)); + ErrorCodes::LOGICAL_ERROR, "Incorrect mark rows for mark #{} (compressed offset {}, decompressed offset {}), actually in bin file {}, in mrk file {}, total marks {}", + mark_num, offset_in_compressed_file, offset_in_decompressed_block, column->size(), index_granularity.getMarkRows(mark_num), index_granularity.getMarksCount()); } } @@ -486,7 +502,14 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch serialize_settings.low_cardinality_use_single_dictionary_for_part = global_settings.low_cardinality_use_single_dictionary_for_part != 0; WrittenOffsetColumns offset_columns; if (rows_written_in_last_mark > 0) + { + if (settings.blocks_are_granules_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Incomplete granule is not allowed while blocks are granules size even for last granule. " + "Mark number {} (rows {}), rows written for last mark {}, total marks {}", + getCurrentMark(), index_granularity.getMarkRows(getCurrentMark()), rows_written_in_last_mark, index_granularity.getMarksCount()); + adjustLastMarkIfNeedAndFlushToDisk(rows_written_in_last_mark); + } bool write_final_mark = (with_final_mark && data_written); @@ -517,7 +540,7 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch #ifndef NDEBUG /// Heavy weight validation of written data. Checks that we are able to read - /// data according to marks. Otherwise throws LOGICAL_ERROR (equal to about in debug mode) + /// data according to marks. Otherwise throws LOGICAL_ERROR (equal to abort in debug mode) for (const auto & column : columns_list) { if (column.type->isValueRepresentedByNumber() && !column.type->haveSubtypes()) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 8c76c10abef..d897503a033 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -85,8 +85,7 @@ private: void addStreams( const String & name, const IDataType & type, - const ASTPtr & effective_codec_desc, - size_t estimated_size); + const ASTPtr & effective_codec_desc); /// Method for self check (used in debug-build only). Checks that written /// data and corresponding marks are consistent. Otherwise throws logical diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 75760e145d3..4e1f307137a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -709,12 +709,12 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( auto max_partitions_to_read = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data.getSettings()->max_partitions_to_read; - if (max_partitions_to_read) + if (max_partitions_to_read > 0) { std::set partitions; for (auto & part_with_ranges : parts_with_ranges) partitions.insert(part_with_ranges.data_part->info.partition_id); - if (partitions.size() > max_partitions_to_read) + if (partitions.size() > size_t(max_partitions_to_read)) throw Exception( ErrorCodes::TOO_MANY_PARTITIONS, "Too many partitions to read. Current {}, max {}", diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index c93d4bceba0..de4d70d5e3e 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include @@ -362,7 +362,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa new_data_part->minmax_idx = std::move(minmax_idx); new_data_part->is_temp = true; - std::optional sync_guard; + std::optional sync_guard; if (new_data_part->isStoredOnDisk()) { /// The name could be non-unique in case of stale files from previous runs. diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 2dec16e7d10..9e315c08681 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -24,7 +24,6 @@ struct MergeTreeWriterSettings const Settings & global_settings, const MergeTreeSettingsPtr & storage_settings, bool can_use_adaptive_granularity_, - size_t aio_threshold_, bool rewrite_primary_key_, bool blocks_are_granules_size_ = false) : min_compress_block_size( @@ -32,7 +31,6 @@ struct MergeTreeWriterSettings , max_compress_block_size( storage_settings->max_compress_block_size ? storage_settings->max_compress_block_size : global_settings.max_compress_block_size) - , aio_threshold(aio_threshold_) , can_use_adaptive_granularity(can_use_adaptive_granularity_) , rewrite_primary_key(rewrite_primary_key_) , blocks_are_granules_size(blocks_are_granules_size_) @@ -41,14 +39,9 @@ struct MergeTreeWriterSettings size_t min_compress_block_size; size_t max_compress_block_size; - size_t aio_threshold; bool can_use_adaptive_granularity; bool rewrite_primary_key; bool blocks_are_granules_size; - - /// Used for AIO threshold comparison - /// FIXME currently doesn't work because WriteBufferAIO contain obscure bug(s) - size_t estimated_size = 0; }; } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 97c6114a0be..7f23a1a42ab 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -110,11 +110,12 @@ struct Settings; M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \ M(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \ - M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ + M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ M(UInt64, check_delay_period, 60, "Obsolete setting, does nothing.", 0) \ + M(Bool, allow_floating_point_partition_key, false, "Allow floating point as partition key", 0) \ /// Settings that should not change after the creation of a table. #define APPLY_FOR_IMMUTABLE_MERGE_TREE_SETTINGS(M) \ M(index_granularity) diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index feaf46194d1..1605ec693cb 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -21,25 +21,6 @@ MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeIndices & skip_indices, CompressionCodecPtr default_codec_, bool blocks_are_granules_size) - : MergedBlockOutputStream( - data_part, - metadata_snapshot_, - columns_list_, - skip_indices, - default_codec_, - data_part->storage.global_context.getSettings().min_bytes_to_use_direct_io, - blocks_are_granules_size) -{ -} - -MergedBlockOutputStream::MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part, - const StorageMetadataPtr & metadata_snapshot_, - const NamesAndTypesList & columns_list_, - const MergeTreeIndices & skip_indices, - CompressionCodecPtr default_codec_, - size_t aio_threshold, - bool blocks_are_granules_size) : IMergedBlockOutputStream(data_part, metadata_snapshot_) , columns_list(columns_list_) , default_codec(default_codec_) @@ -48,7 +29,6 @@ MergedBlockOutputStream::MergedBlockOutputStream( storage.global_context.getSettings(), storage.getSettings(), data_part->index_granularity_info.is_adaptive, - aio_threshold, /* rewrite_primary_key = */ true, blocks_are_granules_size); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index 3db0e45d207..d04df598218 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -21,15 +21,6 @@ public: CompressionCodecPtr default_codec_, bool blocks_are_granules_size = false); - MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part, - const StorageMetadataPtr & metadata_snapshot_, - const NamesAndTypesList & columns_list_, - const MergeTreeIndices & skip_indices, - CompressionCodecPtr default_codec_, - size_t aio_threshold, - bool blocks_are_granules_size = false); - Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } /// If the data is pre-sorted. diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 87f67ee92a5..41479f104f3 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -28,7 +28,6 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( global_settings, storage_settings, index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity(), - global_settings.min_bytes_to_use_direct_io, /* rewrite_primary_key = */false); writer = data_part->getWriter( diff --git a/src/Storages/MergeTree/ReplicatedFetchList.h b/src/Storages/MergeTree/ReplicatedFetchList.h index 81d538abf9c..0ab631e53b4 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.h +++ b/src/Storages/MergeTree/ReplicatedFetchList.h @@ -3,9 +3,9 @@ #include #include #include -#include #include + namespace CurrentMetrics { extern const Metric ReplicatedFetch; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 54248e6032c..7046a510f75 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -254,6 +254,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( part->info.min_block = block_number; part->info.max_block = block_number; part->info.level = 0; + part->info.mutation = 0; part->name = part->getNewName(part->info); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 6d5fab744a5..00ef3ee7292 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -12,7 +12,6 @@ #include #include -#include namespace DB diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index a2429cead3d..44c2bcefcd8 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -718,6 +718,15 @@ static StoragePtr create(const StorageFactory::Arguments & args) ++arg_num; } + DataTypes data_types = metadata.partition_key.data_types; + if (!args.attach && !storage_settings->allow_floating_point_partition_key) + { + for (size_t i = 0; i < data_types.size(); ++i) + if (isFloat(data_types[i])) + throw Exception( + "Donot support float point as partition key: " + metadata.partition_key.column_names[i], ErrorCodes::BAD_ARGUMENTS); + } + if (arg_num != arg_cnt) throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS); @@ -756,6 +765,7 @@ void registerStorageMergeTree(StorageFactory & factory) .supports_skipping_indices = true, .supports_sort_order = true, .supports_ttl = true, + .supports_parallel_insert = true, }; factory.registerStorage("MergeTree", create, features); diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index 2aa90a039bc..8902707ab64 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -1,5 +1,6 @@ #include -#include +#include +#include #include #include #include @@ -133,13 +134,13 @@ void MutationCommands::writeText(WriteBuffer & out) const { WriteBufferFromOwnString commands_buf; formatAST(*ast(), commands_buf, /* hilite = */ false, /* one_line = */ true); - out << escape << commands_buf.str(); + writeEscapedString(commands_buf.str(), out); } void MutationCommands::readText(ReadBuffer & in) { String commands_str; - in >> escape >> commands_str; + readEscapedString(commands_str, in); ParserAlterCommandList p_alter_commands; auto commands_ast = parseQuery( diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index a46da6072af..893c5167a97 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -29,7 +29,6 @@ class StorageRabbitMQ final: public ext::shared_ptr_helper, pub public: std::string getName() const override { return "RabbitMQ"; } - bool supportsSettings() const override { return true; } bool noPushingToViews() const override { return true; } void startup() override; diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index ae49433dbc6..2b751329208 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -1,8 +1,11 @@ #include +#include +#include +#include +#include #include #include -#include #include #include @@ -28,11 +31,10 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( /// Do not analyze joined columns. /// They may have aliases and come to description as is. /// We can mismatch them with order key columns at stage of fetching columns. - for (const auto & elem : syntax_result->array_join_result_to_source) - forbidden_columns.insert(elem.first); + forbidden_columns = syntax_result->getArrayJoinSourceNameSet(); } -InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot) const +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, const Context & context) const { Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); if (!metadata_snapshot->hasSortingKey()) @@ -42,6 +44,7 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & int read_direction = required_sort_description.at(0).direction; size_t prefix_size = std::min(required_sort_description.size(), sorting_key_columns.size()); + auto aliase_columns = metadata_snapshot->getColumns().getAliases(); for (size_t i = 0; i < prefix_size; ++i) { @@ -50,65 +53,100 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & /// Optimize in case of exact match with order key element /// or in some simple cases when order key element is wrapped into monotonic function. - int current_direction = required_sort_description[i].direction; - if (required_sort_description[i].column_name == sorting_key_columns[i] && current_direction == read_direction) + auto apply_order_judge = [&] (const ExpressionActions::Actions & actions, const String & sort_column) + { + int current_direction = required_sort_description[i].direction; + /// For the path: order by (sort_column, ...) + if (sort_column == sorting_key_columns[i] && current_direction == read_direction) + { + return true; + } + /// For the path: order by (function(sort_column), ...) + /// Allow only one simple monotonic functions with one argument + /// Why not allow multi monotonic functions? + else + { + bool found_function = false; + + for (const auto & action : actions) + { + if (action.node->type != ActionsDAG::ActionType::FUNCTION) + { + continue; + } + + if (found_function) + { + current_direction = 0; + break; + } + else + found_function = true; + + if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_columns[i]) + { + current_direction = 0; + break; + } + + const auto & func = *action.node->function_base; + if (!func.hasInformationAboutMonotonicity()) + { + current_direction = 0; + break; + } + + auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); + if (!monotonicity.is_monotonic) + { + current_direction = 0; + break; + } + else if (!monotonicity.is_positive) + current_direction *= -1; + } + + if (!found_function) + current_direction = 0; + + if (!current_direction || (i > 0 && current_direction != read_direction)) + return false; + + if (i == 0) + read_direction = current_direction; + + return true; + } + }; + + const auto & actions = elements_actions[i]->getActions(); + bool ok; + /// check if it's alias column + /// currently we only support alias column without any function wrapper + /// ie: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. + /// This suits most cases. + if (context.getSettingsRef().optimize_respect_aliases && aliase_columns.contains(required_sort_description[i].column_name)) + { + auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); + replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), forbidden_columns, context); + + auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); + const auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context).getActions(true); + const auto & alias_actions = expression_analyzer->getActions(); + + ok = apply_order_judge(alias_actions, column_expr->getColumnName()); + } + else + ok = apply_order_judge(actions, required_sort_description[i].column_name); + + if (ok) order_key_prefix_descr.push_back(required_sort_description[i]); else - { - /// Allow only one simple monotonic functions with one argument - bool found_function = false; - for (const auto & action : elements_actions[i]->getActions()) - { - if (action.node->type != ActionsDAG::ActionType::FUNCTION) - continue; - - if (found_function) - { - current_direction = 0; - break; - } - else - found_function = true; - - if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_columns[i]) - { - current_direction = 0; - break; - } - - const auto & func = *action.node->function_base; - if (!func.hasInformationAboutMonotonicity()) - { - current_direction = 0; - break; - } - - auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); - if (!monotonicity.is_monotonic) - { - current_direction = 0; - break; - } - else if (!monotonicity.is_positive) - current_direction *= -1; - } - - if (!found_function) - current_direction = 0; - - if (!current_direction || (i > 0 && current_direction != read_direction)) - break; - - if (i == 0) - read_direction = current_direction; - - order_key_prefix_descr.push_back(required_sort_description[i]); - } + break; } if (order_key_prefix_descr.empty()) return {}; - return std::make_shared(std::move(order_key_prefix_descr), read_direction); } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 7a268189222..3676f4cc88c 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -12,6 +12,8 @@ namespace DB * common prefix, which is needed for * performing reading in order of PK. */ +class Context; + class ReadInOrderOptimizer { public: @@ -20,7 +22,7 @@ public: const SortDescription & required_sort_description, const TreeRewriterResultPtr & syntax_result); - InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot) const; + InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot, const Context & context) const; private: /// Actions for every element of order expression to analyze functions for monotonicity @@ -28,5 +30,4 @@ private: NameSet forbidden_columns; SortDescription required_sort_description; }; - } diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 80b25793806..249026d1011 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -367,6 +367,7 @@ void registerStorageEmbeddedRocksDB(StorageFactory & factory) { StorageFactory::StorageFeatures features{ .supports_sort_order = true, + .supports_parallel_insert = true, }; factory.registerStorage("EmbeddedRocksDB", create, features); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index e24db51688e..13bab7a00d9 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -195,7 +195,7 @@ void StorageBuffer::read( if (dst_has_same_structure) { if (query_info.order_optimizer) - query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination_metadata_snapshot); + query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination_metadata_snapshot, context); /// The destination table has the same structure of the requested columns and we can simply read blocks from there. destination->read( @@ -996,6 +996,9 @@ void registerStorageBuffer(StorageFactory & factory) StorageBuffer::Thresholds{max_time, max_rows, max_bytes}, destination_id, static_cast(args.local_context.getSettingsRef().insert_allow_materialized_columns)); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 1390455edb1..afd7d6b876e 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -363,6 +363,7 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, + const DistributedSettings & distributed_settings_, bool attach_, ClusterPtr owned_cluster_) : IStorage(id_) @@ -374,6 +375,7 @@ StorageDistributed::StorageDistributed( , cluster_name(global_context.getMacros()->expand(cluster_name_)) , has_sharding_key(sharding_key_) , relative_data_path(relative_data_path_) + , distributed_settings(distributed_settings_) , rng(randomSeed()) { StorageInMemoryMetadata storage_metadata; @@ -417,9 +419,10 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, + const DistributedSettings & distributed_settings_, bool attach, ClusterPtr owned_cluster_) - : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, storage_policy_name_, relative_data_path_, attach, std::move(owned_cluster_)) + : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, storage_policy_name_, relative_data_path_, distributed_settings_, attach, std::move(owned_cluster_)) { remote_table_function_ptr = std::move(remote_table_function_ptr_); } @@ -601,7 +604,7 @@ void StorageDistributed::startup() return; for (const DiskPtr & disk : data_volume->getDisks()) - createDirectoryMonitors(disk->getPath()); + createDirectoryMonitors(disk); for (const String & path : getDataPaths()) { @@ -681,9 +684,9 @@ StoragePolicyPtr StorageDistributed::getStoragePolicy() const return storage_policy; } -void StorageDistributed::createDirectoryMonitors(const std::string & disk) +void StorageDistributed::createDirectoryMonitors(const DiskPtr & disk) { - const std::string path(disk + relative_data_path); + const std::string path(disk->getPath() + relative_data_path); Poco::File{path}.createDirectories(); std::filesystem::directory_iterator begin(path); @@ -714,10 +717,10 @@ void StorageDistributed::createDirectoryMonitors(const std::string & disk) } -StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(const std::string & disk, const std::string & name) +StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor(const DiskPtr & disk, const std::string & name) { - const std::string path(disk + relative_data_path + name); - const std::string key(disk + name); + const std::string & disk_path = disk->getPath(); + const std::string key(disk_path + name); std::lock_guard lock(cluster_nodes_mutex); auto & node_data = cluster_nodes_data[key]; @@ -725,7 +728,10 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor( { node_data.connection_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); node_data.directory_monitor = std::make_unique( - *this, path, node_data.connection_pool, monitors_blocker, global_context.getDistributedSchedulePool()); + *this, disk, relative_data_path + name, + node_data.connection_pool, + monitors_blocker, + global_context.getDistributedSchedulePool()); } return *node_data.directory_monitor; } @@ -932,7 +938,7 @@ void StorageDistributed::renameOnDisk(const String & new_path_to_table_data) std::lock_guard lock(cluster_nodes_mutex); for (auto & node : cluster_nodes_data) - node.second.directory_monitor->updatePath(new_path); + node.second.directory_monitor->updatePath(new_path_to_table_data); } relative_data_path = new_path_to_table_data; @@ -954,6 +960,8 @@ void registerStorageDistributed(StorageFactory & factory) * - constant expression with string result, like currentDatabase(); * -- string literal as specific case; * - empty string means 'use default database from cluster'. + * + * Distributed engine also supports SETTINGS clause. */ ASTs & engine_args = args.engine_args; @@ -995,6 +1003,13 @@ void registerStorageDistributed(StorageFactory & factory) ", but should be one of integer type", ErrorCodes::TYPE_MISMATCH); } + /// TODO: move some arguments from the arguments to the SETTINGS. + DistributedSettings distributed_settings; + if (args.storage_def->settings) + { + distributed_settings.loadFromQuery(*args.storage_def); + } + return StorageDistributed::create( args.table_id, args.columns, args.constraints, remote_database, remote_table, cluster_name, @@ -1002,9 +1017,12 @@ void registerStorageDistributed(StorageFactory & factory) sharding_key, storage_policy, args.relative_data_path, + distributed_settings, args.attach); }, { + .supports_settings = true, + .supports_parallel_insert = true, .source_access_type = AccessType::REMOTE, }); } diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index ce7e48c85a9..585efafddfb 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,9 @@ class Context; class IVolume; using VolumePtr = std::shared_ptr; +class IDisk; +using DiskPtr = std::shared_ptr; + class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; @@ -103,9 +107,9 @@ public: std::string getClusterName() const { return cluster_name; } /// Returns empty string if tables is used by TableFunctionRemote /// create directory monitors for each existing subdirectory - void createDirectoryMonitors(const std::string & disk); + void createDirectoryMonitors(const DiskPtr & disk); /// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name - StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const std::string & disk, const std::string & name); + StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const DiskPtr & disk, const std::string & name); /// Return list of metrics for all created monitors /// (note that monitors are created lazily, i.e. until at least one INSERT executed) std::vector getDirectoryMonitorsStatuses() const; @@ -127,6 +131,8 @@ public: size_t getRandomShardIndex(const Cluster::ShardsInfo & shards); + const DistributedSettings & getDistributedSettingsRef() const { return distributed_settings; } + String remote_database; String remote_table; ASTPtr remote_table_function_ptr; @@ -162,6 +168,7 @@ protected: const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, + const DistributedSettings & distributed_settings_, bool attach_, ClusterPtr owned_cluster_ = {}); @@ -175,6 +182,7 @@ protected: const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, + const DistributedSettings & distributed_settings_, bool attach, ClusterPtr owned_cluster_ = {}); @@ -188,6 +196,8 @@ protected: /// Other volumes will be ignored. It's needed to allow using the same multi-volume policy both for Distributed and other engines. VolumePtr data_volume; + DistributedSettings distributed_settings; + struct ClusterNodeData { std::unique_ptr directory_monitor; diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index de9060769cb..18dd24e10db 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -47,14 +47,20 @@ public: bool has_force_restore_data_flag; }; + /// Analog of the IStorage::supports*() helpers + /// (But the former cannot be replaced with StorageFeatures due to nesting) struct StorageFeatures { bool supports_settings = false; bool supports_skipping_indices = false; bool supports_sort_order = false; bool supports_ttl = false; + /// See also IStorage::supportsReplication() bool supports_replication = false; + /// See also IStorage::supportsDeduplication() bool supports_deduplication = false; + /// See also IStorage::supportsParallelInsert() + bool supports_parallel_insert = false; AccessType source_access_type = AccessType::NONE; }; @@ -85,6 +91,7 @@ public: .supports_ttl = false, .supports_replication = false, .supports_deduplication = false, + .supports_parallel_insert = false, .source_access_type = AccessType::NONE, }); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 1a75a585c85..61fdbc0198b 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -142,7 +142,7 @@ void StorageMaterializedView::read( auto metadata_snapshot = storage->getInMemoryMetadataPtr(); if (query_info.order_optimizer) - query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot); + query_info.input_order_info = query_info.order_optimizer->getInputOrder(metadata_snapshot, context); storage->read(query_plan, column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 93f00206e6b..8651caecdfa 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -303,6 +303,9 @@ void registerStorageMemory(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageMemory::create(args.table_id, args.columns, args.constraints); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index bc156b22abd..74df6dd185b 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -239,7 +239,7 @@ Pipe StorageMerge::read( { auto storage_ptr = std::get<0>(*it); auto storage_metadata_snapshot = storage_ptr->getInMemoryMetadataPtr(); - auto current_info = query_info.order_optimizer->getInputOrder(storage_metadata_snapshot); + auto current_info = query_info.order_optimizer->getInputOrder(storage_metadata_snapshot, context); if (it == selected_tables.begin()) input_sorting_info = current_info; else if (!current_info || (input_sorting_info && *current_info != *input_sorting_info)) diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 3263f124afa..9dd62439814 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -14,7 +14,6 @@ #include #include #include -#include #include diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 499f7329cd9..f324d502834 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -29,6 +29,9 @@ void registerStorageNull(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageNull::create(args.table_id, args.columns, args.constraints); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index b7b948af4ba..fed9dd04e76 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -25,7 +25,6 @@ public: bool supportsReplication() const override { return getNested()->supportsReplication(); } bool supportsParallelInsert() const override { return getNested()->supportsParallelInsert(); } bool supportsDeduplication() const override { return getNested()->supportsDeduplication(); } - bool supportsSettings() const override { return getNested()->supportsSettings(); } bool noPushingToViews() const override { return getNested()->noPushingToViews(); } bool hasEvenlyDistributedRead() const override { return getNested()->hasEvenlyDistributedRead(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8a802037f61..afa25c728a2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4958,8 +4958,13 @@ void StorageReplicatedMergeTree::fetchPartition( const String & from_, const Context & query_context) { - String auxiliary_zookeeper_name = extractZooKeeperName(from_); - String from = extractZooKeeperPath(from_); + Macros::MacroExpansionInfo info; + info.expand_special_macros_only = false; + info.table_id = getStorageID(); + info.table_id.uuid = UUIDHelpers::Nil; + auto expand_from = query_context.getMacros()->expand(from_, info); + String auxiliary_zookeeper_name = extractZooKeeperName(expand_from); + String from = extractZooKeeperPath(expand_from); if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -5556,9 +5561,9 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOps( continue; ReadBufferFromString buf(result.data); - Int64 block_num = 0; - bool parsed = tryReadIntText(block_num, buf) && buf.eof(); - if (!parsed || (min_block_num <= block_num && block_num <= max_block_num)) + MergeTreePartInfo part_info; + bool parsed = MergeTreePartInfo::tryParsePartName(result.data, &part_info, format_version); + if (!parsed || (min_block_num <= part_info.min_block && part_info.max_block <= max_block_num)) ops.emplace_back(zkutil::makeRemoveRequest(path, -1)); } } @@ -6145,7 +6150,7 @@ bool StorageReplicatedMergeTree::dropPart( Coordination::Requests ops; getClearBlocksInPartitionOps(ops, *zookeeper, part_info.partition_id, part_info.min_block, part_info.max_block); - size_t clean_block_ops_size = ops.size(); + size_t clear_block_ops_size = ops.size(); /// Set fake level to treat this part as virtual in queue. auto drop_part_info = part->info; @@ -6170,10 +6175,15 @@ bool StorageReplicatedMergeTree::dropPart( LOG_TRACE(log, "A new log entry appeared while trying to commit DROP RANGE. Retry."); continue; } + else if (rc == Coordination::Error::ZNONODE) + { + LOG_TRACE(log, "Other replica already removing same part {} or part deduplication node was removed by background thread. Retry.", part_name); + continue; + } else zkutil::KeeperMultiException::check(rc, ops, responses); - String log_znode_path = dynamic_cast(*responses[clean_block_ops_size + 1]).path_created; + String log_znode_path = dynamic_cast(*responses[clear_block_ops_size + 1]).path_created; entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); return true; diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index fe8a25ba13b..6e3e9c612bb 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -294,11 +294,17 @@ IDataType::OutputStreamGetter TinyLogBlockOutputStream::createStreamGetter( void TinyLogBlockOutputStream::writeData(const String & name, const IDataType & type, const IColumn & column, WrittenStreams & written_streams) { IDataType::SerializeBinaryBulkSettings settings; - settings.getter = createStreamGetter(name, written_streams); if (serialize_states.count(name) == 0) + { + /// Some stream getters may be called form `serializeBinaryBulkStatePrefix`. + /// Use different WrittenStreams set, or we get nullptr for them in `serializeBinaryBulkWithMultipleStreams` + WrittenStreams prefix_written_streams; + settings.getter = createStreamGetter(name, prefix_written_streams); type.serializeBinaryBulkStatePrefix(settings, serialize_states[name]); + } + settings.getter = createStreamGetter(name, written_streams); type.serializeBinaryBulkWithMultipleStreams(column, 0, 0, settings, serialize_states[name]); } diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 39dc74822e0..ee39390a0f5 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -1,4 +1,4 @@ -// autogenerated by ./StorageSystemContributors.sh +// autogenerated by src/Storages/System/StorageSystemContributors.sh const char * auto_contributors[] { "0xflotus", "20018712", @@ -11,6 +11,7 @@ const char * auto_contributors[] { "Alberto", "Aleksandr Karo", "Aleksandra (Ася)", + "Aleksandrov Vladimir", "Aleksei Levushkin", "Aleksey", "Aleksey Akulovich", @@ -117,6 +118,7 @@ const char * auto_contributors[] { "BanyRule", "Baudouin Giard", "BayoNet", + "Bertrand Junqua", "Bharat Nallan", "Big Elephant", "Bill", @@ -132,6 +134,7 @@ const char * auto_contributors[] { "Carbyn", "Chao Wang", "Chen Yufei", + "Chienlung Cheung", "Ciprian Hacman", "Clement Rodriguez", "Clément Rodriguez", @@ -178,6 +181,8 @@ const char * auto_contributors[] { "Elizaveta Mironyuk", "Emmanuel Donin de Rosière", "Eric", + "Eric Daniel", + "Erixonich", "Ernest Poletaev", "Eugene Klimov", "Eugene Konkov", @@ -187,6 +192,7 @@ const char * auto_contributors[] { "Evgeniy Gatov", "Evgeniy Udodov", "Evgeny Konkov", + "Evgeny Markov", "Ewout", "Fabian Stäber", "Fabiano Francesconi", @@ -218,6 +224,7 @@ const char * auto_contributors[] { "Hamoon", "Hasnat", "Hiroaki Nakamura", + "HuFuwang", "Hui Wang", "Igor", "Igor Hatarist", @@ -239,6 +246,7 @@ const char * auto_contributors[] { "Ilya Skrypitsa", "Ilya Yatsishin", "ImgBotApp", + "Islam Israfilov (Islam93)", "Ivan", "Ivan A. Torgashov", "Ivan Babrou", @@ -264,6 +272,8 @@ const char * auto_contributors[] { "Kang Liu", "Karl Pietrzak", "Keiji Yoshida", + "Ken Chen", + "Kevin Chiang", "Kiran", "Kirill Danshin", "Kirill Malev", @@ -299,12 +309,14 @@ const char * auto_contributors[] { "Marek Vavrusa", "Marek Vavruša", "Marek Vavruša", + "Mark Andreev", "Mark Papadakis", "Maroun Maroun", "Marsel Arduanov", "Marti Raudsepp", "Martijn Bakker", "Masha", + "Matthew Peveler", "Matwey V. Kornilov", "Max", "Max Akhmedov", @@ -349,6 +361,7 @@ const char * auto_contributors[] { "Milad Arabi", "Mohammad Hossein Sekhavat", "MovElb", + "Mr.General", "Murat Kabilov", "MyroTk", "NIKITA MIKHAILOV", @@ -359,6 +372,7 @@ const char * auto_contributors[] { "Nico Piderman", "Nicolae Vartolomei", "Nik", + "Nikhil Nadig", "Nikhil Raman", "Nikita Lapkov", "Nikita Mikhailov", @@ -384,6 +398,7 @@ const char * auto_contributors[] { "Orivej Desh", "Oskar Wojciski", "OuO", + "PHO", "Paramtamtam", "Patrick Zippenfenig", "Pavel", @@ -442,6 +457,7 @@ const char * auto_contributors[] { "Sergey Zaikin", "Sergi Vladykin", "SevaCode", + "Sherry Wang", "Silviu Caragea", "Simon Liu", "Simon Podlipsky", @@ -452,13 +468,17 @@ const char * auto_contributors[] { "Stanislav Pavlovichev", "Stas Pavlovichev", "Stefan Thies", + "Stepan", "Stepan Herold", + "Steve-金勇", + "Stig Bakken", "Stupnikov Andrey", "SuperBot", "Sébastien Launay", "TAC", "TCeason", "Tagir Kuskarov", + "Tai White", "Tangaev", "Tema Novikov", "The-Alchemist", @@ -467,6 +487,7 @@ const char * auto_contributors[] { "Tsarkova Anastasia", "Ubuntu", "Ubus", + "UnamedRus", "V", "VDimir", "Vadim", @@ -499,6 +520,7 @@ const char * auto_contributors[] { "Vladimir Chebotarev", "Vladimir Golovchenko", "Vladimir Goncharov", + "Vladimir Klimontovich", "Vladimir Kolobaev", "Vladimir Kopysov", "Vladimir Kozbin", @@ -564,16 +586,20 @@ const char * auto_contributors[] { "asiana21", "avasiliev", "avsharapov", + "awesomeleo", "benamazing", "bgranvea", "bharatnc", "blazerer", "bluebirddm", "bobrovskij artemij", + "booknouse", "bseng", "cekc", "champtar", + "chang.chen", "chengy8934", + "chenqi", "chenxing-xc", "chenxing.xc", "chertus", @@ -591,6 +617,7 @@ const char * auto_contributors[] { "dgrr", "dimarub2000", "dinosaur", + "dkxiaohei", "dmi-feo", "dmitrii", "dmitriiut", @@ -605,8 +632,10 @@ const char * auto_contributors[] { "exprmntr", "ezhaka", "f1yegor", + "fastio", "favstovol", "felixoid", + "felixxdu", "feng lv", "fenglv", "fessmage", @@ -622,8 +651,10 @@ const char * auto_contributors[] { "ggerogery", "giordyb", "glockbender", + "glushkovds", "gyuton", "hao.he", + "hchen9", "hcz", "heng zhao", "hexiaoting", @@ -640,6 +671,8 @@ const char * auto_contributors[] { "javartisan", "javi", "javi santana", + "jetgm", + "jianmei zhang", "kmeaw", "koshachy", "kreuzerkrieg", @@ -653,6 +686,7 @@ const char * auto_contributors[] { "levysh", "liangqian", "libenwang", + "lichengxiang", "linceyou", "litao91", "liu-bov", @@ -685,6 +719,8 @@ const char * auto_contributors[] { "moscas", "myrrc", "nagorny", + "nauta", + "nautaa", "never lee", "nicelulu", "nikitamikhaylov", @@ -693,6 +729,7 @@ const char * auto_contributors[] { "nvartolomei", "oandrew", "objatie_groba", + "ocadaruma", "ogorbacheva", "olegkv", "olgarev", @@ -700,6 +737,7 @@ const char * auto_contributors[] { "palasonicq", "peshkurov", "philip.han", + "pingyu", "potya", "proller", "pufit", @@ -716,7 +754,9 @@ const char * auto_contributors[] { "roman", "romanzhukov", "root", + "roverxu", "santaux", + "satanson", "sdk2", "serebrserg", "sev7e0", @@ -725,6 +765,7 @@ const char * auto_contributors[] { "shangshujie", "shedx", "simon-says", + "spongedc", "spyros87", "stavrolia", "stepenhu", @@ -738,6 +779,7 @@ const char * auto_contributors[] { "taiyang-li", "tao jiang", "tavplubix", + "tiger.yan", "topvisor", "tyrionhuang", "ubuntu", @@ -755,10 +797,13 @@ const char * auto_contributors[] { "vxider", "vzakaznikov", "wangchao", + "weeds085490", "xPoSx", + "yangshuai", "yhgcn", "ylchou", "yonesko", + "yuefoo", "yulu86", "yuluxu", "zamulla", @@ -768,6 +813,7 @@ const char * auto_contributors[] { "zhen ni", "zhukai", "zvrr", + "zvvr", "zzsmdfj", "Šimon Podlipský", "Артем Стрельцов", @@ -781,6 +827,7 @@ const char * auto_contributors[] { "小路", "张健", "张风啸", + "徐炘", "极客青年", "谢磊", "贾顺名(Jarvis)", diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp new file mode 100644 index 00000000000..229325313e2 --- /dev/null +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -0,0 +1,231 @@ +#include +#include + +#include "StorageSystemDDLWorkerQueue.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace fs = std::filesystem; + +enum Status +{ + ACTIVE, + FINISHED, + UNKNOWN, + ERRORED +}; + +namespace DB +{ +std::vector> getStatusEnumsAndValues() +{ + return std::vector>{ + {"Active", static_cast(Status::ACTIVE)}, + {"Finished", static_cast(Status::FINISHED)}, + {"Unknown", static_cast(Status::UNKNOWN)}, + {"Errored", static_cast(Status::ERRORED)}, + }; +} + + +std::vector> getZooKeeperErrorEnumsAndValues() +{ + return std::vector>{ + {"ZOK", static_cast(Coordination::Error::ZOK)}, + {"ZSYSTEMERROR", static_cast(Coordination::Error::ZSYSTEMERROR)}, + {"ZRUNTIMEINCONSISTENCY", static_cast(Coordination::Error::ZRUNTIMEINCONSISTENCY)}, + {"ZDATAINCONSISTENCY", static_cast(Coordination::Error::ZDATAINCONSISTENCY)}, + {"ZCONNECTIONLOSS", static_cast(Coordination::Error::ZCONNECTIONLOSS)}, + {"ZMARSHALLINGERROR", static_cast(Coordination::Error::ZMARSHALLINGERROR)}, + {"ZUNIMPLEMENTED", static_cast(Coordination::Error::ZUNIMPLEMENTED)}, + {"ZOPERATIONTIMEOUT", static_cast(Coordination::Error::ZOPERATIONTIMEOUT)}, + {"ZBADARGUMENTS", static_cast(Coordination::Error::ZBADARGUMENTS)}, + {"ZINVALIDSTATE", static_cast(Coordination::Error::ZINVALIDSTATE)}, + {"ZAPIERROR", static_cast(Coordination::Error::ZAPIERROR)}, + {"ZNONODE", static_cast(Coordination::Error::ZNONODE)}, + {"ZNOAUTH", static_cast(Coordination::Error::ZNOAUTH)}, + {"ZBADVERSION", static_cast(Coordination::Error::ZBADVERSION)}, + {"ZNOCHILDRENFOREPHEMERALS", static_cast(Coordination::Error::ZNOCHILDRENFOREPHEMERALS)}, + {"ZNODEEXISTS", static_cast(Coordination::Error::ZNODEEXISTS)}, + {"ZNOTEMPTY", static_cast(Coordination::Error::ZNOTEMPTY)}, + {"ZSESSIONEXPIRED", static_cast(Coordination::Error::ZSESSIONEXPIRED)}, + {"ZINVALIDCALLBACK", static_cast(Coordination::Error::ZINVALIDCALLBACK)}, + {"ZINVALIDACL", static_cast(Coordination::Error::ZINVALIDACL)}, + {"ZAUTHFAILED", static_cast(Coordination::Error::ZAUTHFAILED)}, + {"ZCLOSING", static_cast(Coordination::Error::ZCLOSING)}, + {"ZNOTHING", static_cast(Coordination::Error::ZNOTHING)}, + {"ZSESSIONMOVED", static_cast(Coordination::Error::ZSESSIONMOVED)}, + }; +} + + +NamesAndTypesList StorageSystemDDLWorkerQueue::getNamesAndTypes() +{ + return { + {"entry", std::make_shared()}, + {"host_name", std::make_shared()}, + {"host_address", std::make_shared()}, + {"port", std::make_shared()}, + {"status", std::make_shared(getStatusEnumsAndValues())}, + {"cluster", std::make_shared()}, + {"query", std::make_shared()}, + {"initiator", std::make_shared()}, + {"query_start_time", std::make_shared()}, + {"query_finish_time", std::make_shared()}, + {"query_duration_ms", std::make_shared()}, + {"exception_code", std::make_shared(getZooKeeperErrorEnumsAndValues())}, + }; +} + +static String clusterNameFromDDLQuery(const Context & context, const DDLLogEntry & entry) +{ + const char * begin = entry.query.data(); + const char * end = begin + entry.query.size(); + ASTPtr query; + ASTQueryWithOnCluster * query_on_cluster; + String cluster_name; + ParserQuery parser_query(end); + String description; + query = parseQuery(parser_query, begin, end, description, 0, context.getSettingsRef().max_parser_depth); + if (query && (query_on_cluster = dynamic_cast(query.get()))) + cluster_name = query_on_cluster->cluster; + return cluster_name; +} + +void StorageSystemDDLWorkerQueue::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + zkutil::ZooKeeperPtr zookeeper = context.getZooKeeper(); + Coordination::Error zk_exception_code = Coordination::Error::ZOK; + String ddl_zookeeper_path = config.getString("distributed_ddl.path", "/clickhouse/task_queue/ddl/"); + String ddl_query_path; + + // this is equivalent to query zookeeper at the `ddl_zookeeper_path` + /* [zk: localhost:2181(CONNECTED) 51] ls /clickhouse/task_queue/ddl + [query-0000000000, query-0000000001, query-0000000002, query-0000000003, query-0000000004] + */ + + zkutil::Strings queries; + + Coordination::Error code = zookeeper->tryGetChildren(ddl_zookeeper_path, queries); + // if there is an error here, just register the code. + // the queries will be empty and so there will be nothing to fill the table + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) + zk_exception_code = code; + + const auto & clusters = context.getClusters(); + for (const auto & name_and_cluster : clusters.getContainer()) + { + const ClusterPtr & cluster = name_and_cluster.second; + const auto & shards_info = cluster->getShardsInfo(); + const auto & addresses_with_failover = cluster->getShardsAddresses(); + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) + { + const auto & shard_addresses = addresses_with_failover[shard_index]; + const auto & shard_info = shards_info[shard_index]; + const auto pool_status = shard_info.pool->getStatus(); + for (size_t replica_index = 0; replica_index < shard_addresses.size(); ++replica_index) + { + /* Dir contents of every query will be similar to + [zk: localhost:2181(CONNECTED) 53] ls /clickhouse/task_queue/ddl/query-0000000004 + [active, finished] + */ + std::vector> futures; + futures.reserve(queries.size()); + for (const String & q : queries) + { + futures.push_back(zookeeper->asyncTryGet(fs::path(ddl_zookeeper_path) / q)); + } + for (size_t query_id = 0; query_id < queries.size(); query_id++) + { + Int64 query_finish_time = 0; + size_t i = 0; + res_columns[i++]->insert(queries[query_id]); // entry + const auto & address = shard_addresses[replica_index]; + res_columns[i++]->insert(address.host_name); + auto resolved = address.getResolvedAddress(); + res_columns[i++]->insert(resolved ? resolved->host().toString() : String()); // host_address + res_columns[i++]->insert(address.port); + ddl_query_path = fs::path(ddl_zookeeper_path) / queries[query_id]; + + zkutil::Strings active_nodes; + zkutil::Strings finished_nodes; + + code = zookeeper->tryGetChildren(fs::path(ddl_query_path) / "active", active_nodes); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) + zk_exception_code = code; + + code = zookeeper->tryGetChildren(fs::path(ddl_query_path) / "finished", finished_nodes); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE) + zk_exception_code = code; + + /* status: + * active: If the hostname:port entry is present under active path. + * finished: If the hostname:port entry is present under the finished path. + * errored: If the hostname:port entry is present under the finished path but the error count is not 0. + * unknown: If the above cases don't hold true, then status is unknown. + */ + if (std::find(active_nodes.begin(), active_nodes.end(), address.toString()) != active_nodes.end()) + { + res_columns[i++]->insert(static_cast(Status::ACTIVE)); + } + else if (std::find(finished_nodes.begin(), finished_nodes.end(), address.toString()) != finished_nodes.end()) + { + if (pool_status[replica_index].error_count != 0) + { + res_columns[i++]->insert(static_cast(Status::ERRORED)); + } + else + { + res_columns[i++]->insert(static_cast(Status::FINISHED)); + } + // regardless of the status finished or errored, the node host_name:port entry was found under the /finished path + // & should be able to get the contents of the znode at /finished path. + auto res_fn = zookeeper->asyncTryGet(fs::path(ddl_query_path) / "finished"); + auto stat_fn = res_fn.get().stat; + query_finish_time = stat_fn.mtime; + } + else + { + res_columns[i++]->insert(static_cast(Status::UNKNOWN)); + } + + Coordination::GetResponse res; + res = futures[query_id].get(); + + auto query_start_time = res.stat.mtime; + + DDLLogEntry entry; + entry.parse(res.data); + String cluster_name = clusterNameFromDDLQuery(context, entry); + + res_columns[i++]->insert(cluster_name); + res_columns[i++]->insert(entry.query); + res_columns[i++]->insert(entry.initiator); + res_columns[i++]->insert(UInt64(query_start_time / 1000)); + res_columns[i++]->insert(UInt64(query_finish_time / 1000)); + res_columns[i++]->insert(UInt64(query_finish_time - query_start_time)); + res_columns[i++]->insert(static_cast(zk_exception_code)); + } + } + } + } +} +} diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.h b/src/Storages/System/StorageSystemDDLWorkerQueue.h new file mode 100644 index 00000000000..9326d4dcb26 --- /dev/null +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include +#include +#include + + +namespace DB +{ +class Context; + + +/** System table "distributed_ddl_queue" with list of queries that are currently in the DDL worker queue. + */ +class StorageSystemDDLWorkerQueue final : public ext::shared_ptr_helper, + public IStorageSystemOneBlock +{ + friend struct ext::shared_ptr_helper; + Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); + +protected: + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; + + using IStorageSystemOneBlock::IStorageSystemOneBlock; + +public: + std::string getName() const override { return "SystemDDLWorkerQueue"; } + + static NamesAndTypesList getNamesAndTypes(); +}; +} diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index 0b5e82a1f3d..abb2fdf54ed 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -60,6 +60,7 @@ namespace void signalHandler(int, siginfo_t * info, void * context) { + DENY_ALLOCATIONS_IN_SCOPE; auto saved_errno = errno; /// We must restore previous value of errno in signal handler. /// In case malicious user is sending signals manually (for unknown reason). diff --git a/src/Storages/System/StorageSystemTableEngines.cpp b/src/Storages/System/StorageSystemTableEngines.cpp index e63923f69b6..3f06faf6736 100644 --- a/src/Storages/System/StorageSystemTableEngines.cpp +++ b/src/Storages/System/StorageSystemTableEngines.cpp @@ -8,26 +8,31 @@ namespace DB NamesAndTypesList StorageSystemTableEngines::getNamesAndTypes() { - return {{"name", std::make_shared()}, - {"supports_settings", std::make_shared()}, - {"supports_skipping_indices", std::make_shared()}, - {"supports_sort_order", std::make_shared()}, - {"supports_ttl", std::make_shared()}, - {"supports_replication", std::make_shared()}, - {"supports_deduplication", std::make_shared()}}; + return { + {"name", std::make_shared()}, + {"supports_settings", std::make_shared()}, + {"supports_skipping_indices", std::make_shared()}, + {"supports_sort_order", std::make_shared()}, + {"supports_ttl", std::make_shared()}, + {"supports_replication", std::make_shared()}, + {"supports_deduplication", std::make_shared()}, + {"supports_parallel_insert", std::make_shared()}, + }; } void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const { for (const auto & pair : StorageFactory::instance().getAllStorages()) { - res_columns[0]->insert(pair.first); - res_columns[1]->insert(pair.second.features.supports_settings); - res_columns[2]->insert(pair.second.features.supports_skipping_indices); - res_columns[3]->insert(pair.second.features.supports_sort_order); - res_columns[4]->insert(pair.second.features.supports_ttl); - res_columns[5]->insert(pair.second.features.supports_replication); - res_columns[6]->insert(pair.second.features.supports_deduplication); + int i = 0; + res_columns[i++]->insert(pair.first); + res_columns[i++]->insert(pair.second.features.supports_settings); + res_columns[i++]->insert(pair.second.features.supports_skipping_indices); + res_columns[i++]->insert(pair.second.features.supports_sort_order); + res_columns[i++]->insert(pair.second.features.supports_ttl); + res_columns[i++]->insert(pair.second.features.supports_replication); + res_columns[i++]->insert(pair.second.features.supports_deduplication); + res_columns[i++]->insert(pair.second.features.supports_parallel_insert); } } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 5e1a830f2a1..673cf671548 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -39,6 +39,8 @@ #include #include #include +#include + #if !defined(ARCADIA_BUILD) #include #include @@ -134,6 +136,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) attach(system_database, "mutations"); attach(system_database, "replicas"); attach(system_database, "replication_queue"); + attach(system_database, "distributed_ddl_queue"); attach(system_database, "distribution_queue"); attach(system_database, "dictionaries"); attach(system_database, "models"); diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 835aebab900..99dfc55ed69 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -80,6 +80,24 @@ TEST(TransformQueryForExternalDatabase, InWithSingleElement) state.context, state.columns); } +TEST(TransformQueryForExternalDatabase, InWithTable) +{ + const State & state = State::instance(); + + check("SELECT column FROM test.table WHERE 1 IN external_table", + R"(SELECT "column" FROM "test"."table")", + state.context, state.columns); + check("SELECT column FROM test.table WHERE 1 IN (x)", + R"(SELECT "column" FROM "test"."table")", + state.context, state.columns); + check("SELECT column, field, value FROM test.table WHERE column IN (field, value)", + R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN ("field", "value"))", + state.context, state.columns); + check("SELECT column FROM test.table WHERE column NOT IN hello AND column = 123", + R"(SELECT "column" FROM "test"."table" WHERE ("column" = 123))", + state.context, state.columns); +} + TEST(TransformQueryForExternalDatabase, Like) { const State & state = State::instance(); diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index f35fb1c8a34..42daf8cfc26 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -138,6 +138,12 @@ bool isCompatible(const IAST & node) if (name == "tuple" && function->arguments->children.size() <= 1) return false; + /// If the right hand side of IN is an identifier (example: x IN table), then it's not compatible. + if ((name == "in" || name == "notIn") + && (function->arguments->children.size() != 2 + || function->arguments->children[1]->as())) + return false; + for (const auto & expr : function->arguments->children) if (!isCompatible(*expr)) return false; diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 27aa9e3ac3f..69e319cbad5 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -17,6 +17,7 @@ SRCS( ConstraintsDescription.cpp Distributed/DirectoryMonitor.cpp Distributed/DistributedBlockOutputStream.cpp + Distributed/DistributedSettings.cpp IStorage.cpp IndicesDescription.cpp JoinSettings.cpp @@ -147,6 +148,7 @@ SRCS( System/StorageSystemContributors.cpp System/StorageSystemContributors.generated.cpp System/StorageSystemCurrentRoles.cpp + System/StorageSystemDDLWorkerQueue.cpp System/StorageSystemDataTypeFamilies.cpp System/StorageSystemDatabases.cpp System/StorageSystemDetachedParts.cpp diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index a031490b88b..914b7083fca 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -211,6 +211,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, con ASTPtr{}, String{}, String{}, + DistributedSettings{}, false, cluster) : StorageDistributed::create( @@ -224,6 +225,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, con ASTPtr{}, String{}, String{}, + DistributedSettings{}, false, cluster); diff --git a/tests/integration/test_alter_settings_on_cluster/__init__.py b/tests/integration/test_alter_settings_on_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml b/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml new file mode 100644 index 00000000000..26c9caa63db --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml @@ -0,0 +1,17 @@ + + + + + true + + ch1 + 9000 + + + ch2 + 9000 + + + + + diff --git a/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml b/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml new file mode 100644 index 00000000000..6a88929c8ac --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml @@ -0,0 +1,5 @@ + + + /clickhouse/task_queue/ddl + + diff --git a/tests/integration/test_alter_settings_on_cluster/test.py b/tests/integration/test_alter_settings_on_cluster/test.py new file mode 100644 index 00000000000..6ab3d446b59 --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/test.py @@ -0,0 +1,54 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) +ch2 = cluster.add_instance( + "ch2", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + ch1.query("CREATE DATABASE test_default_database ON CLUSTER 'cluster';") + yield cluster + + finally: + cluster.shutdown() + + +def test_default_database_on_cluster(started_cluster): + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r1') ORDER BY tuple();", + ) + + ch2.query( + database="test_default_database", + sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r2') ORDER BY tuple();", + ) + + ch1.query( + database="test_default_database", + sql="ALTER TABLE test_local_table ON CLUSTER 'cluster' MODIFY SETTING old_parts_lifetime = 100;", + ) + + for node in [ch1, ch2]: + assert node.query( + database="test_default_database", + sql="SHOW CREATE test_local_table FORMAT TSV", + ).endswith("old_parts_lifetime = 100\n") diff --git a/tests/integration/test_fetch_partition_should_reset_mutation/__init__.py b/tests/integration/test_fetch_partition_should_reset_mutation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_fetch_partition_should_reset_mutation/configs/zookeeper_config.xml b/tests/integration/test_fetch_partition_should_reset_mutation/configs/zookeeper_config.xml new file mode 100644 index 00000000000..17cbd681c41 --- /dev/null +++ b/tests/integration/test_fetch_partition_should_reset_mutation/configs/zookeeper_config.xml @@ -0,0 +1,8 @@ + + + + zoo1 + 2181 + + + diff --git a/tests/integration/test_fetch_partition_should_reset_mutation/test.py b/tests/integration/test_fetch_partition_should_reset_mutation/test.py new file mode 100644 index 00000000000..14a91a42031 --- /dev/null +++ b/tests/integration/test_fetch_partition_should_reset_mutation/test.py @@ -0,0 +1,60 @@ +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/zookeeper_config.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_part_should_reset_mutation(start_cluster): + node.query( + "CREATE TABLE test (i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test', 'node') ORDER BY i;" + ) + node.query("INSERT INTO test SELECT 1, 'a'") + node.query("optimize table test final") + node.query("optimize table test final") + + + expected = TSV('''all_0_0_2\t1\ta''') + assert TSV(node.query('SELECT _part, * FROM test')) == expected + + node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) + node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) + node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) + node.query("ALTER TABLE test UPDATE s='xxx' WHERE 1", settings={"mutations_sync": "2"}) + + expected = TSV('''all_0_0_2_4\t1\txxx''') + assert TSV(node.query('SELECT _part, * FROM test')) == expected + + node.query( + "CREATE TABLE restore (i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/restore', 'node') ORDER BY i;" + ) + node.query("ALTER TABLE restore FETCH PARTITION tuple() FROM '/clickhouse/tables/test/'") + node.query("ALTER TABLE restore ATTACH PART 'all_0_0_2_4'") + node.query("INSERT INTO restore select 2, 'a'") + + print(TSV(node.query('SELECT _part, * FROM restore'))) + expected = TSV('''all_0_0_0\t1\txxx\nall_1_1_0\t2\ta''') + assert TSV(node.query('SELECT _part, * FROM restore ORDER BY i')) == expected + + node.query("ALTER TABLE restore UPDATE s='yyy' WHERE 1", settings={"mutations_sync": "2"}) + + + expected = TSV('''all_0_0_0_2\t1\tyyy\nall_1_1_0_2\t2\tyyy''') + assert TSV(node.query('SELECT _part, * FROM restore ORDER BY i')) == expected + + node.query("ALTER TABLE restore DELETE WHERE 1", settings={"mutations_sync": "2"}) + + + assert node.query("SELECT count() FROM restore").strip() == "0" diff --git a/tests/integration/test_insert_into_distributed/test.py b/tests/integration/test_insert_into_distributed/test.py index 52beaf06ec2..d71d1075c70 100644 --- a/tests/integration/test_insert_into_distributed/test.py +++ b/tests/integration/test_insert_into_distributed/test.py @@ -36,7 +36,7 @@ CREATE TABLE distributed (x UInt32) ENGINE = Distributed('test_cluster', 'defaul remote.query("CREATE TABLE local2 (d Date, x UInt32, s String) ENGINE = MergeTree(d, x, 8192)") instance_test_inserts_batching.query(''' -CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') +CREATE TABLE distributed (d Date, x UInt32) ENGINE = Distributed('test_cluster', 'default', 'local2') SETTINGS fsync_after_insert=1, fsync_directories=1 ''') instance_test_inserts_local_cluster.query( diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/__init__.py b/tests/integration/test_passing_max_partitions_to_read_remotely/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py new file mode 100644 index 00000000000..45b3dd00b2a --- /dev/null +++ b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py @@ -0,0 +1,28 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance("ch1") +ch2 = cluster.add_instance("ch2") + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + ch1.query("CREATE DATABASE test_default_database;") + yield cluster + + finally: + cluster.shutdown() + + +def test_default_database_on_cluster(started_cluster): + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_local_table ENGINE MergeTree PARTITION BY i ORDER BY tuple() SETTINGS max_partitions_to_read = 1 AS SELECT arrayJoin([1, 2]) i;", + ) + + assert ch2.query( + sql="SELECT * FROM remote('ch1:9000', test_default_database, test_local_table) ORDER BY i FORMAT TSV SETTINGS max_partitions_to_read = 0;", + ) == "1\n2\n" diff --git a/tests/integration/test_system_ddl_worker_queue/__init__.py b/tests/integration/test_system_ddl_worker_queue/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml b/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml new file mode 100644 index 00000000000..4c3de4b3905 --- /dev/null +++ b/tests/integration/test_system_ddl_worker_queue/configs/remote_servers.xml @@ -0,0 +1,28 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/tests/integration/test_system_ddl_worker_queue/test.py b/tests/integration/test_system_ddl_worker_queue/test.py new file mode 100644 index 00000000000..c5037fc400e --- /dev/null +++ b/tests/integration/test_system_ddl_worker_queue/test.py @@ -0,0 +1,48 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node3 = cluster.add_instance('node3', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', main_configs=['configs/remote_servers.xml'], with_zookeeper=True) + +nodes = [node1, node2, node3, node4] + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for i, node in enumerate([node1, node2]): + node.query("CREATE DATABASE testdb") + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table1', '{}') ORDER BY id;'''.format( + i)) + for i, node in enumerate([node3, node4]): + node.query("CREATE DATABASE testdb") + node.query( + '''CREATE TABLE testdb.test_table(id UInt32, val String) ENGINE = ReplicatedMergeTree('/clickhouse/test/test_table2', '{}') ORDER BY id;'''.format( + i)) + yield cluster + + finally: + cluster.shutdown() + + +def test_distributed_ddl_queue(started_cluster): + node1.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node3.query("INSERT INTO testdb.test_table SELECT number, toString(number) FROM numbers(100)") + node2.query("SYSTEM SYNC REPLICA testdb.test_table") + node4.query("SYSTEM SYNC REPLICA testdb.test_table") + + node1.query("ALTER TABLE testdb.test_table ON CLUSTER test_cluster ADD COLUMN somecolumn UInt8 AFTER val", + settings={"replication_alter_partitions_sync": "2"}) + for node in nodes: + node.query("SYSTEM SYNC REPLICA testdb.test_table") + assert node.query("SELECT somecolumn FROM testdb.test_table LIMIT 1") == "0\n" + assert node.query( + "SELECT If((SELECT count(*) FROM system.distributed_ddl_queue WHERE cluster='test_cluster' AND entry='query-0000000000') > 0, 'ok', 'fail')") == "ok\n" diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 68c2c65cbf2..79e5dece174 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -58,7 +58,7 @@ def test_mutate_and_upgrade(start_cluster): assert node1.query("SELECT COUNT() FROM mt") == "2\n" assert node2.query("SELECT COUNT() FROM mt") == "2\n" - node1.query("ALTER TABLE mt MODIFY COLUMN id String DEFAULT '0'", + node1.query("ALTER TABLE mt MODIFY COLUMN id Int32 DEFAULT 0", settings={"replication_alter_partitions_sync": "2"}) node2.query("OPTIMIZE TABLE mt FINAL") diff --git a/tests/performance/if_to_multiif.xml b/tests/performance/if_to_multiif.xml index 65bda1b23c6..085a604649b 100644 --- a/tests/performance/if_to_multiif.xml +++ b/tests/performance/if_to_multiif.xml @@ -2,7 +2,7 @@ - + diff --git a/tests/performance/optimize_window_funnel.xml b/tests/performance/optimize_window_funnel.xml new file mode 100644 index 00000000000..0d928fd0f4e --- /dev/null +++ b/tests/performance/optimize_window_funnel.xml @@ -0,0 +1,12 @@ + + CREATE TABLE action(uid UInt64, event String, time DateTime) ENGINE = MergeTree ORDER BY uid + + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'a', now() from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'b', now() + INTERVAL 6 hour from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'c', now() + INTERVAL 12 hour from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'd', now() + INTERVAL 18 hour from numbers(1000000) + + SELECT level, count() from (select windowFunnel(86400)(time, event='a', event='b', event='c', event='d') level from action group by uid) group by level FORMAT Null + + DROP TABLE IF EXISTS action + diff --git a/tests/performance/url_hits.xml b/tests/performance/url_hits.xml index f0ad6a786e0..a699ef6ba97 100644 --- a/tests/performance/url_hits.xml +++ b/tests/performance/url_hits.xml @@ -32,6 +32,7 @@ extractURLParameters extractURLParameterNames decodeURLComponent + decodeXMLComponent cutWWW cutQueryString cutQueryStringAndFragment diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 8945fc3e56b..5aa445858db 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -151,7 +151,7 @@ $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r2 $CLICKHOUSE_CLIENT --query="SELECT 'After restart';" $CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICA dst_r1;" -$CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICAS;" +$CLICKHOUSE_CLIENT --query="SYSTEM RESTART REPLICA dst_r2;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r1;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d) FROM dst_r2;" diff --git a/tests/queries/0_stateless/00829_bitmap64_function.sql b/tests/queries/0_stateless/00829_bitmap64_function.sql index 9d33dcefff9..c4e0293e9d5 100644 --- a/tests/queries/0_stateless/00829_bitmap64_function.sql +++ b/tests/queries/0_stateless/00829_bitmap64_function.sql @@ -70,4 +70,5 @@ SELECT count(*) FROM bitmap_test WHERE 0 = bitmapHasAny((SELECT groupBitmapState SELECT bitmapToArray(bitmapAnd(groupBitmapState(uid), bitmapBuild(CAST([4294967296, 4294967297, 4294967298], 'Array(UInt64)')))) FROM bitmap_test GROUP BY city_id; - +DROP TABLE bitmap_state_test; +DROP TABLE bitmap_test; diff --git a/tests/queries/0_stateless/00829_bitmap_function.sql b/tests/queries/0_stateless/00829_bitmap_function.sql index 1217fbefc71..b9e9664a56e 100644 --- a/tests/queries/0_stateless/00829_bitmap_function.sql +++ b/tests/queries/0_stateless/00829_bitmap_function.sql @@ -137,10 +137,10 @@ DROP TABLE IF EXISTS bitmap_column_expr_test3; CREATE TABLE bitmap_column_expr_test3 ( tag_id String, - z AggregateFunction(groupBitmap, UInt32), + z AggregateFunction(groupBitmap, UInt64), replace Nested ( - from UInt32, - to UInt32 + from UInt16, + to UInt64 ) ) ENGINE = MergeTree @@ -149,10 +149,10 @@ ORDER BY tag_id; DROP TABLE IF EXISTS numbers10; CREATE VIEW numbers10 AS SELECT number FROM system.numbers LIMIT 10; -INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag1', groupBitmapState(toUInt32(number)), cast([] as Array(UInt32)), cast([] as Array(UInt32)) FROM numbers10; -INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag2', groupBitmapState(toUInt32(number)), cast([0] as Array(UInt32)), cast([2] as Array(UInt32)) FROM numbers10; -INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag3', groupBitmapState(toUInt32(number)), cast([0,7] as Array(UInt32)), cast([3,101] as Array(UInt32)) FROM numbers10; -INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag4', groupBitmapState(toUInt32(number)), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)) FROM numbers10; +INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag1', groupBitmapState(toUInt64(number)), cast([] as Array(UInt16)), cast([] as Array(UInt64)) FROM numbers10; +INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag2', groupBitmapState(toUInt64(number)), cast([0] as Array(UInt16)), cast([2] as Array(UInt64)) FROM numbers10; +INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag3', groupBitmapState(toUInt64(number)), cast([0,7] as Array(UInt16)), cast([3,101] as Array(UInt64)) FROM numbers10; +INSERT INTO bitmap_column_expr_test3(tag_id, z, replace.from, replace.to) SELECT 'tag4', groupBitmapState(toUInt64(number)), cast([5,999,2] as Array(UInt16)), cast([2,888,20] as Array(UInt64)) FROM numbers10; SELECT tag_id, bitmapToArray(z), replace.from, replace.to, bitmapToArray(bitmapTransform(z, replace.from, replace.to)) FROM bitmap_column_expr_test3 ORDER BY tag_id; @@ -232,11 +232,11 @@ select bitmapHasAll(bitmapBuild([ -- bitmapContains: ---- Empty -SELECT bitmapContains(bitmapBuild(emptyArrayUInt32()), toUInt32(0)); -SELECT bitmapContains(bitmapBuild(emptyArrayUInt16()), toUInt32(5)); +SELECT bitmapContains(bitmapBuild(emptyArrayUInt32()), toUInt8(0)); +SELECT bitmapContains(bitmapBuild(emptyArrayUInt16()), toUInt16(5)); ---- Small select bitmapContains(bitmapBuild([1,5,7,9]),toUInt32(0)); -select bitmapContains(bitmapBuild([1,5,7,9]),toUInt32(9)); +select bitmapContains(bitmapBuild([1,5,7,9]),toUInt64(9)); ---- Large select bitmapContains(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, @@ -250,31 +250,31 @@ select bitmapContains(bitmapBuild([ -- bitmapSubsetInRange: ---- Empty -SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild(emptyArrayUInt32()), toUInt32(0), toUInt32(10))); -SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild(emptyArrayUInt16()), toUInt32(0), toUInt32(10))); +SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild(emptyArrayUInt32()), toUInt64(0), toUInt32(10))); +SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild(emptyArrayUInt16()), toUInt32(0), toUInt64(10))); ---- Small -select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt32(0), toUInt32(4))); -select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt32(10), toUInt32(10))); -select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt32(3), toUInt32(7))); +select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt8(0), toUInt16(4))); +select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt32(10), toUInt64(10))); +select bitmapToArray(bitmapSubsetInRange(bitmapBuild([1,5,7,9]), toUInt64(3), toUInt32(7))); ---- Large select bitmapToArray(bitmapSubsetInRange(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, - 100,200,500]), toUInt32(0), toUInt32(100))); + 100,200,500]), toUInt8(0), toUInt32(100))); select bitmapToArray(bitmapSubsetInRange(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, - 100,200,500]), toUInt32(30), toUInt32(200))); + 100,200,500]), toUInt64(30), toUInt32(200))); select bitmapToArray(bitmapSubsetInRange(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, - 100,200,500]), toUInt32(100), toUInt32(200))); + 100,200,500]), toUInt32(100), toUInt64(200))); -- bitmapSubsetLimit: ---- Empty -SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild(emptyArrayUInt32()), toUInt32(0), toUInt32(10))); -SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild(emptyArrayUInt16()), toUInt32(0), toUInt32(10))); +SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild(emptyArrayUInt32()), toUInt8(0), toUInt32(10))); +SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild(emptyArrayUInt16()), toUInt32(0), toUInt64(10))); ---- Small -select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt32(0), toUInt32(4))); -select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt32(10), toUInt32(10))); -select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt32(3), toUInt32(7))); +select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt8(0), toUInt32(4))); +select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt32(10), toUInt64(10))); +select bitmapToArray(bitmapSubsetLimit(bitmapBuild([1,5,7,9]), toUInt16(3), toUInt32(7))); ---- Large select bitmapToArray(bitmapSubsetLimit(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, @@ -284,7 +284,7 @@ select bitmapToArray(bitmapSubsetLimit(bitmapBuild([ 100,200,500]), toUInt32(30), toUInt32(200))); select bitmapToArray(bitmapSubsetLimit(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, - 100,200,500]), toUInt32(100), toUInt32(200))); + 100,200,500]), toUInt32(100), toUInt16(200))); -- bitmapMin: ---- Empty @@ -309,3 +309,39 @@ select bitmapMax(bitmapBuild([1,5,7,9])); select bitmapMax(bitmapBuild([ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33, 100,200,500])); + + +-- reproduce #18911 +CREATE TABLE bitmap_test(pickup_date Date, city_id UInt32, uid UInt32)ENGINE = Memory; +INSERT INTO bitmap_test SELECT '2019-01-01', 1, number FROM numbers(1,50); +INSERT INTO bitmap_test SELECT '2019-01-02', 1, number FROM numbers(11,60); +INSERT INTO bitmap_test SELECT '2019-01-03', 2, number FROM numbers(1,10); + +SELECT + bitmapCardinality(day_today) AS today_users, + bitmapCardinality(day_before) AS before_users, + bitmapOrCardinality(day_today, day_before) AS all_users, + bitmapAndCardinality(day_today, day_before) AS old_users, + bitmapAndnotCardinality(day_today, day_before) AS new_users, + bitmapXorCardinality(day_today, day_before) AS diff_users +FROM +( + SELECT + city_id, + groupBitmapState(uid) AS day_today + FROM bitmap_test + WHERE pickup_date = '2019-01-02' + GROUP BY + rand((rand((rand('') % nan) = NULL) % 7) % rand(NULL)), + city_id +) AS js1 +ALL LEFT JOIN +( + SELECT + city_id, + groupBitmapState(uid) AS day_before + FROM bitmap_test + WHERE pickup_date = '2019-01-01' + GROUP BY city_id +) AS js2 USING (city_id) FORMAT Null; +drop table bitmap_test; diff --git a/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.reference b/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.reference index d00491fd7e5..98fb6a68656 100644 --- a/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.reference +++ b/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.reference @@ -1 +1,4 @@ 1 +1 +1 +1 diff --git a/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.sql b/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.sql index 81dd7cab9f4..520b4a03057 100644 --- a/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.sql +++ b/tests/queries/0_stateless/00974_bitmapContains_with_primary_key.sql @@ -1,5 +1,8 @@ DROP TABLE IF EXISTS test; CREATE TABLE test (num UInt64, str String) ENGINE = MergeTree ORDER BY num; INSERT INTO test (num) VALUES (1), (2), (10), (15), (23); +SELECT count(*) FROM test WHERE bitmapContains(bitmapBuild([1, 5, 7, 9]), toUInt8(num)); +SELECT count(*) FROM test WHERE bitmapContains(bitmapBuild([1, 5, 7, 9]), toUInt16(num)); SELECT count(*) FROM test WHERE bitmapContains(bitmapBuild([1, 5, 7, 9]), toUInt32(num)); +SELECT count(*) FROM test WHERE bitmapContains(bitmapBuild([1, 5, 7, 9]), toUInt64(num)); DROP TABLE test; diff --git a/tests/queries/0_stateless/01048_exists_query.reference b/tests/queries/0_stateless/01048_exists_query.reference index f1db7c70e71..ede3b4cdea7 100644 --- a/tests/queries/0_stateless/01048_exists_query.reference +++ b/tests/queries/0_stateless/01048_exists_query.reference @@ -21,6 +21,13 @@ 0 0 0 +1 +0 +0 +0 +0 +0 +0 0 0 0 diff --git a/tests/queries/0_stateless/01048_exists_query.sql b/tests/queries/0_stateless/01048_exists_query.sql index fca2c233c64..239f865fa99 100644 --- a/tests/queries/0_stateless/01048_exists_query.sql +++ b/tests/queries/0_stateless/01048_exists_query.sql @@ -40,6 +40,20 @@ EXISTS db_01048.t_01048; EXISTS TABLE db_01048.t_01048; EXISTS DICTIONARY db_01048.t_01048; + +CREATE TABLE db_01048.t_01048_2 (x UInt8) ENGINE = Memory; +CREATE VIEW db_01048.v_01048 AS SELECT * FROM db_01048.t_01048_2; +EXISTS VIEW db_01048.v_01048; +EXISTS VIEW db_01048.t_01048_2; +EXISTS VIEW db_01048.v_not_exist; +DROP VIEW db_01048.v_01048; +EXISTS VIEW db_01048.v_01048; +EXISTS VIEW db_01048.t_01048_2; +EXISTS VIEW db_01048.v_not_exist; +EXISTS VIEW db_not_exists.v_not_exist; +DROP TABLE db_01048.t_01048_2; + + DROP DATABASE db_01048; EXISTS db_01048.t_01048; EXISTS TABLE db_01048.t_01048; diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index b0cba465c3d..cddf1ebcda6 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -19,7 +19,7 @@ function rename_thread_1() replica_01108_3 TO replica_01108_3_tmp, replica_01108_4 TO replica_01108_4_tmp"; sleep 0.$RANDOM; - done + done } function rename_thread_2() @@ -30,23 +30,27 @@ function rename_thread_2() replica_01108_3_tmp TO replica_01108_4, replica_01108_4_tmp TO replica_01108_1"; sleep 0.$RANDOM; - done + done } -function restart_thread_1() +function restart_replicas_loop() { while true; do - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICAS"; - sleep 0.$RANDOM; + for i in $(seq 4); do + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}"; + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}_tmp"; done + sleep 0.$RANDOM; + done +} +function restart_thread_1() +{ + restart_replicas_loop } function restart_thread_2() { - while true; do - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICAS"; - sleep 0.$RANDOM; - done + restart_replicas_loop } export -f rename_thread_1; diff --git a/tests/queries/0_stateless/01184_insert_values_huge_strings.reference b/tests/queries/0_stateless/01184_insert_values_huge_strings.reference new file mode 100644 index 00000000000..1c42cb6d5ed --- /dev/null +++ b/tests/queries/0_stateless/01184_insert_values_huge_strings.reference @@ -0,0 +1,3 @@ +1000100 +1000100 +1000100 diff --git a/tests/queries/0_stateless/01184_insert_values_huge_strings.sh b/tests/queries/0_stateless/01184_insert_values_huge_strings.sh new file mode 100755 index 00000000000..9b63f401a59 --- /dev/null +++ b/tests/queries/0_stateless/01184_insert_values_huge_strings.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "drop table if exists huge_strings" +$CLICKHOUSE_CLIENT -q "create table huge_strings (n UInt64, l UInt64, s String, h UInt64) engine=MergeTree order by n" + +for _ in {1..10}; do + $CLICKHOUSE_CLIENT -q "select number, (rand() % 100*1000*1000) as l, repeat(randomString(l/1000/1000), 1000*1000) as s, cityHash64(s) from numbers(10) format Values" | $CLICKHOUSE_CLIENT -q "insert into huge_strings values" & + $CLICKHOUSE_CLIENT -q "select number % 10, (rand() % 100) as l, randomString(l) as s, cityHash64(s) from numbers(100000)" | $CLICKHOUSE_CLIENT -q "insert into huge_strings format TSV" & +done; +wait + +$CLICKHOUSE_CLIENT -q "select count() from huge_strings" +$CLICKHOUSE_CLIENT -q "select sum(l = length(s)) from huge_strings" +$CLICKHOUSE_CLIENT -q "select sum(h = cityHash64(s)) from huge_strings" + +$CLICKHOUSE_CLIENT -q "drop table huge_strings" diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index 2e28c2c0165..143af3679a5 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -50,4 +50,4 @@ $CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(query_duration_ms) < $max_time_ms $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" -$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $db" diff --git a/tests/queries/0_stateless/01267_alter_default_key_columns_zookeeper.sql b/tests/queries/0_stateless/01267_alter_default_key_columns_zookeeper.sql index 3d7a3227f0c..d96085bc086 100644 --- a/tests/queries/0_stateless/01267_alter_default_key_columns_zookeeper.sql +++ b/tests/queries/0_stateless/01267_alter_default_key_columns_zookeeper.sql @@ -17,7 +17,9 @@ ALTER TABLE test_alter_r2 MODIFY COLUMN x DEFAULT '2000-01-01' SETTINGS replicat DESCRIBE TABLE test_alter_r1; DESCRIBE TABLE test_alter_r2; -SYSTEM RESTART REPLICAS; +SYSTEM RESTART REPLICA test_alter_r1; +SYSTEM RESTART REPLICA test_alter_r2; + DESCRIBE TABLE test_alter_r1; DESCRIBE TABLE test_alter_r2; diff --git a/tests/queries/0_stateless/01398_any_with_alias.sql b/tests/queries/0_stateless/01398_any_with_alias.sql index 32e67f3f4b6..a65b8132c67 100644 --- a/tests/queries/0_stateless/01398_any_with_alias.sql +++ b/tests/queries/0_stateless/01398_any_with_alias.sql @@ -1,3 +1,5 @@ +SET optimize_move_functions_out_of_any = 1; + SELECT any(number * number) AS n FROM numbers(100) FORMAT CSVWithNames; EXPLAIN SYNTAX SELECT any(number * number) AS n FROM numbers(100); diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index a333d9be8ae..376ee58859e 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -15,10 +15,12 @@ for i in $(seq 1 $NUM_REPLICAS); do " done +valid_exceptions_to_retry='Quorum for previous write has not been satisfied yet|Another quorum insert has been already started|Unexpected logical error while adding block' + function thread { for x in {0..99}; do while true; do - $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 0 --query "INSERT INTO r$1 SELECT $x" 2>&1 | grep -qF 'Quorum for previous write has not been satisfied yet' || break + $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 0 --query "INSERT INTO r$1 SELECT $x" 2>&1 | grep -qE "$valid_exceptions_to_retry" || break done done } @@ -32,7 +34,9 @@ wait for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " SYSTEM SYNC REPLICA r$i; - SELECT count(), min(x), max(x), sum(x) FROM r$i; - DROP TABLE IF EXISTS r$i; -" + SELECT count(), min(x), max(x), sum(x) FROM r$i;" +done + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS r$i;" done diff --git a/tests/queries/0_stateless/01470_columns_transformers.reference b/tests/queries/0_stateless/01470_columns_transformers.reference index a103d62167b..ae0adb3ba60 100644 --- a/tests/queries/0_stateless/01470_columns_transformers.reference +++ b/tests/queries/0_stateless/01470_columns_transformers.reference @@ -27,9 +27,9 @@ SELECT avg(k) FROM columns_transformers SELECT - toDate(any(i)), - toDate(any(j)), - toDate(any(k)) + any(toDate(i)), + any(toDate(j)), + any(toDate(k)) FROM columns_transformers AS a SELECT length(toString(j)), @@ -44,9 +44,9 @@ SELECT avg(k) FROM columns_transformers SELECT - toDate(any(i)), - toDate(any(j)), - toDate(any(k)) + any(toDate(i)), + any(toDate(j)), + any(toDate(k)) FROM columns_transformers AS a SELECT sum(i + 1 AS i), @@ -59,9 +59,9 @@ SELECT avg(k) FROM columns_transformers SELECT - toDate(any(i)), - toDate(any(j)), - toDate(any(k)) + any(toDate(i)), + any(toDate(j)), + any(toDate(k)) FROM columns_transformers AS a SELECT (i + 1) + 1 AS i, diff --git a/tests/queries/0_stateless/01480_binary_operator_monotonicity.sql b/tests/queries/0_stateless/01480_binary_operator_monotonicity.sql index 20c3b542e18..61313de4669 100644 --- a/tests/queries/0_stateless/01480_binary_operator_monotonicity.sql +++ b/tests/queries/0_stateless/01480_binary_operator_monotonicity.sql @@ -8,13 +8,13 @@ DROP TABLE IF EXISTS binary_op_mono7; DROP TABLE IF EXISTS binary_op_mono8; CREATE TABLE binary_op_mono1(i int, j int) ENGINE MergeTree PARTITION BY toDate(i / 1000) ORDER BY j; -CREATE TABLE binary_op_mono2(i int, j int) ENGINE MergeTree PARTITION BY 1000 / i ORDER BY j; +CREATE TABLE binary_op_mono2(i int, j int) ENGINE MergeTree PARTITION BY 1000 / i ORDER BY j settings allow_floating_point_partition_key=true;; CREATE TABLE binary_op_mono3(i int, j int) ENGINE MergeTree PARTITION BY i + 1000 ORDER BY j; CREATE TABLE binary_op_mono4(i int, j int) ENGINE MergeTree PARTITION BY 1000 + i ORDER BY j; CREATE TABLE binary_op_mono5(i int, j int) ENGINE MergeTree PARTITION BY i - 1000 ORDER BY j; CREATE TABLE binary_op_mono6(i int, j int) ENGINE MergeTree PARTITION BY 1000 - i ORDER BY j; -CREATE TABLE binary_op_mono7(i int, j int) ENGINE MergeTree PARTITION BY i / 1000.0 ORDER BY j; -CREATE TABLE binary_op_mono8(i int, j int) ENGINE MergeTree PARTITION BY 1000.0 / i ORDER BY j; +CREATE TABLE binary_op_mono7(i int, j int) ENGINE MergeTree PARTITION BY i / 1000.0 ORDER BY j settings allow_floating_point_partition_key=true;; +CREATE TABLE binary_op_mono8(i int, j int) ENGINE MergeTree PARTITION BY 1000.0 / i ORDER BY j settings allow_floating_point_partition_key=true;; INSERT INTO binary_op_mono1 VALUES (toUnixTimestamp('2020-09-01 00:00:00') * 1000, 1), (toUnixTimestamp('2020-09-01 00:00:00') * 1000, 2); INSERT INTO binary_op_mono2 VALUES (1, 1), (10000, 2); diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh index 856f4c1516f..9f31fcb6da1 100755 --- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh @@ -84,3 +84,6 @@ function test_with_engine { test_with_engine TinyLog test_with_engine StripeLog test_with_engine Log + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t2" diff --git a/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.reference b/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.reference new file mode 100644 index 00000000000..9972842f982 --- /dev/null +++ b/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.reference @@ -0,0 +1 @@ +1 1 diff --git a/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.sql b/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.sql new file mode 100644 index 00000000000..849a4378e45 --- /dev/null +++ b/tests/queries/0_stateless/01552_impl_aggfunc_cloneresize.sql @@ -0,0 +1,57 @@ +drop table if EXISTS test_bm; + +drop table if EXISTS test_bm_join; + +create table test_bm( + dim UInt64, + id UInt64 ) +ENGINE = MergeTree() +ORDER BY( dim, id ) +SETTINGS index_granularity = 8192; + +create table test_bm_join( + dim UInt64, + id UInt64 ) +ENGINE = MergeTree() +ORDER BY(dim,id) +SETTINGS index_granularity = 8192; + +insert into test_bm VALUES (1,1),(2,2),(3,3),(4,4); + +select + dim , + sum(idnum) +from + test_bm_join +right join( + select + dim, + bitmapOrCardinality(ids,ids2) as idnum + from + ( + select + dim, + groupBitmapState(toUInt64(id)) as ids + FROM + test_bm + where + dim >2 + group by + dim ) A all + right join ( + select + dim, + groupBitmapState(toUInt64(id)) as ids2 + FROM + test_bm + where + dim < 2 + group by + dim ) B + using(dim) ) C +using(dim) +group by dim; + +drop table test_bm; + +drop table test_bm_join; diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.reference b/tests/queries/0_stateless/01576_alias_column_rewrite.reference new file mode 100644 index 00000000000..ebc8be4f79b --- /dev/null +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.reference @@ -0,0 +1,74 @@ +test-partition-prune +1 +1 +1 +1 +1 +test-join +1 +1 +alias2alias +1 +1 +1 +1 +1 +1 +1 +array-join +1 +0 0 +lambda +1 +optimize_read_in_order +Expression (Projection) + Limit (preliminary LIMIT) + MergingSorted (Merge sorted streams for ORDER BY) + MergeSorting (Merge sorted blocks for ORDER BY) + PartialSorting (Sort each block for ORDER BY) + Expression (Before ORDER BY + Add table aliases) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (MergeTree) +Expression (Projection) + Limit (preliminary LIMIT) + FinishSorting + Expression (Before ORDER BY + Add table aliases) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Union + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) +Expression (Projection) + Limit (preliminary LIMIT) + FinishSorting + Expression (Before ORDER BY) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Union + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) +optimize_aggregation_in_order +Expression (Projection + Before ORDER BY) + Aggregating + Expression (Before GROUP BY + Add table aliases) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + ReadFromStorage (MergeTree) +Expression (Projection + Before ORDER BY) + Aggregating + Expression (Before GROUP BY + Add table aliases) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Union + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) +Expression (Projection + Before ORDER BY) + Aggregating + Expression (Before GROUP BY) + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Union + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) + ReadFromStorage (MergeTree with order) +second-index +1 +1 diff --git a/tests/queries/0_stateless/01576_alias_column_rewrite.sql b/tests/queries/0_stateless/01576_alias_column_rewrite.sql new file mode 100644 index 00000000000..fabe20010af --- /dev/null +++ b/tests/queries/0_stateless/01576_alias_column_rewrite.sql @@ -0,0 +1,117 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + `timestamp` DateTime, + `value` UInt64, + `day` Date ALIAS toDate(timestamp), + `day1` Date ALIAS day + 1, + `day2` Date ALIAS day1 + 1, + `time` DateTime ALIAS timestamp +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY timestamp SETTINGS index_granularity = 1; + + +INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-01 12:00:00'), 1 FROM numbers(10); +INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-02 12:00:00'), 1 FROM numbers(10); +INSERT INTO test_table(timestamp, value) SELECT toDateTime('2020-01-03 12:00:00'), 1 FROM numbers(10); + +set optimize_respect_aliases = 1; +SELECT 'test-partition-prune'; + +SELECT COUNT() = 10 FROM test_table WHERE day = '2020-01-01' SETTINGS max_rows_to_read = 10; +SELECT t = '2020-01-03' FROM (SELECT day AS t FROM test_table WHERE t = '2020-01-03' GROUP BY t SETTINGS max_rows_to_read = 10); +SELECT COUNT() = 10 FROM test_table WHERE day = '2020-01-01' UNION ALL SELECT 1 FROM numbers(1) SETTINGS max_rows_to_read = 11; +SELECT COUNT() = 0 FROM (SELECT toDate('2019-01-01') AS day, day AS t FROM test_table PREWHERE t = '2020-01-03' WHERE t = '2020-01-03' GROUP BY t ); + + + +SELECT 'test-join'; +SELECT day = '2020-01-03' +FROM +( + SELECT toDate('2020-01-03') AS day + FROM numbers(1) +) AS a +INNER JOIN +( + SELECT day + FROM test_table + WHERE day = '2020-01-03' + GROUP BY day +) AS b ON a.day = b.day SETTINGS max_rows_to_read = 11; + +SELECT day = '2020-01-01' +FROM +( + SELECT day + FROM test_table + WHERE day = '2020-01-01' + GROUP BY day +) AS a +INNER JOIN +( + SELECT toDate('2020-01-01') AS day + FROM numbers(1) +) AS b ON a.day = b.day SETTINGS max_rows_to_read = 11; + + +SELECT 'alias2alias'; +SELECT COUNT() = 10 FROM test_table WHERE day1 = '2020-01-02' SETTINGS max_rows_to_read = 10; +SELECT t = '2020-01-03' FROM (SELECT day1 AS t FROM test_table WHERE t = '2020-01-03' GROUP BY t SETTINGS max_rows_to_read = 10); +SELECT t = '2020-01-03' FROM (SELECT day2 AS t FROM test_table WHERE t = '2020-01-03' GROUP BY t SETTINGS max_rows_to_read = 10); +SELECT COUNT() = 10 FROM test_table WHERE day1 = '2020-01-03' UNION ALL SELECT 1 FROM numbers(1) SETTINGS max_rows_to_read = 11; +SELECT COUNT() = 0 FROM (SELECT toDate('2019-01-01') AS day1, day1 AS t FROM test_table PREWHERE t = '2020-01-03' WHERE t = '2020-01-03' GROUP BY t ); +SELECT day1 = '2020-01-04' FROM test_table PREWHERE day1 = '2020-01-04' WHERE day1 = '2020-01-04' GROUP BY day1 SETTINGS max_rows_to_read = 10; + + +ALTER TABLE test_table add column array Array(UInt8) default [1, 2, 3]; +ALTER TABLE test_table add column struct.key Array(UInt8) default [2, 4, 6], add column struct.value Array(UInt8) alias array; + + +SELECT 'array-join'; +set max_rows_to_read = 10; +SELECT count() == 10 FROM test_table WHERE day = '2020-01-01'; +SELECT sum(struct.key) == 30, sum(struct.value) == 30 FROM (SELECT struct.key, struct.value FROM test_table array join struct WHERE day = '2020-01-01'); + + +SELECT 'lambda'; +-- lambda parameters in filter should not be rewrite +SELECT count() == 10 FROM test_table WHERE arrayMap((day) -> day + 1, [1,2,3]) [1] = 2 AND day = '2020-01-03'; + +set max_rows_to_read = 0; + +SELECT 'optimize_read_in_order'; +EXPLAIN SELECT day AS s FROM test_table ORDER BY s LIMIT 1 SETTINGS optimize_read_in_order = 0; +EXPLAIN SELECT day AS s FROM test_table ORDER BY s LIMIT 1 SETTINGS optimize_read_in_order = 1; +EXPLAIN SELECT toDate(timestamp) AS s FROM test_table ORDER BY toDate(timestamp) LIMIT 1 SETTINGS optimize_read_in_order = 1; + + +SELECT 'optimize_aggregation_in_order'; +EXPLAIN SELECT day, count() AS s FROM test_table GROUP BY day SETTINGS optimize_aggregation_in_order = 0; +EXPLAIN SELECT day, count() AS s FROM test_table GROUP BY day SETTINGS optimize_aggregation_in_order = 1; +EXPLAIN SELECT toDate(timestamp), count() AS s FROM test_table GROUP BY toDate(timestamp) SETTINGS optimize_aggregation_in_order = 1; + +DROP TABLE test_table; + + +SELECT 'second-index'; +DROP TABLE IF EXISTS test_index; +CREATE TABLE test_index +( + `key_string` String, + `key_uint32` ALIAS toUInt32(key_string), + INDEX idx toUInt32(key_string) TYPE set(0) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY tuple() +PRIMARY KEY tuple() +ORDER BY key_string SETTINGS index_granularity = 1; + +INSERT INTO test_index SELECT * FROM numbers(10); +set max_rows_to_read = 1; +SELECT COUNT() == 1 FROM test_index WHERE key_uint32 = 1; +SELECT COUNT() == 1 FROM test_index WHERE toUInt32(key_string) = 1; +DROP TABLE IF EXISTS test_index; + diff --git a/tests/queries/0_stateless/01582_any_join_supertype.reference b/tests/queries/0_stateless/01582_any_join_supertype.reference new file mode 100644 index 00000000000..a45cb12d3af --- /dev/null +++ b/tests/queries/0_stateless/01582_any_join_supertype.reference @@ -0,0 +1,5 @@ +1 +2020-01-01 10:00:00 +2020-01-01 10:00:00 +1 +2020-01-01 10:00:00 diff --git a/tests/queries/0_stateless/01582_any_join_supertype.sql b/tests/queries/0_stateless/01582_any_join_supertype.sql new file mode 100644 index 00000000000..6b06d78c83c --- /dev/null +++ b/tests/queries/0_stateless/01582_any_join_supertype.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS foo; +DROP TABLE IF EXISTS bar; + +CREATE TABLE foo (server_date Date, server_time Datetime('Europe/Moscow'), dimension_1 String) ENGINE = MergeTree() PARTITION BY toYYYYMM(server_date) ORDER BY (server_date); +CREATE TABLE bar (server_date Date, dimension_1 String) ENGINE = MergeTree() PARTITION BY toYYYYMM(server_date) ORDER BY (server_date); + +INSERT INTO foo VALUES ('2020-01-01', '2020-01-01 12:00:00', 'test1'), ('2020-01-01', '2020-01-01 13:00:00', 'test2'); +INSERT INTO bar VALUES ('2020-01-01', 'test2'), ('2020-01-01', 'test3'); + +SET optimize_move_to_prewhere = 1; +SET any_join_distinct_right_table_keys = 0; + +SELECT count() +FROM foo ANY INNER JOIN bar USING (dimension_1) +WHERE (foo.server_date <= '2020-11-07') AND (toDate(foo.server_time, 'Asia/Yekaterinburg') <= '2020-11-07'); + +SELECT toDateTime(foo.server_time, 'UTC') +FROM foo +ANY INNER JOIN bar USING (dimension_1) +WHERE toDate(foo.server_time, 'UTC') <= toDate('2020-04-30'); + +SELECT toDateTime(foo.server_time, 'UTC') FROM foo +SEMI JOIN bar USING (dimension_1) WHERE toDate(foo.server_time, 'UTC') <= toDate('2020-04-30'); + +SET any_join_distinct_right_table_keys = 1; + +SELECT count() +FROM foo ANY INNER JOIN bar USING (dimension_1) +WHERE (foo.server_date <= '2020-11-07') AND (toDate(foo.server_time, 'Asia/Yekaterinburg') <= '2020-11-07'); + +SELECT toDateTime(foo.server_time, 'UTC') +FROM foo +ANY INNER JOIN bar USING (dimension_1) +WHERE toDate(foo.server_time, 'UTC') <= toDate('2020-04-30'); + +DROP TABLE foo; +DROP TABLE bar; diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index ce56860ed8b..aad784b1ac1 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -122,7 +122,7 @@ select * from (select * from numbers(5) order by rand()) order by count() over ( 2 3 4 -select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc; +select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number) + 1) over (order by min(number) desc) desc; -- different windows -- an explain test would also be helpful, but it's too immature now and I don't diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index 082a6652a65..30094690a92 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -41,7 +41,7 @@ select * from (select * from numbers(5) order by rand()) order by count() over ( -- Aggregate functions as window function arguments. This query is semantically -- the same as the above one, only we replace `number` with -- `any(number) group by number` and so on. -select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number + 1)) over (order by min(number) desc) desc; +select * from (select * from numbers(5) order by rand()) group by number order by sum(any(number) + 1) over (order by min(number) desc) desc; -- different windows -- an explain test would also be helpful, but it's too immature now and I don't diff --git a/tests/queries/0_stateless/01621_decode_XML.reference b/tests/queries/0_stateless/01621_decode_XML.reference new file mode 100644 index 00000000000..d4fa75bbf94 --- /dev/null +++ b/tests/queries/0_stateless/01621_decode_XML.reference @@ -0,0 +1,17 @@ +Hello, "world"! +<123> +&clickhouse +\'foo\' +Hello, && world +Hello, &;& world +Hello, &a;& world +Hello, <t;& world +Hello, <t& world +Hello, &t;& world + !"#$%&\'()*+,-./012 +)*+,-./0123456789:;< +=>?@ABCDEFGHIJKLMNOP +为 +为 +�\'123 +ЦЦЮЮЫㄱ diff --git a/tests/queries/0_stateless/01621_decode_XML.sql b/tests/queries/0_stateless/01621_decode_XML.sql new file mode 100644 index 00000000000..b111520db4c --- /dev/null +++ b/tests/queries/0_stateless/01621_decode_XML.sql @@ -0,0 +1,20 @@ +SELECT decodeXMLComponent('Hello, "world"!'); +SELECT decodeXMLComponent('<123>'); +SELECT decodeXMLComponent('&clickhouse'); +SELECT decodeXMLComponent(''foo''); +SELECT decodeXMLComponent('Hello, && world'); +SELECT decodeXMLComponent('Hello, &;& world'); +SELECT decodeXMLComponent('Hello, &a;& world'); +SELECT decodeXMLComponent('Hello, <t;& world'); +SELECT decodeXMLComponent('Hello, <t& world'); +SELECT decodeXMLComponent('Hello, &t;& world'); + +--decode numeric entities + +SELECT decodeXMLComponent(' !"#$%&'()*+,-./012'); +SELECT decodeXMLComponent(')*+,-./0123456789:;<'); +SELECT decodeXMLComponent('=>?@ABCDEFGHIJKLMNOP'); +SELECT decodeXMLComponent('为'); +SELECT decodeXMLComponent('为'); +SELECT decodeXMLComponent('�'123'); +SELECT decodeXMLComponent('ЦЦЮЮЫㄱ'); \ No newline at end of file diff --git a/tests/queries/0_stateless/01622_codec_zstd_long.reference b/tests/queries/0_stateless/01622_codec_zstd_long.reference new file mode 100644 index 00000000000..53bbf843449 --- /dev/null +++ b/tests/queries/0_stateless/01622_codec_zstd_long.reference @@ -0,0 +1,4 @@ +10000 +10000 +10000 +10000 diff --git a/tests/queries/0_stateless/01622_codec_zstd_long.sql b/tests/queries/0_stateless/01622_codec_zstd_long.sql new file mode 100644 index 00000000000..7045699d73f --- /dev/null +++ b/tests/queries/0_stateless/01622_codec_zstd_long.sql @@ -0,0 +1,29 @@ +DROP TABLE IF EXISTS zstd_1_00; +DROP TABLE IF EXISTS zstd_1_24; +DROP TABLE IF EXISTS zstd_9_00; +DROP TABLE IF EXISTS zstd_9_24; +DROP TABLE IF EXISTS words; + +CREATE TABLE words(i Int, word String) ENGINE = Memory; +INSERT INTO words SELECT * FROM generateRandom('i Int, word String',1,10) LIMIT 10000; + +CREATE TABLE zstd_1_00(n Int, b String CODEC(ZSTD(1))) ENGINE = MergeTree ORDER BY n; +CREATE TABLE zstd_1_24(n Int, b String CODEC(ZSTD(1,24))) ENGINE = MergeTree ORDER BY n; +CREATE TABLE zstd_9_00(n Int, b String CODEC(ZSTD(9))) ENGINE = MergeTree ORDER BY n; +CREATE TABLE zstd_9_24(n Int, b String CODEC(ZSTD(9,24))) ENGINE = MergeTree ORDER BY n; + +INSERT INTO zstd_1_00 SELECT * FROM words; +INSERT INTO zstd_1_24 SELECT * FROM words; +INSERT INTO zstd_9_00 SELECT * FROM words; +INSERT INTO zstd_9_24 SELECT * FROM words; + +SELECT COUNT(n) FROM zstd_1_00 LEFT JOIN words ON i == n WHERE b == word; +SELECT COUNT(n) FROM zstd_1_24 LEFT JOIN words ON i == n WHERE b == word; +SELECT COUNT(n) FROM zstd_9_00 LEFT JOIN words ON i == n WHERE b == word; +SELECT COUNT(n) FROM zstd_9_24 LEFT JOIN words ON i == n WHERE b == word; + +DROP TABLE zstd_1_00; +DROP TABLE zstd_1_24; +DROP TABLE zstd_9_00; +DROP TABLE zstd_9_24; +DROP TABLE words; diff --git a/tests/queries/0_stateless/01622_defaults_for_file_engine.reference b/tests/queries/0_stateless/01622_defaults_for_file_engine.reference new file mode 100644 index 00000000000..f04ee88dabd --- /dev/null +++ b/tests/queries/0_stateless/01622_defaults_for_file_engine.reference @@ -0,0 +1 @@ +1 7 8 diff --git a/tests/queries/0_stateless/01622_defaults_for_file_engine.sql b/tests/queries/0_stateless/01622_defaults_for_file_engine.sql new file mode 100644 index 00000000000..203486fe71c --- /dev/null +++ b/tests/queries/0_stateless/01622_defaults_for_file_engine.sql @@ -0,0 +1,7 @@ +insert into table function file("data1622.json", "TSV", "value String") VALUES ('{"a":1}'); +drop table if exists json; +create table json(a int, b int default 7, c default a + b) engine File(JSONEachRow, 'data1622.json'); +set input_format_defaults_for_omitted_fields = 1; +select * from json; +truncate table json; +drop table if exists json; diff --git a/tests/queries/0_stateless/01622_defaults_for_url_engine.reference b/tests/queries/0_stateless/01622_defaults_for_url_engine.reference new file mode 100644 index 00000000000..7326d960397 --- /dev/null +++ b/tests/queries/0_stateless/01622_defaults_for_url_engine.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/01622_defaults_for_url_engine.sh b/tests/queries/0_stateless/01622_defaults_for_url_engine.sh new file mode 100755 index 00000000000..e7deace8b46 --- /dev/null +++ b/tests/queries/0_stateless/01622_defaults_for_url_engine.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +PORT="$(($RANDOM%63000+2001))" + +TEMP_FILE="$CURDIR/01622_defaults_for_url_engine.tmp" + +function thread1 +{ + while true; do + echo -e "HTTP/1.1 200 OK\n\n{\"a\": 1}" | nc -l -p $1 -q 1; + done +} + +function thread2 +{ + while true; do + $CLICKHOUSE_CLIENT --input_format_defaults_for_omitted_fields=1 -q "SELECT * FROM url('http://127.0.0.1:$1/', JSONEachRow, 'a int, b int default 7, c default a + b') format Values" + done +} + +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; + +TIMEOUT=5 + +timeout $TIMEOUT bash -c "thread1 $PORT" > /dev/null 2>&1 & +timeout $TIMEOUT bash -c "thread2 $PORT" 2> /dev/null > $TEMP_FILE & + +wait + +grep -q '(1,7,8)' $TEMP_FILE && echo "Ok" diff --git a/tests/queries/0_stateless/01630_disallow_floating_point_as_partition_key.reference b/tests/queries/0_stateless/01630_disallow_floating_point_as_partition_key.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01630_disallow_floating_point_as_partition_key.sql b/tests/queries/0_stateless/01630_disallow_floating_point_as_partition_key.sql new file mode 100644 index 00000000000..0c33bce6068 --- /dev/null +++ b/tests/queries/0_stateless/01630_disallow_floating_point_as_partition_key.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Float32, b int) Engine = MergeTree() ORDER BY tuple() PARTITION BY a; -- { serverError 36 } +CREATE TABLE test (a Float32, b int) Engine = MergeTree() ORDER BY tuple() PARTITION BY a settings allow_floating_point_partition_key=true; +DROP TABLE IF EXISTS test; +CREATE TABLE test (a Float32, b int, c String, d Float64) Engine = MergeTree() ORDER BY tuple() PARTITION BY (b, c, d) settings allow_floating_point_partition_key=false; -- { serverError 36 } +CREATE TABLE test (a Float32, b int, c String, d Float64) Engine = MergeTree() ORDER BY tuple() PARTITION BY (b, c, d) settings allow_floating_point_partition_key=true; +DROP TABLE IF EXISTS test; diff --git a/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.reference b/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.reference new file mode 100644 index 00000000000..b8cf9277350 --- /dev/null +++ b/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.reference @@ -0,0 +1,2 @@ +42.10 +\N diff --git a/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.sql b/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.sql new file mode 100644 index 00000000000..b8fa08edb68 --- /dev/null +++ b/tests/queries/0_stateless/01632_nullable_string_type_convert_to_decimal_type.sql @@ -0,0 +1,2 @@ +SELECT CAST(arrayJoin(['42.1', NULL]) AS Nullable(Decimal(10,2))); + diff --git a/tests/queries/0_stateless/01632_select_all_syntax.reference b/tests/queries/0_stateless/01632_select_all_syntax.reference new file mode 100644 index 00000000000..c836beb205d --- /dev/null +++ b/tests/queries/0_stateless/01632_select_all_syntax.reference @@ -0,0 +1,35 @@ +a +a +1 +1 +2 +45 +45 +45 +2 +1 +1 +1 + +a +aa +aaa +aaaa +aaaaa +aaaaaa +aaaaaaa +aaaaaaaa +aaaaaaaaa + +a +aa +aaa +aaaa +aaaaa +aaaaaa +aaaaaaa +aaaaaaaa +aaaaaaaaa +aaaaa +aaaaa +aaaaa diff --git a/tests/queries/0_stateless/01632_select_all_syntax.sql b/tests/queries/0_stateless/01632_select_all_syntax.sql new file mode 100644 index 00000000000..f5e96a5cb4e --- /dev/null +++ b/tests/queries/0_stateless/01632_select_all_syntax.sql @@ -0,0 +1,25 @@ +SELECT ALL 'a'; +SELECT DISTINCT 'a'; +SELECT ALL * FROM (SELECT 1 UNION ALL SELECT 1); +SELECT DISTINCT * FROM (SELECT 2 UNION ALL SELECT 2); + +SELECT sum(number) FROM numbers(10); +SELECT sum(ALL number) FROM numbers(10); +SELECT sum(DISTINCT number) FROM numbers(10); + +SELECT sum(ALL x) FROM (SELECT 1 x UNION ALL SELECT 1); +SELECT sum(DISTINCT x) FROM (SELECT 1 x UNION ALL SELECT 1); + +SELECT sum(ALL) FROM (SELECT 1 AS ALL); + +SELECT sum(DISTINCT) FROM (SELECT 1 AS DISTINCT); + +SELECT repeat('a', ALL) FROM (SELECT number AS ALL FROM numbers(10)); + +SELECT repeat('a', DISTINCT) FROM (SELECT number AS DISTINCT FROM numbers(10)); + +SELECT repeat(ALL, 5) FROM (SELECT 'a' AS ALL); + +SELECT repeat(DISTINCT, 5) FROM (SELECT 'a' AS DISTINCT); + +SELECT repeat(ALL, DISTINCT) FROM (SELECT 'a' AS ALL, 5 AS DISTINCT); diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference new file mode 100644 index 00000000000..654db9dbc86 --- /dev/null +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference @@ -0,0 +1,12 @@ +default +1 +compact fsync_after_insert +1 +compact fsync_after_insert,fsync_part_directory +1 +wide fsync_after_insert +1 +wide fsync_after_insert,fsync_part_directory +1 +memory in_memory_parts_insert_sync +1 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql new file mode 100644 index 00000000000..21ebb607693 --- /dev/null +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -0,0 +1,37 @@ +drop table if exists data_01643; + +select 'default'; +create table data_01643 (key Int) engine=MergeTree() order by key; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'compact fsync_after_insert'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'compact fsync_after_insert,fsync_part_directory'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'wide fsync_after_insert'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'wide fsync_after_insert,fsync_part_directory'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'memory in_memory_parts_insert_sync'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; diff --git a/tests/queries/0_stateless/01643_system_suspend.reference b/tests/queries/0_stateless/01643_system_suspend.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01643_system_suspend.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01643_system_suspend.sql b/tests/queries/0_stateless/01643_system_suspend.sql new file mode 100644 index 00000000000..c2cd37e6156 --- /dev/null +++ b/tests/queries/0_stateless/01643_system_suspend.sql @@ -0,0 +1,5 @@ +CREATE TEMPORARY TABLE t (x DateTime); +INSERT INTO t VALUES (now()); +SYSTEM SUSPEND FOR 1 SECOND; +INSERT INTO t VALUES (now()); +SELECT max(x) - min(x) >= 1 FROM t; diff --git a/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.reference b/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.reference new file mode 100644 index 00000000000..fc919524275 --- /dev/null +++ b/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.reference @@ -0,0 +1,6 @@ +no fsync +0 +90 +fsync +90 +180 diff --git a/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.sql b/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.sql new file mode 100644 index 00000000000..87ae96aa451 --- /dev/null +++ b/tests/queries/0_stateless/01644_distributed_async_insert_fsync_smoke.sql @@ -0,0 +1,24 @@ +drop table if exists dist_01643; +drop table if exists data_01643; + +create table data_01643 (key Int) engine=Memory(); + +select 'no fsync'; +create table dist_01643 as data_01643 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01643, key); +system stop distributed sends dist_01643; +insert into dist_01643 select * from numbers(10) settings prefer_localhost_replica=0; +select sum(*) from dist_01643; +system flush distributed dist_01643; +select sum(*) from dist_01643; +drop table dist_01643; + +select 'fsync'; +create table dist_01643 as data_01643 engine=Distributed(test_cluster_two_shards, currentDatabase(), data_01643, key) settings fsync_after_insert=1, fsync_directories=1; +system stop distributed sends dist_01643; +insert into dist_01643 select * from numbers(10) settings prefer_localhost_replica=0; +select sum(*) from dist_01643; +system flush distributed dist_01643; +select sum(*) from dist_01643; +drop table dist_01643; + +drop table if exists data_01643; diff --git a/tests/queries/0_stateless/01645_system_table_engines.reference b/tests/queries/0_stateless/01645_system_table_engines.reference new file mode 100644 index 00000000000..afe0584bea1 --- /dev/null +++ b/tests/queries/0_stateless/01645_system_table_engines.reference @@ -0,0 +1,4 @@ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ diff --git a/tests/queries/0_stateless/01645_system_table_engines.sql b/tests/queries/0_stateless/01645_system_table_engines.sql new file mode 100644 index 00000000000..5e8eef5508b --- /dev/null +++ b/tests/queries/0_stateless/01645_system_table_engines.sql @@ -0,0 +1 @@ +SELECT * FROM system.table_engines WHERE name in ('MergeTree', 'ReplicatedCollapsingMergeTree') FORMAT PrettyCompactNoEscapes; diff --git a/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.reference b/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.reference new file mode 100644 index 00000000000..3b39d6021a2 --- /dev/null +++ b/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.reference @@ -0,0 +1,3 @@ +2 1 +--- +2 1 diff --git a/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.sql b/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.sql new file mode 100644 index 00000000000..51f599a2b09 --- /dev/null +++ b/tests/queries/0_stateless/01646_fix_window_funnel_inconistency.sql @@ -0,0 +1,49 @@ +DROP TABLE IF EXISTS trend; +CREATE TABLE trend +( + `event_date` Date, + `user_id` Int32, + `timestamp` DateTime, + `eventID` Int32, + `product` String +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(event_date) +ORDER BY user_id; + +insert into trend values ('2019-01-28', 1, '2019-01-29 10:00:00', 1004, 'phone') ('2019-01-28', 1, '2019-01-29 10:00:00', 1003, 'phone') ('2019-01-28', 1, '2019-01-28 10:00:00', 1002, 'phone'); + +SELECT + level, + count() AS c +FROM +( + SELECT + user_id, + windowFunnel(6048000000000000)(timestamp, eventID = 1004, eventID = 1003, eventID = 1002) AS level + FROM trend + GROUP BY user_id +) +GROUP BY level +ORDER BY level ASC; + +SELECT '---'; + +TRUNCATE TABLE trend; +insert into trend values ('2019-01-28', 1, '2019-01-29 10:00:00', 1003, 'phone') ('2019-01-28', 1, '2019-01-29 10:00:00', 1004, 'phone') ('2019-01-28', 1, '2019-01-28 10:00:00', 1002, 'phone'); + +SELECT + level, + count() AS c +FROM +( + SELECT + user_id, + windowFunnel(6048000000000000)(timestamp, eventID = 1004, eventID = 1003, eventID = 1002) AS level + FROM trend + GROUP BY user_id +) +GROUP BY level +ORDER BY level ASC; + +DROP TABLE trend; diff --git a/tests/queries/0_stateless/01646_system_restart_replicas_smoke.reference b/tests/queries/0_stateless/01646_system_restart_replicas_smoke.reference new file mode 100644 index 00000000000..3c5ce58bfb2 --- /dev/null +++ b/tests/queries/0_stateless/01646_system_restart_replicas_smoke.reference @@ -0,0 +1,2 @@ +x Date +s String diff --git a/tests/queries/0_stateless/01646_system_restart_replicas_smoke.sql b/tests/queries/0_stateless/01646_system_restart_replicas_smoke.sql new file mode 100644 index 00000000000..cfd70df8dd4 --- /dev/null +++ b/tests/queries/0_stateless/01646_system_restart_replicas_smoke.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS data_01646; +CREATE TABLE data_01646 (x Date, s String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01646/data_01646', 'r') ORDER BY s PARTITION BY x; +SYSTEM RESTART REPLICAS; +DESCRIBE TABLE data_01646; +DROP TABLE data_01646; diff --git a/tests/queries/0_stateless/01647_clickhouse_local_hung.reference b/tests/queries/0_stateless/01647_clickhouse_local_hung.reference new file mode 100644 index 00000000000..8925955b88e --- /dev/null +++ b/tests/queries/0_stateless/01647_clickhouse_local_hung.reference @@ -0,0 +1 @@ + 100 14 diff --git a/tests/queries/0_stateless/01647_clickhouse_local_hung.sh b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh new file mode 100755 index 00000000000..04f32055ab6 --- /dev/null +++ b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +for _ in {1..100}; do echo 'Hello, world!' | ${CLICKHOUSE_LOCAL} --query "SELECT * FROM table" --structure 's String' | wc -c; done | uniq -c diff --git a/tests/queries/0_stateless/01648_mutations_and_escaping.reference b/tests/queries/0_stateless/01648_mutations_and_escaping.reference new file mode 100644 index 00000000000..0d55bed3a35 --- /dev/null +++ b/tests/queries/0_stateless/01648_mutations_and_escaping.reference @@ -0,0 +1,2 @@ +foo +foo diff --git a/tests/queries/0_stateless/01648_mutations_and_escaping.sql b/tests/queries/0_stateless/01648_mutations_and_escaping.sql new file mode 100644 index 00000000000..689da842f16 --- /dev/null +++ b/tests/queries/0_stateless/01648_mutations_and_escaping.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS mutations_and_escaping_1648; + +CREATE TABLE mutations_and_escaping_1648 (d Date, e Enum8('foo'=1, 'bar'=2)) Engine = MergeTree(d, (d), 8192); +INSERT INTO mutations_and_escaping_1648 (d, e) VALUES ('2018-01-01', 'foo'); +INSERT INTO mutations_and_escaping_1648 (d, e) VALUES ('2018-01-02', 'bar'); + +-- slow mutation +ALTER TABLE mutations_and_escaping_1648 UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1, \'bar\' = 2)') WHERE d='2018-01-02' and sleepEachRow(1) = 0; + +-- check that we able to read mutation text after serialization +DETACH TABLE mutations_and_escaping_1648; +ATTACH TABLE mutations_and_escaping_1648; + +ALTER TABLE mutations_and_escaping_1648 UPDATE e = CAST('foo', 'Enum8(\'foo\' = 1, \'bar\' = 2)') WHERE d='2018-01-02' SETTINGS mutations_sync = 1; + +SELECT e FROM mutations_and_escaping_1648 ORDER BY d; + +DROP TABLE mutations_and_escaping_1648; diff --git a/tests/queries/0_stateless/01649_with_alias_key_condition.reference b/tests/queries/0_stateless/01649_with_alias_key_condition.reference new file mode 100644 index 00000000000..5816b4eb49b --- /dev/null +++ b/tests/queries/0_stateless/01649_with_alias_key_condition.reference @@ -0,0 +1 @@ +3 4 diff --git a/tests/queries/0_stateless/01649_with_alias_key_condition.sql b/tests/queries/0_stateless/01649_with_alias_key_condition.sql new file mode 100644 index 00000000000..b813e6ee84f --- /dev/null +++ b/tests/queries/0_stateless/01649_with_alias_key_condition.sql @@ -0,0 +1,11 @@ +drop table if exists alias_key_condition; + +create table alias_key_condition ( i int, j int ) engine MergeTree order by i; + +insert into alias_key_condition values (1, 2), (3, 4); + +set force_primary_key = 1; + +with i as k select * from alias_key_condition where k = 3; + +drop table if exists alias_key_condition; diff --git a/tests/queries/0_stateless/01650_any_null_if.reference b/tests/queries/0_stateless/01650_any_null_if.reference new file mode 100644 index 00000000000..e965047ad7c --- /dev/null +++ b/tests/queries/0_stateless/01650_any_null_if.reference @@ -0,0 +1 @@ +Hello diff --git a/tests/queries/0_stateless/01650_any_null_if.sql b/tests/queries/0_stateless/01650_any_null_if.sql new file mode 100644 index 00000000000..17f57e92032 --- /dev/null +++ b/tests/queries/0_stateless/01650_any_null_if.sql @@ -0,0 +1,6 @@ +SELECT any(nullIf(s, '')) FROM (SELECT arrayJoin(['', 'Hello']) AS s); + +SET optimize_move_functions_out_of_any = 0; +EXPLAIN SYNTAX select any(nullIf('', ''), 'some text'); -- { serverError 42 } +SET optimize_move_functions_out_of_any = 1; +EXPLAIN SYNTAX select any(nullIf('', ''), 'some text'); -- { serverError 42 } diff --git a/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.reference b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.reference new file mode 100644 index 00000000000..ebb0b033d5b --- /dev/null +++ b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.reference @@ -0,0 +1,46 @@ +1 1_0_0_0 +1 1_1_1_0 +2 2_0_0_0 +2 2_1_1_0 +3 3_0_0_0 +3 3_1_1_0 +1_ 1_0_0_0 +1_ 1_1_1_0 +2_ 2_0_0_0 +2_ 2_1_1_0 +3_ 3_0_0_0 +3_ 3_1_1_0 +1 1_0_0_0 +1 1_1_1_0 +2 2_0_0_0 +2 2_1_1_0 +3 3_0_0_0 +3 3_1_1_0 +1_ 1_0_0_0 +1_ 1_1_1_0 +2_ 2_0_0_0 +2_ 2_1_1_0 +3_ 3_0_0_0 +3_ 3_1_1_0 +1 1_0_0_0 +1 1_1_1_0 +2 2_0_0_0 +2 2_1_1_0 +3 3_0_0_0 +1_ 1_0_0_0 +1_ 1_1_1_0 +2_ 2_0_0_0 +2_ 2_1_1_0 +3_ 3_0_0_0 +1 1_0_0_0 +1 1_1_1_0 +2 2_0_0_0 +2 2_1_1_0 +3 3_0_0_0 +3 3_2_2_0 +1_ 1_0_0_0 +1_ 1_1_1_0 +2_ 2_0_0_0 +2_ 2_1_1_0 +3_ 3_0_0_0 +3_ 3_2_2_0 diff --git a/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql new file mode 100644 index 00000000000..50596680618 --- /dev/null +++ b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql @@ -0,0 +1,39 @@ +DROP TABLE IF EXISTS partitioned_table; + +CREATE TABLE partitioned_table ( + key UInt64, + partitioner UInt8, + value String +) +ENGINE ReplicatedMergeTree('/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table', '1') +ORDER BY key +PARTITION BY partitioner; + +SYSTEM STOP MERGES partitioned_table; + +INSERT INTO partitioned_table VALUES (1, 1, 'A'), (2, 2, 'B'), (3, 3, 'C'); +INSERT INTO partitioned_table VALUES (11, 1, 'AA'), (22, 2, 'BB'), (33, 3, 'CC'); + +SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; + +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; + +INSERT INTO partitioned_table VALUES (33, 3, 'CC'); -- must be deduplicated + +SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; + +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; + +ALTER TABLE partitioned_table DROP PART '3_1_1_0'; + +SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; + +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; + +INSERT INTO partitioned_table VALUES (33, 3, 'CC'); -- mustn't be deduplicated + +SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; + +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; + +DROP TABLE IF EXISTS partitioned_table; diff --git a/tests/queries/0_stateless/01650_expressions_merge_bug.reference b/tests/queries/0_stateless/01650_expressions_merge_bug.reference new file mode 100644 index 00000000000..3a48788e754 --- /dev/null +++ b/tests/queries/0_stateless/01650_expressions_merge_bug.reference @@ -0,0 +1 @@ +\N \N diff --git a/tests/queries/0_stateless/01650_expressions_merge_bug.sql b/tests/queries/0_stateless/01650_expressions_merge_bug.sql new file mode 100644 index 00000000000..f28f663b2d4 --- /dev/null +++ b/tests/queries/0_stateless/01650_expressions_merge_bug.sql @@ -0,0 +1,21 @@ + +SELECT + NULL IN + ( + SELECT + 9223372036854775807, + 9223372036854775807 + ), + NULL +FROM +( + SELECT DISTINCT + NULL, + NULL, + NULL IN + ( + SELECT (NULL, '-1') + ), + NULL + FROM numbers(1024) +) diff --git a/tests/queries/0_stateless/01651_group_uniq_array_enum.reference b/tests/queries/0_stateless/01651_group_uniq_array_enum.reference new file mode 100644 index 00000000000..ba4401b6afd --- /dev/null +++ b/tests/queries/0_stateless/01651_group_uniq_array_enum.reference @@ -0,0 +1,3 @@ +['Hello','World','Упячка'] +['Hello','World','World','Упячка','Упячка','Упячка'] +['world','hello'] Array(Enum8(\'world\' = 0, \'hello\' = 1)) ['world','hello'] Array(Enum8(\'world\' = 0, \'hello\' = 1)) diff --git a/tests/queries/0_stateless/01651_group_uniq_array_enum.sql b/tests/queries/0_stateless/01651_group_uniq_array_enum.sql new file mode 100644 index 00000000000..19de51f9681 --- /dev/null +++ b/tests/queries/0_stateless/01651_group_uniq_array_enum.sql @@ -0,0 +1,13 @@ +SELECT arraySort(groupUniqArray(x)) FROM (SELECT CAST(arrayJoin([1, 2, 3, 2, 3, 3]) AS Enum('Hello' = 1, 'World' = 2, 'Упячка' = 3)) AS x); +SELECT arraySort(groupArray(x)) FROM (SELECT CAST(arrayJoin([1, 2, 3, 2, 3, 3]) AS Enum('Hello' = 1, 'World' = 2, 'Упячка' = 3)) AS x); + +SELECT + arraySort(groupUniqArray(val)) AS uniq, + toTypeName(uniq), + arraySort(groupArray(val)) AS arr, + toTypeName(arr) +FROM +( + SELECT CAST(number % 2, 'Enum(\'hello\' = 1, \'world\' = 0)') AS val + FROM numbers(2) +); diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference new file mode 100644 index 00000000000..3da44c57b27 --- /dev/null +++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference @@ -0,0 +1,12 @@ +10000000 +10000000 1274991808 +20000000 +20000000 2549983616 +10000000 +10000000 1274991808 +20000000 +20000000 2549983616 +10000000 +10000000 1274991808 +20000000 +20000000 2549983616 diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql new file mode 100644 index 00000000000..22532529812 --- /dev/null +++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql @@ -0,0 +1,47 @@ +drop table if exists perf_lc_num; + +CREATE TABLE perf_lc_num(　 num UInt8,　 arr Array(LowCardinality(Int64)) default [num]　 ) ENGINE = TinyLog; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +drop table if exists perf_lc_num; + + +CREATE TABLE perf_lc_num(　 num UInt8,　 arr Array(LowCardinality(Int64)) default [num]　 ) ENGINE = Log; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +drop table if exists perf_lc_num; + + +CREATE TABLE perf_lc_num(　 num UInt8,　 arr Array(LowCardinality(Int64)) default [num]　 ) ENGINE = StripeLog; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000); + +select sum(length(arr)) from perf_lc_num; +select sum(length(arr)), sum(num) from perf_lc_num; + +drop table if exists perf_lc_num; + + diff --git a/tests/queries/0_stateless/01652_ignore_and_low_cardinality.reference b/tests/queries/0_stateless/01652_ignore_and_low_cardinality.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01652_ignore_and_low_cardinality.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01652_ignore_and_low_cardinality.sql b/tests/queries/0_stateless/01652_ignore_and_low_cardinality.sql new file mode 100644 index 00000000000..b3d3ad81834 --- /dev/null +++ b/tests/queries/0_stateless/01652_ignore_and_low_cardinality.sql @@ -0,0 +1,6 @@ +set allow_suspicious_low_cardinality_types = 1; +CREATE TABLE lc_null_int8_defnull (val LowCardinality(Nullable(Int8)) DEFAULT NULL) ENGINE = MergeTree order by tuple(); +SELECT ignore(10, ignore(*), ignore(ignore(-2, 1025, *)), NULL, *), * FROM lc_null_int8_defnull AS values; + + +SELECT ignore(toLowCardinality(1), toLowCardinality(2), 3); diff --git a/tests/queries/0_stateless/01652_ttl_old_syntax.reference b/tests/queries/0_stateless/01652_ttl_old_syntax.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01652_ttl_old_syntax.sql b/tests/queries/0_stateless/01652_ttl_old_syntax.sql new file mode 100644 index 00000000000..05c391b85e5 --- /dev/null +++ b/tests/queries/0_stateless/01652_ttl_old_syntax.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS ttl_old_syntax; + +CREATE TABLE ttl_old_syntax (d Date, i Int) ENGINE = MergeTree(d, i, 8291); +ALTER TABLE ttl_old_syntax MODIFY TTL toDate('2020-01-01'); -- { serverError 36 } + +DROP TABLE ttl_old_syntax; diff --git a/tests/queries/0_stateless/01653_tuple_hamming_distance_2.reference b/tests/queries/0_stateless/01653_tuple_hamming_distance_2.reference new file mode 100644 index 00000000000..613ab096c8c --- /dev/null +++ b/tests/queries/0_stateless/01653_tuple_hamming_distance_2.reference @@ -0,0 +1,18 @@ +0 +1 +\N +\N +2 +1 +1 +1 +0 +1 +2 +\N +\N +\N +1 +0 +1 +2 diff --git a/tests/queries/0_stateless/01653_tuple_hamming_distance_2.sql b/tests/queries/0_stateless/01653_tuple_hamming_distance_2.sql new file mode 100644 index 00000000000..81afb1e1201 --- /dev/null +++ b/tests/queries/0_stateless/01653_tuple_hamming_distance_2.sql @@ -0,0 +1,23 @@ +SELECT tupleHammingDistance(tuple(1), tuple(1)); +SELECT tupleHammingDistance(tuple(1), tuple(2)); +SELECT tupleHammingDistance(tuple(1), tuple(Null)); +SELECT tupleHammingDistance(tuple(Null), tuple(Null)); +SELECT tupleHammingDistance((1, 2), (3, 4)); +SELECT tupleHammingDistance((1, 2), (1, 4)); +SELECT tupleHammingDistance(materialize((1, 2)), (1, 4)); +SELECT tupleHammingDistance(materialize((1, 2)),materialize ((1, 4))); +SELECT tupleHammingDistance((1, 2), (1, 2)); +SELECT tupleHammingDistance((1, 2), (1, 257)); +SELECT tupleHammingDistance((1, 2, 3), (1, 257, 65537)); +SELECT tupleHammingDistance((1, 2), (1, Null)); +SELECT tupleHammingDistance((1, Null), (1, Null)); +SELECT tupleHammingDistance((Null, Null), (Null, Null)); +SELECT tupleHammingDistance(('abc', 2), ('abc', 257)); +SELECT tupleHammingDistance(('abc', (1, 2)), ('abc', (1, 2))); +SELECT tupleHammingDistance(('abc', (1, 2)), ('def', (1, 2))); +SELECT tupleHammingDistance(('abc', (1, 2)), ('def', (1, 3))); + + +SELECT tupleHammingDistance(tuple(1), tuple(1, 1)); --{serverError 43} +SELECT tupleHammingDistance(tuple(1), tuple('a')); --{serverError 386} +SELECT tupleHammingDistance((1, 3), (3, 'a')); --{serverError 386} diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6c6636b923d..116d0beb80b 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -182,3 +182,5 @@ 01601_custom_tld 01636_nullable_fuzz2 01639_distributed_sync_insert_zero_rows +01644_distributed_async_insert_fsync_smoke +01552_impl_aggfunc_cloneresize diff --git a/tests/queries/1_stateful/00160_decode_xml_component.reference b/tests/queries/1_stateful/00160_decode_xml_component.reference new file mode 100644 index 00000000000..96234a2262e --- /dev/null +++ b/tests/queries/1_stateful/00160_decode_xml_component.reference @@ -0,0 +1 @@ +10601114492838968014 diff --git a/tests/queries/1_stateful/00160_decode_xml_component.sql b/tests/queries/1_stateful/00160_decode_xml_component.sql new file mode 100644 index 00000000000..0194eb330a9 --- /dev/null +++ b/tests/queries/1_stateful/00160_decode_xml_component.sql @@ -0,0 +1 @@ +SELECT sum(DISTINCT sipHash64(decodeXMLComponent(Title) AS decoded)) FROM test.hits WHERE Title != decoded; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index be1ee954136..cfbac463932 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -15,7 +15,8 @@ "01526_max_untracked_memory", /// requires TraceCollector, does not available under sanitizers "01474_executable_dictionary", /// informational stderr from sanitizer at start "functions_bad_arguments", /// Too long for TSan - "01603_read_with_backoff_bug" /// Too long for TSan + "01603_read_with_backoff_bug", /// Too long for TSan + "01646_system_restart_replicas_smoke" /// RESTART REPLICAS can acquire too much locks, while only 64 is possible from one thread under TSan ], "address-sanitizer": [ "00877", @@ -103,14 +104,37 @@ "01482_move_to_prewhere_and_cast" /// bug, shoud be fixed ], "antlr": [ + "00186_very_long_arrays", + "00233_position_function_sql_comparibilty", + "00417_kill_query", + "00534_functions_bad_arguments12", + "00534_functions_bad_arguments2", + "00534_functions_bad_arguments4", + "00534_functions_bad_arguments9", + "00564_temporary_table_management", + "00626_replace_partition_from_table_zookeeper", + "00652_replicated_mutations_zookeeper", + "00687_top_and_offset", + "00746_sql_fuzzy", "00763_create_query_as_table_engine_bug", "00765_sql_compatibility_aliases", + "00825_protobuf_format_input", "00826_cross_to_inner_join", "00834_not_between", + "00909_kill_not_initialized_query", + "00938_template_input_format", "00939_limit_by_offset", + "00943_materialize_index", + "00944_clear_index_in_partition", + "00952_input_function", + "00953_constraints_operations", + "00954_client_prepared_statements", + "00956_sensitive_data_masking", "00969_columns_clause", + "00975_indices_mutation_replicated_zookeeper", "00975_values_list", "00976_system_stop_ttl_merges", + "00977_int_div", "00978_table_function_values_alias", "00980_merge_alter_settings", "00980_zookeeper_merge_tree_alter_settings", @@ -121,16 +145,22 @@ "01001_enums_in_in_section", "01011_group_uniq_array_memsan", "01011_test_create_as_skip_indices", + "01014_format_custom_separated", "01015_attach_part", "01015_database_bad_tables", "01017_uniqCombined_memory_usage", + "01019_alter_materialized_view_atomic", + "01019_alter_materialized_view_consistent", "01019_alter_materialized_view_query", "01021_tuple_parser", "01025_array_compact_generic", + "01030_limit_by_with_ties_error", "01033_quota_dcl", "01034_with_fill_and_push_down_predicate", + "01035_avg_weighted_long", "01039_row_policy_dcl", "01039_test_setting_parse", + "01042_system_reload_dictionary_reloads_completely", "01045_dictionaries_restrictions", "01048_exists_query", "01055_compact_parts_1", @@ -142,11 +172,15 @@ "01073_show_tables_not_like", "01074_partial_revokes", "01075_allowed_client_hosts", + "01085_regexp_input_format", + "01086_regexp_input_format_skip_unmatched", "01089_alter_settings_old_format", "01095_tpch_like_smoke", + "01107_atomic_db_detach_attach", "01109_exchange_tables", "01109_sc0rp10_string_hash_map_zero_bytes", "01110_dictionary_layout_without_arguments", + "01114_database_atomic", "01114_materialize_clear_index_compact_parts", "01115_join_with_dictionary", "01117_comma_and_others_join_mix", @@ -156,9 +190,13 @@ "01144_multiword_data_types", "01145_with_fill_const", "01149_zookeeper_mutation_stuck_after_replace_partition", + "01150_ddl_guard_rwr", + "01185_create_or_replace_table", + "01187_set_profile_as_setting", "01188_attach_table_from_path", "01190_full_attach_syntax", "01191_rename_dictionary", + "01192_rename_database_zookeeper", "01210_drop_view", "01213_alter_rename_column", "01232_untuple", @@ -173,19 +211,23 @@ "01272_offset_without_limit", "01275_parallel_mv", "01277_alter_rename_column_constraint_zookeeper", + "01278_min_insert_block_size_rows_for_materialized_views", "01280_min_map_max_map", "01280_null_in", "01280_ttl_where_group_by_negative", + "01280_ttl_where_group_by", "01280_unicode_whitespaces_lexer", "01292_create_user", "01293_create_role", "01293_pretty_max_value_width", "01293_show_settings", "01294_create_settings_profile", + "01294_lazy_database_concurrent_recreate_reattach_and_show_tables", "01295_create_row_policy", "01296_create_row_policy_in_current_database", "01297_create_quota", "01308_row_policy_and_trivial_count_query", + "01317_no_password_in_command_line", "01318_map_add_map_subtract", "01322_any_input_optimize", "01324_if_transform_strings_to_enum", @@ -197,7 +239,10 @@ "01378_alter_rename_with_ttl_zookeeper", "01379_with_fill_several_columns", "01397_in_bad_arguments", + "01412_mod_float", "01415_table_function_view", + "01417_freeze_partition_verbose_zookeeper", + "01417_freeze_partition_verbose", "01419_merge_tree_settings_sanity_check", "01430_modify_sample_by_zookeeper", "01447_json_strings", @@ -210,19 +255,25 @@ "01463_test_alter_live_view_refresh", "01465_ttl_recompression", "01470_columns_transformers", + "01470_columns_transformers2", "01470_explain", "01470_show_databases_like", "01470_test_insert_select_asterisk", "01491_nested_multiline_comments", "01493_table_function_null", - "01495_subqueries_in_with_statement", "01495_subqueries_in_with_statement_2", "01495_subqueries_in_with_statement_3", + "01495_subqueries_in_with_statement", + "01501_clickhouse_client_INSERT_exception", "01504_compression_multiple_streams", "01508_explain_header", + "01508_partition_pruning", + "01509_check_parallel_quorum_inserts", + "01509_parallel_quorum_and_merge", "01515_mv_and_array_join_optimisation_bag", "01516_create_table_primary_key", "01517_drop_mv_with_inner_table", + "01523_client_local_queries_file_parameter", "01523_interval_operator_support_string_literal", "01525_select_with_offset_fetch_clause", "01526_client_start_and_exit", @@ -230,6 +281,7 @@ "01530_drop_database_atomic_sync", "01532_execute_merges_on_single_replica", "01532_primary_key_without_order_by_zookeeper", + "01541_max_memory_usage_for_user", "01551_mergetree_read_in_order_spread", "01552_dict_fixedstring", "01554_bloom_filter_index_big_integer_uuid", @@ -238,14 +290,26 @@ "01562_optimize_monotonous_functions_in_order_by", "01581_deduplicate_by_columns_local", "01581_deduplicate_by_columns_replicated", + "01582_distinct_optimization", "01590_countSubstrings", "01593_insert_settings", + "01594_too_low_memory_limits", "01596_setting_limit_offset", + "01600_log_queries_with_extensive_info", + "01600_quota_by_forwarded_ip", "01601_detach_permanently", + "01602_show_create_view", "01603_read_with_backoff_bug", "01604_explain_ast_of_nonselect_query", + "01605_drop_settings_profile_while_assigned", "01605_skip_idx_compact_parts", - "01606_git_import" + "01606_git_import", + "01606_merge_from_wide_to_compact", + "01614_with_fill_with_limit", + "01632_max_partitions_to_read", + "01638_div_mod_ambiguities", + "01642_if_nullable_regression", + "01643_system_suspend" ], "parallel": [ diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 8229947adf7..b47a85e3e92 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -874,7 +874,7 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, "enable_tls": "no", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" + "verification_cooldown": "300" }} self.context.ldap_node = self.context.cluster.node(server) @@ -898,11 +898,6 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, with Then("when I try to login again with the server offline it should work"): login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with And("when I sleep for 2 seconds and try to log in, it should fail"): - time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - finally: with Finally("I start the ldap server back up"): self.context.ldap_node.start() @@ -957,7 +952,7 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False): RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance("1.0") ) def verification_cooldown_performance(self, server, rbac=False, iterations=5000): - """Check that login performance is better when the verification cooldown + """Check login performance when the verification cooldown parameter is set to a positive value when comparing to the case when the verification cooldown parameter is turned off. """ @@ -973,10 +968,7 @@ def verification_cooldown_performance(self, server, rbac=False, iterations=5000) no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) - with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): - assert no_vcd_time > vcd_time, error() - - with And("Log the performance improvement as a percentage."): + with Then("Log the performance improvement as a percentage"): metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) @TestOutline diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 0e9a821cae0..ef2008aa173 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -14,10 +14,10 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path, "stress": stress, "parallel": parallel} - Feature(test=load("example.regression", "regression"))(**args) - Feature(test=load("ldap.regression", "regression"))(**args) - Feature(test=load("rbac.regression", "regression"))(**args) - Feature(test=load("aes_encryption.regression", "regression"))(**args) +# Feature(test=load("example.regression", "regression"))(**args) +# Feature(test=load("ldap.regression", "regression"))(**args) +# Feature(test=load("rbac.regression", "regression"))(**args) +# Feature(test=load("aes_encryption.regression", "regression"))(**args) if main(): regression() diff --git a/utils/github/backport.py b/utils/github/backport.py index c51c84e6680..576e3b069c2 100644 --- a/utils/github/backport.py +++ b/utils/github/backport.py @@ -32,8 +32,8 @@ class Backport: branches.append(pull_request['headRefName']) return branches - def execute(self, repo, until_commit, number, run_cherrypick, find_lts=False): - repo = LocalRepo(repo, 'origin', self.default_branch_name) + def execute(self, repo, upstream, until_commit, number, run_cherrypick, find_lts=False): + repo = LocalRepo(repo, upstream, self.default_branch_name) all_branches = repo.get_release_branches() # [(branch_name, base_commit)] last_branches = set([branch[0] for branch in all_branches[-number:]]) @@ -42,7 +42,7 @@ class Backport: branches = [] # iterate over all branches to preserve their precedence. for branch in all_branches: - if branch in last_branches or branch in lts_branches: + if branch[0] in last_branches or branch[0] in lts_branches: branches.append(branch) if not branches: @@ -119,6 +119,7 @@ if __name__ == "__main__": parser.add_argument('--lts', action='store_true', help='consider branches with LTS') parser.add_argument('--dry-run', action='store_true', help='do not create or merge any PRs', default=False) parser.add_argument('--verbose', '-v', action='store_true', help='more verbose output', default=False) + parser.add_argument('--upstream', '-u', type=str, help='remote name of upstream in repository', default='origin') args = parser.parse_args() if args.verbose: @@ -128,4 +129,4 @@ if __name__ == "__main__": cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run) bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core') - bp.execute(args.repo, args.til, args.number, cherrypick_run, args.lts) + bp.execute(args.repo, args.upstream, args.til, args.number, cherrypick_run, args.lts) diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index c4dd177b931..896fa7f08a0 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -48,11 +48,12 @@ function gen_revision_author { VERSION_REVISION=$(($VERSION_REVISION + 1)) VERSION_MAJOR=$(($VERSION_MAJOR + 1)) VERSION_MINOR=1 - VERSION_PATCH=0 + # Version cannot be zero, otherwise is breaks CMake + VERSION_PATCH=1 elif [ "$TYPE" == "minor" ] || [ "$TYPE" == "" ]; then VERSION_REVISION=$(($VERSION_REVISION + 1)) VERSION_MINOR=$(($VERSION_MINOR + 1)) - VERSION_PATCH=0 + VERSION_PATCH=1 elif [ "$TYPE" == "patch" ] || [ "$TYPE" == "bugfix" ]; then # VERSION_REVISION not incremented in new scheme. if [ "$VERSION_MAJOR" -eq "1" ] && [ "$VERSION_MINOR" -eq "1" ]; then @@ -126,15 +127,6 @@ function gen_revision_author { fi fi - - # Reset testing branch to current commit. - git checkout testing - git reset --hard "$tag" - - if [ -z $NO_PUSH ]; then - git push - fi - else get_version echo reusing old version $VERSION_STRING diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html index 7fbb3b6d505..6e6664af55a 100644 --- a/website/benchmark/hardware/index.html +++ b/website/benchmark/hardware/index.html @@ -72,7 +72,8 @@ Results for Android phones for "cold cache" are done without cache flushing, so Results for Digital Ocean are from Zimin Aleksey.
Results for 2x EPYC 7642 w/ 512 GB RAM (192 Cores) + 12X 1TB SSD (RAID6) are from Yiğit Konur and Metehan Çetinkaya of seo.do.
Results for Raspberry Pi and Digital Ocean CPU-optimized are from Fritz Wijaya.
-Results for Digitalocean (Storage-intesinve VMs) + (CPU/GP) are from Yiğit Konur and Metehan Çetinkaya of seo.do. +Results for Digitalocean (Storage-intesinve VMs) + (CPU/GP) are from Yiğit Konur and Metehan Çetinkaya of seo.do.
+Results for 2x AMD EPYC 7F72 3.2 Ghz (Total 96 Cores, IBM Cloud's Bare Metal Service) from Yiğit Konur and Metehan Çetinkaya of seo.do.

diff --git a/website/benchmark/hardware/results/ibm_cloud_baremetal.json b/website/benchmark/hardware/results/ibm_cloud_baremetal.json new file mode 100644 index 00000000000..46d319c6c7b --- /dev/null +++ b/website/benchmark/hardware/results/ibm_cloud_baremetal.json @@ -0,0 +1,210 @@ +[ + { + "system": "2x AMD EPYC 3.2 GHz, Micron 5100 MAX 960 GB", + "system_full": "2x AMD EPYC 7F72 3.2 Ghz - Total 96 Cores - IBM Cloud's Bare Metal Service - 512 GB RAM - Micron 5100 MAX 960 GB SSD", + "time": "2021-01-13 00:00:00", + "kind": "server", + "result": + [ +[0.002, 0.002, 0.002], +[0.027, 0.013, 0.012], +[0.116, 0.017, 0.016], +[0.509, 0.020, 0.018], +[0.535, 0.071, 0.069], +[0.155, 0.111, 0.108], +[0.028, 0.016, 0.015], +[0.016, 0.013, 0.013], +[0.700, 0.124, 0.123], +[0.832, 0.144, 0.149], +[0.567, 0.074, 0.075], +[0.571, 0.078, 0.076], +[0.241, 0.176, 0.171], +[0.728, 0.223, 0.223], +[0.333, 0.227, 0.221], +[0.634, 0.235, 0.241], +[0.983, 0.577, 0.566], +[0.692, 0.340, 0.319], +[2.222, 1.081, 1.117], +[0.500, 0.034, 0.033], +[5.333, 0.263, 0.193], +[5.484, 0.221, 0.200], +[9.963, 0.623, 0.614], +[11.277, 0.331, 0.299], +[0.940, 0.075, 0.070], +[0.124, 0.060, 0.056], +[0.942, 0.072, 0.066], +[5.445, 0.307, 0.269], +[4.305, 0.313, 0.301], +[0.613, 0.558, 0.573], +[0.732, 0.190, 0.186], +[2.433, 0.301, 0.293], +[3.473, 1.907, 1.782], +[6.202, 1.048, 0.976], +[6.193, 1.013, 0.991], +[0.499, 0.304, 0.300], +[0.192, 0.134, 0.129], +[0.072, 0.054, 0.055], +[0.030, 0.029, 0.029], +[0.295, 0.263, 0.300], +[0.052, 0.017, 0.022], +[0.021, 0.016, 0.011], +[0.006, 0.005, 0.008] + ] + }, + { + "system": "2x AMD EPYC 3.2 GHz, Intel P4610 NVM SSD", + "system_full": "2x AMD EPYC 7F72 3.2 Ghz - Total 96 Cores - IBM Cloud's Bare Metal Service - 512 GB RAM - 1.8 TB Intel P4610 NVM SSD", + "time": "2021-01-13 00:00:00", + "kind": "server", + "result": + [ +[0.002, 0.001, 0.001], +[0.023, 0.013, 0.013], +[0.036, 0.018, 0.018], +[0.078, 0.021, 0.021], +[0.113, 0.070, 0.070], +[0.171, 0.106, 0.103], +[0.026, 0.017, 0.017], +[0.016, 0.015, 0.015], +[0.174, 0.119, 0.120], +[0.190, 0.135, 0.142], +[0.128, 0.075, 0.087], +[0.132, 0.078, 0.077], +[0.254, 0.172, 0.169], +[0.303, 0.215, 0.210], +[0.275, 0.221, 0.218], +[0.251, 0.223, 0.220], +[0.787, 0.550, 0.536], +[0.373, 0.298, 0.307], +[1.149, 1.050, 1.061], +[0.081, 0.025, 0.019], +[0.820, 0.193, 0.163], +[0.923, 0.204, 0.179], +[1.723, 0.639, 0.625], +[2.089, 0.300, 0.282], +[0.239, 0.068, 0.064], +[0.126, 0.063, 0.055], +[0.237, 0.073, 0.063], +[0.829, 0.267, 0.254], +[0.721, 0.284, 0.270], +[0.463, 0.450, 0.453], +[0.281, 0.177, 0.172], +[0.573, 0.276, 0.272], +[2.388, 1.731, 1.699], +[1.254, 0.927, 0.992], +[1.280, 0.975, 0.963], +[0.315, 0.285, 0.285], +[0.188, 0.126, 0.136], +[0.070, 0.049, 0.051], +[0.031, 0.029, 0.030], +[0.326, 0.304, 0.337], +[0.034, 0.021, 0.015], +[0.017, 0.025, 0.020], +[0.010, 0.005, 0.005] + ] + }, + { + "system": "2x AMD EPYC 7642, 7.2 TB NVM", + "system_full": "2x AMD EPYC 7642 - 512 GB RAM - 7.2 TB NVM", + "time": "2021-01-13 00:00:00", + "kind": "server", + "result": + [ +[0.003, 0.002, 0.002], +[0.056, 0.057, 0.041], +[0.081, 0.051, 0.059], +[0.080, 0.039, 0.034], +[0.149, 0.109, 0.105], +[0.192, 0.124, 0.125], +[0.031, 0.026, 0.027], +[0.029, 0.022, 0.025], +[0.181, 0.122, 0.126], +[0.206, 0.136, 0.136], +[0.133, 0.079, 0.081], +[0.140, 0.079, 0.080], +[0.207, 0.150, 0.148], +[0.281, 0.182, 0.180], +[0.228, 0.150, 0.149], +[0.183, 0.151, 0.151], +[0.431, 0.348, 0.349], +[0.316, 0.237, 0.244], +[1.091, 0.794, 0.787], +[0.081, 0.037, 0.024], +[0.833, 0.183, 0.152], +[0.950, 0.203, 0.164], +[1.735, 0.503, 0.491], +[1.656, 0.290, 0.270], +[0.251, 0.063, 0.057], +[0.134, 0.050, 0.056], +[0.250, 0.061, 0.063], +[0.844, 0.244, 0.238], +[0.735, 0.224, 0.225], +[0.532, 0.505, 0.512], +[0.259, 0.128, 0.134], +[0.523, 0.215, 0.199], +[1.380, 1.031, 1.029], +[1.122, 0.665, 0.638], +[1.115, 0.651, 0.646], +[0.261, 0.222, 0.227], +[0.151, 0.147, 0.134], +[0.063, 0.060, 0.063], +[0.035, 0.038, 0.035], +[0.372, 0.310, 0.351], +[0.031, 0.025, 0.026], +[0.019, 0.016, 0.016], +[0.008, 0.007, 0.012] + ] + }, + { + "system": "2x AMD EPYC 3.2 GHz, 4x1.8 TB Intel P4610 NVM SSD", + "system_full": "2x AMD EPYC 7F72 3.2 Ghz - Total 96 Cores - IBM Cloud's Bare Metal Service - 512 GB RAM - RAID 0 4X 1.8 TB Intel P4610 NVM SSD", + "time": "2021-01-13 00:00:00", + "kind": "server", + "result": + [ +[0.002, 0.002, 0.001], +[0.052, 0.037, 0.043], +[0.045, 0.047, 0.034], +[0.074, 0.024, 0.025], +[0.102, 0.073, 0.065], +[0.135, 0.106, 0.101], +[0.021, 0.017, 0.017], +[0.020, 0.014, 0.014], +[0.140, 0.115, 0.116], +[0.156, 0.132, 0.131], +[0.113, 0.074, 0.071], +[0.106, 0.075, 0.080], +[0.221, 0.194, 0.192], +[0.270, 0.247, 0.240], +[0.253, 0.236, 0.237], +[0.265, 0.247, 0.246], +[0.748, 0.576, 0.583], +[0.349, 0.318, 0.335], +[1.101, 1.096, 1.103], +[0.083, 0.024, 0.025], +[0.613, 0.209, 0.225], +[0.606, 0.208, 0.191], +[0.803, 0.666, 0.648], +[0.647, 0.324, 0.304], +[0.137, 0.069, 0.068], +[0.118, 0.055, 0.052], +[0.138, 0.064, 0.064], +[0.597, 0.270, 0.258], +[0.560, 0.302, 0.293], +[0.607, 0.629, 0.620], +[0.212, 0.176, 0.181], +[0.393, 0.290, 0.281], +[2.137, 1.832, 1.779], +[1.125, 0.990, 1.007], +[1.193, 1.016, 0.985], +[0.309, 0.294, 0.327], +[0.149, 0.135, 0.150], +[0.058, 0.057, 0.055], +[0.037, 0.037, 0.029], +[0.346, 0.315, 0.323], +[0.020, 0.024, 0.021], +[0.020, 0.019, 0.015], +[0.009, 0.009, 0.005] + ] + } +] diff --git a/website/templates/index/performance.html b/website/templates/index/performance.html index 75e2a43f902..cea327001bf 100644 --- a/website/templates/index/performance.html +++ b/website/templates/index/performance.html @@ -3,8 +3,7 @@

ClickHouse works 100-1000x faster than traditional approaches

-

ClickHouse's performance exceeds comparable column-oriented database management systems currently available - on the market. It processes hundreds of millions to more than a billion rows and tens of gigabytes of data per single server per second.

+

ClickHouse's performance exceeds comparable column-oriented database management systems that are available on the market. It processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second.

Detailed comparison
@@ -37,7 +36,7 @@
  • ClickHouse and Vertica comparison
    zhtsh (machine translation from Chinese)
  • ClickHouse, Redshift and 2.5 Billion Rows of Time Series Data
    Brandon Harris
    • + rel="external nofollow noreferrer" target="_blank" class="text-reset">ClickHouse, Redshift and 2.5 Billion Rows of Time Series Data
    Brandon Harris