Merge branch 'master' of https://github.com/yandex/ClickHouse into distributed_insert_select

.clang-tidy

@@ -1,2 +1,205 @@
-Checks: '-*,google-readability-avoid-underscore-in-googletest-name,misc-throw-by-value-catch-by-reference,misc-misplaced-const,misc-unconventional-assign-operator,modernize-avoid-bind,modernize-loop-convert,modernize-make-shared,modernize-make-unique,modernize-raw-string-literal,modernize-redundant-void-arg,modernize-replace-auto-ptr,modernize-replace-random-shuffle,modernize-use-bool-literals,modernize-use-nullptr,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-avoid-const-params-in-decls,readability-const-return-type,readability-container-size-empty,readability-convert-member-functions-to-static,readability-delete-null-pointer,readability-deleted-default,readability-make-member-function-const,readability-misplaced-array-index,readability-non-const-parameter,readability-qualified-auto,readability-redundant-access-specifiers,readability-redundant-control-flow,readability-redundant-function-ptr-dereference,readability-redundant-smartptr-get,readability-redundant-string-cstr,readability-redundant-string-init,readability-static-definition-in-anonymous-namespace,readability-string-compare,readability-uniqueptr-delete-release,modernize-use-equals-default,modernize-use-equals-delete,bugprone-undelegated-constructor,readability-redundant-member-init,readability-simplify-subscript-expr,readability-simplify-boolean-expr,readability-inconsistent-declaration-parameter-name'
+Checks: '-*,
+    misc-throw-by-value-catch-by-reference,
+    misc-misplaced-const,
+    misc-unconventional-assign-operator,
+    misc-redundant-expression,
+    misc-static-assert,
+    misc-unconventional-assign-operator,
+    misc-uniqueptr-reset-release,
+    misc-unused-alias-decls,
+    misc-unused-parameters,
+    misc-unused-using-decls,
+    
+    modernize-avoid-bind,
+    modernize-loop-convert,
+    modernize-make-shared,
+    modernize-make-unique,
+    modernize-raw-string-literal,
+    modernize-redundant-void-arg,
+    modernize-replace-auto-ptr,
+    modernize-replace-random-shuffle,
+    modernize-use-bool-literals,
+    modernize-use-nullptr,
+    modernize-use-using,
+    modernize-use-equals-default,
+    modernize-use-equals-delete,
+
+    performance-faster-string-find,
+    performance-for-range-copy,
+    performance-implicit-conversion-in-loop,
+    performance-inefficient-algorithm,
+    performance-inefficient-vector-operation,
+    performance-move-constructor-init,
+    performance-no-automatic-move,
+    performance-trivially-destructible,
+    performance-unnecessary-copy-initialization,
+    
+    readability-avoid-const-params-in-decls,
+    readability-const-return-type,
+    readability-container-size-empty,
+    readability-convert-member-functions-to-static,
+    readability-delete-null-pointer,
+    readability-deleted-default,
+    readability-make-member-function-const,
+    readability-misplaced-array-index,
+    readability-non-const-parameter,
+    readability-qualified-auto,
+    readability-redundant-access-specifiers,
+    readability-redundant-control-flow,
+    readability-redundant-function-ptr-dereference,
+    readability-redundant-smartptr-get,
+    readability-redundant-string-cstr,
+    readability-redundant-string-init,
+    readability-static-definition-in-anonymous-namespace,
+    readability-string-compare,
+    readability-uniqueptr-delete-release,
+    readability-redundant-member-init,
+    readability-simplify-subscript-expr,
+    readability-simplify-boolean-expr,
+    readability-inconsistent-declaration-parameter-name,
+    readability-identifier-naming,
+    
+    bugprone-undelegated-constructor,
+    bugprone-argument-comment,
+    bugprone-bad-signal-to-kill-thread,
+    bugprone-bool-pointer-implicit-conversion,
+    bugprone-copy-constructor-init,
+    bugprone-dangling-handle,
+    bugprone-forward-declaration-namespace,
+    bugprone-fold-init-type,
+    bugprone-inaccurate-erase,
+    bugprone-incorrect-roundings,
+    bugprone-infinite-loop,
+    bugprone-integer-division,
+    bugprone-macro-parentheses,
+    bugprone-macro-repeated-side-effects,
+    bugprone-misplaced-operator-in-strlen-in-alloc,
+    bugprone-misplaced-pointer-artithmetic-in-alloc,
+    bugprone-misplaced-widening-cast,
+    bugprone-move-forwarding-reference,
+    bugprone-multiple-statement-macro,
+    bugprone-parent-virtual-call,
+    bugprone-posix-return,
+    bugprone-reserved-identifier,
+    bugprone-signed-char-misuse,
+    bugprone-sizeof-container,
+    bugprone-sizeof-expression,
+    bugprone-string-constructor,
+    bugprone-string-integer-assignment,
+    bugprone-string-literal-with-embedded-nul,
+    bugprone-suspicious-enum-usage,
+    bugprone-suspicious-include,
+    bugprone-suspicious-memset-usage,
+    bugprone-suspicious-missing-comma,
+    bugprone-suspicious-string-compare,
+    bugprone-swapped-arguments,
+    bugprone-terminating-continue,
+    bugprone-throw-keyword-missing,
+    bugprone-too-small-loop-variable,
+    bugprone-undefined-memory-manipulation,
+    bugprone-unhandled-self-assignment,
+    bugprone-unused-raii,
+    bugprone-unused-return-value,
+    bugprone-use-after-move,
+    bugprone-virtual-near-miss,
+    
+    cert-dcl21-cpp,
+    cert-dcl50-cpp,
+    cert-env33-c,
+    cert-err34-c,
+    cert-err52-cpp,
+    cert-flp30-c,
+    cert-mem57-cpp,
+    cert-msc50-cpp,
+    cert-oop58-cpp,
+    
+    google-build-explicit-make-pair,
+    google-build-namespaces,
+    google-default-arguments,
+    google-explicit-constructor,
+    google-readability-casting,
+    google-readability-avoid-underscore-in-googletest-name,
+    google-runtime-int,
+    google-runtime-operator,
+    
+    hicpp-exception-baseclass,
+    
+    clang-analyzer-core.CallAndMessage,
+    clang-analyzer-core.DivideZero,
+    clang-analyzer-core.NonNullParamChecker,
+    clang-analyzer-core.NullDereference,
+    clang-analyzer-core.StackAddressEscape,
+    clang-analyzer-core.UndefinedBinaryOperatorResult,
+    clang-analyzer-core.VLASize,
+    clang-analyzer-core.uninitialized.ArraySubscript,
+    clang-analyzer-core.uninitialized.Assign,
+    clang-analyzer-core.uninitialized.Branch,
+    clang-analyzer-core.uninitialized.CapturedBlockVariable,
+    clang-analyzer-core.uninitialized.UndefReturn,
+    clang-analyzer-cplusplus.InnerPointer,
+    clang-analyzer-cplusplus.NewDelete,
+    clang-analyzer-cplusplus.NewDeleteLeaks,
+    clang-analyzer-cplusplus.PlacementNewChecker,
+    clang-analyzer-cplusplus.SelfAssignment,
+    clang-analyzer-deadcode.DeadStores,
+    clang-analyzer-optin.cplusplus.VirtualCall,
+    clang-analyzer-security.insecureAPI.UncheckedReturn,
+    clang-analyzer-security.insecureAPI.bcmp,
+    clang-analyzer-security.insecureAPI.bcopy,
+    clang-analyzer-security.insecureAPI.bzero,
+    clang-analyzer-security.insecureAPI.getpw,
+    clang-analyzer-security.insecureAPI.gets,
+    clang-analyzer-security.insecureAPI.mkstemp,
+    clang-analyzer-security.insecureAPI.mktemp,
+    clang-analyzer-security.insecureAPI.rand,
+    clang-analyzer-security.insecureAPI.strcpy,
+    clang-analyzer-unix.Malloc,
+    clang-analyzer-unix.MallocSizeof,
+    clang-analyzer-unix.MismatchedDeallocator,
+    clang-analyzer-unix.Vfork,
+    clang-analyzer-unix.cstring.BadSizeArg,
+    clang-analyzer-unix.cstring.NullArg,
+
+    boost-use-to-string,
+'
 WarningsAsErrors: '*'
+
+CheckOptions:
+  - key: readability-identifier-naming.ClassCase
+    value: CamelCase
+  - key: readability-identifier-naming.EnumCase
+    value: CamelCase
+  - key: readability-identifier-naming.LocalVariableCase
+    value: lower_case
+  - key: readability-identifier-naming.StaticConstantCase
+    value: aNy_CasE
+  - key: readability-identifier-naming.MemberCase
+    value: lower_case
+  - key: readability-identifier-naming.PrivateMemberPrefix
+    value: ''
+  - key: readability-identifier-naming.ProtectedMemberPrefix
+    value: ''
+  - key: readability-identifier-naming.PublicMemberCase
+    value: lower_case
+  - key: readability-identifier-naming.MethodCase
+    value: camelBack
+  - key: readability-identifier-naming.PrivateMethodPrefix
+    value: ''
+  - key: readability-identifier-naming.ProtectedMethodPrefix
+    value: ''
+  - key: readability-identifier-naming.ParameterPackCase
+    value: lower_case
+  - key: readability-identifier-naming.StructCase
+    value: CamelCase
+  - key: readability-identifier-naming.TemplateTemplateParameterCase
+    value: CamelCase
+  - key: readability-identifier-naming.TemplateUsingCase
+    value: lower_case
+  - key: readability-identifier-naming.TypeTemplateParameterCase
+    value: CamelCase
+  - key: readability-identifier-naming.TypedefCase
+    value: CamelCase
+  - key: readability-identifier-naming.UnionCase
+    value: CamelCase
+  - key: readability-identifier-naming.UsingCase
+    value: CamelCase

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 docs/* @ClickHouse/docs
 docs/zh/* @ClickHouse/docs-zh
+website/* @ClickHouse/docs

.gitignore

@@ -16,7 +16,10 @@
 /docs/publish
 /docs/edit
 /docs/website
+/docs/venv/
 /docs/tools/venv/
+/docs/tools/translate/venv/
+/docs/tools/translate/output.md
 /docs/en/single.md
 /docs/ru/single.md
 /docs/zh/single.md

CMakeLists.txt

@@ -186,11 +186,13 @@ if (COMPILER_GCC OR COMPILER_CLANG)
 endif ()
 
 option(WITH_COVERAGE "Build with coverage." 0)
+
 if(WITH_COVERAGE AND COMPILER_CLANG)
    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
    # If we want to disable coverage for specific translation units
    set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping")
 endif()
+
 if(WITH_COVERAGE AND COMPILER_GCC)
    set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-arcs -ftest-coverage")
    set(COVERAGE_OPTION "-lgcov")
@@ -352,7 +354,6 @@ endif ()
 
 # Need to process before "contrib" dir:
 include (cmake/find/jemalloc.cmake)
-include (cmake/find/cctz.cmake)
 include (cmake/find/mysqlclient.cmake)
 
 # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.

README.md

@@ -8,6 +8,7 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
 * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
+* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
 * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.

base/common/CMakeLists.txt

@@ -6,6 +6,7 @@ set (SRCS
     DateLUT.cpp
     DateLUTImpl.cpp
     demangle.cpp
+    getFQDNOrHostName.cpp
     getMemoryAmount.cpp
     getThreadId.cpp
     JSON.cpp
@@ -20,15 +21,9 @@ set (SRCS
 )
 
 if (ENABLE_REPLXX)
-    set (SRCS ${SRCS}
+    list (APPEND SRCS ReplxxLineReader.cpp)
-        ReplxxLineReader.cpp
-        ReplxxLineReader.h
-    )
 elseif (ENABLE_READLINE)
-    set (SRCS ${SRCS}
+    list (APPEND SRCS ReadlineLineReader.cpp)
-        ReadlineLineReader.cpp
-        ReadlineLineReader.h
-    )
 endif ()
 
 if (USE_DEBUG_HELPERS)
@@ -38,12 +33,14 @@ endif ()
 
 add_library (common ${SRCS})
 
-target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
+if (WITH_COVERAGE)
-
+    target_compile_definitions(common PUBLIC WITH_COVERAGE=1)
-if(CCTZ_INCLUDE_DIR)
+else ()
-    target_include_directories(common BEFORE PRIVATE ${CCTZ_INCLUDE_DIR})
+    target_compile_definitions(common PUBLIC WITH_COVERAGE=0)
 endif ()
 
+target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
+
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
     target_include_directories (common SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()
@@ -52,12 +49,6 @@ if(NOT USE_INTERNAL_POCO_LIBRARY)
     target_include_directories (common SYSTEM BEFORE PUBLIC ${Poco_Foundation_INCLUDE_DIR})
 endif()
 
-if(CCTZ_LIBRARY)
-    target_link_libraries(common PRIVATE ${CCTZ_LIBRARY})
-endif()
-
-target_link_libraries(common PUBLIC replxx)
-
 # allow explicitly fallback to readline
 if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
     message (STATUS "Attempt to fallback to readline explicitly")
@@ -82,11 +73,16 @@ endif ()
 
 target_link_libraries (common
     PUBLIC
+        ${Poco_Net_LIBRARY}
         ${Poco_Util_LIBRARY}
         ${Poco_Foundation_LIBRARY}
         ${CITYHASH_LIBRARIES}
         ${Boost_SYSTEM_LIBRARY}
         FastMemcpy
+        replxx
+
+    PRIVATE
+        cctz
 )
 
 if (ENABLE_TESTS)

base/common/DateLUT.cpp

@@ -1,9 +1,10 @@
-#include <common/DateLUT.h>
+#include "DateLUT.h"
 
-#include <filesystem>
+#include <Poco/DigestStream.h>
 #include <Poco/Exception.h>
 #include <Poco/SHA1Engine.h>
-#include <Poco/DigestStream.h>
+
+#include <filesystem>
 #include <fstream>
 
 

base/common/DateLUT.h

@@ -1,20 +1,15 @@
 #pragma once
 
 #include "DateLUTImpl.h"
-#include <unordered_map>
+
-#include <atomic>
+#include "defines.h"
-#include <mutex>
+
-#include <memory>
 #include <boost/noncopyable.hpp>
 
-// Also defined in Core/Defines.h
+#include <atomic>
-#if !defined(ALWAYS_INLINE)
+#include <memory>
-#if defined(_MSC_VER)
+#include <mutex>
-    #define ALWAYS_INLINE __forceinline
+#include <unordered_map>
-#else
-    #define ALWAYS_INLINE __attribute__((__always_inline__))
-#endif
-#endif
 
 
 /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone.

base/common/DateLUTImpl.cpp

@@ -1,23 +1,14 @@
-#if __has_include(<cctz/civil_time.h>)
+#include "DateLUTImpl.h"
-#include <cctz/civil_time.h> // bundled, debian
-#else
-#include <civil_time.h> // freebsd
-#endif
 
-#if __has_include(<cctz/time_zone.h>)
+#include <cctz/civil_time.h>
 #include <cctz/time_zone.h>
-#else
-#include <time_zone.h>
-#endif
-
-#include <common/DateLUTImpl.h>
 #include <Poco/Exception.h>
 
-#include <memory>
+#include <cassert>
 #include <chrono>
 #include <cstring>
-#include <cassert>
 #include <iostream>
+#include <memory>
 
 #define DATE_LUT_MIN 0
 

base/common/DateLUTImpl.h

@@ -1,8 +1,9 @@
 #pragma once
 
-#include "Types.h"
 #include "DayNum.h"
-#include "likely.h"
+#include "defines.h"
+#include "types.h"
+
 #include <ctime>
 #include <string>
 

base/common/DayNum.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/Types.h>
+#include <common/types.h>
 #include <common/strong_typedef.h>
 
 /** Represents number of days since 1970-01-01.

base/common/JSON.cpp

@@ -215,7 +215,7 @@ JSON::ElementType JSON::getType() const
 
 void JSON::checkPos(Pos pos) const
 {
-    if (pos >= ptr_end)
+    if (pos >= ptr_end || ptr_begin == nullptr)
         throw JSONException("JSON: unexpected end of data.");
 }
 
@@ -776,7 +776,7 @@ JSON::iterator & JSON::iterator::operator++()
     return *this;
 }
 
-JSON::iterator JSON::iterator::operator++(int)
+JSON::iterator JSON::iterator::operator++(int) // NOLINT
 {
     iterator copy(*this);
     ++*this;

base/common/JSON.h

@@ -3,7 +3,7 @@
 #include <typeinfo>
 #include <Poco/Exception.h>
 #include <common/StringRef.h>
-#include <common/Types.h>
+#include <common/types.h>
 
 
 /** Очень простой класс для чтения JSON (или его кусочков).

base/common/LineReader.cpp

@@ -30,7 +30,7 @@ void trim(String & s)
 bool hasInputData()
 {
     timeval timeout = {0, 0};
-    fd_set fds;
+    fd_set fds{};
     FD_ZERO(&fds);
     FD_SET(STDIN_FILENO, &fds);
     return select(1, &fds, nullptr, nullptr, &timeout) == 1;
@@ -53,15 +53,15 @@ LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String
 
     /// last_word can be empty.
 
-    if (case_insensitive)
+    /// Only perform case sensitive completion when the prefix string contains any uppercase characters
+    if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; }))
         return std::equal_range(
-                words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
+            words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
             {
                 return strncasecmp(s.data(), prefix_searched.data(), prefix_length) < 0;
             });
     else
-        return std::equal_range(
+        return std::equal_range(words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
-                words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
         {
             return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0;
         });

base/common/LineReader.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/Types.h>
+#include <common/types.h>
 
 #include <atomic>
 #include <vector>
@@ -14,13 +14,11 @@ public:
         using WordsRange = std::pair<Words::const_iterator, Words::const_iterator>;
 
         Words words;
+        Words words_no_case;
         std::atomic<bool> ready{false};
 
         /// Get iterators for the matched range of words if any.
         WordsRange getCompletions(const String & prefix, size_t prefix_length) const;
-
-        /// case sensitive suggestion
-        bool case_insensitive = false;
     };
 
     LineReader(const String & history_file_path, char extender, char delimiter = 0);  /// if delimiter != 0, then it's multiline mode

base/common/ReadlineLineReader.cpp

@@ -104,6 +104,8 @@ ReadlineLineReader::ReadlineLineReader(const Suggest & suggest_, const String &
 
     if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
         throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno));
+
+    rl_variable_bind("completion-ignore-case", "on");
 }
 
 ReadlineLineReader::~ReadlineLineReader()

base/common/StringRef.h

@@ -5,7 +5,7 @@
 #include <functional>
 #include <ostream>
 
-#include <common/Types.h>
+#include <common/types.h>
 #include <common/unaligned.h>
 
 #include <city.h>
@@ -27,17 +27,17 @@ struct StringRef
     size_t size = 0;
 
     template <typename CharT, typename = std::enable_if_t<sizeof(CharT) == 1>>
-    StringRef(const CharT * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_) {}
+    constexpr StringRef(const CharT * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_) {}
 
     StringRef(const std::string & s) : data(s.data()), size(s.size()) {}
-    StringRef(const std::string_view & s) : data(s.data()), size(s.size()) {}
+    constexpr StringRef(const std::string_view & s) : data(s.data()), size(s.size()) {}
-    explicit StringRef(const char * data_) : data(data_), size(strlen(data_)) {}
+    constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {}
-    StringRef() = default;
+    constexpr StringRef() = default;
 
     std::string toString() const { return std::string(data, size); }
 
     explicit operator std::string() const { return toString(); }
-    explicit operator std::string_view() const { return {data, size}; }
+    constexpr explicit operator std::string_view() const { return {data, size}; }
 };
 
 using StringRefs = std::vector<StringRef>;

base/common/argsToConfig.cpp

@@ -1,4 +1,4 @@
-#include <common/argsToConfig.h>
+#include "argsToConfig.h"
 
 #include <Poco/Util/Application.h>
 #include <Poco/Util/LayeredConfiguration.h>

base/common/argsToConfig.h

@@ -1,4 +1,5 @@
 #pragma once
+
 #include <Poco/Util/Application.h>
 
 namespace Poco::Util

base/common/config_common.h.in

@@ -4,4 +4,3 @@
 
 #cmakedefine01 USE_JEMALLOC
 #cmakedefine01 UNBUNDLED
-#cmakedefine01 WITH_COVERAGE

base/common/coverage.cpp

@@ -1,11 +1,12 @@
-#include <common/coverage.h>
+#include "coverage.h"
-#include <common/config_common.h>
 
 #if WITH_COVERAGE
 
-#include <unistd.h>
 #    include <mutex>
 
+#    include <unistd.h>
+
+
 #    if defined(__clang__)
 extern "C" void __llvm_profile_dump();
 #    elif defined(__GNUC__) || defined(__GNUG__)

base/common/defines.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#if defined(_MSC_VER)
+#   if !defined(likely)
+#      define likely(x)   (x)
+#   endif
+#   if !defined(unlikely)
+#      define unlikely(x) (x)
+#   endif
+#else
+#   if !defined(likely)
+#       define likely(x)   (__builtin_expect(!!(x), 1))
+#   endif
+#   if !defined(unlikely)
+#       define unlikely(x) (__builtin_expect(!!(x), 0))
+#   endif
+#endif
+
+#if defined(_MSC_VER)
+#    define ALWAYS_INLINE __forceinline
+#    define NO_INLINE static __declspec(noinline)
+#    define MAY_ALIAS
+#else
+#    define ALWAYS_INLINE __attribute__((__always_inline__))
+#    define NO_INLINE __attribute__((__noinline__))
+#    define MAY_ALIAS __attribute__((__may_alias__))
+#endif
+
+#if !defined(__x86_64__) && !defined(__aarch64__) && !defined(__PPC__)
+#    error "The only supported platforms are x86_64 and AArch64, PowerPC (work in progress)"
+#endif
+
+/// Check for presence of address sanitizer
+#if !defined(ADDRESS_SANITIZER)
+#    if defined(__has_feature)
+#        if __has_feature(address_sanitizer)
+#            define ADDRESS_SANITIZER 1
+#        endif
+#    elif defined(__SANITIZE_ADDRESS__)
+#        define ADDRESS_SANITIZER 1
+#    endif
+#endif
+
+#if !defined(THREAD_SANITIZER)
+#    if defined(__has_feature)
+#        if __has_feature(thread_sanitizer)
+#            define THREAD_SANITIZER 1
+#        endif
+#    elif defined(__SANITIZE_THREAD__)
+#        define THREAD_SANITIZER 1
+#    endif
+#endif
+
+#if !defined(MEMORY_SANITIZER)
+#    if defined(__has_feature)
+#        if __has_feature(memory_sanitizer)
+#            define MEMORY_SANITIZER 1
+#        endif
+#    elif defined(__MEMORY_SANITIZER__)
+#        define MEMORY_SANITIZER 1
+#    endif
+#endif
+
+/// TODO: Strange enough, there is no way to detect UB sanitizer.
+
+/// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.
+/// It is useful in case when compiler cannot see (and exploit) it, but UBSan can.
+/// Example: multiplication of signed integers with possibility of overflow when both sides are from user input.
+#if defined(__clang__)
+#    define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined")))
+#    define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address")))
+#    define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread")))
+#else  /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it.
+#    define NO_SANITIZE_UNDEFINED
+#    define NO_SANITIZE_ADDRESS
+#    define NO_SANITIZE_THREAD
+#endif
+
+#if defined __GNUC__ && !defined __clang__
+#    define OPTIMIZE(x) __attribute__((__optimize__(x)))
+#else
+#    define OPTIMIZE(x)
+#endif
+
+/// A macro for suppressing warnings about unused variables or function results.
+/// Useful for structured bindings which have no standard way to declare this.
+#define UNUSED(...) (void)(__VA_ARGS__)

base/common/find_symbols.h

@@ -292,3 +292,26 @@ inline char * find_last_not_symbols_or_null(char * begin, char * end)
 {
     return const_cast<char *>(detail::find_last_symbols_sse2<false, detail::ReturnMode::Nullptr, symbols...>(begin, end));
 }
+
+
+/// Slightly resembles boost::split. The drawback of boost::split is that it fires a false positive in clang static analyzer.
+/// See https://github.com/boostorg/algorithm/issues/63
+/// And https://bugs.llvm.org/show_bug.cgi?id=41141
+template <char... symbols, typename To>
+inline void splitInto(To & to, const std::string & what, bool token_compress = false)
+{
+    const char * pos = what.data();
+    const char * end = pos + what.size();
+    while (pos < end)
+    {
+        const char * delimiter_or_end = find_first_symbols<symbols...>(pos, end);
+
+        if (!token_compress || pos < delimiter_or_end)
+            to.emplace_back(pos, delimiter_or_end);
+
+        if (delimiter_or_end < end)
+            pos = delimiter_or_end + 1;
+        else
+            pos = delimiter_or_end;
+    }
+}

base/common/getFQDNOrHostName.cpp

@@ -1,5 +1,5 @@
 #include <Poco/Net/DNS.h>
-#include <Common/getFQDNOrHostName.h>
+#include <common/getFQDNOrHostName.h>
 
 
 namespace

base/common/getMemoryAmount.cpp

@@ -69,13 +69,13 @@ uint64_t getMemoryAmountOrZero()
 
 #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
     /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
-    return (uint64_t)sysconf(_SC_PHYS_PAGES)
+    return uint64_t(sysconf(_SC_PHYS_PAGES))
-           * (uint64_t)sysconf(_SC_PAGESIZE);
+           *uint64_t(sysconf(_SC_PAGESIZE));
 
 #elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
     /* Legacy. -------------------------------------------------- */
-    return (uint64_t)sysconf(_SC_PHYS_PAGES)
+    return uint64_t(sysconf(_SC_PHYS_PAGES))
-           * (uint64_t)sysconf(_SC_PAGE_SIZE);
+           * uint64_t(sysconf(_SC_PAGE_SIZE));
 
 #elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
     /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */

base/common/itoa.h

@@ -30,7 +30,6 @@
 #include <cstddef>
 #include <cstring>
 #include <type_traits>
-#include "likely.h"
 
 using int128_t = __int128;
 using uint128_t = unsigned __int128;

base/common/likely.h

@@ -1,15 +0,0 @@
-#if defined(_MSC_VER)
-#   if !defined(likely)
-#      define likely(x)   (x)
-#   endif
-#   if !defined(unlikely)
-#      define unlikely(x) (x)
-#   endif
-#else
-#   if !defined(likely)
-#       define likely(x)   (__builtin_expect(!!(x), 1))
-#   endif
-#   if !defined(unlikely)
-#       define unlikely(x) (__builtin_expect(!!(x), 0))
-#   endif
-#endif

base/common/memory.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <new>
-#include "likely.h"
+#include "defines.h"
 
 #if __has_include(<common/config_common.h>)
 #include <common/config_common.h>
@@ -19,27 +19,11 @@
 #include <cstdlib>
 #endif
 
-// Also defined in Core/Defines.h
-#if !defined(ALWAYS_INLINE)
-#if defined(_MSC_VER)
-    #define ALWAYS_INLINE inline __forceinline
-#else
-    #define ALWAYS_INLINE inline __attribute__((__always_inline__))
-#endif
-#endif
-
-#if !defined(NO_INLINE)
-#if defined(_MSC_VER)
-    #define NO_INLINE static __declspec(noinline)
-#else
-    #define NO_INLINE __attribute__((__noinline__))
-#endif
-#endif
 
 namespace Memory
 {
 
-ALWAYS_INLINE void * newImpl(std::size_t size)
+inline ALWAYS_INLINE void * newImpl(std::size_t size)
 {
     auto * ptr = malloc(size);
     if (likely(ptr != nullptr))
@@ -49,19 +33,19 @@ ALWAYS_INLINE void * newImpl(std::size_t size)
     throw std::bad_alloc{};
 }
 
-ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
+inline ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
 {
     return malloc(size);
 }
 
-ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
+inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
 {
     free(ptr);
 }
 
 #if USE_JEMALLOC
 
-ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept
+inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept
 {
     if (unlikely(ptr == nullptr))
         return;
@@ -71,7 +55,7 @@ ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept
 
 #else
 
-ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept
+inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept
 {
     free(ptr);
 }

base/common/shift10.cpp

@@ -1,6 +1,6 @@
 #include <common/shift10.h>
 
-#include <common/likely.h>
+#include "defines.h"
 
 #include <limits>
 
@@ -8,8 +8,8 @@
 template <typename T>
 static T shift10Impl(T x, int exponent)
 {
-    static constexpr ssize_t MIN_EXPONENT = -323;
+    static constexpr ssize_t min_exponent = -323;
-    static constexpr ssize_t MAX_EXPONENT = 308;
+    static constexpr ssize_t max_exponent = 308;
 
     static const long double powers10[] =
     {
@@ -47,12 +47,12 @@ static T shift10Impl(T x, int exponent)
         1e291L,1e292L,1e293L,1e294L,1e295L,1e296L,1e297L,1e298L,1e299L,1e300L,1e301L,1e302L,1e303L,1e304L,1e305L,1e306L,1e307L,1e308L
     };
 
-    if (unlikely(exponent < MIN_EXPONENT))  /// Note: there are some values below MIN_EXPONENT that is greater than zero.
+    if (unlikely(exponent < min_exponent))  /// Note: there are some values below MIN_EXPONENT that is greater than zero.
         x *= 0;     /// Multiplying to keep the sign of zero.
-    else if (unlikely(exponent > MAX_EXPONENT))
+    else if (unlikely(exponent > max_exponent))
         x *= std::numeric_limits<T>::infinity();  /// Multiplying to keep the sign of infinity.
     else
-        x *= powers10[exponent - MIN_EXPONENT];
+        x *= powers10[exponent - min_exponent];
 
     return x;
 }

base/common/shift10.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <common/Types.h>
+#include <common/types.h>
 
 /** Almost the same as x = x * exp10(exponent), but gives more accurate result.
   * Example:

base/common/terminalColors.h

@@ -1,5 +1,5 @@
 #include <string>
-#include <common/Types.h>
+#include <common/types.h>
 
 
 /** Set color in terminal based on 64-bit hash value.

base/common/tests/date_lut2.cpp

@@ -43,7 +43,7 @@ void loop(time_t begin, time_t end, int step)
 }
 
 
-int main(int argc, char ** argv)
+int main(int, char **)
 {
     loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60);
     loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60);

base/common/tests/date_lut3.cpp

@@ -53,7 +53,7 @@ void loop(time_t begin, time_t end, int step)
 }
 
 
-int main(int argc, char ** argv)
+int main(int, char **)
 {
     loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60);
     loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60);

base/common/tests/date_lut4.cpp

@@ -2,15 +2,15 @@
 #include <common/DateLUT.h>
 
 
-int main(int argc, char ** argv)
+int main(int, char **)
 {
     /** В DateLUT был глюк - для времён из дня 1970-01-01, возвращался номер часа больше 23. */
-    static const time_t TIME = 66130;
+    static const time_t time = 66130;
 
     const auto & date_lut = DateLUT::instance();
 
-    std::cerr << date_lut.toHour(TIME) << std::endl;
+    std::cerr << date_lut.toHour(time) << std::endl;
-    std::cerr << date_lut.toDayNum(TIME) << std::endl;
+    std::cerr << date_lut.toDayNum(time) << std::endl;
 
     const auto * values = reinterpret_cast<const DateLUTImpl::Values *>(&date_lut);
 

base/common/tests/date_lut_default_timezone.cpp

@@ -2,7 +2,7 @@
 #include <common/DateLUT.h>
 #include <Poco/Exception.h>
 
-int main(int argc, char ** argv)
+int main(int, char **)
 {
     try
     {

base/common/tests/date_lut_init.cpp

@@ -1,7 +1,7 @@
 #include <common/DateLUT.h>
 
 /// Позволяет проверить время инициализации DateLUT.
-int main(int argc, char ** argv)
+int main(int, char **)
 {
     DateLUT::instance();
     return 0;

base/common/tests/gtest_find_symbols.cpp

@@ -1,4 +1,5 @@
 #include <string>
+#include <vector>
 #include <common/find_symbols.h>
 #include <gtest/gtest.h>
 
@@ -22,4 +23,16 @@ TEST(FindSymbols, SimpleTest)
     ASSERT_EQ(find_last_symbols_or_null<' '>(begin, end), end - 11);
     ASSERT_EQ(find_last_symbols_or_null<'H'>(begin, end), begin);
     ASSERT_EQ((find_last_symbols_or_null<'a', 'e'>(begin, end)), end - 4);
+
+    {
+        std::vector<std::string> vals;
+        splitInto<' ', ','>(vals, "hello, world", true);
+        ASSERT_EQ(vals, (std::vector<std::string>{"hello", "world"}));
+    }
+
+    {
+        std::vector<std::string> vals;
+        splitInto<' ', ','>(vals, "s String", true);
+        ASSERT_EQ(vals, (std::vector<std::string>{"s", "String"}));
+    }
 }

base/common/tests/gtest_strong_typedef.cpp

@@ -54,7 +54,7 @@ TEST(StrongTypedefSuite, NoDefaultCtor)
 {
     struct NoDefaultCtor
     {
-        NoDefaultCtor(int i) {}
+        NoDefaultCtor(int) {} // NOLINT
     };
 
     STRONG_TYPEDEF(NoDefaultCtor, MyStruct);

base/common/ya.make

@@ -0,0 +1,12 @@
+LIBRARY()
+
+PEERDIR(
+    contrib/libs/poco/Util
+)
+
+SRCS(
+    argsToConfig.cpp
+    coverage.cpp
+)
+
+END()

base/daemon/BaseDaemon.cpp

@@ -99,12 +99,12 @@ static void writeSignalIDtoSignalPipe(int sig)
 }
 
 /** Signal handler for HUP / USR1 */
-static void closeLogsSignalHandler(int sig, siginfo_t * info, void * context)
+static void closeLogsSignalHandler(int sig, siginfo_t *, void *)
 {
     writeSignalIDtoSignalPipe(sig);
 }
 
-static void terminateRequestedSignalHandler(int sig, siginfo_t * info, void * context)
+static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
 {
     writeSignalIDtoSignalPipe(sig);
 }
@@ -174,6 +174,10 @@ public:
         {
             int sig = 0;
             DB::readBinary(sig, in);
+            // We may log some specific signals afterwards, with different log
+            // levels and more info, but for completeness we log all signals
+            // here at trace level.
+            LOG_TRACE(log, "Received signal " << strsignal(sig) << " (" << sig << ")");
 
             if (sig == Signals::StopThread)
             {
@@ -362,19 +366,8 @@ void BaseDaemon::reloadConfiguration()
 }
 
 
-BaseDaemon::BaseDaemon()
+namespace
 {
-    checkRequiredInstructions();
-}
-
-
-BaseDaemon::~BaseDaemon()
-{
-    writeSignalIDtoSignalPipe(SignalListener::StopThread);
-    signal_listener_thread.join();
-    signal_pipe.close();
-}
-
 
 enum class InstructionFail
 {
@@ -388,7 +381,7 @@ enum class InstructionFail
     AVX512 = 7
 };
 
-static std::string instructionFailToString(InstructionFail fail)
+std::string instructionFailToString(InstructionFail fail)
 {
     switch (fail)
     {
@@ -413,16 +406,16 @@ static std::string instructionFailToString(InstructionFail fail)
 }
 
 
-static sigjmp_buf jmpbuf;
+sigjmp_buf jmpbuf;
 
-static void sigIllCheckHandler(int sig, siginfo_t * info, void * context)
+void sigIllCheckHandler(int, siginfo_t *, void *)
 {
     siglongjmp(jmpbuf, 1);
 }
 
 /// Check if necessary sse extensions are available by trying to execute some sse instructions.
 /// If instruction is unavailable, SIGILL will be sent by kernel.
-static void checkRequiredInstructions(volatile InstructionFail & fail)
+void checkRequiredInstructionsImpl(volatile InstructionFail & fail)
 {
 #if __SSE3__
     fail = InstructionFail::SSE3;
@@ -463,8 +456,9 @@ static void checkRequiredInstructions(volatile InstructionFail & fail)
     fail = InstructionFail::NONE;
 }
 
-
+/// Check SSE and others instructions availability
-void BaseDaemon::checkRequiredInstructions()
+/// Calls exit on fail
+void checkRequiredInstructions()
 {
     struct sigaction sa{};
     struct sigaction sa_old{};
@@ -487,7 +481,7 @@ void BaseDaemon::checkRequiredInstructions()
         exit(1);
     }
 
-    ::checkRequiredInstructions(fail);
+    checkRequiredInstructionsImpl(fail);
 
     if (sigaction(signal, &sa_old, nullptr))
     {
@@ -496,6 +490,22 @@ void BaseDaemon::checkRequiredInstructions()
     }
 }
 
+}
+
+
+BaseDaemon::BaseDaemon()
+{
+    checkRequiredInstructions();
+}
+
+
+BaseDaemon::~BaseDaemon()
+{
+    writeSignalIDtoSignalPipe(SignalListener::StopThread);
+    signal_listener_thread.join();
+    signal_pipe.close();
+}
+
 
 void BaseDaemon::terminate()
 {
@@ -784,7 +794,9 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
 
 void BaseDaemon::logRevision() const
 {
-    Logger::root().information("Starting " + std::string{VERSION_FULL} + " with revision " + std::to_string(ClickHouseRevision::get()));
+    Logger::root().information("Starting " + std::string{VERSION_FULL}
+        + " with revision " + std::to_string(ClickHouseRevision::get())
+        + ", PID " + std::to_string(getpid()));
 }
 
 /// Makes server shutdown if at least one Poco::Task have failed.

base/daemon/BaseDaemon.h

@@ -17,7 +17,7 @@
 #include <Poco/Util/ServerApplication.h>
 #include <Poco/Net/SocketAddress.h>
 #include <Poco/Version.h>
-#include <common/Types.h>
+#include <common/types.h>
 #include <common/logger_useful.h>
 #include <common/getThreadId.h>
 #include <daemon/GraphiteWriter.h>
@@ -128,7 +128,7 @@ public:
     /// close all process FDs except
     /// 0-2 -- stdin, stdout, stderr
     /// also doesn't close global internal pipes for signal handling
-    void closeFDs();
+    static void closeFDs();
 
 protected:
     /// Возвращает TaskManager приложения
@@ -198,12 +198,6 @@ protected:
     std::string config_path;
     DB::ConfigProcessor::LoadedConfig loaded_config;
     Poco::Util::AbstractConfiguration * last_configuration = nullptr;
-
-private:
-
-    /// Check SSE and others instructions availability
-    /// Calls exit on fail
-    void checkRequiredInstructions();
 };
 
 

base/daemon/GraphiteWriter.cpp

@@ -2,7 +2,7 @@
 #include <daemon/BaseDaemon.h>
 #include <Poco/Util/LayeredConfiguration.h>
 #include <Poco/Util/Application.h>
-#include <Common/getFQDNOrHostName.h>
+#include <common/getFQDNOrHostName.h>
 
 #include <mutex>
 #include <iomanip>

base/ext/chrono_io.h

@@ -1,16 +1,30 @@
 #pragma once
 
 #include <chrono>
+#include <ctime>
 #include <string>
-#include <common/DateLUT.h>
+#include <iomanip>
 
 
 namespace ext
 {
+    inline std::string to_string(const std::time_t & time)
+    {
+        std::stringstream ss;
+        ss << std::put_time(std::localtime(&time), "%Y-%m-%d %X");
+        return ss.str();
+    }
+
     template <typename Clock, typename Duration = typename Clock::duration>
     std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
     {
-        return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
+        // Don't use DateLUT because it shows weird characters for
+        // TimePoint::max(). I wish we could use C++20 format, but it's not
+        // there yet.
+        // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
+
+        auto in_time_t = std::chrono::system_clock::to_time_t(tp);
+        return to_string(in_time_t);
     }
 
     template <typename Rep, typename Period = std::ratio<1>>

base/ext/scope_guard.h

@@ -12,20 +12,20 @@ class [[nodiscard]] basic_scope_guard
 {
 public:
     constexpr basic_scope_guard() = default;
-    constexpr basic_scope_guard(basic_scope_guard && src) : function{std::exchange(src.function, {})} {}
+    constexpr basic_scope_guard(basic_scope_guard && src) : function{src.release()} {}
 
     constexpr basic_scope_guard & operator=(basic_scope_guard && src)
     {
         if (this != &src)
         {
             invoke();
-            function = std::exchange(src.function, {});
+            function = src.release();
         }
         return *this;
     }
 
     template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
-    constexpr basic_scope_guard(basic_scope_guard<G> && src) : function{std::exchange(src.function, {})} {}
+    constexpr basic_scope_guard(basic_scope_guard<G> && src) : function{src.release()} {}
 
     template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
     constexpr basic_scope_guard & operator=(basic_scope_guard<G> && src)
@@ -33,7 +33,7 @@ public:
         if (this != &src)
         {
             invoke();
-            function = std::exchange(src.function, {});
+            function = src.release();
         }
         return *this;
     }
@@ -46,14 +46,26 @@ public:
 
     ~basic_scope_guard() { invoke(); }
 
+    static constexpr bool is_nullable = std::is_constructible_v<bool, F>;
+
     explicit operator bool() const
     {
-        if constexpr (std::is_constructible_v<bool, F>)
+        if constexpr (is_nullable)
             return static_cast<bool>(function);
         return true;
     }
 
-    void reset() { function = {}; }
+    void reset()
+    {
+        invoke();
+        release();
+    }
+
+    F release()
+    {
+        static_assert(is_nullable);
+        return std::exchange(function, {});
+    }
 
     template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
     basic_scope_guard<F> & join(basic_scope_guard<G> && other)
@@ -62,14 +74,14 @@ public:
         {
             if (function)
             {
-                function = [x = std::make_shared<std::pair<F, G>>(std::move(function), std::exchange(other.function, {}))]()
+                function = [x = std::make_shared<std::pair<F, G>>(std::move(function), other.release())]()
                 {
                     std::move(x->first)();
                     std::move(x->second)();
                 };
             }
             else
-                function = std::exchange(other.function, {});
+                function = other.release();
         }
         return *this;
     }
@@ -77,7 +89,7 @@ public:
 private:
     void invoke()
     {
-        if constexpr (std::is_constructible_v<bool, F>)
+        if constexpr (is_nullable)
         {
             if (!function)
                 return;

base/loggers/OwnSplitChannel.cpp

@@ -71,7 +71,8 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
 
 
     /// Also log to system.text_log table, if message is not too noisy
-    if (text_log_max_priority && msg.getPriority() <= text_log_max_priority)
+    auto text_log_max_priority_loaded = text_log_max_priority.load(std::memory_order_relaxed);
+    if (text_log_max_priority_loaded && msg.getPriority() <= text_log_max_priority_loaded)
     {
         TextLogElement elem;
 
@@ -108,7 +109,7 @@ void OwnSplitChannel::addTextLog(std::shared_ptr<DB::TextLog> log, int max_prior
 {
     std::lock_guard<std::mutex> lock(text_log_mutex);
     text_log = log;
-    text_log_max_priority = max_priority;
+    text_log_max_priority.store(max_priority, std::memory_order_relaxed);
 }
 
 }

base/loggers/OwnSplitChannel.h

@@ -33,7 +33,7 @@ private:
     std::mutex text_log_mutex;
 
     std::weak_ptr<DB::TextLog> text_log;
-    int text_log_max_priority = -1;
+    std::atomic<int> text_log_max_priority = -1;
 };
 
 }

base/mysqlxx/Pool.cpp

@@ -119,7 +119,7 @@ Pool::~Pool()
 }
 
 
-Pool::Entry Pool::Get()
+Pool::Entry Pool::get()
 {
     std::unique_lock<std::mutex> lock(mutex);
 

base/mysqlxx/Pool.h

@@ -185,7 +185,7 @@ public:
     ~Pool();
 
     /// Allocates connection.
-    Entry Get();
+    Entry get();
 
     /// Allocates connection.
     /// If database is not accessible, returns empty Entry object.

base/mysqlxx/PoolFactory.cpp

@@ -17,10 +17,10 @@ struct PoolFactory::Impl
     std::mutex mutex;
 };
 
-PoolWithFailover PoolFactory::Get(const std::string & config_name, unsigned default_connections,
+PoolWithFailover PoolFactory::get(const std::string & config_name, unsigned default_connections,
     unsigned max_connections, size_t max_tries)
 {
-    return Get(Poco::Util::Application::instance().config(), config_name, default_connections, max_connections, max_tries);
+    return get(Poco::Util::Application::instance().config(), config_name, default_connections, max_connections, max_tries);
 }
 
 /// Duplicate of code from StringUtils.h. Copied here for less dependencies.
@@ -72,7 +72,7 @@ static std::string getPoolEntryName(const Poco::Util::AbstractConfiguration & co
     return entry_name;
 }
 
-PoolWithFailover PoolFactory::Get(const Poco::Util::AbstractConfiguration & config,
+PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & config,
         const std::string & config_name, unsigned default_connections, unsigned max_connections, size_t max_tries)
 {
 

base/mysqlxx/PoolFactory.h

@@ -27,13 +27,13 @@ public:
     PoolFactory(const PoolFactory &) = delete;
 
     /** Allocates a PoolWithFailover to connect to MySQL. */
-    PoolWithFailover Get(const std::string & config_name,
+    PoolWithFailover get(const std::string & config_name,
         unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
         unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
         size_t max_tries = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
 
     /** Allocates a PoolWithFailover to connect to MySQL. */
-    PoolWithFailover Get(const Poco::Util::AbstractConfiguration & config,
+    PoolWithFailover get(const Poco::Util::AbstractConfiguration & config,
         const std::string & config_name,
         unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
         unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,

base/mysqlxx/PoolWithFailover.cpp

@@ -68,7 +68,7 @@ PoolWithFailover::PoolWithFailover(const PoolWithFailover & other)
     }
 }
 
-PoolWithFailover::Entry PoolWithFailover::Get()
+PoolWithFailover::Entry PoolWithFailover::get()
 {
     Poco::Util::Application & app = Poco::Util::Application::instance();
     std::lock_guard<std::mutex> locker(mutex);
@@ -89,7 +89,7 @@ PoolWithFailover::Entry PoolWithFailover::Get()
 
                 try
                 {
-                    Entry entry = shareable ? pool->Get() : pool->tryGet();
+                    Entry entry = shareable ? pool->get() : pool->tryGet();
 
                     if (!entry.isNull())
                     {
@@ -121,7 +121,7 @@ PoolWithFailover::Entry PoolWithFailover::Get()
     if (full_pool)
     {
         app.logger().error("All connections failed, trying to wait on a full pool " + (*full_pool)->getDescription());
-        return (*full_pool)->Get();
+        return (*full_pool)->get();
     }
 
     std::stringstream message;

base/mysqlxx/PoolWithFailover.h

@@ -105,6 +105,6 @@ namespace mysqlxx
         PoolWithFailover(const PoolWithFailover & other);
 
         /** Allocates a connection to use. */
-        Entry Get();
+        Entry get();
     };
 }

base/mysqlxx/Value.cpp

@@ -154,7 +154,7 @@ double Value::readFloatText(const char * buf, size_t length) const
 
 void Value::throwException(const char * text) const
 {
-    static constexpr size_t MYSQLXX_QUERY_PREVIEW_LENGTH = 1000;
+    static constexpr size_t preview_length = 1000;
 
     std::stringstream info;
     info << text;
@@ -166,7 +166,7 @@ void Value::throwException(const char * text) const
     }
 
     if (res && res->getQuery())
-        info << ", query: " << res->getQuery()->str().substr(0, MYSQLXX_QUERY_PREVIEW_LENGTH);
+        info << ", query: " << res->getQuery()->str().substr(0, preview_length);
 
     throw CannotParseValue(info.str());
 }

base/mysqlxx/Value.h

@@ -9,7 +9,7 @@
 #include <limits>
 
 #include <common/preciseExp10.h>
-#include <common/Types.h>
+#include <common/types.h>
 #include <common/DateLUT.h>
 
 #include <mysqlxx/Types.h>

base/ya.make

@@ -1 +1,3 @@
-
+RECURSE(
+    common
+)

cmake/find/cctz.cmake

@@ -1,23 +0,0 @@
-option (USE_INTERNAL_CCTZ_LIBRARY "Set to FALSE to use system cctz library instead of bundled" ${NOT_UNBUNDLED})
-
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cctz/include/cctz/time_zone.h")
-    if(USE_INTERNAL_CCTZ_LIBRARY)
-       message(WARNING "submodule contrib/cctz is missing. to fix try run: \n git submodule update --init --recursive")
-    endif()
-    set(USE_INTERNAL_CCTZ_LIBRARY 0)
-    set(MISSING_INTERNAL_CCTZ_LIBRARY 1)
-endif()
-
-if (NOT USE_INTERNAL_CCTZ_LIBRARY)
-    find_library (CCTZ_LIBRARY cctz)
-    find_path (CCTZ_INCLUDE_DIR NAMES cctz/civil_time.h civil_time.h PATHS ${CCTZ_INCLUDE_PATHS})
-endif ()
-
-if (CCTZ_LIBRARY AND CCTZ_INCLUDE_DIR)
-elseif (NOT MISSING_INTERNAL_CCTZ_LIBRARY)
-    set (USE_INTERNAL_CCTZ_LIBRARY 1)
-    set (CCTZ_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cctz/include")
-    set (CCTZ_LIBRARY cctz)
-endif ()
-
-message (STATUS "Using cctz: ${CCTZ_INCLUDE_DIR} : ${CCTZ_LIBRARY}")

contrib/CMakeLists.txt

@@ -75,9 +75,7 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
     endif ()
 endif ()
 
-if (USE_INTERNAL_CCTZ_LIBRARY)
 add_subdirectory (cctz-cmake)
-endif ()
 
 if (ENABLE_JEMALLOC AND USE_INTERNAL_JEMALLOC_LIBRARY)
     add_subdirectory (jemalloc-cmake)
@@ -310,12 +308,13 @@ if (USE_BASE64)
 endif()
 
 if (USE_INTERNAL_HYPERSCAN_LIBRARY)
-    add_subdirectory (hyperscan)
-
     # The library is large - avoid bloat.
     if (USE_STATIC_LIBRARIES)
+        add_subdirectory (hyperscan)
         target_compile_options (hs PRIVATE -g0)
     else ()
+        set(BUILD_SHARED_LIBS 1 CACHE INTERNAL "")
+        add_subdirectory (hyperscan)
         target_compile_options (hs_shared PRIVATE -g0)
     endif ()
 endif()

contrib/cctz-cmake/CMakeLists.txt

@@ -1,6 +1,9 @@
+option (USE_INTERNAL_CCTZ "Use internal cctz library" ${NOT_UNBUNDLED})
+
+if (USE_INTERNAL_CCTZ)
     SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz)
 
-add_library(cctz
+    SET (SRCS
         ${LIBRARY_DIR}/src/civil_time_detail.cc
         ${LIBRARY_DIR}/src/time_zone_fixed.cc
         ${LIBRARY_DIR}/src/time_zone_format.cc
@@ -11,21 +14,37 @@ add_library(cctz
         ${LIBRARY_DIR}/src/time_zone_lookup.cc
         ${LIBRARY_DIR}/src/time_zone_posix.cc
         ${LIBRARY_DIR}/src/zone_info_source.cc
+    )
 
-    ${LIBRARY_DIR}/src/time_zone_libc.h
+    add_library (cctz ${SRCS})
-    ${LIBRARY_DIR}/src/time_zone_if.h
+    target_include_directories (cctz PUBLIC ${LIBRARY_DIR}/include)
-    ${LIBRARY_DIR}/src/tzfile.h
-    ${LIBRARY_DIR}/src/time_zone_impl.h
-    ${LIBRARY_DIR}/src/time_zone_posix.h
-    ${LIBRARY_DIR}/src/time_zone_info.h
 
-    ${LIBRARY_DIR}/include/cctz/time_zone.h
+    if (OS_FREEBSD)
-    ${LIBRARY_DIR}/include/cctz/civil_time_detail.h
-    ${LIBRARY_DIR}/include/cctz/civil_time.h)
-
-if (CMAKE_SYSTEM MATCHES "FreeBSD")
         # yes, need linux, because bsd check inside linux in time_zone_libc.cc:24
         target_compile_definitions (cctz PRIVATE __USE_BSD linux _XOPEN_SOURCE=600)
     endif ()
+else ()
+    find_library (LIBRARY_CCTZ cctz)
+    find_path (INCLUDE_CCTZ NAMES cctz/civil_time.h)
 
-target_include_directories (cctz PUBLIC ${LIBRARY_DIR}/include)
+    add_library (cctz UNKNOWN IMPORTED)
+    set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ})
+    set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ})
+
+    set(CMAKE_REQUIRED_LIBRARIES cctz)
+    check_cxx_source_compiles(
+        "
+        #include <cctz/civil_time.h>
+        int main() {
+            cctz::civil_day date;
+        }
+        "
+        EXTERNAL_CCTZ_WORKS
+    )
+
+    if (NOT EXTERNAL_CCTZ_WORKS)
+        message (FATAL_ERROR "cctz is unusable: ${LIBRARY_CCTZ} ${INCLUDE_CCTZ}")
+    endif ()
+endif ()
+
+message (STATUS "Using cctz")

contrib/poco

@@ -1 +1 @@
-Subproject commit 5da7ce676c20e36ee9a2547eb48a50083b3d5237
+Subproject commit ddca76ba4956cb57150082394536cc43ff28f6fa

contrib/replxx

@@ -1 +1 @@
-Subproject commit 07cbfbec550133b88c91c4073fa5af2ae2ae6a9a
+Subproject commit f1332626639d6492eaf170758642da14fbbda7bf

dbms/programs/CMakeLists.txt

@@ -7,7 +7,6 @@ option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_
 option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL})
-option (ENABLE_CLICKHOUSE_PERFORMANCE_TEST "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL})
@@ -76,7 +75,6 @@ add_subdirectory (server)
 add_subdirectory (client)
 add_subdirectory (local)
 add_subdirectory (benchmark)
-add_subdirectory (performance-test)
 add_subdirectory (extract-from-config)
 add_subdirectory (compressor)
 add_subdirectory (copier)
@@ -88,16 +86,15 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
 endif ()
 
 if (CLICKHOUSE_ONE_SHARED)
-    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
+    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
-    target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
+    target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
-    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
+    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
     set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
     install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
 endif()
 
 if (CLICKHOUSE_SPLIT_BINARY)
-    set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
+    set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-copier)
-            clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-copier)
 
     if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
         list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
@@ -126,9 +123,6 @@ else ()
     if (ENABLE_CLICKHOUSE_BENCHMARK)
         clickhouse_target_link_split_lib(clickhouse benchmark)
     endif ()
-    if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
-        clickhouse_target_link_split_lib(clickhouse performance-test)
-    endif ()
     if (ENABLE_CLICKHOUSE_COPIER)
         clickhouse_target_link_split_lib(clickhouse copier)
     endif ()
@@ -166,11 +160,6 @@ else ()
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
     endif ()
-    if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
-        add_custom_target (clickhouse-performance-test ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-performance-test DEPENDS clickhouse)
-        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-performance-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-        list(APPEND CLICKHOUSE_BUNDLE clickhouse-performance-test)
-    endif ()
     if (ENABLE_CLICKHOUSE_COPIER)
         add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse)
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)

dbms/programs/client/Client.cpp

@@ -225,11 +225,11 @@ private:
         context.setQueryParameters(query_parameters);
 
         /// settings and limits could be specified in config file, but passed settings has higher priority
-        for (auto && setting : context.getSettingsRef())
+        for (const auto & setting : context.getSettingsRef())
         {
             const String & name = setting.getName().toString();
             if (config().has(name) && !setting.isChanged())
-                setting.setValue(config().getString(name));
+                context.setSetting(name, config().getString(name));
         }
 
         /// Set path for format schema files
@@ -281,7 +281,7 @@ private:
     }
 
     /// Should we celebrate a bit?
-    bool isNewYearMode()
+    static bool isNewYearMode()
     {
         time_t current_time = time(nullptr);
 
@@ -294,7 +294,7 @@ private:
             || (now.month() == 1 && now.day() <= 5);
     }
 
-    bool isChineseNewYearMode(const String & local_tz)
+    static bool isChineseNewYearMode(const String & local_tz)
     {
         /// Days of Dec. 20 in Chinese calendar starting from year 2019 to year 2105
         static constexpr UInt16 chineseNewYearIndicators[]
@@ -481,8 +481,6 @@ private:
 
             if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false))
             {
-                if (config().has("case_insensitive_suggestion"))
-                    Suggest::instance().setCaseInsensitive();
                 /// Load suggestion data from the server.
                 Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
             }
@@ -1124,7 +1122,7 @@ private:
                 /// to avoid losing sync.
                 if (!cancelled)
                 {
-                    auto cancelQuery = [&] {
+                    auto cancel_query = [&] {
                         connection->sendCancel();
                         cancelled = true;
                         if (is_interactive)
@@ -1136,7 +1134,7 @@ private:
 
                     if (interrupt_listener.check())
                     {
-                        cancelQuery();
+                        cancel_query();
                     }
                     else
                     {
@@ -1147,7 +1145,7 @@ private:
                                       << " Waited for " << static_cast<size_t>(elapsed) << " seconds,"
                                       << " timeout is " << receive_timeout.totalSeconds() << " seconds." << std::endl;
 
-                            cancelQuery();
+                            cancel_query();
                         }
                     }
                 }
@@ -1594,7 +1592,7 @@ private:
             std::cout << "Ok." << std::endl;
     }
 
-    void showClientVersion()
+    static void showClientVersion()
     {
         std::cout << DBMS_NAME << " client version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
     }
@@ -1720,7 +1718,6 @@ public:
             ("always_load_suggestion_data", "Load suggestion data even if clickhouse-client is run in non-interactive mode. Used for testing.")
             ("suggestion_limit", po::value<int>()->default_value(10000),
                 "Suggestion limit for how many databases, tables and columns to fetch.")
-            ("case_insensitive_suggestion", "Case sensitive suggestions.")
             ("multiline,m", "multiline")
             ("multiquery,n", "multiquery")
             ("format,f", po::value<std::string>(), "default output format")
@@ -1739,8 +1736,8 @@ public:
             ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
         ;
 
-        context.makeGlobalContext();
+        Settings cmd_settings;
-        context.getSettingsRef().addProgramOptions(main_description);
+        cmd_settings.addProgramOptions(main_description);
 
         /// Commandline options related to external tables.
         po::options_description external_description = createOptionsDescription("External tables options", terminal_width);
@@ -1808,6 +1805,9 @@ public:
             }
         }
 
+        context.makeGlobalContext();
+        context.setSettings(cmd_settings);
+
         /// Copy settings-related program options to config.
         /// TODO: Is this code necessary?
         for (const auto & setting : context.getSettingsRef())

dbms/programs/client/Suggest.cpp

@@ -50,16 +50,15 @@ void Suggest::load(const ConnectionParameters & connection_parameters, size_t su
 
         /// Note that keyword suggestions are available even if we cannot load data from server.
 
-        if (case_insensitive)
+        std::sort(words.begin(), words.end());
-            std::sort(words.begin(), words.end(), [](const std::string & str1, const std::string & str2)
+        words_no_case = words;
+        std::sort(words_no_case.begin(), words_no_case.end(), [](const std::string & str1, const std::string & str2)
         {
             return std::lexicographical_compare(begin(str1), end(str1), begin(str2), end(str2), [](const char char1, const char char2)
             {
                 return std::tolower(char1) < std::tolower(char2);
             });
         });
-        else
-            std::sort(words.begin(), words.end());
 
         ready = true;
     });

dbms/programs/client/Suggest.h

@@ -23,9 +23,6 @@ public:
         return instance;
     }
 
-    /// Need to set before load
-    void setCaseInsensitive() { case_insensitive = true; }
-
     void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit);
 
     /// Older server versions cannot execute the query above.

dbms/programs/copier/ClusterCopier.cpp

@@ -216,7 +216,7 @@ void ClusterCopier::reloadTaskDescription()
 
     /// Setup settings
     task_cluster->reloadSettings(*config);
-    context.getSettingsRef() = task_cluster->settings_common;
+    context.setSettings(task_cluster->settings_common);
 
     task_cluster_current_config = config;
     task_descprtion_current_stat = stat;
@@ -499,7 +499,10 @@ ASTPtr ClusterCopier::removeAliasColumnsFromCreateQuery(const ASTPtr & query_ast
     return new_query_ast;
 }
 
-std::shared_ptr<ASTCreateQuery> ClusterCopier::rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast)
+
+/// Replaces ENGINE and table name in a create query
+static std::shared_ptr<ASTCreateQuery> rewriteCreateQueryStorage(
+    const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast)
 {
     const auto & create = create_query_ast->as<ASTCreateQuery &>();
     auto res = std::make_shared<ASTCreateQuery>(create);
@@ -961,8 +964,8 @@ PartitionTaskStatus ClusterCopier::processPartitionTaskImpl(const ConnectionTime
         {
             Context local_context = context;
             // Use pull (i.e. readonly) settings, but fetch data from destination servers
-            local_context.getSettingsRef() = task_cluster->settings_pull;
+            local_context.setSettings(task_cluster->settings_pull);
-            local_context.getSettingsRef().skip_unavailable_shards = true;
+            local_context.setSetting("skip_unavailable_shards", true);
 
             Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().in);
             count = (block) ? block.safeGetByPosition(0).column->getUInt(0) : 0;
@@ -1050,10 +1053,10 @@ PartitionTaskStatus ClusterCopier::processPartitionTaskImpl(const ConnectionTime
         {
             /// Custom INSERT SELECT implementation
             Context context_select = context;
-            context_select.getSettingsRef() = task_cluster->settings_pull;
+            context_select.setSettings(task_cluster->settings_pull);
 
             Context context_insert = context;
-            context_insert.getSettingsRef() = task_cluster->settings_push;
+            context_insert.setSettings(task_cluster->settings_push);
 
             BlockInputStreamPtr input;
             BlockOutputStreamPtr output;
@@ -1178,7 +1181,7 @@ String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, C
 ASTPtr ClusterCopier::getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard)
 {
     /// Fetch and parse (possibly) new definition
-    auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull);
+    auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull, true);
     String create_query_pull_str = getRemoteCreateTable(
         task_shard.task_table.table_pull,
         *connection_entry,

dbms/programs/copier/ClusterCopier.h

@@ -88,11 +88,6 @@ protected:
     /// Removes MATERIALIZED and ALIAS columns from create table query
     static ASTPtr removeAliasColumnsFromCreateQuery(const ASTPtr &query_ast);
 
-    /// Replaces ENGINE and table name in a create query
-    std::shared_ptr<ASTCreateQuery>
-    rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table,
-                              const ASTPtr & new_storage_ast);
-
     bool tryDropPartition(ShardPartition & task_partition,
                           const zkutil::ZooKeeperPtr & zookeeper,
                           const CleanStateClock & clean_state_clock);
@@ -119,8 +114,7 @@ protected:
                                 Connection & connection,
                                 const Settings * settings = nullptr);
 
-    ASTPtr getCreateTableForPullShard(const ConnectionTimeouts & timeouts,
+    ASTPtr getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard);
-                                      TaskShard & task_shard);
 
     void createShardInternalTables(const ConnectionTimeouts & timeouts,
                                    TaskShard & task_shard,

dbms/programs/copier/ClusterCopierApp.cpp

@@ -43,11 +43,11 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self)
 
 void ClusterCopierApp::handleHelp(const std::string &, const std::string &)
 {
-    Poco::Util::HelpFormatter helpFormatter(options());
+    Poco::Util::HelpFormatter help_formatter(options());
-    helpFormatter.setCommand(commandName());
+    help_formatter.setCommand(commandName());
-    helpFormatter.setHeader("Copies tables from one cluster to another");
+    help_formatter.setHeader("Copies tables from one cluster to another");
-    helpFormatter.setUsage("--config-file <config-file> --task-path <task-path>");
+    help_formatter.setUsage("--config-file <config-file> --task-path <task-path>");
-    helpFormatter.format(std::cerr);
+    help_formatter.format(std::cerr);
 
     stopOptionsProcessing();
 }

dbms/programs/copier/Internals.h

@@ -19,7 +19,7 @@
 #include <Common/Exception.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/ZooKeeper/KeeperException.h>
-#include <Common/getFQDNOrHostName.h>
+#include <common/getFQDNOrHostName.h>
 #include <Common/isLocalAddress.h>
 #include <Common/typeid_cast.h>
 #include <Common/ClickHouseRevision.h>

dbms/programs/local/LocalServer.cpp

@@ -21,6 +21,7 @@
 #include <Common/ThreadStatus.h>
 #include <Common/config_version.h>
 #include <Common/quoteString.h>
+#include <Common/SettingsChanges.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/UseSSL.h>
@@ -92,7 +93,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
 
 void LocalServer::applyCmdSettings()
 {
-    context->getSettingsRef().copyChangesFrom(cmd_settings);
+    context->applySettingsChanges(cmd_settings.changes());
 }
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
@@ -118,6 +119,20 @@ void LocalServer::tryInitPath()
 }
 
 
+static void attachSystemTables()
+{
+    DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE);
+    if (!system_database)
+    {
+        /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
+        system_database = std::make_shared<DatabaseMemory>(DatabaseCatalog::SYSTEM_DATABASE);
+        DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database);
+    }
+
+    attachSystemTablesLocal(*system_database);
+}
+
+
 int LocalServer::main(const std::vector<std::string> & /*args*/)
 try
 {
@@ -248,20 +263,6 @@ std::string LocalServer::getInitialCreateTableQuery()
 }
 
 
-void LocalServer::attachSystemTables()
-{
-    DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE);
-    if (!system_database)
-    {
-        /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
-        system_database = std::make_shared<DatabaseMemory>(DatabaseCatalog::SYSTEM_DATABASE);
-        DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database);
-    }
-
-    attachSystemTablesLocal(*system_database);
-}
-
-
 void LocalServer::processQueries()
 {
     String initial_create_query = getInitialCreateTableQuery();
@@ -375,7 +376,7 @@ static void showClientVersion()
     std::cout << DBMS_NAME << " client version " << VERSION_STRING << VERSION_OFFICIAL << "." << '\n';
 }
 
-std::string LocalServer::getHelpHeader() const
+static std::string getHelpHeader()
 {
     return
         "usage: clickhouse-local [initial table definition] [--query <query>]\n"
@@ -390,7 +391,7 @@ std::string LocalServer::getHelpHeader() const
         "Either through corresponding command line parameters --table --structure --input-format and --file.";
 }
 
-std::string LocalServer::getHelpFooter() const
+static std::string getHelpFooter()
 {
     return
         "Example printing memory used by each Unix user:\n"

dbms/programs/local/LocalServer.h

@@ -37,13 +37,9 @@ private:
     void tryInitPath();
     void applyCmdOptions();
     void applyCmdSettings();
-    void attachSystemTables();
     void processQueries();
     void setupUsers();
 
-    std::string getHelpHeader() const;
-    std::string getHelpFooter() const;
-
 protected:
     std::unique_ptr<Context> context;
 

dbms/programs/main.cpp

@@ -33,9 +33,6 @@ int mainEntryClickHouseLocal(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK)
 int mainEntryClickHouseBenchmark(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
-int mainEntryClickHousePerformanceTest(int argc, char ** argv);
-#endif
 #if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
 int mainEntryClickHouseExtractFromConfig(int argc, char ** argv);
 #endif
@@ -74,9 +71,6 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER)
     {"server", mainEntryClickHouseServer},
 #endif
-#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
-    {"performance-test", mainEntryClickHousePerformanceTest},
-#endif
 #if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
     {"extract-from-config", mainEntryClickHouseExtractFromConfig},
 #endif

dbms/programs/obfuscator/Obfuscator.cpp

@@ -181,7 +181,7 @@ private:
     UInt64 seed;
 
 public:
-    UnsignedIntegerModel(UInt64 seed_) : seed(seed_) {}
+    explicit UnsignedIntegerModel(UInt64 seed_) : seed(seed_) {}
 
     void train(const IColumn &) override {}
     void finalize() override {}
@@ -222,7 +222,7 @@ private:
     UInt64 seed;
 
 public:
-    SignedIntegerModel(UInt64 seed_) : seed(seed_) {}
+    explicit SignedIntegerModel(UInt64 seed_) : seed(seed_) {}
 
     void train(const IColumn &) override {}
     void finalize() override {}
@@ -271,7 +271,7 @@ private:
     Float res_prev_value = 0;
 
 public:
-    FloatModel(UInt64 seed_) : seed(seed_) {}
+    explicit FloatModel(UInt64 seed_) : seed(seed_) {}
 
     void train(const IColumn &) override {}
     void finalize() override {}
@@ -372,7 +372,7 @@ private:
     UInt64 seed;
 
 public:
-    FixedStringModel(UInt64 seed_) : seed(seed_) {}
+    explicit FixedStringModel(UInt64 seed_) : seed(seed_) {}
 
     void train(const IColumn &) override {}
     void finalize() override {}
@@ -414,7 +414,7 @@ private:
     const DateLUTImpl & date_lut;
 
 public:
-    DateTimeModel(UInt64 seed_) : seed(seed_), date_lut(DateLUT::instance()) {}
+    explicit DateTimeModel(UInt64 seed_) : seed(seed_), date_lut(DateLUT::instance()) {}
 
     void train(const IColumn &) override {}
     void finalize() override {}
@@ -529,13 +529,13 @@ private:
     static constexpr CodePoint END = -2;
 
 
-    NGramHash hashContext(const CodePoint * begin, const CodePoint * end) const
+    static NGramHash hashContext(const CodePoint * begin, const CodePoint * end)
     {
         return CRC32Hash()(StringRef(reinterpret_cast<const char *>(begin), (end - begin) * sizeof(CodePoint)));
     }
 
     /// By the way, we don't have to use actual Unicode numbers. We use just arbitrary bijective mapping.
-    CodePoint readCodePoint(const char *& pos, const char * end)
+    static CodePoint readCodePoint(const char *& pos, const char * end)
     {
         size_t length = UTF8::seqLength(*pos);
 
@@ -550,7 +550,7 @@ private:
         return res;
     }
 
-    bool writeCodePoint(CodePoint code, char *& pos, const char * end)
+    static bool writeCodePoint(CodePoint code, char *& pos, const char * end)
     {
         size_t length
             = (code & 0xFF000000) ? 4
@@ -567,7 +567,7 @@ private:
     }
 
 public:
-    MarkovModel(MarkovModelParameters params_)
+    explicit MarkovModel(MarkovModelParameters params_)
         : params(std::move(params_)), code_points(params.order, BEGIN) {}
 
     void consume(const char * data, size_t size)
@@ -677,7 +677,7 @@ public:
                 if (!histogram.total)
                     continue;
 
-                double average = histogram.total / histogram.buckets.size();
+                double average = double(histogram.total) / histogram.buckets.size();
 
                 UInt64 new_total = 0;
                 for (auto & bucket : histogram.buckets)
@@ -836,7 +836,7 @@ private:
     ModelPtr nested_model;
 
 public:
-    ArrayModel(ModelPtr nested_model_) : nested_model(std::move(nested_model_)) {}
+    explicit ArrayModel(ModelPtr nested_model_) : nested_model(std::move(nested_model_)) {}
 
     void train(const IColumn & column) override
     {
@@ -874,7 +874,7 @@ private:
     ModelPtr nested_model;
 
 public:
-    NullableModel(ModelPtr nested_model_) : nested_model(std::move(nested_model_)) {}
+    explicit NullableModel(ModelPtr nested_model_) : nested_model(std::move(nested_model_)) {}
 
     void train(const IColumn & column) override
     {

dbms/programs/odbc-bridge/ODBCBridge.cpp

@@ -5,6 +5,13 @@
 #include <errno.h>
 #include <IO/ReadHelpers.h>
 #include <boost/program_options.hpp>
+
+#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
+// It doesn't make much sense to build this bridge without ODBC, but we
+// still do this.
+#include <Poco/Data/ODBC/Connector.h>
+#endif
+
 #include <Poco/Net/HTTPServer.h>
 #include <Poco/Net/NetException.h>
 #include <Poco/String.h>
@@ -72,11 +79,11 @@ namespace
 
 void ODBCBridge::handleHelp(const std::string &, const std::string &)
 {
-    Poco::Util::HelpFormatter helpFormatter(options());
+    Poco::Util::HelpFormatter help_formatter(options());
-    helpFormatter.setCommand(commandName());
+    help_formatter.setCommand(commandName());
-    helpFormatter.setHeader("HTTP-proxy for odbc requests");
+    help_formatter.setHeader("HTTP-proxy for odbc requests");
-    helpFormatter.setUsage("--http-port <port>");
+    help_formatter.setUsage("--http-port <port>");
-    helpFormatter.format(std::cerr);
+    help_formatter.format(std::cerr);
 
     stopOptionsProcessing();
 }
@@ -111,7 +118,7 @@ void ODBCBridge::defineOptions(Poco::Util::OptionSet & options)
                           .binding("help")
                           .callback(Poco::Util::OptionCallback<Me>(this, &Me::handleHelp)));
 
-    ServerApplication::defineOptions(options); /// Don't need complex BaseDaemon's .xml config
+    ServerApplication::defineOptions(options); // NOLINT Don't need complex BaseDaemon's .xml config
 }
 
 void ODBCBridge::initialize(Application & self)
@@ -126,6 +133,8 @@ void ODBCBridge::initialize(Application & self)
 
     buildLoggers(config(), logger(), self.commandName());
 
+    BaseDaemon::logRevision();
+
     log = &logger();
     hostname = config().getString("listen-host", "localhost");
     port = config().getUInt("http-port");
@@ -138,7 +147,13 @@ void ODBCBridge::initialize(Application & self)
 
     initializeTerminationAndSignalProcessing();
 
-    ServerApplication::initialize(self);
+#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
+    // It doesn't make much sense to build this bridge without ODBC, but we
+    // still do this.
+    Poco::Data::ODBC::Connector::registerConnector();
+#endif
+
+    ServerApplication::initialize(self); // NOLINT
 }
 
 void ODBCBridge::uninitialize()

dbms/programs/performance-test/CMakeLists.txt

@@ -1,18 +0,0 @@
-set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES
-  ${CMAKE_CURRENT_SOURCE_DIR}/JSONString.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/StopConditionsSet.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/TestStopConditions.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/TestStats.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/ConfigPreprocessor.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTest.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestInfo.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/executeQuery.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/applySubstitutions.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/ReportBuilder.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestSuite.cpp
- )
-
-set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_config ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY})
-set(CLICKHOUSE_PERFORMANCE_TEST_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
-
-clickhouse_program_add(performance-test)

dbms/programs/performance-test/ConfigPreprocessor.cpp

@@ -1,95 +0,0 @@
-#include "ConfigPreprocessor.h"
-#include <Core/Types.h>
-#include <Poco/Path.h>
-#include <regex>
-namespace DB
-{
-std::vector<XMLConfigurationPtr> ConfigPreprocessor::processConfig(
-    const Strings & tests_tags,
-    const Strings & tests_names,
-    const Strings & tests_names_regexp,
-    const Strings & skip_tags,
-    const Strings & skip_names,
-    const Strings & skip_names_regexp) const
-{
-
-    std::vector<XMLConfigurationPtr> result;
-    for (const auto & path_str : paths)
-    {
-        auto test = XMLConfigurationPtr(new XMLConfiguration(path_str));
-        result.push_back(test);
-
-        const auto path = Poco::Path(path_str);
-        test->setString("path", path.absolute().toString());
-        if (test->getString("name", "").empty())
-            test->setString("name", path.getBaseName());
-    }
-
-    /// Leave tests:
-    removeConfigurationsIf(result, FilterType::Tag, tests_tags, true);
-    removeConfigurationsIf(result, FilterType::Name, tests_names, true);
-    removeConfigurationsIf(result, FilterType::Name_regexp, tests_names_regexp, true);
-
-    /// Skip tests
-    removeConfigurationsIf(result, FilterType::Tag, skip_tags, false);
-    removeConfigurationsIf(result, FilterType::Name, skip_names, false);
-    removeConfigurationsIf(result, FilterType::Name_regexp, skip_names_regexp, false);
-    return result;
-}
-
-void ConfigPreprocessor::removeConfigurationsIf(
-    std::vector<XMLConfigurationPtr> & configs,
-    ConfigPreprocessor::FilterType filter_type,
-    const Strings & values,
-    bool leave) const
-{
-    auto checker = [&filter_type, &values, &leave] (XMLConfigurationPtr & config)
-    {
-        if (values.empty())
-            return false;
-
-        bool remove_or_not = false;
-
-        if (filter_type == FilterType::Tag)
-        {
-            Strings tags_keys;
-            config->keys("tags", tags_keys);
-
-            Strings tags(tags_keys.size());
-            for (size_t i = 0; i != tags_keys.size(); ++i)
-                tags[i] = config->getString("tags.tag[" + std::to_string(i) + "]");
-
-            for (const std::string & config_tag : tags)
-            {
-                if (std::find(values.begin(), values.end(), config_tag) != values.end())
-                    remove_or_not = true;
-            }
-        }
-
-        if (filter_type == FilterType::Name)
-        {
-            remove_or_not = (std::find(values.begin(), values.end(), config->getString("name", "")) != values.end());
-        }
-
-        if (filter_type == FilterType::Name_regexp)
-        {
-            std::string config_name = config->getString("name", "");
-            auto regex_checker = [&config_name](const std::string & name_regexp)
-            {
-                std::regex pattern(name_regexp);
-                return std::regex_search(config_name, pattern);
-            };
-
-            remove_or_not = config->has("name") ? (std::find_if(values.begin(), values.end(), regex_checker) != values.end()) : false;
-        }
-
-        if (leave)
-            remove_or_not = !remove_or_not;
-        return remove_or_not;
-    };
-
-    auto new_end = std::remove_if(configs.begin(), configs.end(), checker);
-    configs.erase(new_end, configs.end());
-}
-
-}

dbms/programs/performance-test/ConfigPreprocessor.h

@@ -1,50 +0,0 @@
-#pragma once
-
-#include <Poco/DOM/Document.h>
-#include <Poco/Util/XMLConfiguration.h>
-#include <Core/Types.h>
-#include <vector>
-#include <string>
-
-namespace DB
-{
-
-using XMLConfiguration = Poco::Util::XMLConfiguration;
-using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
-using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;
-
-class ConfigPreprocessor
-{
-public:
-    ConfigPreprocessor(const Strings & paths_)
-        : paths(paths_)
-    {}
-
-    std::vector<XMLConfigurationPtr> processConfig(
-        const Strings & tests_tags,
-        const Strings & tests_names,
-        const Strings & tests_names_regexp,
-        const Strings & skip_tags,
-        const Strings & skip_names,
-        const Strings & skip_names_regexp) const;
-
-private:
-
-    enum class FilterType
-    {
-        Tag,
-        Name,
-        Name_regexp
-    };
-
-    /// Removes configurations that has a given value.
-    /// If leave is true, the logic is reversed.
-    void removeConfigurationsIf(
-        std::vector<XMLConfigurationPtr> & configs,
-        FilterType filter_type,
-        const Strings & values,
-        bool leave = false) const;
-
-    const Strings paths;
-};
-}

dbms/programs/performance-test/JSONString.cpp

@@ -1,66 +0,0 @@
-#include "JSONString.h"
-
-#include <regex>
-#include <sstream>
-namespace DB
-{
-
-namespace
-{
-std::string pad(size_t padding)
-{
-    return std::string(padding * 4, ' ');
-}
-
-const std::regex NEW_LINE{"\n"};
-}
-
-void JSONString::set(const std::string & key, std::string value, bool wrap)
-{
-    if (value.empty())
-        value = "null";
-
-    bool reserved = (value[0] == '[' || value[0] == '{' || value == "null");
-    if (!reserved && wrap)
-        value = '"' + std::regex_replace(value, NEW_LINE, "\\n") + '"';
-
-    content[key] = value;
-}
-
-void JSONString::set(const std::string & key, const std::vector<JSONString> & run_infos)
-{
-    std::ostringstream value;
-    value << "[\n";
-
-    for (size_t i = 0; i < run_infos.size(); ++i)
-    {
-        value << pad(padding + 1) + run_infos[i].asString(padding + 2);
-        if (i != run_infos.size() - 1)
-            value << ',';
-
-        value << "\n";
-    }
-
-    value << pad(padding) << ']';
-    content[key] = value.str();
-}
-
-std::string JSONString::asString(size_t cur_padding) const
-{
-    std::ostringstream repr;
-    repr << "{";
-
-    for (auto it = content.begin(); it != content.end(); ++it)
-    {
-        if (it != content.begin())
-            repr << ',';
-        /// construct "key": "value" string with padding
-        repr << "\n" << pad(cur_padding) << '"' << it->first << '"' << ": " << it->second;
-    }
-
-    repr << "\n" << pad(cur_padding - 1) << '}';
-    return repr.str();
-}
-
-
-}

dbms/programs/performance-test/JSONString.h

@@ -1,40 +0,0 @@
-#pragma once
-#include <Core/Types.h>
-
-#include <sys/stat.h>
-#include <type_traits>
-#include <vector>
-#include <map>
-
-namespace DB
-{
-
-/// NOTE The code is totally wrong.
-class JSONString
-{
-private:
-    std::map<std::string, std::string> content;
-    size_t padding;
-
-public:
-    explicit JSONString(size_t padding_ = 1) : padding(padding_) {}
-
-    void set(const std::string & key, std::string value, bool wrap = true);
-
-    template <typename T>
-    std::enable_if_t<is_arithmetic_v<T>> set(const std::string key, T value)
-    {
-        set(key, std::to_string(value), /*wrap= */ false);
-    }
-
-    void set(const std::string & key, const std::vector<JSONString> & run_infos);
-
-    std::string asString() const
-    {
-        return asString(padding);
-    }
-
-    std::string asString(size_t cur_padding) const;
-};
-
-}

dbms/programs/performance-test/PerformanceTest.cpp

@@ -1,340 +0,0 @@
-#include "PerformanceTest.h"
-
-#include <Core/Types.h>
-#include <Common/CpuId.h>
-#include <Common/quoteString.h>
-#include <common/getMemoryAmount.h>
-#include <DataStreams/copyData.h>
-#include <DataStreams/NullBlockOutputStream.h>
-#include <DataStreams/RemoteBlockInputStream.h>
-#include <IO/ConnectionTimeouts.h>
-#include <IO/ReadBufferFromFile.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteBufferFromFile.h>
-
-#include <filesystem>
-
-#include "executeQuery.h"
-
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-extern const int NOT_IMPLEMENTED;
-}
-
-namespace
-{
-void waitQuery(Connection & connection)
-{
-    bool finished = false;
-
-    while (true)
-    {
-        if (!connection.poll(1000000))
-            continue;
-
-        Packet packet = connection.receivePacket();
-        switch (packet.type)
-        {
-            case Protocol::Server::EndOfStream:
-                finished = true;
-                break;
-            case Protocol::Server::Exception:
-                throw Exception(*packet.exception);
-        }
-
-        if (finished)
-            break;
-    }
-}
-}
-
-namespace fs = std::filesystem;
-
-PerformanceTest::PerformanceTest(
-    const XMLConfigurationPtr & config_,
-    Connection & connection_,
-    const ConnectionTimeouts & timeouts_,
-    InterruptListener & interrupt_listener_,
-    const PerformanceTestInfo & test_info_,
-    Context & context_,
-    const std::vector<size_t> & queries_to_run_)
-    : config(config_)
-    , connection(connection_)
-    , timeouts(timeouts_)
-    , interrupt_listener(interrupt_listener_)
-    , test_info(test_info_)
-    , context(context_)
-    , queries_to_run(queries_to_run_)
-    , log(&Poco::Logger::get("PerformanceTest"))
-{
-}
-
-bool PerformanceTest::checkPreconditions() const
-{
-    if (!config->has("preconditions"))
-        return true;
-
-    Strings preconditions;
-    config->keys("preconditions", preconditions);
-    size_t table_precondition_index = 0;
-    size_t cpu_precondition_index = 0;
-
-    for (const std::string & precondition : preconditions)
-    {
-        if (precondition == "ram_size")
-        {
-            size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
-            size_t actual_ram = getMemoryAmount();
-            if (!actual_ram)
-                throw Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
-
-            if (ram_size_needed > actual_ram)
-            {
-                LOG_WARNING(log, "Not enough RAM: need = " << ram_size_needed << ", present = " << actual_ram);
-                return false;
-            }
-        }
-
-        if (precondition == "table_exists")
-        {
-            std::string precondition_key = "preconditions.table_exists[" + std::to_string(table_precondition_index++) + "]";
-            std::string table_to_check = config->getString(precondition_key);
-            std::string query = "EXISTS TABLE " + table_to_check + ";";
-
-            size_t exist = 0;
-
-            connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &test_info.settings, nullptr, false);
-
-            while (true)
-            {
-                Packet packet = connection.receivePacket();
-
-                if (packet.type == Protocol::Server::Data)
-                {
-                    for (const ColumnWithTypeAndName & column : packet.block)
-                    {
-                        if (column.name == "result" && !column.column->empty())
-                        {
-                            exist = column.column->get64(0);
-                            if (exist)
-                                break;
-                        }
-                    }
-                }
-
-                if (packet.type == Protocol::Server::Exception
-                    || packet.type == Protocol::Server::EndOfStream)
-                    break;
-            }
-
-            if (!exist)
-            {
-                LOG_WARNING(log, "Table " << backQuote(table_to_check) << " doesn't exist");
-                return false;
-            }
-        }
-
-        if (precondition == "cpu")
-        {
-            std::string precondition_key = "preconditions.cpu[" + std::to_string(cpu_precondition_index++) + "]";
-            std::string flag_to_check = config->getString(precondition_key);
-
-            #define CHECK_CPU_PRECONDITION(OP) \
-            if (flag_to_check == #OP) \
-            { \
-                if (!Cpu::CpuFlagsCache::have_##OP) \
-                { \
-                    LOG_WARNING(log, "CPU doesn't support " << #OP); \
-                    return false; \
-                } \
-            } else
-
-            CPU_ID_ENUMERATE(CHECK_CPU_PRECONDITION)
-            {
-                LOG_WARNING(log, "CPU doesn't support " << flag_to_check);
-                return false;
-            }
-
-            #undef CHECK_CPU_PRECONDITION
-        }
-    }
-
-    return true;
-}
-
-
-UInt64 PerformanceTest::calculateMaxExecTime() const
-{
-
-    UInt64 result = 0;
-    for (const auto & stop_conditions : test_info.stop_conditions_by_run)
-    {
-        UInt64 condition_max_time = stop_conditions.getMaxExecTime();
-        if (condition_max_time == 0)
-            return 0;
-        result += condition_max_time;
-    }
-    return result;
-}
-
-
-void PerformanceTest::prepare() const
-{
-    for (const auto & query : test_info.create_and_fill_queries)
-    {
-        LOG_INFO(log, "Executing create or fill query \"" << query << '\"');
-        connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &test_info.settings, nullptr, false);
-        waitQuery(connection);
-        LOG_INFO(log, "Query finished");
-    }
-
-}
-
-void PerformanceTest::finish() const
-{
-    for (const auto & query : test_info.drop_queries)
-    {
-        LOG_INFO(log, "Executing drop query \"" << query << '\"');
-        connection.sendQuery(timeouts, query, "", QueryProcessingStage::Complete, &test_info.settings, nullptr, false);
-        waitQuery(connection);
-        LOG_INFO(log, "Query finished");
-    }
-}
-
-std::vector<TestStats> PerformanceTest::execute()
-{
-    std::vector<TestStats> statistics_by_run;
-    size_t query_count;
-    if (queries_to_run.empty())
-        query_count = test_info.queries.size();
-    else
-        query_count = queries_to_run.size();
-    size_t total_runs = test_info.times_to_run * test_info.queries.size();
-    statistics_by_run.resize(total_runs);
-    LOG_INFO(log, "Totally will run cases " << test_info.times_to_run * query_count << " times");
-    UInt64 max_exec_time = calculateMaxExecTime();
-    if (max_exec_time != 0)
-        LOG_INFO(log, "Test will be executed for a maximum of " << max_exec_time / 1000. << " seconds");
-    else
-        LOG_INFO(log, "Test execution time cannot be determined");
-
-    for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
-    {
-        QueriesWithIndexes queries_with_indexes;
-
-        for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
-        {
-            if (queries_to_run.empty() || std::find(queries_to_run.begin(), queries_to_run.end(), query_index) != queries_to_run.end())
-            {
-                size_t statistic_index = number_of_launch * test_info.queries.size() + query_index;
-                queries_with_indexes.push_back({test_info.queries[query_index], statistic_index});
-            }
-            else
-                LOG_INFO(log, "Will skip query " << test_info.queries[query_index] << " by index");
-        }
-
-        if (got_SIGINT)
-            break;
-
-        runQueries(queries_with_indexes, statistics_by_run);
-    }
-
-    if (got_SIGINT)
-    {
-        return statistics_by_run;
-    }
-
-    // Pull memory usage data from query log. The log is normally filled in
-    // background, so we have to flush it synchronously here to see all the
-    // previous queries.
-    {
-        NullBlockOutputStream null_output(Block{});
-        RemoteBlockInputStream flush_log(connection, "system flush logs",
-            {} /* header */, context);
-        copyData(flush_log, null_output);
-    }
-
-    for (auto & statistics : statistics_by_run)
-    {
-        if (statistics.query_id.empty())
-        {
-            // We have statistics structs for skipped queries as well, so we
-            // have to filter them out.
-            continue;
-        }
-
-        // We run some test queries several times, specifying the same query id,
-        // so this query to the log may return several records. Choose the
-        // last one, because this is when the query performance has stabilized.
-        RemoteBlockInputStream log_reader(connection,
-            "select memory_usage, query_start_time from system.query_log "
-            "where type = 2 and query_id = '" + statistics.query_id + "' "
-            "order by query_start_time desc",
-            {} /* header */, context);
-
-        log_reader.readPrefix();
-        Block block = log_reader.read();
-        if (block.columns() == 0)
-        {
-            LOG_WARNING(log, "Query '" << statistics.query_id << "' is not found in query log.");
-            continue;
-        }
-
-        auto column = block.getByName("memory_usage").column;
-        statistics.memory_usage = column->get64(0);
-
-        log_reader.readSuffix();
-    }
-
-    return statistics_by_run;
-}
-
-void PerformanceTest::runQueries(
-    const QueriesWithIndexes & queries_with_indexes,
-    std::vector<TestStats> & statistics_by_run)
-{
-    for (const auto & [query, run_index] : queries_with_indexes)
-    {
-        LOG_INFO(log, "[" << run_index<< "] Run query '" << query << "'");
-        TestStopConditions & stop_conditions = test_info.stop_conditions_by_run[run_index];
-        TestStats & statistics = statistics_by_run[run_index];
-        statistics.startWatches();
-        try
-        {
-            LOG_INFO(log, "Will run query in loop");
-            for (size_t iteration = 0; !statistics.got_SIGINT; ++iteration)
-            {
-                stop_conditions.reportIterations(iteration);
-                if (stop_conditions.areFulfilled())
-                {
-                    LOG_INFO(log, "Stop conditions fulfilled");
-                    break;
-                }
-
-                executeQuery(connection, query, statistics, stop_conditions, interrupt_listener, context, test_info.settings);
-            }
-        }
-        catch (const Exception & e)
-        {
-            statistics.exception = "Code: " + std::to_string(e.code()) + ", e.displayText() = " + e.displayText();
-            LOG_WARNING(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText()
-                << ", Stack trace:\n\n" << e.getStackTraceString());
-        }
-
-        if (!statistics.got_SIGINT)
-            statistics.ready = true;
-        else
-        {
-            got_SIGINT = true;
-            LOG_INFO(log, "Got SIGINT, will terminate as soon as possible");
-            break;
-        }
-    }
-}
-
-
-}

dbms/programs/performance-test/PerformanceTest.h

@@ -1,62 +0,0 @@
-#pragma once
-
-#include <Client/Connection.h>
-#include <Common/InterruptListener.h>
-#include <common/logger_useful.h>
-#include <Poco/Util/XMLConfiguration.h>
-
-#include <IO/ConnectionTimeouts.h>
-#include "PerformanceTestInfo.h"
-
-namespace DB
-{
-
-using XMLConfiguration = Poco::Util::XMLConfiguration;
-using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
-using QueriesWithIndexes = std::vector<std::pair<std::string, size_t>>;
-
-class PerformanceTest
-{
-public:
-    PerformanceTest(
-        const XMLConfigurationPtr & config_,
-        Connection & connection_,
-        const ConnectionTimeouts & timeouts_,
-        InterruptListener & interrupt_listener_,
-        const PerformanceTestInfo & test_info_,
-        Context & context_,
-        const std::vector<size_t> & queries_to_run_);
-
-    bool checkPreconditions() const;
-    void prepare() const;
-    std::vector<TestStats> execute();
-    void finish() const;
-
-    bool checkSIGINT() const
-    {
-        return got_SIGINT;
-    }
-
-private:
-    void runQueries(
-        const QueriesWithIndexes & queries_with_indexes,
-        std::vector<TestStats> & statistics_by_run);
-
-    UInt64 calculateMaxExecTime() const;
-
-private:
-    XMLConfigurationPtr config;
-    Connection & connection;
-    const ConnectionTimeouts & timeouts;
-    InterruptListener & interrupt_listener;
-
-    PerformanceTestInfo test_info;
-    Context & context;
-
-    std::vector<size_t> queries_to_run;
-    Poco::Logger * log;
-
-    bool got_SIGINT = false;
-};
-
-}

dbms/programs/performance-test/PerformanceTestInfo.cpp

@@ -1,183 +0,0 @@
-#include "PerformanceTestInfo.h"
-#include <Common/getMultipleKeysFromConfig.h>
-#include <Common/SettingsChanges.h>
-#include <IO/ReadBufferFromFile.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteBufferFromFile.h>
-#include "applySubstitutions.h"
-#include <filesystem>
-#include <iostream>
-
-
-namespace DB
-{
-namespace ErrorCodes
-{
-extern const int BAD_ARGUMENTS;
-}
-
-namespace
-{
-
-void extractSettings(
-    const XMLConfigurationPtr & config,
-    const std::string & key,
-    const Strings & settings_list,
-    SettingsChanges & settings_to_apply)
-{
-    for (const std::string & setup : settings_list)
-    {
-        if (setup == "profile")
-            continue;
-
-        std::string value = config->getString(key + "." + setup);
-        if (value.empty())
-            value = "true";
-
-        settings_to_apply.emplace_back(SettingChange{setup, value});
-    }
-}
-
-}
-
-
-namespace fs = std::filesystem;
-
-PerformanceTestInfo::PerformanceTestInfo(
-    XMLConfigurationPtr config,
-    const Settings & global_settings_)
-    : settings(global_settings_)
-{
-    path = config->getString("path");
-    test_name = fs::path(path).stem().string();
-    applySettings(config);
-    extractQueries(config);
-    extractAuxiliaryQueries(config);
-    processSubstitutions(config);
-    getStopConditions(config);
-}
-
-void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
-{
-    if (config->has("settings"))
-    {
-        SettingsChanges settings_to_apply;
-        Strings config_settings;
-        config->keys("settings", config_settings);
-        extractSettings(config, "settings", config_settings, settings_to_apply);
-        settings.applyChanges(settings_to_apply);
-    }
-}
-
-void PerformanceTestInfo::extractQueries(XMLConfigurationPtr config)
-{
-    if (config->has("query"))
-        queries = getMultipleValuesFromConfig(*config, "", "query");
-
-    if (config->has("query_file"))
-    {
-        const std::string filename = config->getString("query_file");
-        if (filename.empty())
-            throw Exception("Empty file name", ErrorCodes::BAD_ARGUMENTS);
-
-        bool tsv = fs::path(filename).extension().string() == ".tsv";
-
-        ReadBufferFromFile query_file(filename);
-        std::string query;
-
-        if (tsv)
-        {
-            while (!query_file.eof())
-            {
-                readEscapedString(query, query_file);
-                assertChar('\n', query_file);
-                queries.push_back(query);
-            }
-        }
-        else
-        {
-            readStringUntilEOF(query, query_file);
-            queries.push_back(query);
-        }
-    }
-
-    if (queries.empty())
-        throw Exception("Did not find any query to execute: " + test_name,
-            ErrorCodes::BAD_ARGUMENTS);
-}
-
-void PerformanceTestInfo::processSubstitutions(XMLConfigurationPtr config)
-{
-    if (config->has("substitutions"))
-    {
-        /// Make "subconfig" of inner xml block
-        ConfigurationPtr substitutions_view(config->createView("substitutions"));
-        constructSubstitutions(substitutions_view, substitutions);
-
-        auto create_and_fill_queries_preformat = create_and_fill_queries;
-        create_and_fill_queries.clear();
-        for (const auto & query : create_and_fill_queries_preformat)
-        {
-            auto formatted = formatQueries(query, substitutions);
-            create_and_fill_queries.insert(create_and_fill_queries.end(), formatted.begin(), formatted.end());
-        }
-
-        auto queries_preformat = queries;
-        queries.clear();
-        for (const auto & query : queries_preformat)
-        {
-            auto formatted = formatQueries(query, substitutions);
-            queries.insert(queries.end(), formatted.begin(), formatted.end());
-        }
-
-        auto drop_queries_preformat = drop_queries;
-        drop_queries.clear();
-        for (const auto & query : drop_queries_preformat)
-        {
-            auto formatted = formatQueries(query, substitutions);
-            drop_queries.insert(drop_queries.end(), formatted.begin(), formatted.end());
-        }
-    }
-}
-
-
-void PerformanceTestInfo::getStopConditions(XMLConfigurationPtr config)
-{
-    TestStopConditions stop_conditions_template;
-    if (config->has("stop_conditions"))
-    {
-        ConfigurationPtr stop_conditions_config(config->createView("stop_conditions"));
-        stop_conditions_template.loadFromConfig(stop_conditions_config);
-    }
-
-    if (stop_conditions_template.empty())
-        throw Exception("No termination conditions were found in config",
-            ErrorCodes::BAD_ARGUMENTS);
-
-    times_to_run = config->getUInt("times_to_run", 1);
-
-    for (size_t i = 0; i < times_to_run * queries.size(); ++i)
-        stop_conditions_by_run.push_back(stop_conditions_template);
-
-}
-
-void PerformanceTestInfo::extractAuxiliaryQueries(XMLConfigurationPtr config)
-{
-    if (config->has("create_query"))
-    {
-        create_and_fill_queries = getMultipleValuesFromConfig(*config, "", "create_query");
-    }
-
-    if (config->has("fill_query"))
-    {
-        auto fill_queries = getMultipleValuesFromConfig(*config, "", "fill_query");
-        create_and_fill_queries.insert(create_and_fill_queries.end(), fill_queries.begin(), fill_queries.end());
-    }
-
-    if (config->has("drop_query"))
-    {
-        drop_queries = getMultipleValuesFromConfig(*config, "", "drop_query");
-    }
-}
-
-}

dbms/programs/performance-test/PerformanceTestInfo.h

@@ -1,48 +0,0 @@
-#pragma once
-#include <string>
-#include <vector>
-#include <map>
-#include <Core/Settings.h>
-#include <Poco/Util/XMLConfiguration.h>
-#include <Poco/AutoPtr.h>
-
-#include "StopConditionsSet.h"
-#include "TestStopConditions.h"
-#include "TestStats.h"
-
-namespace DB
-{
-
-using XMLConfiguration = Poco::Util::XMLConfiguration;
-using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
-using StringToVector = std::map<std::string, Strings>;
-
-/// Class containing all info to run performance test
-class PerformanceTestInfo
-{
-public:
-    PerformanceTestInfo(XMLConfigurationPtr config, const Settings & global_settings_);
-
-    std::string test_name;
-    std::string path;
-
-    Strings queries;
-
-    Settings settings;
-    StringToVector substitutions;
-    size_t times_to_run;
-
-    std::vector<TestStopConditions> stop_conditions_by_run;
-
-    Strings create_and_fill_queries;
-    Strings drop_queries;
-
-private:
-    void applySettings(XMLConfigurationPtr config);
-    void extractQueries(XMLConfigurationPtr config);
-    void processSubstitutions(XMLConfigurationPtr config);
-    void getStopConditions(XMLConfigurationPtr config);
-    void extractAuxiliaryQueries(XMLConfigurationPtr config);
-};
-
-}

dbms/programs/performance-test/PerformanceTestSuite.cpp

@@ -1,416 +0,0 @@
-#include <algorithm>
-#include <iostream>
-#include <limits>
-#include <regex>
-#include <thread>
-#include <memory>
-#include <filesystem>
-
-#include <unistd.h>
-#include <sys/stat.h>
-
-#include <boost/program_options.hpp>
-
-#include <Poco/AutoPtr.h>
-#include <Poco/ConsoleChannel.h>
-#include <Poco/FormattingChannel.h>
-#include <Poco/Logger.h>
-#include <Poco/Path.h>
-#include <Poco/PatternFormatter.h>
-#include <Poco/Util/XMLConfiguration.h>
-
-#include <common/logger_useful.h>
-#include <Client/Connection.h>
-#include <Core/Types.h>
-#include <Interpreters/Context.h>
-#include <IO/ConnectionTimeouts.h>
-#include <IO/UseSSL.h>
-#include <Core/Settings.h>
-#include <Common/Exception.h>
-#include <Common/InterruptListener.h>
-#include <Common/TerminalSize.h>
-
-#include "TestStopConditions.h"
-#include "TestStats.h"
-#include "ConfigPreprocessor.h"
-#include "PerformanceTest.h"
-#include "ReportBuilder.h"
-
-
-namespace fs = std::filesystem;
-namespace po = boost::program_options;
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-    extern const int FILE_DOESNT_EXIST;
-}
-
-/** Tests launcher for ClickHouse.
- * The tool walks through given or default folder in order to find files with
- * tests' descriptions and launches it.
- */
-class PerformanceTestSuite
-{
-public:
-
-    PerformanceTestSuite(const std::string & host_,
-        const UInt16 port_,
-        const bool secure_,
-        const std::string & default_database_,
-        const std::string & user_,
-        const std::string & password_,
-        const Settings & cmd_settings,
-        const bool lite_output_,
-        Strings && input_files_,
-        Strings && tests_tags_,
-        Strings && skip_tags_,
-        Strings && tests_names_,
-        Strings && skip_names_,
-        Strings && tests_names_regexp_,
-        Strings && skip_names_regexp_,
-        const std::unordered_map<std::string, std::vector<size_t>> query_indexes_,
-        const ConnectionTimeouts & timeouts_)
-        : connection(host_, port_, default_database_, user_,
-            password_, "performance-test", Protocol::Compression::Enable,
-            secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable)
-        , timeouts(timeouts_)
-        , tests_tags(std::move(tests_tags_))
-        , tests_names(std::move(tests_names_))
-        , tests_names_regexp(std::move(tests_names_regexp_))
-        , skip_tags(std::move(skip_tags_))
-        , skip_names(std::move(skip_names_))
-        , skip_names_regexp(std::move(skip_names_regexp_))
-        , query_indexes(query_indexes_)
-        , lite_output(lite_output_)
-        , input_files(input_files_)
-        , log(&Poco::Logger::get("PerformanceTestSuite"))
-    {
-        global_context.makeGlobalContext();
-        global_context.getSettingsRef().copyChangesFrom(cmd_settings);
-        if (input_files.empty())
-            throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS);
-    }
-
-    int run()
-    {
-        std::string name;
-        UInt64 version_major;
-        UInt64 version_minor;
-        UInt64 version_patch;
-        UInt64 version_revision;
-        connection.getServerVersion(timeouts, name, version_major, version_minor, version_patch, version_revision);
-
-        std::stringstream ss;
-        ss << version_major << "." << version_minor << "." << version_patch;
-        server_version = ss.str();
-
-        report_builder = std::make_shared<ReportBuilder>(server_version);
-
-        processTestsConfigurations(input_files);
-
-        return 0;
-    }
-
-private:
-    Connection connection;
-    const ConnectionTimeouts & timeouts;
-
-    const Strings & tests_tags;
-    const Strings & tests_names;
-    const Strings & tests_names_regexp;
-    const Strings & skip_tags;
-    const Strings & skip_names;
-    const Strings & skip_names_regexp;
-    std::unordered_map<std::string, std::vector<size_t>> query_indexes;
-
-    Context global_context = Context::createGlobal();
-    std::shared_ptr<ReportBuilder> report_builder;
-
-    std::string server_version;
-
-    InterruptListener interrupt_listener;
-
-    using XMLConfiguration = Poco::Util::XMLConfiguration;
-    using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
-
-    bool lite_output;
-
-    Strings input_files;
-    std::vector<XMLConfigurationPtr> tests_configurations;
-    Poco::Logger * log;
-
-    void processTestsConfigurations(const Strings & paths)
-    {
-        LOG_INFO(log, "Preparing test configurations");
-        ConfigPreprocessor config_prep(paths);
-        tests_configurations = config_prep.processConfig(
-            tests_tags,
-            tests_names,
-            tests_names_regexp,
-            skip_tags,
-            skip_names,
-            skip_names_regexp);
-
-        LOG_INFO(log, "Test configurations prepared");
-
-        if (!tests_configurations.empty())
-        {
-            Strings outputs;
-
-            for (auto & test_config : tests_configurations)
-            {
-                auto [output, signal] = runTest(test_config);
-                if (!output.empty())
-                {
-                    if (lite_output)
-                        std::cout << output;
-                    else
-                        outputs.push_back(output);
-                }
-                if (signal)
-                    break;
-            }
-
-            if (!lite_output && !outputs.empty())
-            {
-                std::cout << "[" << std::endl;
-
-                for (size_t i = 0; i != outputs.size(); ++i)
-                {
-                    std::cout << outputs[i];
-                    if (i != outputs.size() - 1)
-                        std::cout << ",";
-
-                    std::cout << std::endl;
-                }
-
-                std::cout << "]" << std::endl;
-            }
-        }
-    }
-
-    std::pair<std::string, bool> runTest(XMLConfigurationPtr & test_config)
-    {
-        PerformanceTestInfo info(test_config, global_context.getSettingsRef());
-        LOG_INFO(log, "Config for test '" << info.test_name << "' parsed");
-        PerformanceTest current(test_config, connection, timeouts, interrupt_listener, info, global_context, query_indexes[info.path]);
-
-        if (current.checkPreconditions())
-        {
-            LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fulfilled");
-            LOG_INFO(
-                log,
-                "Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries");
-            current.prepare();
-            LOG_INFO(log, "Prepared");
-            LOG_INFO(log, "Running test '" << info.test_name << "'");
-            auto result = current.execute();
-            LOG_INFO(log, "Test '" << info.test_name << "' finished");
-
-            LOG_INFO(log, "Running post run queries");
-            current.finish();
-            LOG_INFO(log, "Postqueries finished");
-            if (lite_output)
-                return {report_builder->buildCompactReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
-            else
-                return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
-        }
-        else
-            LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fulfilled, skip run");
-
-        return {"", current.checkSIGINT()};
-    }
-};
-
-}
-
-static void getFilesFromDir(const fs::path & dir, std::vector<std::string> & input_files, const bool recursive = false)
-{
-    Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
-    if (dir.extension().string() == ".xml")
-        LOG_WARNING(log, dir.string() + "' is a directory, but has .xml extension");
-
-    fs::directory_iterator end;
-    for (fs::directory_iterator it(dir); it != end; ++it)
-    {
-        const fs::path file = (*it);
-        if (recursive && fs::is_directory(file))
-            getFilesFromDir(file, input_files, recursive);
-        else if (!fs::is_directory(file) && file.extension().string() == ".xml")
-            input_files.push_back(file.string());
-    }
-}
-
-static std::vector<std::string> getInputFiles(const po::variables_map & options, Poco::Logger * log)
-{
-    std::vector<std::string> input_files;
-    bool recursive = options.count("recursive");
-
-    if (!options.count("input-files"))
-    {
-        LOG_INFO(log, "Trying to find test scenario files in the current folder...");
-        fs::path curr_dir(".");
-
-        getFilesFromDir(curr_dir, input_files, recursive);
-
-        if (input_files.empty())
-            throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
-    }
-    else
-    {
-        input_files = options["input-files"].as<std::vector<std::string>>();
-
-        std::vector<std::string> collected_files;
-        for (const std::string & filename : input_files)
-        {
-            fs::path file(filename);
-
-            if (!fs::exists(file))
-                throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
-
-            if (fs::is_directory(file))
-            {
-                getFilesFromDir(file, collected_files, recursive);
-            }
-            else
-            {
-                if (file.extension().string() != ".xml")
-                    throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
-                collected_files.push_back(filename);
-            }
-        }
-
-        input_files = std::move(collected_files);
-    }
-
-    LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files");
-    std::sort(input_files.begin(), input_files.end());
-    return input_files;
-}
-
-static std::unordered_map<std::string, std::vector<std::size_t>> getTestQueryIndexes(const po::basic_parsed_options<char> & parsed_opts)
-{
-    std::unordered_map<std::string, std::vector<std::size_t>> result;
-    const auto & options = parsed_opts.options;
-    if (options.empty())
-        return result;
-    for (size_t i = 0; i < options.size() - 1; ++i)
-    {
-        const auto & opt = options[i];
-        if (opt.string_key == "input-files")
-        {
-            if (options[i + 1].string_key == "query-indexes")
-            {
-                const std::string & test_path = Poco::Path(opt.value[0]).absolute().toString();
-                for (const auto & query_num_str : options[i + 1].value)
-                {
-                    size_t query_num = std::stoul(query_num_str);
-                    result[test_path].push_back(query_num);
-                }
-            }
-        }
-    }
-    return result;
-}
-
-#pragma GCC diagnostic ignored "-Wunused-function"
-#pragma GCC diagnostic ignored "-Wmissing-declarations"
-
-int mainEntryClickHousePerformanceTest(int argc, char ** argv)
-try
-{
-    using po::value;
-    using Strings = DB::Strings;
-
-    po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
-    desc.add_options()
-        ("help", "produce help message")
-        ("lite", "use lite version of output")
-        ("host,h", value<std::string>()->default_value("localhost"), "")
-        ("port", value<UInt16>()->default_value(9000), "")
-        ("secure,s", "Use TLS connection")
-        ("database", value<std::string>()->default_value("default"), "")
-        ("user", value<std::string>()->default_value("default"), "")
-        ("password", value<std::string>()->default_value(""), "")
-        ("log-level", value<std::string>()->default_value("information"), "Set log level")
-        ("tags", value<Strings>()->multitoken(), "Run only tests with tag")
-        ("skip-tags", value<Strings>()->multitoken(), "Do not run tests with tag")
-        ("names", value<Strings>()->multitoken(), "Run tests with specific name")
-        ("skip-names", value<Strings>()->multitoken(), "Do not run tests with name")
-        ("names-regexp", value<Strings>()->multitoken(), "Run tests with names matching regexp")
-        ("skip-names-regexp", value<Strings>()->multitoken(), "Do not run tests with names matching regexp")
-        ("input-files", value<Strings>()->multitoken(), "Input .xml files")
-        ("query-indexes", value<std::vector<size_t>>()->multitoken(), "Input query indexes")
-        ("recursive,r", "Recurse in directories to find all xml's")
-    ;
-
-    DB::Settings cmd_settings;
-    cmd_settings.addProgramOptions(desc);
-
-    po::options_description cmdline_options;
-    cmdline_options.add(desc);
-
-    po::variables_map options;
-    po::basic_parsed_options<char> parsed = po::command_line_parser(argc, argv).options(cmdline_options).run();
-    auto queries_with_indexes = getTestQueryIndexes(parsed);
-    po::store(parsed, options);
-
-    po::notify(options);
-
-    Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t"));
-    Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
-    Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_channel));
-
-    Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
-    Poco::Logger::root().setChannel(channel);
-
-    Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
-    if (options.count("help"))
-    {
-        std::cout << "Usage: " << argv[0] << " [options]\n";
-        std::cout << desc << "\n";
-        return 0;
-    }
-
-    Strings input_files = getInputFiles(options, log);
-
-    Strings tests_tags = options.count("tags") ? options["tags"].as<Strings>() : Strings({});
-    Strings skip_tags = options.count("skip-tags") ? options["skip-tags"].as<Strings>() : Strings({});
-    Strings tests_names = options.count("names") ? options["names"].as<Strings>() : Strings({});
-    Strings skip_names = options.count("skip-names") ? options["skip-names"].as<Strings>() : Strings({});
-    Strings tests_names_regexp = options.count("names-regexp") ? options["names-regexp"].as<Strings>() : Strings({});
-    Strings skip_names_regexp = options.count("skip-names-regexp") ? options["skip-names-regexp"].as<Strings>() : Strings({});
-
-    auto timeouts = DB::ConnectionTimeouts::getTCPTimeoutsWithoutFailover(DB::Settings());
-
-    DB::UseSSL use_ssl;
-
-    DB::PerformanceTestSuite performance_test_suite(
-        options["host"].as<std::string>(),
-        options["port"].as<UInt16>(),
-        options.count("secure"),
-        options["database"].as<std::string>(),
-        options["user"].as<std::string>(),
-        options["password"].as<std::string>(),
-        cmd_settings,
-        options.count("lite") > 0,
-        std::move(input_files),
-        std::move(tests_tags),
-        std::move(skip_tags),
-        std::move(tests_names),
-        std::move(skip_names),
-        std::move(tests_names_regexp),
-        std::move(skip_names_regexp),
-        queries_with_indexes,
-        timeouts);
-    return performance_test_suite.run();
-}
-catch (...)
-{
-    std::cout << DB::getCurrentExceptionMessage(/*with stacktrace = */ true) << std::endl;
-    int code = DB::getCurrentExceptionCode();
-    return code ? code : 1;
-}

dbms/programs/performance-test/ReportBuilder.cpp

@@ -1,193 +0,0 @@
-#include "ReportBuilder.h"
-
-#include <algorithm>
-#include <regex>
-#include <sstream>
-#include <thread>
-
-#include <Common/getNumberOfPhysicalCPUCores.h>
-#include <Common/getFQDNOrHostName.h>
-#include <common/getMemoryAmount.h>
-#include <Common/StringUtils/StringUtils.h>
-
-#include "JSONString.h"
-
-namespace DB
-{
-
-namespace
-{
-
-bool isASCIIString(const std::string & str)
-{
-    return std::all_of(str.begin(), str.end(), isASCII);
-}
-
-String jsonString(const String & str, FormatSettings & settings)
-{
-    WriteBufferFromOwnString buffer;
-    writeJSONString(str, buffer, settings);
-    return std::move(buffer.str());
-}
-}
-
-ReportBuilder::ReportBuilder(const std::string & server_version_)
-    : server_version(server_version_)
-    , hostname(getFQDNOrHostName())
-    , num_cores(getNumberOfPhysicalCPUCores())
-    , num_threads(std::thread::hardware_concurrency())
-    , ram(getMemoryAmount())
-{
-}
-
-std::string ReportBuilder::getCurrentTime() const
-{
-    return DateLUT::instance().timeToString(time(nullptr));
-}
-
-std::string ReportBuilder::buildFullReport(
-    const PerformanceTestInfo & test_info,
-    std::vector<TestStats> & stats,
-    const std::vector<std::size_t> & queries_to_run) const
-{
-    FormatSettings settings;
-
-    JSONString json_output;
-
-    json_output.set("hostname", hostname);
-    json_output.set("num_cores", num_cores);
-    json_output.set("num_threads", num_threads);
-    json_output.set("ram", ram);
-    json_output.set("server_version", server_version);
-    json_output.set("time", getCurrentTime());
-    json_output.set("test_name", test_info.test_name);
-    json_output.set("path", test_info.path);
-
-    if (!test_info.substitutions.empty())
-    {
-        JSONString json_parameters(2); /// here, 2 is the size of \t padding
-
-        for (auto & [parameter, values] : test_info.substitutions)
-        {
-            std::ostringstream array_string;
-            array_string << "[";
-            for (size_t i = 0; i != values.size(); ++i)
-            {
-                array_string << jsonString(values[i], settings);
-                if (i != values.size() - 1)
-                {
-                    array_string << ", ";
-                }
-            }
-            array_string << ']';
-
-            json_parameters.set(parameter, array_string.str());
-        }
-
-        json_output.set("parameters", json_parameters.asString());
-    }
-
-    std::vector<JSONString> run_infos;
-    for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
-    {
-        if (!queries_to_run.empty() && std::find(queries_to_run.begin(), queries_to_run.end(), query_index) == queries_to_run.end())
-            continue;
-
-        for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
-        {
-            size_t stat_index = number_of_launch * test_info.queries.size() + query_index;
-            TestStats & statistics = stats[stat_index];
-
-            if (!statistics.ready)
-                continue;
-
-            JSONString runJSON;
-
-            runJSON.set("query", jsonString(test_info.queries[query_index], settings), false);
-            runJSON.set("query_index", query_index);
-            if (!statistics.exception.empty())
-            {
-                if (isASCIIString(statistics.exception))
-                    runJSON.set("exception", jsonString(statistics.exception, settings), false);
-                else
-                    runJSON.set("exception", "Some exception occurred with non ASCII message. This may produce invalid JSON. Try reproduce locally.");
-            }
-
-            /// in seconds
-            runJSON.set("min_time", statistics.min_time / double(1000));
-
-            if (statistics.sampler.size() != 0)
-            {
-                JSONString quantiles(4); /// here, 4 is the size of \t padding
-                for (double percent = 10; percent <= 90; percent += 10)
-                {
-                    std::string quantile_key = std::to_string(percent / 100.0);
-                    while (quantile_key.back() == '0')
-                        quantile_key.pop_back();
-
-                    quantiles.set(quantile_key,
-                        statistics.sampler.quantileInterpolated(percent / 100.0));
-                }
-                quantiles.set("0.95",
-                    statistics.sampler.quantileInterpolated(95 / 100.0));
-                quantiles.set("0.99",
-                    statistics.sampler.quantileInterpolated(99 / 100.0));
-                quantiles.set("0.999",
-                    statistics.sampler.quantileInterpolated(99.9 / 100.0));
-                quantiles.set("0.9999",
-                    statistics.sampler.quantileInterpolated(99.99 / 100.0));
-
-                runJSON.set("quantiles", quantiles.asString());
-            }
-
-            runJSON.set("total_time", statistics.total_time);
-
-            if (statistics.total_time != 0)
-            {
-                runJSON.set("queries_per_second", static_cast<double>(statistics.queries) / statistics.total_time);
-                runJSON.set("rows_per_second", static_cast<double>(statistics.total_rows_read) / statistics.total_time);
-                runJSON.set("bytes_per_second", static_cast<double>(statistics.total_bytes_read) / statistics.total_time);
-            }
-
-            runJSON.set("memory_usage", statistics.memory_usage);
-
-            run_infos.push_back(runJSON);
-        }
-    }
-
-    json_output.set("runs", run_infos);
-
-    return json_output.asString();
-}
-
-std::string ReportBuilder::buildCompactReport(
-    const PerformanceTestInfo & test_info,
-    std::vector<TestStats> & stats,
-    const std::vector<std::size_t> & queries_to_run) const
-{
-    FormatSettings settings;
-    std::ostringstream output;
-
-    for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
-    {
-        if (!queries_to_run.empty() && std::find(queries_to_run.begin(), queries_to_run.end(), query_index) == queries_to_run.end())
-            continue;
-
-        for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
-        {
-            if (test_info.queries.size() > 1)
-                output << "query " << jsonString(test_info.queries[query_index], settings) << ", ";
-
-            output << "run " << std::to_string(number_of_launch + 1) << ": ";
-
-            std::string main_metric = "min_time";
-
-            output << main_metric << " = ";
-            size_t index = number_of_launch * test_info.queries.size() + query_index;
-            output << stats[index].getStatisticByName(main_metric);
-            output << "\n";
-        }
-    }
-    return output.str();
-}
-}

dbms/programs/performance-test/ReportBuilder.h

@@ -1,36 +0,0 @@
-#pragma once
-#include "PerformanceTestInfo.h"
-#include <vector>
-#include <string>
-
-namespace DB
-{
-
-class ReportBuilder
-{
-public:
-    ReportBuilder(const std::string & server_version_);
-    std::string buildFullReport(
-        const PerformanceTestInfo & test_info,
-        std::vector<TestStats> & stats,
-        const std::vector<std::size_t> & queries_to_run) const;
-
-
-    std::string buildCompactReport(
-        const PerformanceTestInfo & test_info,
-        std::vector<TestStats> & stats,
-        const std::vector<std::size_t> & queries_to_run) const;
-
-private:
-    std::string server_version;
-    std::string hostname;
-    size_t num_cores;
-    size_t num_threads;
-    size_t ram;
-
-private:
-    std::string getCurrentTime() const;
-
-};
-
-}

dbms/programs/performance-test/StopConditionsSet.cpp

@@ -1,60 +0,0 @@
-#include "StopConditionsSet.h"
-#include <Common/Exception.h>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-extern const int LOGICAL_ERROR;
-}
-
-void StopConditionsSet::loadFromConfig(const ConfigurationPtr & stop_conditions_view)
-{
-    Strings keys;
-    stop_conditions_view->keys(keys);
-
-    for (const std::string & key : keys)
-    {
-        if (key == "total_time_ms")
-            total_time_ms.value = stop_conditions_view->getUInt64(key);
-        else if (key == "rows_read")
-            rows_read.value = stop_conditions_view->getUInt64(key);
-        else if (key == "bytes_read_uncompressed")
-            bytes_read_uncompressed.value = stop_conditions_view->getUInt64(key);
-        else if (key == "iterations")
-            iterations.value = stop_conditions_view->getUInt64(key);
-        else if (key == "min_time_not_changing_for_ms")
-            min_time_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
-        else if (key == "max_speed_not_changing_for_ms")
-            max_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
-        else
-            throw Exception("Met unknown stop condition: " + key, ErrorCodes::LOGICAL_ERROR);
-
-        ++initialized_count;
-    }
-}
-
-void StopConditionsSet::reset()
-{
-    total_time_ms.fulfilled = false;
-    rows_read.fulfilled = false;
-    bytes_read_uncompressed.fulfilled = false;
-    iterations.fulfilled = false;
-    min_time_not_changing_for_ms.fulfilled = false;
-    max_speed_not_changing_for_ms.fulfilled = false;
-
-    fulfilled_count = 0;
-}
-
-void StopConditionsSet::report(UInt64 value, StopConditionsSet::StopCondition & condition)
-{
-    if (condition.value && !condition.fulfilled && value >= condition.value)
-    {
-        condition.fulfilled = true;
-        ++fulfilled_count;
-    }
-}
-
-
-}

dbms/programs/performance-test/StopConditionsSet.h

@@ -1,38 +0,0 @@
-#pragma once
-
-#include <Core/Types.h>
-#include <Poco/Util/XMLConfiguration.h>
-
-namespace DB
-{
-
-using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
-
-/// A set of supported stop conditions.
-struct StopConditionsSet
-{
-    void loadFromConfig(const ConfigurationPtr & stop_conditions_view);
-    void reset();
-
-    /// Note: only conditions with UInt64 minimal thresholds are supported.
-    /// I.e. condition is fulfilled when value is exceeded.
-    struct StopCondition
-    {
-        UInt64 value = 0;
-        bool fulfilled = false;
-    };
-
-    void report(UInt64 value, StopCondition & condition);
-
-    StopCondition total_time_ms;
-    StopCondition rows_read;
-    StopCondition bytes_read_uncompressed;
-    StopCondition iterations;
-    StopCondition min_time_not_changing_for_ms;
-    StopCondition max_speed_not_changing_for_ms;
-
-    size_t initialized_count = 0;
-    size_t fulfilled_count = 0;
-};
-
-}

dbms/programs/performance-test/TestStats.cpp

@@ -1,102 +0,0 @@
-#include "TestStats.h"
-#include <algorithm>
-namespace DB
-{
-
-namespace
-{
-const std::string FOUR_SPACES = "    ";
-}
-
-std::string TestStats::getStatisticByName(const std::string & statistic_name)
-{
-    if (statistic_name == "min_time")
-        return std::to_string(min_time) + "ms";
-
-    if (statistic_name == "quantiles")
-    {
-        std::string result = "\n";
-
-        for (double percent = 10; percent <= 90; percent += 10)
-        {
-            result += FOUR_SPACES + std::to_string((percent / 100));
-            result += ": " + std::to_string(sampler.quantileInterpolated(percent / 100.0));
-            result += "\n";
-        }
-        result += FOUR_SPACES + "0.95:   " + std::to_string(sampler.quantileInterpolated(95 / 100.0)) + "\n";
-        result += FOUR_SPACES + "0.99: " + std::to_string(sampler.quantileInterpolated(99 / 100.0)) + "\n";
-        result += FOUR_SPACES + "0.999: " + std::to_string(sampler.quantileInterpolated(99.9 / 100.)) + "\n";
-        result += FOUR_SPACES + "0.9999: " + std::to_string(sampler.quantileInterpolated(99.99 / 100.));
-
-        return result;
-    }
-    if (statistic_name == "total_time")
-        return std::to_string(total_time) + "s";
-
-    if (statistic_name == "queries_per_second")
-        return std::to_string(queries / total_time);
-
-    if (statistic_name == "rows_per_second")
-        return std::to_string(total_rows_read / total_time);
-
-    if (statistic_name == "bytes_per_second")
-        return std::to_string(total_bytes_read / total_time);
-
-    if (statistic_name == "max_rows_per_second")
-        return std::to_string(max_rows_speed);
-
-    if (statistic_name == "max_bytes_per_second")
-        return std::to_string(max_bytes_speed);
-
-    if (statistic_name == "avg_rows_per_second")
-        return std::to_string(avg_rows_speed_value);
-
-    if (statistic_name == "avg_bytes_per_second")
-        return std::to_string(avg_bytes_speed_value);
-
-    return "";
-}
-
-
-void TestStats::update_min_time(UInt64 min_time_candidate)
-{
-    if (min_time_candidate < min_time)
-    {
-        min_time = min_time_candidate;
-        min_time_watch.restart();
-    }
-}
-
-
-void TestStats::add(size_t rows_read_inc, size_t bytes_read_inc)
-{
-    total_rows_read += rows_read_inc;
-    total_bytes_read += bytes_read_inc;
-    last_query_rows_read += rows_read_inc;
-    last_query_bytes_read += bytes_read_inc;
-}
-
-void TestStats::updateQueryInfo()
-{
-    ++queries;
-    sampler.insert(watch_per_query.elapsedSeconds());
-    update_min_time(watch_per_query.elapsed() / (1000 * 1000)); /// ns to ms
-}
-
-
-TestStats::TestStats()
-{
-    watch.reset();
-    watch_per_query.reset();
-    min_time_watch.reset();
-}
-
-
-void TestStats::startWatches()
-{
-    watch.start();
-    watch_per_query.start();
-    min_time_watch.start();
-}
-
-}

dbms/programs/performance-test/TestStats.h

@@ -1,73 +0,0 @@
-#pragma once
-
-#include <Core/Types.h>
-#include <limits>
-#include <Common/Stopwatch.h>
-#include <AggregateFunctions/ReservoirSampler.h>
-
-namespace DB
-{
-struct TestStats
-{
-    TestStats();
-    Stopwatch watch;
-    Stopwatch watch_per_query;
-    Stopwatch min_time_watch;
-
-    bool last_query_was_cancelled = false;
-    std::string query_id;
-
-    size_t queries = 0;
-
-    size_t total_rows_read = 0;
-    size_t total_bytes_read = 0;
-
-    size_t last_query_rows_read = 0;
-    size_t last_query_bytes_read = 0;
-
-    using Sampler = ReservoirSampler<double>;
-    Sampler sampler{1 << 16};
-
-    /// min_time in ms
-    UInt64 min_time = std::numeric_limits<UInt64>::max();
-    double total_time = 0;
-
-    UInt64 max_rows_speed = 0;
-    UInt64 max_bytes_speed = 0;
-
-    double avg_rows_speed_value = 0;
-    double avg_rows_speed_first = 0;
-    static inline double avg_rows_speed_precision = 0.005;
-
-    double avg_bytes_speed_value = 0;
-    double avg_bytes_speed_first = 0;
-    static inline double avg_bytes_speed_precision = 0.005;
-
-    size_t number_of_rows_speed_info_batches = 0;
-    size_t number_of_bytes_speed_info_batches = 0;
-
-    UInt64 memory_usage = 0;
-
-    bool ready = false; // check if a query wasn't interrupted by SIGINT
-    std::string exception;
-
-    /// Hack, actually this field doesn't required for statistics
-    bool got_SIGINT = false;
-
-    std::string getStatisticByName(const std::string & statistic_name);
-
-    void update_min_time(UInt64 min_time_candidate);
-
-    void add(size_t rows_read_inc, size_t bytes_read_inc);
-
-    void updateQueryInfo();
-
-    void setTotalTime()
-    {
-        total_time = watch.elapsedSeconds();
-    }
-
-    void startWatches();
-};
-
-}

dbms/programs/performance-test/TestStopConditions.cpp

@@ -1,38 +0,0 @@
-#include "TestStopConditions.h"
-
-namespace DB
-{
-
-void TestStopConditions::loadFromConfig(ConfigurationPtr & stop_conditions_config)
-{
-    if (stop_conditions_config->has("all_of"))
-    {
-        ConfigurationPtr config_all_of(stop_conditions_config->createView("all_of"));
-        conditions_all_of.loadFromConfig(config_all_of);
-    }
-    if (stop_conditions_config->has("any_of"))
-    {
-        ConfigurationPtr config_any_of(stop_conditions_config->createView("any_of"));
-        conditions_any_of.loadFromConfig(config_any_of);
-    }
-}
-
-bool TestStopConditions::areFulfilled() const
-{
-    return (conditions_all_of.initialized_count && conditions_all_of.fulfilled_count >= conditions_all_of.initialized_count)
-        || (conditions_any_of.initialized_count && conditions_any_of.fulfilled_count);
-}
-
-UInt64 TestStopConditions::getMaxExecTime() const
-{
-    UInt64 all_of_time = conditions_all_of.total_time_ms.value;
-    if (all_of_time == 0 && conditions_all_of.initialized_count != 0) /// max time is not set in all conditions
-        return 0;
-    else if(all_of_time != 0 && conditions_all_of.initialized_count > 1) /// max time is set, but we have other conditions
-        return 0;
-
-    UInt64 any_of_time = conditions_any_of.total_time_ms.value;
-    return std::max(all_of_time, any_of_time);
-}
-
-}

dbms/programs/performance-test/TestStopConditions.h

@@ -1,55 +0,0 @@
-#pragma once
-#include "StopConditionsSet.h"
-#include <Poco/Util/XMLConfiguration.h>
-
-namespace DB
-{
-/// Stop conditions for a test run. The running test will be terminated in either of two conditions:
-/// 1. All conditions marked 'all_of' are fulfilled
-/// or
-/// 2. Any condition  marked 'any_of' is  fulfilled
-
-using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
-
-class TestStopConditions
-{
-public:
-    void loadFromConfig(ConfigurationPtr & stop_conditions_config);
-    inline bool empty() const
-    {
-        return !conditions_all_of.initialized_count && !conditions_any_of.initialized_count;
-    }
-
-#define DEFINE_REPORT_FUNC(FUNC_NAME, CONDITION)                      \
-    void FUNC_NAME(UInt64 value)                                      \
-    {                                                                 \
-        conditions_all_of.report(value, conditions_all_of.CONDITION); \
-        conditions_any_of.report(value, conditions_any_of.CONDITION); \
-    }
-
-    DEFINE_REPORT_FUNC(reportTotalTime, total_time_ms)
-    DEFINE_REPORT_FUNC(reportRowsRead, rows_read)
-    DEFINE_REPORT_FUNC(reportBytesReadUncompressed, bytes_read_uncompressed)
-    DEFINE_REPORT_FUNC(reportIterations, iterations)
-    DEFINE_REPORT_FUNC(reportMinTimeNotChangingFor, min_time_not_changing_for_ms)
-
-#undef REPORT
-
-    bool areFulfilled() const;
-
-    void reset()
-    {
-        conditions_all_of.reset();
-        conditions_any_of.reset();
-    }
-
-    /// Return max exec time for these conditions
-    /// Return zero if max time cannot be determined
-    UInt64 getMaxExecTime() const;
-
-private:
-    StopConditionsSet conditions_all_of;
-    StopConditionsSet conditions_any_of;
-};
-
-}

dbms/programs/performance-test/applySubstitutions.cpp

@@ -1,82 +0,0 @@
-#include "applySubstitutions.h"
-#include <algorithm>
-#include <vector>
-
-namespace DB
-{
-
-void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & out_substitutions)
-{
-    Strings xml_substitutions;
-    substitutions_view->keys(xml_substitutions);
-
-    for (size_t i = 0; i != xml_substitutions.size(); ++i)
-    {
-        const ConfigurationPtr xml_substitution(substitutions_view->createView("substitution[" + std::to_string(i) + "]"));
-
-        /// Property values for substitution will be stored in a vector
-        /// accessible by property name
-        Strings xml_values;
-        xml_substitution->keys("values", xml_values);
-
-        std::string name = xml_substitution->getString("name");
-
-        for (size_t j = 0; j != xml_values.size(); ++j)
-        {
-            out_substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]"));
-        }
-    }
-}
-
-/// Recursive method which goes through all substitution blocks in xml
-/// and replaces property {names} by their values
-static void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left,
-    StringToVector::iterator substitutions_right,
-    const std::string & template_query,
-    Strings & out_queries)
-{
-    if (substitutions_left == substitutions_right)
-    {
-        out_queries.push_back(template_query); /// completely substituted query
-        return;
-    }
-
-    std::string substitution_mask = "{" + substitutions_left->first + "}";
-
-    if (template_query.find(substitution_mask) == std::string::npos) /// nothing to substitute here
-    {
-        runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, out_queries);
-        return;
-    }
-
-    for (const std::string & value : substitutions_left->second)
-    {
-        /// Copy query string for each unique permutation
-        std::string query = template_query;
-        size_t substr_pos = 0;
-
-        while (substr_pos != std::string::npos)
-        {
-            substr_pos = query.find(substitution_mask);
-
-            if (substr_pos != std::string::npos)
-                query.replace(substr_pos, substitution_mask.length(), value);
-        }
-
-        runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, out_queries);
-    }
-}
-
-Strings formatQueries(const std::string & query, StringToVector substitutions_to_generate)
-{
-    Strings queries_res;
-    runThroughAllOptionsAndPush(
-        substitutions_to_generate.begin(),
-        substitutions_to_generate.end(),
-        query,
-        queries_res);
-    return queries_res;
-}
-
-
-}

dbms/programs/performance-test/applySubstitutions.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include <Poco/Util/XMLConfiguration.h>
-#include <Core/Types.h>
-#include <vector>
-#include <string>
-#include <map>
-
-namespace DB
-{
-
-using StringToVector = std::map<std::string, Strings>;
-using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
-
-void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & out_substitutions);
-
-Strings formatQueries(const std::string & query, StringToVector substitutions_to_generate);
-
-}

dbms/programs/performance-test/clickhouse-performance-test.cpp

@@ -1,2 +0,0 @@
-int mainEntryClickHousePerformanceTest(int argc, char ** argv);
-int main(int argc_, char ** argv_) { return mainEntryClickHousePerformanceTest(argc_, argv_); }

dbms/programs/performance-test/executeQuery.cpp

@@ -1,80 +0,0 @@
-#include "executeQuery.h"
-#include <IO/Progress.h>
-#include <DataStreams/RemoteBlockInputStream.h>
-#include <Core/Block.h>
-#include <Poco/UUIDGenerator.h>
-
-namespace DB
-{
-
-namespace
-{
-
-void checkFulfilledConditionsAndUpdate(
-    const Progress & progress, RemoteBlockInputStream & stream,
-    TestStats & statistics, TestStopConditions & stop_conditions,
-    InterruptListener & interrupt_listener)
-{
-    statistics.add(progress.read_rows, progress.read_bytes);
-
-    stop_conditions.reportRowsRead(statistics.total_rows_read);
-    stop_conditions.reportBytesReadUncompressed(statistics.total_bytes_read);
-    stop_conditions.reportTotalTime(statistics.watch.elapsed() / (1000 * 1000));
-    stop_conditions.reportMinTimeNotChangingFor(statistics.min_time_watch.elapsed() / (1000 * 1000));
-
-    if (stop_conditions.areFulfilled())
-    {
-        statistics.last_query_was_cancelled = true;
-        stream.cancel(false);
-    }
-
-    if (interrupt_listener.check())
-    {
-        statistics.got_SIGINT = true;
-        statistics.last_query_was_cancelled = true;
-        stream.cancel(false);
-    }
-}
-
-} // anonymous namespace
-
-void executeQuery(
-    Connection & connection,
-    const std::string & query,
-    TestStats & statistics,
-    TestStopConditions & stop_conditions,
-    InterruptListener & interrupt_listener,
-    Context & context,
-    const Settings & settings)
-{
-    static const std::string query_id_prefix
-        = Poco::UUIDGenerator::defaultGenerator().create().toString() + "-";
-    static int next_query_id = 1;
-
-    statistics.watch_per_query.restart();
-    statistics.last_query_was_cancelled = false;
-    statistics.last_query_rows_read = 0;
-    statistics.last_query_bytes_read = 0;
-    statistics.query_id = query_id_prefix + std::to_string(next_query_id++);
-
-    RemoteBlockInputStream stream(connection, query, {}, context, &settings);
-    stream.setQueryId(statistics.query_id);
-
-    stream.setProgressCallback(
-        [&](const Progress & value)
-        {
-            checkFulfilledConditionsAndUpdate(
-                value, stream, statistics,
-                stop_conditions, interrupt_listener);
-        });
-    stream.readPrefix();
-    while (Block block = stream.read());
-    stream.readSuffix();
-
-    if (!statistics.last_query_was_cancelled)
-        statistics.updateQueryInfo();
-
-    statistics.setTotalTime();
-}
-
-}

dbms/programs/performance-test/executeQuery.h

@@ -1,20 +0,0 @@
-#pragma once
-#include <string>
-#include "TestStats.h"
-#include "TestStopConditions.h"
-#include <Common/InterruptListener.h>
-#include <Interpreters/Context.h>
-#include <Core/Settings.h>
-#include <Client/Connection.h>
-
-namespace DB
-{
-void executeQuery(
-    Connection & connection,
-    const std::string & query,
-    TestStats & statistics,
-    TestStopConditions & stop_conditions,
-    InterruptListener & interrupt_listener,
-    Context & context,
-    const Settings & settings);
-}

dbms/programs/server/HTTPHandler.cpp

@@ -12,7 +12,7 @@
 #include <Core/ExternalTable.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
-#include <Common/getFQDNOrHostName.h>
+#include <common/getFQDNOrHostName.h>
 #include <Common/CurrentThread.h>
 #include <Common/setThreadName.h>
 #include <Common/config.h>
@@ -439,13 +439,13 @@ void HTTPHandler::processQuery(
 
     /// In theory if initially readonly = 0, the client can change any setting and then set readonly
     /// to some other value.
-    auto & settings = context.getSettingsRef();
+    const auto & settings = context.getSettingsRef();
 
     /// Only readonly queries are allowed for HTTP GET requests.
     if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_GET)
     {
         if (settings.readonly == 0)
-            settings.readonly = 2;
+            context.setSetting("readonly", 2);
     }
 
     bool has_external_data = startsWith(request.getContentType(), "multipart/form-data");
@@ -546,7 +546,7 @@ void HTTPHandler::processQuery(
     client_info.http_method = http_method;
     client_info.http_user_agent = request.get("User-Agent", "");
 
-    auto appendCallback = [&context] (ProgressCallback callback)
+    auto append_callback = [&context] (ProgressCallback callback)
     {
         auto prev = context.getProgressCallback();
 
@@ -561,13 +561,13 @@ void HTTPHandler::processQuery(
 
     /// While still no data has been sent, we will report about query execution progress by sending HTTP headers.
     if (settings.send_progress_in_http_headers)
-        appendCallback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); });
+        append_callback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); });
 
     if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close)
     {
         Poco::Net::StreamSocket & socket = dynamic_cast<Poco::Net::HTTPServerRequestImpl &>(request).socket();
 
-        appendCallback([&context, &socket](const Progress &)
+        append_callback([&context, &socket](const Progress &)
         {
             /// Assume that at the point this method is called no one is reading data from the socket any more.
             /// True for read-only queries.