diff --git a/.gitmodules b/.gitmodules index 37b22527eb4..74d1049ce01 100644 --- a/.gitmodules +++ b/.gitmodules @@ -246,3 +246,6 @@ [submodule "contrib/bzip2"] path = contrib/bzip2 url = https://github.com/ClickHouse-Extras/bzip2.git +[submodule "contrib/magic_enum"] + path = contrib/magic_enum + url = https://github.com/Neargye/magic_enum diff --git a/CMakeLists.txt b/CMakeLists.txt index e0fa6d8e197..9c8903d853c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,7 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") endif () +include (cmake/check_flags.cmake) include (cmake/add_warning.cmake) if (NOT MSVC) @@ -166,7 +167,8 @@ if (COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges") endif () - if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0) + if (HAS_USE_CTOR_HOMING) + # For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") @@ -192,7 +194,7 @@ endif () # Make sure the final executable has symbols exported set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") -find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") +find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") if (NOT OBJCOPY_PATH AND OS_DARWIN) find_program (BREW_PATH NAMES "brew") @@ -379,7 +381,7 @@ if (COMPILER_CLANG) endif () # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled - find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") + find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") if (LLVM_AR_PATH) message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.") @@ -388,7 +390,7 @@ if (COMPILER_CLANG) message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.") endif () - find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") + find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") if (LLVM_RANLIB_PATH) message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.") @@ -629,9 +631,6 @@ include_directories(${ConfigIncludePath}) # Add as many warnings as possible for our own code. include (cmake/warnings.cmake) -# Check if needed compiler flags are supported -include (cmake/check_flags.cmake) - add_subdirectory (base) add_subdirectory (src) add_subdirectory (programs) diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index e5e18669ebe..202a2dff56f 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -85,6 +85,7 @@ target_link_libraries (common replxx cctz fmt + magic_enum ) if (ENABLE_TESTS) diff --git a/base/common/Decimal.h b/base/common/Decimal.h new file mode 100644 index 00000000000..e856abf23c7 --- /dev/null +++ b/base/common/Decimal.h @@ -0,0 +1,157 @@ +#pragma once +#include "common/extended_types.h" + +#if !defined(NO_SANITIZE_UNDEFINED) +#if defined(__clang__) + #define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) +#else + #define NO_SANITIZE_UNDEFINED +#endif +#endif + +namespace DB +{ +template struct Decimal; +class DateTime64; + +using Decimal32 = Decimal; +using Decimal64 = Decimal; +using Decimal128 = Decimal; +using Decimal256 = Decimal; + +template +concept is_decimal = + std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; + +template +concept is_over_big_int = + std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; + +template struct NativeTypeT { using Type = T; }; +template struct NativeTypeT { using Type = typename T::NativeType; }; +template using NativeType = typename NativeTypeT::Type; + +/// Own FieldType for Decimal. +/// It is only a "storage" for decimal. +/// To perform operations, you also have to provide a scale (number of digits after point). +template +struct Decimal +{ + using NativeType = T; + + constexpr Decimal() = default; + constexpr Decimal(Decimal &&) = default; + constexpr Decimal(const Decimal &) = default; + + constexpr Decimal(const T & value_): value(value_) {} + + template + constexpr Decimal(const Decimal & x): value(x.value) {} + + constexpr Decimal & operator = (Decimal &&) = default; + constexpr Decimal & operator = (const Decimal &) = default; + + constexpr operator T () const { return value; } + + template + constexpr U convertTo() const + { + if constexpr (is_decimal) + return convertTo(); + else + return static_cast(value); + } + + const Decimal & operator += (const T & x) { value += x; return *this; } + const Decimal & operator -= (const T & x) { value -= x; return *this; } + const Decimal & operator *= (const T & x) { value *= x; return *this; } + const Decimal & operator /= (const T & x) { value /= x; return *this; } + const Decimal & operator %= (const T & x) { value %= x; return *this; } + + template const Decimal & operator += (const Decimal & x) { value += x.value; return *this; } + template const Decimal & operator -= (const Decimal & x) { value -= x.value; return *this; } + template const Decimal & operator *= (const Decimal & x) { value *= x.value; return *this; } + template const Decimal & operator /= (const Decimal & x) { value /= x.value; return *this; } + template const Decimal & operator %= (const Decimal & x) { value %= x.value; return *this; } + + /// This is to avoid UB for sumWithOverflow() + void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; } + + T value; +}; + +template inline bool operator< (const Decimal & x, const Decimal & y) { return x.value < y.value; } +template inline bool operator> (const Decimal & x, const Decimal & y) { return x.value > y.value; } +template inline bool operator<= (const Decimal & x, const Decimal & y) { return x.value <= y.value; } +template inline bool operator>= (const Decimal & x, const Decimal & y) { return x.value >= y.value; } +template inline bool operator== (const Decimal & x, const Decimal & y) { return x.value == y.value; } +template inline bool operator!= (const Decimal & x, const Decimal & y) { return x.value != y.value; } + +template inline Decimal operator+ (const Decimal & x, const Decimal & y) { return x.value + y.value; } +template inline Decimal operator- (const Decimal & x, const Decimal & y) { return x.value - y.value; } +template inline Decimal operator* (const Decimal & x, const Decimal & y) { return x.value * y.value; } +template inline Decimal operator/ (const Decimal & x, const Decimal & y) { return x.value / y.value; } +template inline Decimal operator- (const Decimal & x) { return -x.value; } + +/// Distinguishable type to allow function resolution/deduction based on value type, +/// but also relatively easy to convert to/from Decimal64. +class DateTime64 : public Decimal64 +{ +public: + using Base = Decimal64; + using Base::Base; + using NativeType = Base::NativeType; + + constexpr DateTime64(const Base & v): Base(v) {} +}; +} + +constexpr DB::UInt64 max_uint_mask = std::numeric_limits::max(); + +namespace std +{ + template + struct hash> + { + size_t operator()(const DB::Decimal & x) const { return hash()(x.value); } + }; + + template <> + struct hash + { + size_t operator()(const DB::Decimal128 & x) const + { + return std::hash()(x.value >> 64) + ^ std::hash()(x.value & max_uint_mask); + } + }; + + template <> + struct hash + { + size_t operator()(const DB::DateTime64 & x) const + { + return std::hash()(x); + } + }; + + template <> + struct hash + { + size_t operator()(const DB::Decimal256 & x) const + { + // FIXME temp solution + return std::hash()(static_cast(x.value >> 64 & max_uint_mask)) + ^ std::hash()(static_cast(x.value & max_uint_mask)); + } + }; +} diff --git a/base/common/EnumReflection.h b/base/common/EnumReflection.h new file mode 100644 index 00000000000..0d1f8ae0a40 --- /dev/null +++ b/base/common/EnumReflection.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +template concept is_enum = std::is_enum_v; + +namespace detail +{ +template +constexpr void static_for(F && f, std::index_sequence) +{ + (std::forward(f)(std::integral_constant(I)>()) , ...); +} +} + +/** + * Iterate over enum values in compile-time (compile-time switch/case, loop unrolling). + * + * @example static_for([](auto enum_value) { return template_func(); } + * ^ enum_value can be used as a template parameter + */ +template +constexpr void static_for(F && f) +{ + constexpr size_t count = magic_enum::enum_count(); + detail::static_for(std::forward(f), std::make_index_sequence()); +} + +/// Enable printing enum values as strings via fmt + magic_enum +template +struct fmt::formatter : fmt::formatter +{ + constexpr auto format(T value, auto& format_context) + { + return formatter::format(magic_enum::enum_name(value), format_context); + } +}; diff --git a/base/common/LineReader.cpp b/base/common/LineReader.cpp index a32906dd5a5..7dc37322bb0 100644 --- a/base/common/LineReader.cpp +++ b/base/common/LineReader.cpp @@ -16,6 +16,10 @@ extern "C" } #endif +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + namespace { diff --git a/base/common/extended_types.h b/base/common/extended_types.h index 79209568ef5..cbd5688c491 100644 --- a/base/common/extended_types.h +++ b/base/common/extended_types.h @@ -41,22 +41,14 @@ template <> struct is_unsigned { static constexpr bool value = true; }; template inline constexpr bool is_unsigned_v = is_unsigned::value; +template concept is_integer = + std::is_integral_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; -/// TODO: is_integral includes char, char8_t and wchar_t. -template -struct is_integer -{ - static constexpr bool value = std::is_integral_v; -}; - -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; - -template -inline constexpr bool is_integer_v = is_integer::value; - +template concept is_floating_point = std::is_floating_point_v; template struct is_arithmetic diff --git a/base/common/find_symbols.h b/base/common/find_symbols.h index a5921b813a1..b28749afda6 100644 --- a/base/common/find_symbols.h +++ b/base/common/find_symbols.h @@ -36,18 +36,7 @@ namespace detail { - -template -inline bool is_in(char x) -{ - return x == s0; -} - -template -inline bool is_in(char x) -{ - return x == s0 || is_in(x); -} +template constexpr bool is_in(char x) { return ((x == chars) || ...); } #if defined(__SSE2__) template @@ -67,16 +56,10 @@ inline __m128i mm_is_in(__m128i bytes) #endif template -bool maybe_negate(bool x) -{ - if constexpr (positive) - return x; - else - return !x; -} +constexpr bool maybe_negate(bool x) { return x == positive; } template -uint16_t maybe_negate(uint16_t x) +constexpr uint16_t maybe_negate(uint16_t x) { if constexpr (positive) return x; @@ -149,12 +132,13 @@ template -inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end) +inline const char * find_first_symbols_sse42(const char * const begin, const char * const end) { const char * pos = begin; #if defined(__SSE4_2__) -#define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT) + constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; + __m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16); for (; pos + 15 < end; pos += 16) @@ -163,16 +147,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons if constexpr (positive) { - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) - return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + if (_mm_cmpestrc(set, num_chars, bytes, 16, mode)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode); } else { - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY)) - return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY); + if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY); } } -#undef MODE #endif for (; pos < end; ++pos) @@ -197,20 +180,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons } -template -inline const char * find_first_symbols_sse42(const char * begin, const char * end) -{ - return find_first_symbols_sse42_impl(begin, end); -} - /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do. template inline const char * find_first_symbols_dispatch(const char * begin, const char * end) + requires(0 <= sizeof...(symbols) && sizeof...(symbols) <= 16) { #if defined(__SSE4_2__) if (sizeof...(symbols) >= 5) - return find_first_symbols_sse42(begin, end); + return find_first_symbols_sse42(begin, end); else #endif return find_first_symbols_sse2(begin, end); diff --git a/base/common/phdr_cache.cpp b/base/common/phdr_cache.cpp index 49d566dac19..8ca9137cd29 100644 --- a/base/common/phdr_cache.cpp +++ b/base/common/phdr_cache.cpp @@ -1,3 +1,7 @@ +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + /// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex. #include diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index a1e2b253aa7..0c2e9ca7e8e 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -15,15 +15,15 @@ private: public: using UnderlyingType = T; template ::type> - explicit StrongTypedef(const T & t_) : t(t_) {} + constexpr explicit StrongTypedef(const T & t_) : t(t_) {} template ::type> - explicit StrongTypedef(T && t_) : t(std::move(t_)) {} + constexpr explicit StrongTypedef(T && t_) : t(std::move(t_)) {} template ::type> - StrongTypedef(): t() {} + constexpr StrongTypedef(): t() {} - StrongTypedef(const Self &) = default; - StrongTypedef(Self &&) = default; + constexpr StrongTypedef(const Self &) = default; + constexpr StrongTypedef(Self &&) = default; Self & operator=(const Self &) = default; Self & operator=(Self &&) = default; diff --git a/base/common/unit.h b/base/common/unit.h index d5c8d5c9027..5bf5e00c049 100644 --- a/base/common/unit.h +++ b/base/common/unit.h @@ -1,6 +1,10 @@ #pragma once #include +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + constexpr size_t KiB = 1024; constexpr size_t MiB = 1024 * KiB; constexpr size_t GiB = 1024 * MiB; diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 745e020c8bb..141b7b7bbc6 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -1,3 +1,7 @@ +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + #include #include diff --git a/base/mysqlxx/CMakeLists.txt b/base/mysqlxx/CMakeLists.txt index c5230c2b49f..947f61f9051 100644 --- a/base/mysqlxx/CMakeLists.txt +++ b/base/mysqlxx/CMakeLists.txt @@ -49,6 +49,8 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR) target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) endif () +target_no_warning(mysqlxx reserved-macro-identifier) + if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES) message(WARNING "Statically linking with system mysql/mariadb only works " "if mysql client libraries are built with same openssl version as " diff --git a/base/mysqlxx/PoolFactory.cpp b/base/mysqlxx/PoolFactory.cpp index f0a5543d723..1d2f9fc74c7 100644 --- a/base/mysqlxx/PoolFactory.cpp +++ b/base/mysqlxx/PoolFactory.cpp @@ -79,7 +79,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf std::lock_guard lock(impl->mutex); if (auto entry = impl->pools.find(config_name); entry != impl->pools.end()) { - return *(entry->second.get()); + return *(entry->second); } else { @@ -100,7 +100,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf impl->pools.insert_or_assign(config_name, pool); impl->pools_by_ids.insert_or_assign(entry_name, config_name); } - return *(pool.get()); + return *pool; } } diff --git a/base/readpassphrase/readpassphrase.c b/base/readpassphrase/readpassphrase.c index 8a7d3153915..3bcad34e481 100644 --- a/base/readpassphrase/readpassphrase.c +++ b/base/readpassphrase/readpassphrase.c @@ -27,6 +27,10 @@ #define _PATH_TTY "/dev/tty" #endif +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + #include #include #include diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake index 24d8168e2c0..d1b9c86f15f 100644 --- a/cmake/analysis.cmake +++ b/cmake/analysis.cmake @@ -6,7 +6,7 @@ if (ENABLE_CLANG_TIDY) message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") endif() - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-13" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") if (CLANG_TIDY_PATH) message(STATUS diff --git a/cmake/check_flags.cmake b/cmake/check_flags.cmake index 5a4ff472868..58605543dff 100644 --- a/cmake/check_flags.cmake +++ b/cmake/check_flags.cmake @@ -4,3 +4,4 @@ include (CheckCCompilerFlag) check_cxx_compiler_flag("-Wsuggest-destructor-override" HAS_SUGGEST_DESTRUCTOR_OVERRIDE) check_cxx_compiler_flag("-Wshadow" HAS_SHADOW) check_cxx_compiler_flag("-Wsuggest-override" HAS_SUGGEST_OVERRIDE) +check_cxx_compiler_flag("-Xclang -fuse-ctor-homing" HAS_USE_CTOR_HOMING) diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index a85fe8963c7..0b6bbe36cf9 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -192,4 +192,29 @@ elseif (COMPILER_GCC) # For some reason (bug in gcc?) macro 'GCC diagnostic ignored "-Wstringop-overflow"' doesn't help. add_cxx_compile_options(-Wno-stringop-overflow) endif() + + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11) + # reinterpretAs.cpp:182:31: error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type + # ‘using ToFieldType = using FieldType = using UUID = struct StrongTypedef, DB::UUIDTag>’ + # {aka ‘struct StrongTypedef, DB::UUIDTag>’} from an array of ‘const char8_t’ + add_cxx_compile_options(-Wno-error=class-memaccess) + + # Maybe false positive... + # In file included from /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:673, + # In function ‘void std::__1::__libcpp_operator_delete(_Args ...) [with _Args = {void*, long unsigned int}]’, + # inlined from ‘void std::__1::__do_deallocate_handle_size(void*, size_t, _Args ...) [with _Args = {}]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:271:34, + # inlined from ‘void std::__1::__libcpp_deallocate(void*, size_t, size_t)’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:285:41, + # inlined from ‘constexpr void std::__1::allocator<_Tp>::deallocate(_Tp*, size_t) [with _Tp = char]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:849:39, + # inlined from ‘static constexpr void std::__1::allocator_traits<_Alloc>::deallocate(std::__1::allocator_traits<_Alloc>::allocator_type&, std::__1::allocator_traits<_Alloc>::pointer, std::__1::allocator_traits<_Alloc>::size_type) [with _Alloc = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/__memory/allocator_traits.h:476:24, + # inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits; _Allocator = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2219:35, + # inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits; _Allocator = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2213:1, + # inlined from ‘DB::JSONBuilder::JSONMap::Pair::~Pair()’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:90:12, + # inlined from ‘void DB::JSONBuilder::JSONMap::add(std::__1::string, DB::JSONBuilder::ItemPtr)’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:97:68, + # inlined from ‘virtual void DB::ExpressionStep::describeActions(DB::JSONBuilder::JSONMap&) const’ at /home/jakalletti/ClickHouse/ClickHouse/src/Processors/QueryPlan/ExpressionStep.cpp:102:12: + # /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:247:20: error: ‘void operator delete(void*, size_t)’ called on a pointer to an unallocated object ‘7598543875853023301’ [-Werror=free-nonheap-object] + add_cxx_compile_options(-Wno-error=free-nonheap-object) + + # AggregateFunctionAvg.h:203:100: error: ‘this’ pointer is null [-Werror=nonnull] + add_cxx_compile_options(-Wno-error=nonnull) + endif() endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 383a8510035..140cc0846ec 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -33,6 +33,7 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) add_subdirectory (abseil-cpp-cmake) +add_subdirectory (magic-enum-cmake) add_subdirectory (boost-cmake) add_subdirectory (cctz-cmake) add_subdirectory (consistent-hashing) diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 4f3b686f86c..b004a8a0241 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 4f3b686f86c3ebaba7e4e926e62a79cb1c659a54 +Subproject commit b004a8a02418b83de8b686caa0b0f6e39ac2191f diff --git a/contrib/fastops b/contrib/fastops index 88752a5e03c..012b777df9e 160000 --- a/contrib/fastops +++ b/contrib/fastops @@ -1 +1 @@ -Subproject commit 88752a5e03cf34639a4a37a4b41d8b463fffd2b5 +Subproject commit 012b777df9e2d145a24800a6c8c3d4a0249bb09e diff --git a/contrib/llvm b/contrib/llvm index e5751459412..f30bbecef78 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1 +Subproject commit f30bbecef78b75b527e257c1304d0be2f2f95975 diff --git a/contrib/magic-enum-cmake/CMakeLists.txt b/contrib/magic-enum-cmake/CMakeLists.txt new file mode 100644 index 00000000000..142f9c7c755 --- /dev/null +++ b/contrib/magic-enum-cmake/CMakeLists.txt @@ -0,0 +1,3 @@ +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/magic_enum") +add_library (magic_enum INTERFACE) +target_include_directories(magic_enum INTERFACE ${LIBRARY_DIR}/include) diff --git a/contrib/magic_enum b/contrib/magic_enum new file mode 160000 index 00000000000..38f86e4d093 --- /dev/null +++ b/contrib/magic_enum @@ -0,0 +1 @@ +Subproject commit 38f86e4d093cfc9034a140d37de2168e3951bef3 diff --git a/contrib/rocksdb b/contrib/rocksdb index b6480c69bf3..5ea892c8673 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit b6480c69bf3ab6e298e0d019a07fd4f69029b26a +Subproject commit 5ea892c8673e6c5a052887653673b967d44cc59b diff --git a/debian/rules b/debian/rules index 73d1f3d3b34..4562d24bec4 100755 --- a/debian/rules +++ b/debian/rules @@ -36,8 +36,8 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -DEB_CC ?= $(shell which gcc-10 gcc-9 gcc | head -n1) -DEB_CXX ?= $(shell which g++-10 g++-9 g++ | head -n1) +DEB_CC ?= $(shell which gcc-11 gcc-10 gcc-9 gcc | head -n1) +DEB_CXX ?= $(shell which g++-11 g++-10 g++-9 g++ | head -n1) ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 9a1041ee743..5a998ffbe3e 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 7c7a8893751..1025af3f96e 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -4,7 +4,7 @@ set -e #ccache -s # uncomment to display CCache statistics mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-12)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-12)" +cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-13)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-13)" # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 832a913e34c..23012a38f9d 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,7 +1,7 @@ # docker build -t clickhouse/binary-builder . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list @@ -41,18 +41,14 @@ RUN apt-get update \ ccache \ cmake \ curl \ - g++-10 \ - gcc-10 \ gdb \ git \ gperf \ - libicu-dev \ - libreadline-dev \ - clang-12 \ - clang-tidy-12 \ - lld-12 \ - llvm-12 \ - llvm-12-dev \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + lld-${LLVM_VERSION} \ + llvm-${LLVM_VERSION} \ + llvm-${LLVM_VERSION}-dev \ libicu-dev \ libreadline-dev \ moreutils \ @@ -104,15 +100,10 @@ RUN wget -nv "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.0 # Download toolchain for FreeBSD 11.3 RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz -# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable. -# Current workaround is to use latest version proposed repo. Remove as soon as -# gcc-10.2 appear in stable repo. -RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list - -RUN apt-get update \ - && apt-get install gcc-10 g++-10 --yes - -RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update +# NOTE: Seems like gcc-11 is too new for ubuntu20 repository +RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ + && apt-get update \ + && apt-get install gcc-11 g++-11 --yes COPY build.sh / diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 0a535ed2a71..318b960e0b4 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,7 +1,7 @@ # docker build -t clickhouse/deb-builder . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list @@ -37,17 +37,17 @@ RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \ RUN apt-get update \ && apt-get install \ alien \ - clang-12 \ - clang-tidy-12 \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ cmake \ debhelper \ devscripts \ gdb \ git \ gperf \ - lld-12 \ - llvm-12 \ - llvm-12-dev \ + lld-${LLVM_VERSION} \ + llvm-${LLVM_VERSION} \ + llvm-${LLVM_VERSION}-dev \ moreutils \ ninja-build \ perl \ @@ -57,15 +57,11 @@ RUN apt-get update \ tzdata \ --yes --no-install-recommends -# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable. -# Current workaround is to use latest version proposed repo. Remove as soon as -# gcc-10.2 appear in stable repo. -RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list +# NOTE: Seems like gcc-11 is too new for ubuntu20 repository +RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ + && apt-get update \ + && apt-get install gcc-11 g++-11 --yes -RUN apt-get update \ - && apt-get install gcc-10 g++-10 --yes --no-install-recommends - -RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld diff --git a/docker/packager/packager b/docker/packager/packager index ec423bf2c76..f8e5fd717cf 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -205,7 +205,8 @@ if __name__ == "__main__": parser.add_argument("--build-type", choices=("debug", ""), default="") parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64", "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64", - "clang-11-freebsd", "clang-12-freebsd", "gcc-10"), default="clang-12") + "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", + "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index a0345406400..2d49c207ac3 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,7 @@ # docker build -t clickhouse/test-base . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index b87ad9336c9..94aa321252b 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libl # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser -RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-12 -DCMAKE_C_COMPILER=clang-12 && make -j +RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-13 -DCMAKE_C_COMPILER=clang-13 && make -j ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator @@ -24,7 +24,7 @@ ENV SHA=nosha ENV DATA="data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ - cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-12 -DCMAKE_C_COMPILER=/usr/bin/clang-12 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ mkdir -p $HTML_RESULT_DIRECTORY && \ $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ diff --git a/docker/test/coverage/run.sh b/docker/test/coverage/run.sh index 11b6ce13ea1..807efdf1e47 100755 --- a/docker/test/coverage/run.sh +++ b/docker/test/coverage/run.sh @@ -80,7 +80,7 @@ LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAM LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test" -LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time --use-skip-list 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt") @@ -97,7 +97,7 @@ then echo "Going to run again: ${FAILED_TESTS[*]}" - LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper --use-skip-list "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt + LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt else echo "No failed tests" fi diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 1daab8b80f1..3119b2f1bb3 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,7 @@ # docker build -t clickhouse/fasttest . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 8a6b7f70cba..0cda2f8f17f 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} # Compiler version, normally set by Dockerfile -export LLVM_VERSION=${LLVM_VERSION:-12} +export LLVM_VERSION=${LLVM_VERSION:-13} # A variable to pass additional flags to CMake. # Here we explicitly default it to nothing so that bash doesn't complain about @@ -159,6 +159,7 @@ function clone_submodules cd "$FASTTEST_SOURCE" SUBMODULES_TO_UPDATE=( + contrib/magic_enum contrib/abseil-cpp contrib/boost contrib/zlib-ng @@ -261,153 +262,8 @@ function run_tests start_server - TESTS_TO_SKIP=( - 00105_shard_collations - 00109_shard_totals_after_having - 00110_external_sort - 00302_http_compression - 00417_kill_query - 00436_convert_charset - 00490_special_line_separators_and_characters_outside_of_bmp - 00652_replicated_mutations_zookeeper - 00682_empty_parts_merge - 00701_rollup - 00834_cancel_http_readonly_queries_on_client_close - 00911_tautological_compare - - # Hyperscan - 00926_multimatch - 00929_multi_match_edit_distance - 01681_hyperscan_debug_assertion - 02004_max_hyperscan_regex_length - - 01176_mysql_client_interactive # requires mysql client - 01031_mutations_interpreter_and_context - 01053_ssd_dictionary # this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled - 01083_expressions_in_engine_arguments - 01092_memory_profiler - 01098_msgpack_format - 01098_temporary_and_external_tables - 01103_check_cpu_instructions_at_startup # avoid dependency on qemu -- invonvenient when running locally - 01193_metadata_loading - 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently - 01251_dict_is_in_infinite_loop - 01259_dictionary_custom_settings_ddl - 01268_dictionary_direct_layout - 01280_ssd_complex_key_dictionary - 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently - 01318_encrypt # Depends on OpenSSL - 01318_decrypt # Depends on OpenSSL - 01663_aes_msan # Depends on OpenSSL - 01667_aes_args_check # Depends on OpenSSL - 01683_codec_encrypted # Depends on OpenSSL - 01776_decrypt_aead_size_check # Depends on OpenSSL - 01811_filter_by_null # Depends on OpenSSL - 02012_sha512_fixedstring # Depends on OpenSSL - 01281_unsucceeded_insert_select_queries_counter - 01292_create_user - 01294_lazy_database_concurrent - 01305_replica_create_drop_zookeeper - 01354_order_by_tuple_collate_const - 01355_ilike - 01411_bayesian_ab_testing - 01798_uniq_theta_sketch - 01799_long_uniq_theta_sketch - 01890_stem # depends on libstemmer_c - 02003_compress_bz2 # depends on bzip2 - 01059_storage_file_compression # depends on brotli and bzip2 - collate - collation - _orc_ - arrow - avro - base64 - brotli - capnproto - client - ddl_dictionaries - h3 - hashing - hdfs - java_hash - json - limit_memory - live_view - memory_leak - memory_limit - mysql - odbc - parallel_alter - parquet - protobuf - secure - sha256 - xz - - # Not sure why these two fail even in sequential mode. Disabled for now - # to make some progress. - 00646_url_engine - 00974_query_profiler - - # In fasttest, ENABLE_LIBRARIES=0, so rocksdb engine is not enabled by default - 01504_rocksdb - 01686_rocksdb - - # Look at DistributedFilesToInsert, so cannot run in parallel. - 01460_DistributedFilesToInsert - - 01541_max_memory_usage_for_user_long - - # Require python libraries like scipy, pandas and numpy - 01322_ttest_scipy - 01561_mann_whitney_scipy - - 01545_system_errors - # Checks system.errors - 01563_distributed_query_finish - - # nc - command not found - 01601_proxy_protocol - 01622_defaults_for_url_engine - - # JSON functions - 01666_blns - - # Requires postgresql-client - 01802_test_postgresql_protocol_with_row_policy - - # Depends on AWS - 01801_s3_cluster - 02012_settings_clause_for_s3 - - # needs psql - 01889_postgresql_protocol_null_fields - - # needs pv - 01923_network_receive_time_metric_insert - - 01889_sqlite_read_write - - # needs s2 - 01849_geoToS2 - 01851_s2_to_geo - 01852_s2_get_neighbours - 01853_s2_cells_intersect - 01854_s2_cap_contains - 01854_s2_cap_union - - # needs s3 - 01944_insert_partition_by - - # depends on Go - 02013_zlib_read_after_eof - - # Accesses CH via mysql table function (which is unavailable) - 01747_system_session_log_long - ) - - time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \ - --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" \ + time clickhouse-test --hung-check -j 8 --order=random \ + --fast-tests-only --no-long --testname --shard --zookeeper \ -- "$FASTTEST_FOCUS" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee "$FASTTEST_OUTPUT/test_log.txt" diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2247c6a22be..0b26476c20e 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -12,7 +12,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-12_debug_none_bundled_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"} function clone { diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index 8d31b5b7f1c..d7534270e2c 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-12_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index 17bb7d3662b..f4675d35819 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -38,7 +38,7 @@ RUN set -x \ && dpkg -i "${PKG_VERSION}.deb" CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ - && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-12 -DCMAKE_CXX_COMPILER=clang\+\+-12 \ + && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \ && ninja re2_st clickhouse_grpc_protos \ && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \ cp /repo_folder/pvs-studio.log /test_output; \ diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index de058469192..dd5984fd7b5 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -108,7 +108,7 @@ function run_tests() ADDITIONAL_OPTIONS+=('--replicated-database') fi - clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --use-skip-list --print-time "${ADDITIONAL_OPTIONS[@]}" \ + clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \ "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt } diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 154126a3880..ed721690281 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -97,7 +97,7 @@ function run_tests() fi clickhouse-test --testname --shard --zookeeper --hung-check --print-time \ - --use-skip-list --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ + --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee -a test_output/test_result.txt } diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index 8b8612d4211..f8396706ed2 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -13,8 +13,4 @@ dpkg -i package_folder/clickhouse-test_*.deb service clickhouse-server start && sleep 5 -if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then - SKIP_LIST_OPT="--use-skip-list" -fi - -clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt +clickhouse-test --testname --shard --zookeeper "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt diff --git a/docker/test/stress/stress b/docker/test/stress/stress index c71722809d7..73a84ad4c40 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -10,14 +10,6 @@ import logging import time -def get_skip_list_cmd(path): - with open(path, 'r') as f: - for line in f: - if '--use-skip-list' in line: - return '--use-skip-list' - return '' - - def get_options(i): options = [] client_options = [] @@ -56,8 +48,6 @@ def get_options(i): def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit): - skip_list_opt = get_skip_list_cmd(cmd) - global_time_limit_option = '' if global_time_limit: global_time_limit_option = "--global_time_limit={}".format(global_time_limit) @@ -66,7 +56,7 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], 'w') - full_command = "{} {} {} {} {}".format(cmd, skip_list_opt, get_options(i), global_time_limit_option, skip_tests_option) + full_command = "{} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 687e0179e07..91e4902d0d2 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -76,7 +76,7 @@ cd ClickHouse rm -rf build mkdir build cd build -cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo .. +cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo .. cmake --build . --config RelWithDebInfo cd .. ``` diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 8d1aae13957..a246d3dea0a 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build Or cmake3 instead of cmake on older systems. -### Install clang-12 (recommended) {#install-clang-12} +### Install clang-13 (recommended) {#install-clang-13} On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) @@ -33,11 +33,11 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). -#### Use clang-12 for Builds +#### Use clang-13 for Builds ``` bash -$ export CC=clang-12 -$ export CXX=clang++-12 +$ export CC=clang-13 +$ export CXX=clang++-13 ``` Gcc can also be used though it is discouraged. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 277339f9b47..8092baba199 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -288,5 +288,7 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) - [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) [Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index a533d9357e1..f353f0b1d43 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -60,6 +60,7 @@ The supported formats are: | [LineAsString](#lineasstring) | ✔ | ✗ | | [Regexp](#data-format-regexp) | ✔ | ✗ | | [RawBLOB](#rawblob) | ✔ | ✔ | +| [MsgPack](#msgpack) | ✔ | ✔ | You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section. @@ -1551,4 +1552,31 @@ Result: f9725a22f9191e064120d718e26862a9 - ``` -[Original article](https://clickhouse.tech/docs/en/interfaces/formats/) +## MsgPack {#msgpack} + +ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data files. + +### Data Types Matching {#data-types-matching-msgpack} + +| MsgPack data type | ClickHouse data type | +|---------------------------------|----------------------------------------------------------------------------------| +| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | +| `int N` | [IntN](../sql-reference/data-types/int-uint.md) | +| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) | +| `float 32` | [Float32](../sql-reference/data-types/float.md) | +| `float 64` | [Float64](../sql-reference/data-types/float.md) | +| `uint 16` | [Date](../sql-reference/data-types/date.md) | +| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | +| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | +| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) | +| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) | + +Example: + +Writing to a file ".msgpk": + +```sql +$ clickhouse-client --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE = Memory;" +$ clickhouse-client --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])"; +$ clickhouse-client --query="SELECT * FROM msgpack FORMAT MsgPack" > tmp_msgpack.msgpk; +``` \ No newline at end of file diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index f2c6d1ee542..ad39adbd653 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -84,7 +84,7 @@ Features: - Table data preview. - Full-text search. -By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set session_id to a random string (it uses the http connection under the hood). Then you can use any setting from the query window +By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set `session_id` to a random string (it uses the http connection under the hood). Then you can use any setting from the query window. ### clickhouse-cli {#clickhouse-cli} diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index 389cf2361f8..228a7c3666e 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -3,58 +3,58 @@ toc_priority: 66 toc_title: ClickHouse Keeper --- -# [pre-production] clickhouse-keeper +# [pre-production] ClickHouse Keeper -ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper. +ClickHouse server uses [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper. !!! warning "Warning" - This feature currently in pre-production stage. We test it in our CI and on small internal installations. + This feature is currently in the pre-production stage. We test it in our CI and on small internal installations. ## Implementation details -ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages. +ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper ClickHouse Keeper is written in C++ and uses [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages. -By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible. +By default, ClickHouse Keeper provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with ClickHouse Keeper. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to ClickHouse Keeper snapshot. Interserver protocol in ClickHouse Keeper is also incompatible with ZooKeeper so mixed ZooKeeper / ClickHouse Keeper cluster is impossible. ## Configuration -`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is ``. Keeper configuration has the following parameters: +ClickHouse Keeper can be used as a standalone replacement for ZooKeeper or as an internal part of the ClickHouse server, but in both cases configuration is almost the same `.xml` file. The main ClickHouse Keeper configuration tag is ``. Keeper configuration has the following parameters: -- `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`) -- `tcp_port_secure` — the secure port for a client to connect -- `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on) -- `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper) -- `snapshot_storage_path` — path to coordination snapshots +- `tcp_port` — Port for a client to connect (default for ZooKeeper is `2181`). +- `tcp_port_secure` — Secure port for a client to connect. +- `server_id` — Unique server id, each participant of the ClickHouse Keeper cluster must have a unique number (1, 2, 3, and so on). +- `log_storage_path` — Path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper). +- `snapshot_storage_path` — Path to coordination snapshots. -Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on). +Other common parameters are inherited from the ClickHouse server config (`listen_host`, `logger`, and so on). Internal coordination settings are located in `.` section: -- `operation_timeout_ms` — timeout for a single client operation (default: 10000) -- `session_timeout_ms` — timeout for client session (default: 30000) -- `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them (default: 500) -- `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers (default: 500) -- `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election (default: 1000) -- `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election (default: 2000) -- `rotate_log_storage_interval` — how many log records to store in a single file (default: 100000) -- `reserved_log_items` — how many coordination log records to store before compaction (default: 100000) -- `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of records in logs) (default: 100000) -- `snapshots_to_keep` — how many snapshots to keep (default: 3) -- `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs (default: 10000) -- `fresh_log_gap` - when node became fresh (default: 200) -- `max_requests_batch_size` - max size of batch in requests count before it will be sent to RAFT (default: 100) -- `force_sync` — call `fsync` on each write to coordination log (default: true) -- `quorum_reads` - execute read requests as writes through whole RAFT consesus with similar speed (default: false) -- `raft_logs_level` — text logging level about coordination (trace, debug, and so on) (default: system default) -- `auto_forwarding` - allow to forward write requests from followers to leader (default: true) -- `shutdown_timeout` — wait to finish internal connections and shutdown (ms) (default: 5000) -- `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000) +- `operation_timeout_ms` — Timeout for a single client operation (ms) (default: 10000). +- `session_timeout_ms` — Timeout for client session (ms) (default: 30000). +- `dead_session_check_period_ms` — How often ClickHouse Keeper check dead sessions and remove them (ms) (default: 500). +- `heart_beat_interval_ms` — How often a ClickHouse Keeper leader will send heartbeats to followers (ms) (default: 500). +- `election_timeout_lower_bound_ms` — If the follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election (default: 1000). +- `election_timeout_upper_bound_ms` — If the follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election (default: 2000). +- `rotate_log_storage_interval` — How many log records to store in a single file (default: 100000). +- `reserved_log_items` — How many coordination log records to store before compaction (default: 100000). +- `snapshot_distance` — How often ClickHouse Keeper will create new snapshots (in the number of records in logs) (default: 100000). +- `snapshots_to_keep` — How many snapshots to keep (default: 3). +- `stale_log_gap` — Threshold when leader considers follower as stale and sends the snapshot to it instead of logs (default: 10000). +- `fresh_log_gap` — When node became fresh (default: 200). +- `max_requests_batch_size` - Max size of batch in requests count before it will be sent to RAFT (default: 100). +- `force_sync` — Call `fsync` on each write to coordination log (default: true). +- `quorum_reads` — Execute read requests as writes through whole RAFT consensus with similar speed (default: false). +- `raft_logs_level` — Text logging level about coordination (trace, debug, and so on) (default: system default). +- `auto_forwarding` — Allow to forward write requests from followers to the leader (default: true). +- `shutdown_timeout` — Wait to finish internal connections and shutdown (ms) (default: 5000). +- `startup_timeout` — If the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000). Quorum configuration is located in `.` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `` are: -- `id` — server_id in quorum -- `hostname` — hostname where this server placed -- `port` — port where this server listen for connections +- `id` — Server identifier in a quorum. +- `hostname` — Hostname where this server is placed. +- `port` — Port where this server listens for connections. Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1: @@ -94,7 +94,7 @@ Examples of configuration for quorum with three nodes can be found in [integrati ## How to run -`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with: +ClickHouse Keeper is bundled into the ClickHouse server package, just add configuration of `` and start ClickHouse server as always. If you want to run standalone ClickHouse Keeper you can start it in a similar way with: ```bash clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon @@ -102,17 +102,18 @@ clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon ## [experimental] Migration from ZooKeeper -Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration: +Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration: 1. Stop all ZooKeeper nodes. -2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot. +2. Optional, but recommended: find ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create a consistent snapshot. -3. Run `clickhouse-keeper-converter` on leader, example +3. Run `clickhouse-keeper-converter` on a leader, for example: ```bash clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots ``` -4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist on all nodes, otherwise empty nodes can be faster and one of them can becamse leader. +4. Copy snapshot to ClickHouse server nodes with a configured `keeper` or start ClickHouse Keeper instead of ZooKeeper. The snapshot must persist on all nodes, otherwise, empty nodes can be faster and one of them can become a leader. +[Original article](https://clickhouse.tech/docs/en/operations/clickhouse-keeper/) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index be19f476978..8ad450738b7 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -69,29 +69,85 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression. ``` - + +Load key from environment variable: + +```xml + + + + + +``` + +Where current_key_id sets the current key for encryption, and all specified keys can be used for decryption. + +All this methods can be applied for multiple keys: + +```xml + + + 00112233445566778899aabbccddeeff + + 1 + + +``` + +Where `current_key_id` shows current key for encryption. + +Also user can add nonce that must be 12 bytes long (by default encryption and decryption will use nonce consisting of zero bytes): + +```xml + + + 0123456789101 + + +``` + +Or it can be set in hex: + +```xml + + + abcdefabcdef + + +``` + +Everything above can be applied for `aes_256_gcm_siv` (but key must be 32 bytes length). + ## custom_settings_prefixes {#custom_settings_prefixes} List of prefixes for [custom settings](../../operations/settings/index.md#custom_settings). The prefixes must be separated with commas. diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 65d63438aea..a3a258234e1 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -181,6 +181,44 @@ Possible values: Default value: 0. +## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches. This setting is applied to a particular table, unlike the [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server) setting, which is applied to the server. + +You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_fetches_network_bandwidth_for_server` setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating data to add or replace new nodes. + +## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends. This setting is applied to a particular table, unlike the [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server) setting, which is applied to the server. + +You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_sends_network_bandwidth_for_server` setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating data to add or replace new nodes. + ## old_parts_lifetime {#old-parts-lifetime} The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 6c5767d28e0..f286cf4e99e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1140,6 +1140,40 @@ Possible values: Default value: `5`. +## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating the data to add or replace new nodes. + +## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating the data to add or replace new nodes. + ## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms} The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the ‘shard’ and ‘replica’ sections are used in the cluster definition. diff --git a/docs/en/sql-reference/data-types/int-uint.md b/docs/en/sql-reference/data-types/int-uint.md index 95d1120ed3d..b6fff135b78 100644 --- a/docs/en/sql-reference/data-types/int-uint.md +++ b/docs/en/sql-reference/data-types/int-uint.md @@ -25,14 +25,13 @@ Aliases: - `Int32` — `INT`, `INT4`, `INTEGER`. - `Int64` — `BIGINT`. -## Uint Ranges {#uint-ranges} +## UInt Ranges {#uint-ranges} - `UInt8` — \[0 : 255\] - `UInt16` — \[0 : 65535\] - `UInt32` — \[0 : 4294967295\] - `UInt64` — \[0 : 18446744073709551615\] +- `UInt128` — \[0 : 340282366920938463463374607431768211455\] - `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\] -`UInt128` is not supported yet. - [Original article](https://clickhouse.tech/docs/en/data_types/int_uint/) diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md index 5f74c6329d1..f564f15659c 100644 --- a/docs/en/sql-reference/functions/rounding-functions.md +++ b/docs/en/sql-reference/functions/rounding-functions.md @@ -29,7 +29,7 @@ Returns the round number with largest absolute value that has an absolute value Rounds a value to a specified number of decimal places. -The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker’s rounding for float number types and rounds away from zero for the other number types. +The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker’s rounding for float number types and rounds away from zero for the other number types (Decimal). ``` sql round(expression [, decimal_places]) @@ -49,7 +49,7 @@ The rounded number of the same type as the input number. ### Examples {#examples} -**Example of use** +**Example of use with Float** ``` sql SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 @@ -63,6 +63,20 @@ SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 └─────┴──────────────────────────┘ ``` +**Example of use with Decimal** + +``` sql +SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3 +``` + +``` text +┌──────x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐ +│ 0.0000 │ 0.0000 │ +│ 0.5000 │ 1.0000 │ +│ 1.0000 │ 1.0000 │ +└────────┴──────────────────────────────────────────────────┘ +``` + **Examples of rounding** Rounding to the nearest number. diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md index 8d5d06a544c..bdc80322f34 100644 --- a/docs/ru/development/build-osx.md +++ b/docs/ru/development/build-osx.md @@ -74,7 +74,7 @@ $ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/ $ rm -rf build $ mkdir build $ cd build - $ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. $ cmake --build . --config RelWithDebInfo $ cd .. ``` diff --git a/docs/ru/engines/database-engines/postgresql.md b/docs/ru/engines/database-engines/postgresql.md index 06e2b35b002..f68570d40eb 100644 --- a/docs/ru/engines/database-engines/postgresql.md +++ b/docs/ru/engines/database-engines/postgresql.md @@ -15,7 +15,7 @@ toc_title: PostgreSQL ``` sql CREATE DATABASE test_database -ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cache`]); +ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `schema`, `use_table_cache`]); ``` **Параметры движка** @@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac - `database` — имя удаленной БД. - `user` — пользователь PostgreSQL. - `password` — пароль пользователя. + - `schema` — схема PostgreSQL. - `use_table_cache` — определяет кеширование структуры таблиц БД. Необязательный параметр. Значение по умолчанию: `0`. ## Поддерживаемые типы данных {#data_types-support} diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index db749ba455e..a7c1a7d2d1b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -253,4 +253,5 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) - [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) - +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 970c6c36e9f..a02e1436d36 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -59,6 +59,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | [LineAsString](#lineasstring) | ✔ | ✗ | | [Regexp](#data-format-regexp) | ✔ | ✗ | | [RawBLOB](#rawblob) | ✔ | ✔ | +| [MsgPack](#msgpack) | ✔ | ✔ | Вы можете регулировать некоторые параметры работы с форматами с помощью настроек ClickHouse. За дополнительной информацией обращайтесь к разделу [Настройки](../operations/settings/settings.md). @@ -1464,3 +1465,32 @@ $ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum ``` text f9725a22f9191e064120d718e26862a9 - ``` + +## MsgPack {#msgpack} + +ClickHouse поддерживает запись и чтение из файлов в формате [MessagePack](https://msgpack.org/). + +### Соответствие типов данных {#data-types-matching-msgpack} + +| Тип данных MsgPack | Тип данных ClickHouse | +|---------------------------------|------------------------------------------------------------------------------------| +| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | +| `int N` | [IntN](../sql-reference/data-types/int-uint.md) | +| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) | +| `float 32` | [Float32](../sql-reference/data-types/float.md) | +| `float 64` | [Float64](../sql-reference/data-types/float.md) | +| `uint 16` | [Date](../sql-reference/data-types/date.md) | +| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | +| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | +| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) | +| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) | + +Пример: + +Запись в файл ".msgpk": + +```sql +$ clickhouse-client --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE = Memory;" +$ clickhouse-client --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])"; +$ clickhouse-client --query="SELECT * FROM msgpack FORMAT MsgPack" > tmp_msgpack.msgpk; +``` \ No newline at end of file diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index 9cb28a2c9a2..fb14e9f955c 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -75,11 +75,13 @@ toc_title: "Визуальные интерфейсы от сторонних р Основные возможности: -- Построение запросов с подсветкой синтаксиса; -- Просмотр таблиц; -- Автодополнение команд; +- Построение запросов с подсветкой синтаксиса. +- Просмотр таблиц. +- Автодополнение команд. - Полнотекстовый поиск. +По умолчанию DBeaver не использует сессии при подключении (в отличие от CLI, например). Если вам нужна поддержка сессий (например, для установки настроек на сессию), измените настройки подключения драйвера и укажите для настройки `session_id` любое произвольное значение (драйвер использует подключение по http). После этого вы можете использовать любую настройку (setting) в окне запроса. + ### clickhouse-cli {#clickhouse-cli} [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) - это альтернативный клиент командной строки для ClickHouse, написанный на Python 3. diff --git a/docs/ru/operations/clickhouse-keeper.md b/docs/ru/operations/clickhouse-keeper.md new file mode 100644 index 00000000000..5c8661398b5 --- /dev/null +++ b/docs/ru/operations/clickhouse-keeper.md @@ -0,0 +1,119 @@ +--- +toc_priority: 66 +toc_title: ClickHouse Keeper +--- + +# [пре-продакшн] ClickHouse Keeper + +Сервер ClickHouse использует сервис координации [ZooKeeper](https://zookeeper.apache.org/) для [репликации](../engines/table-engines/mergetree-family/replication.md) данных и выполнения [распределенных DDL запросов](../sql-reference/distributed-ddl.md). ClickHouse Keeper — это альтернативный сервис координации, совместимый с ZooKeeper. + +!!! warning "Предупреждение" + ClickHouse Keeper находится в стадии пре-продакшн и тестируется в CI ClickHouse и на нескольких внутренних инсталляциях. + +## Детали реализации + +ZooKeeper — один из первых широко известных сервисов координации с открытым исходным кодом. Он реализован на языке программирования Java, имеет достаточно простую и мощную модель данных. Алгоритм координации Zookeeper называется ZAB (ZooKeeper Atomic Broadcast). Он не гарантирует линеаризуемость операций чтения, поскольку каждый узел ZooKeeper обслуживает чтения локально. В отличие от ZooKeeper, ClickHouse Keeper реализован на C++ и использует алгоритм [RAFT](https://raft.github.io/), [реализация](https://github.com/eBay/NuRaft). Этот алгоритм позволяет достичь линеаризуемости чтения и записи, имеет несколько реализаций с открытым исходным кодом на разных языках. + +По умолчанию ClickHouse Keeper предоставляет те же гарантии, что и ZooKeeper (линеаризуемость записей, последовательная согласованность чтений). У него есть совместимый клиент-серверный протокол, поэтому любой стандартный клиент ZooKeeper может использоваться для взаимодействия с ClickHouse Keeper. Снэпшоты и журналы имеют несовместимый с ZooKeeper формат, однако можно конвертировать данные Zookeeper в снэпшот ClickHouse Keeper с помощью `clickhouse-keeper-converter`. Межсерверный протокол ClickHouse Keeper также несовместим с ZooKeeper, поэтому создание смешанного кластера ZooKeeper / ClickHouse Keeper невозможно. + +## Конфигурация + +ClickHouse Keeper может использоваться как равноценная замена ZooKeeper или как внутренняя часть сервера ClickHouse, но в обоих случаях конфигурация представлена файлом `.xml`. Главный тег конфигурации ClickHouse Keeper — это ``. Параметры конфигурации: + +- `tcp_port` — порт для подключения клиента (по умолчанию для ZooKeeper: `2181`). +- `tcp_port_secure` — зашифрованный порт для подключения клиента. +- `server_id` — уникальный идентификатор сервера, каждый участник кластера должен иметь уникальный номер (1, 2, 3 и т. д.). +- `log_storage_path` — путь к журналам координации, лучше хранить их на незанятом устройстве (актуально и для ZooKeeper). +- `snapshot_storage_path` — путь к снэпшотам координации. + +Другие общие параметры наследуются из конфигурации сервера ClickHouse (`listen_host`, `logger`, и т. д.). + +Настройки внутренней координации находятся в `.`: + +- `operation_timeout_ms` — максимальное время ожидания для одной клиентской операции в миллисекундах (по умолчанию: 10000). +- `session_timeout_ms` — максимальное время ожидания для клиентской сессии в миллисекундах (по умолчанию: 30000). +- `dead_session_check_period_ms` — частота, с которой ClickHouse Keeper проверяет мертвые сессии и удаляет их, в миллисекундах (по умолчанию: 500). +- `heart_beat_interval_ms` — частота, с которой узел-лидер ClickHouse Keeper отправляет хартбиты узлам-последователям, в миллисекундах (по умолчанию: 500). +- `election_timeout_lower_bound_ms` — время, после которого последователь может инициировать выборы лидера, если не получил от него сердцебиения (по умолчанию: 1000). +- `election_timeout_upper_bound_ms` — время, после которого последователь должен инициировать выборы лидера, если не получил от него сердцебиения (по умолчанию: 2000). +- `rotate_log_storage_interval` — количество записей в журнале координации для хранения в одном файле (по умолчанию: 100000). +- `reserved_log_items` — минимальное количество записей в журнале координации которые нужно сохранять после снятия снепшота (по умолчанию: 100000). +- `snapshot_distance` — частота, с которой ClickHouse Keeper делает новые снэпшоты (по количеству записей в журналах), в миллисекундах (по умолчанию: 100000). +- `snapshots_to_keep` — количество снэпшотов для сохранения (по умолчанию: 3). +- `stale_log_gap` — время, после которого лидер считает последователя устаревшим и отправляет ему снэпшот вместо журналов (по умолчанию: 10000). +- `fresh_log_gap` — максимальное отставание от лидера в количестве записей журнала после которого последователь считает себя не отстающим (по умолчанию: 200). +- `max_requests_batch_size` — количество запросов на запись, которые будут сгруппированы в один перед отправкой через RAFT (по умолчанию: 100). +- `force_sync` — вызывать `fsync` при каждой записи в журнал координации (по умолчанию: true). +- `quorum_reads` — выполнять запросы чтения аналогично запросам записи через весь консенсус RAFT с негативным эффектом на производительность и размер журналов (по умолчанию: false). +- `raft_logs_level` — уровень логгирования сообщений в текстовый лог (trace, debug и т. д.) (по умолчанию: information). +- `auto_forwarding` — разрешить пересылку запросов на запись от последователей лидеру (по умолчанию: true). +- `shutdown_timeout` — время ожидания завершения внутренних подключений и выключения, в миллисекундах (по умолчанию: 5000). +- `startup_timeout` — время отключения сервера, если он не подключается к другим участникам кворума, в миллисекундах (по умолчанию: 30000). + +Конфигурация кворума находится в `.` и содержит описание серверов. Единственный параметр для всего кворума — `secure`, который включает зашифрованное соединение для связи между участниками кворума. Параметры для каждого ``: + +- `id` — идентификатор сервера в кворуме. +- `hostname` — имя хоста, на котором размещен сервер. +- `port` — порт, на котором серверу доступны соединения для внутренней коммуникации. + + +Примеры конфигурации кворума с тремя узлами можно найти в [интеграционных тестах](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) с префиксом `test_keeper_`. Пример конфигурации для сервера №1: + +```xml + + 2181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + +``` + +## Как запустить + +ClickHouse Keeper входит в пакет` clickhouse-server`, просто добавьте кофигурацию `` и запустите сервер ClickHouse как обычно. Если вы хотите запустить ClickHouse Keeper автономно, сделайте это аналогичным способом: + +```bash +clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon +``` + +## [экспериментально] Переход с ZooKeeper + +Плавный переход с ZooKeeper на ClickHouse Keeper невозможен, необходимо остановить кластер ZooKeeper, преобразовать данные и запустить ClickHouse Keeper. Утилита `clickhouse-keeper-converter` конвертирует журналы и снэпшоты ZooKeeper в снэпшот ClickHouse Keeper. Работа утилиты проверена только для версий ZooKeeper выше 3.4. Для миграции необходимо выполнить следующие шаги: + +1. Остановите все узлы ZooKeeper. + +2. Необязательно, но рекомендуется: найдите узел-лидер ZooKeeper, запустите и снова остановите его. Это заставит ZooKeeper создать консистентный снэпшот. + +3. Запустите `clickhouse-keeper-converter` на лидере, например: + +```bash +clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots +``` + +4. Скопируйте снэпшот на узлы сервера ClickHouse с настроенным `keeper` или запустите ClickHouse Keeper вместо ZooKeeper. Снэпшот должен сохраняться на всех узлах: в противном случае пустые узлы могут захватить лидерство и сконвертированные данные могут быть отброшены на старте. + +[Original article](https://clickhouse.tech/docs/en/operations/clickhouse-keeper/) \ No newline at end of file diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 88c511d4d80..08ea9979426 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -201,6 +201,44 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: `0`. +## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth} + +Ограничивает максимальную скорость скачивания данных в сети (в байтах в секунду) для синхронизаций между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server), которая применяется к серверу. + +Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_fetches_network_bandwidth_for_server`. + +Настройка соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости передачи данных при репликации данных для добавления или замены новых узлов. + +## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth} + +Ограничивает максимальную скорость отправки данных по сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server), которая применяется к серверу. + +Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_sends_network_bandwidth_for_server`. + +Настройка следуется неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + ## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool} Максимальный суммарный размер кусков (в байтах) в одном слиянии, если есть свободные ресурсы в фоновом пуле. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index f55edec62d1..742d86bdd9e 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1098,6 +1098,40 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test' Значение по умолчанию: `5`. +## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server} + +Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth). + +Значение настройки соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + +## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server} + +Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для [репликационных](../../engines/table-engines/mergetree-family/replication.md) отправок. Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth). + +Значение настройки соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + ## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms} Таймаут в миллисекундах на соединение с удалённым сервером, для движка таблиц Distributed, если используются секции shard и replica в описании кластера. diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index 7879ee707a4..ff58355145d 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -78,10 +78,11 @@ active_replicas: 2 - `log_max_index` (`UInt64`) - максимальный номер записи в общем логе действий. - `log_pointer` (`UInt64`) - максимальный номер записи из общего лога действий, которую реплика скопировала в свою очередь для выполнения, плюс единица. Если log_pointer сильно меньше log_max_index, значит что-то не так. -- `last_queue_update` (`DateTime`) - When the queue was updated last time. -- `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has. +- `last_queue_update` (`DateTime`) - время последнего обновления запроса. +- `absolute_delay` (`UInt64`) - задержка (в секундах) для текущей реплики. - `total_replicas` (`UInt8`) - общее число известных реплик этой таблицы. - `active_replicas` (`UInt8`) - число реплик этой таблицы, имеющих сессию в ZK; то есть, число работающих реплик. +- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — соответствие между именем реплики и признаком активности реплики. Если запрашивать все столбцы, то таблица может работать слегка медленно, так как на каждую строчку делается несколько чтений из ZK. Если не запрашивать последние 4 столбца (log_max_index, log_pointer, total_replicas, active_replicas), то таблица работает быстро. diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index 276f85bf6b7..1eede1ea57c 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -27,7 +27,7 @@ N может быть отрицательным. Округляет значение до указанного десятичного разряда. -Функция возвращает ближайшее значение указанного порядка. В случае, когда заданное число равноудалено от чисел необходимого порядка, функция возвращает то из них, которое имеет ближайшую чётную цифру (банковское округление). +Функция возвращает ближайшее значение указанного порядка. В случае, когда заданное число равноудалено от чисел необходимого порядка, для типов с плавающей точкой (Float32/64) функция возвращает то из них, которое имеет ближайшую чётную цифру (банковское округление), для типов с фиксированной точкой (Decimal) функция использует округление в бо́льшую по модулю сторону (математическое округление). ``` sql round(expression [, decimal_places]) @@ -47,7 +47,7 @@ round(expression [, decimal_places]) ### Примеры {#primery} -**Пример использования** +**Пример использования с Float** ``` sql SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 @@ -61,6 +61,21 @@ SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3 └─────┴──────────────────────────┘ ``` +**Пример использования с Decimal** + + +``` sql +SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3 +``` + +``` text +┌──────x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐ +│ 0.0000 │ 0.0000 │ +│ 0.5000 │ 1.0000 │ +│ 1.0000 │ 1.0000 │ +└────────┴──────────────────────────────────────────────────┘ +``` + **Примеры округления** Округление до ближайшего числа. diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt index 1de5ea88aee..9e774deadf0 100644 --- a/programs/client/CMakeLists.txt +++ b/programs/client/CMakeLists.txt @@ -4,6 +4,7 @@ set (CLICKHOUSE_CLIENT_SOURCES QueryFuzzer.cpp Suggest.cpp TestHint.cpp + TestTags.cpp ) set (CLICKHOUSE_CLIENT_LINK diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c594f01861b..568928c28a3 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -6,6 +6,7 @@ #include "QueryFuzzer.h" #include "Suggest.h" #include "TestHint.h" +#include "TestTags.h" #if USE_REPLXX # include @@ -1078,12 +1079,17 @@ private: bool echo_query = echo_queries; + /// Test tags are started with "--" so they are interpreted as comments anyway. + /// But if the echo is enabled we have to remove the test tags from `all_queries_text` + /// because we don't want test tags to be echoed. + size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0; + /// Several queries separated by ';'. /// INSERT data is ended by the end of line, not ';'. /// An exception is VALUES format where we also support semicolon in /// addition to end of line. - const char * this_query_begin = all_queries_text.data(); + const char * this_query_begin = all_queries_text.data() + test_tags_length; const char * all_queries_end = all_queries_text.data() + all_queries_text.size(); while (this_query_begin < all_queries_end) @@ -2033,8 +2039,21 @@ private: PullingAsyncPipelineExecutor executor(pipeline); Block block; - while (executor.pull(block)) + while (true) { + try + { + if (!executor.pull(block)) + { + break; + } + } + catch (Exception & e) + { + e.addMessage(fmt::format("(in query: {})", full_query)); + throw; + } + /// Check if server send Log packet receiveLogs(); diff --git a/programs/client/TestTags.cpp b/programs/client/TestTags.cpp new file mode 100644 index 00000000000..f3cb49cdabd --- /dev/null +++ b/programs/client/TestTags.cpp @@ -0,0 +1,51 @@ +#include "TestTags.h" + +#include + +namespace DB +{ + +size_t getTestTagsLength(const String & multiline_query) +{ + const String & text = multiline_query; + size_t pos = 0; + bool first_line = true; + + while (true) + { + size_t line_start = pos; + + /// Skip spaces. + while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t')) + ++pos; + + /// Skip comment "--". + static constexpr const char comment[] = "--"; + if (text.compare(pos, strlen(comment), comment) != 0) + return line_start; + pos += strlen(comment); + + /// Skip the prefix "Tags:" if it's the first line. + if (first_line) + { + while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t')) + ++pos; + + static constexpr const char tags_prefix[] = "Tags:"; + if (text.compare(pos, strlen(tags_prefix), tags_prefix) != 0) + return 0; + pos += strlen(tags_prefix); + first_line = false; + } + + /// Skip end-of-line. + size_t eol_pos = text.find_first_of("\r\n", pos); + if (eol_pos == String::npos) + return text.length(); + bool two_chars_eol = (eol_pos + 1 < text.length()) && ((text[eol_pos + 1] == '\r') || (text[eol_pos + 1] == '\n')) && (text[eol_pos + 1] != text[eol_pos]); + size_t eol_length = two_chars_eol ? 2 : 1; + pos = eol_pos + eol_length; + } +} + +} diff --git a/programs/client/TestTags.h b/programs/client/TestTags.h new file mode 100644 index 00000000000..e2e36698cb5 --- /dev/null +++ b/programs/client/TestTags.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace DB +{ + +/// Returns the length of a text looking like +/// -- Tags: x, y, z +/// -- Tag x: explanation of tag x +/// -- Tag y: explanation of tag y +/// -- Tag z: explanation of tag z +/// +/// at the beginning of a multiline query. +/// If there are no test tags in the multiline query the function returns 0. +size_t getTestTagsLength(const String & multiline_query); + +} diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 0c834174519..2d09495c338 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -79,6 +79,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -251,7 +252,6 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int ARGUMENT_OUT_OF_BOUND; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int INCORRECT_DATA; extern const int INVALID_CONFIG_PARAMETER; extern const int SYSTEM_ERROR; extern const int FAILED_TO_GETPWUID; @@ -456,40 +456,6 @@ void checkForUsersNotInMainConfig( } } -static void loadEncryptionKey(const std::string & key_command [[maybe_unused]], Poco::Logger * log) -{ -#if USE_BASE64 && USE_SSL && USE_INTERNAL_SSL_LIBRARY - - auto process = ShellCommand::execute(key_command); - - std::string b64_key; - readStringUntilEOF(b64_key, process->out); - process->wait(); - - // turbob64 doesn't like whitespace characters in input. Strip - // them before decoding. - std::erase_if(b64_key, [](char c) - { - return c == ' ' || c == '\t' || c == '\r' || c == '\n'; - }); - - std::vector buf(b64_key.size()); - const size_t key_size = tb64dec(reinterpret_cast(b64_key.data()), b64_key.size(), - reinterpret_cast(buf.data())); - if (!key_size) - throw Exception("Failed to decode encryption key", ErrorCodes::INCORRECT_DATA); - else if (key_size < 16) - LOG_WARNING(log, "The encryption key should be at least 16 octets long."); - - const std::string_view key = std::string_view(buf.data(), key_size); - CompressionCodecEncrypted::setMasterKey(key); - -#else - LOG_WARNING(log, "Server was built without Base64 or SSL support. Encryption is disabled."); -#endif -} - - [[noreturn]] void forceShutdown() { #if defined(THREAD_SANITIZER) && defined(OS_LINUX) @@ -904,6 +870,8 @@ if (ThreadFuzzer::instance().isEffective()) global_context->updateStorageConfiguration(*config); global_context->updateInterserverCredentials(*config); + + CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs"); }, /* already_loaded = */ false); /// Reload it right now (initial loading) @@ -976,9 +944,9 @@ if (ThreadFuzzer::instance().isEffective()) global_context->getMergeTreeSettings().sanityCheck(settings); global_context->getReplicatedMergeTreeSettings().sanityCheck(settings); - /// Set up encryption. - if (config().has("encryption.key_command")) - loadEncryptionKey(config().getString("encryption.key_command"), log); + + /// try set up encryption. There are some errors in config, error will be printed and server wouldn't start. + CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs"); Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); diff --git a/programs/server/config.xml b/programs/server/config.xml index b8ef17458be..6c98ac740fe 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1026,10 +1026,28 @@ defined, or encryption codecs will be disabled otherwise. The command is executed through /bin/sh and is expected to write a Base64-encoded key to the stdout. --> - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp index 95611771b58..ad3a3090de6 100644 --- a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp +++ b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp @@ -15,8 +15,7 @@ namespace fs = std::filesystem; -#define UUID_PATTERN "[\\w]{8}-[\\w]{4}-[\\w]{4}-[\\w]{4}-[\\w]{12}" -#define EXTRACT_UUID_PATTERN fmt::format(".*/({})/.*", UUID_PATTERN) +#define EXTRACT_PATH_PATTERN ".*\\/store/(.*)" namespace DB @@ -32,39 +31,102 @@ namespace ErrorCodes * If test-mode option is added, files will be put by given url via PUT request. */ -void processTableFiles(const fs::path & path, const String & files_prefix, String uuid, - WriteBuffer & metadata_buf, std::function(const String &)> create_dst_buf) +void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf) { - fs::directory_iterator dir_end; - auto process_file = [&](const String & file_name, const String & file_path) + String remote_path; + RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path); + bool is_directory = fs::is_directory(file_path); + + writeText(file_path.filename().string(), metadata_buf); + writeChar('\t', metadata_buf); + writeBoolText(is_directory, metadata_buf); + if (!is_directory) { - auto remote_file_name = files_prefix + "-" + uuid + "-" + file_name; - writeText(remote_file_name, metadata_buf); writeChar('\t', metadata_buf); writeIntText(fs::file_size(file_path), metadata_buf); - writeChar('\n', metadata_buf); + } + writeChar('\n', metadata_buf); - auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path)); - auto dst_buf = create_dst_buf(remote_file_name); + if (is_directory) + return; - copyData(*src_buf, *dst_buf); - dst_buf->next(); - dst_buf->finalize(); - }; + auto dst_file_path = fs::path(dst_path) / remote_path; - for (fs::directory_iterator dir_it(path); dir_it != dir_end; ++dir_it) + auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path)); + std::shared_ptr dst_buf; + + /// test mode for integration tests. + if (test_mode) + dst_buf = std::make_shared(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT); + else + dst_buf = std::make_shared(dst_file_path); + + copyData(*src_buf, *dst_buf); + dst_buf->next(); + dst_buf->finalize(); +}; + + +void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode) +{ + std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl; + + String prefix; + RE2::FullMatch(data_path.string(), EXTRACT_PATH_PATTERN, &prefix); + + std::shared_ptr root_meta; + if (test_mode) + { + dst_path /= "store"; + auto files_root = dst_path / prefix; + root_meta = std::make_shared(Poco::URI(files_root / ".index"), Poco::Net::HTTPRequest::HTTP_PUT); + } + else + { + dst_path = fs::canonical(dst_path); + auto files_root = dst_path / prefix; + fs::create_directories(files_root); + root_meta = std::make_shared(files_root / ".index"); + } + + fs::directory_iterator dir_end; + for (fs::directory_iterator dir_it(data_path); dir_it != dir_end; ++dir_it) { if (dir_it->is_directory()) { + processFile(dir_it->path(), dst_path, test_mode, *root_meta); + + String directory_prefix; + RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix); + + std::shared_ptr directory_meta; + if (test_mode) + { + auto files_root = dst_path / prefix; + directory_meta = std::make_shared(Poco::URI(dst_path / directory_prefix / ".index"), Poco::Net::HTTPRequest::HTTP_PUT); + } + else + { + dst_path = fs::canonical(dst_path); + auto files_root = dst_path / prefix; + fs::create_directories(dst_path / directory_prefix); + directory_meta = std::make_shared(dst_path / directory_prefix / ".index"); + } + fs::directory_iterator files_end; for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it) - process_file(dir_it->path().filename().string() + "-" + file_it->path().filename().string(), file_it->path()); + processFile(file_it->path(), dst_path, test_mode, *directory_meta); + + directory_meta->next(); + directory_meta->finalize(); } else { - process_file(dir_it->path().filename(), dir_it->path()); + processFile(dir_it->path(), dst_path, test_mode, *root_meta); } } + root_meta->next(); + root_meta->finalize(); } } @@ -80,8 +142,7 @@ try ("metadata-path", po::value(), "Metadata path (select data_paths from system.tables where name='table_name'") ("test-mode", "Use test mode, which will put data on given url via PUT") ("url", po::value(), "Web server url for test mode") - ("output-dir", po::value(), "Directory to put files in non-test mode") - ("files-prefix", po::value(), "Prefix for stored files"); + ("output-dir", po::value(), "Directory to put files in non-test mode"); po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run(); po::variables_map options; @@ -94,18 +155,13 @@ try exit(0); } - String url, metadata_path, files_prefix; + String metadata_path; if (options.count("metadata-path")) metadata_path = options["metadata-path"].as(); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "No metadata-path option passed"); - if (options.count("files-prefix")) - files_prefix = options["files-prefix"].as(); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No files-prefix option passed"); - fs::path fs_path = fs::weakly_canonical(metadata_path); if (!fs::exists(fs_path)) { @@ -113,27 +169,14 @@ try return 1; } - String uuid; - if (!RE2::Extract(metadata_path, EXTRACT_UUID_PATTERN, "\\1", &uuid)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot extract uuid for: {}", metadata_path); - - std::shared_ptr metadata_buf; - std::function(const String &)> create_dst_buf; String root_path; - - if (options.count("test-mode")) + auto test_mode = options.contains("test-mode"); + if (test_mode) { if (options.count("url")) - url = options["url"].as(); + root_path = options["url"].as(); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "No url option passed for test mode"); - - metadata_buf = std::make_shared(Poco::URI(fs::path(url) / (".index-" + uuid)), Poco::Net::HTTPRequest::HTTP_PUT); - - create_dst_buf = [&](const String & remote_file_name) - { - return std::make_shared(Poco::URI(fs::path(url) / remote_file_name), Poco::Net::HTTPRequest::HTTP_PUT); - }; } else { @@ -141,17 +184,9 @@ try root_path = options["output-dir"].as(); else root_path = fs::current_path(); - - metadata_buf = std::make_shared(fs::path(root_path) / (".index-" + uuid)); - create_dst_buf = [&](const String & remote_file_name) - { - return std::make_shared(fs::path(root_path) / remote_file_name); - }; } - processTableFiles(fs_path, files_prefix, uuid, *metadata_buf, create_dst_buf); - metadata_buf->next(); - metadata_buf->finalize(); + processTableFiles(fs_path, root_path, test_mode); return 0; } diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index ad5c67d88d4..fd8f9faa3f8 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -21,11 +21,9 @@ namespace DB { struct Settings; -template -using DecimalOrVectorCol = std::conditional_t, ColumnDecimal, ColumnVector>; template constexpr bool DecimalOrExtendedInt = - IsDecimalNumber + is_decimal || std::is_same_v || std::is_same_v || std::is_same_v @@ -44,7 +42,7 @@ struct AvgFraction /// Invoked only is either Numerator or Denominator are Decimal. Float64 NO_SANITIZE_UNDEFINED divideIfAnyDecimal(UInt32 num_scale, UInt32 denom_scale [[maybe_unused]]) const { - if constexpr (IsDecimalNumber && IsDecimalNumber) + if constexpr (is_decimal && is_decimal) { // According to the docs, num(S1) / denom(S2) would have scale S1 @@ -60,7 +58,7 @@ struct AvgFraction /// Numerator is always casted to Float64 to divide correctly if the denominator is not Float64. Float64 num_converted; - if constexpr (IsDecimalNumber) + if constexpr (is_decimal) num_converted = DecimalUtils::convertTo(numerator, num_scale); else num_converted = static_cast(numerator); /// all other types, including extended integral. @@ -68,7 +66,7 @@ struct AvgFraction std::conditional_t, Float64, Denominator> denom_converted; - if constexpr (IsDecimalNumber) + if constexpr (is_decimal) denom_converted = DecimalUtils::convertTo(denominator, denom_scale); else if constexpr (DecimalOrExtendedInt) /// no way to divide Float64 and extended integral type without an explicit cast. @@ -139,7 +137,7 @@ public: void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if constexpr (IsDecimalNumber || IsDecimalNumber) + if constexpr (is_decimal || is_decimal) assert_cast &>(to).getData().push_back( this->data(place).divideIfAnyDecimal(num_scale, denom_scale)); else @@ -222,7 +220,7 @@ private: }; template -using AvgFieldType = std::conditional_t, +using AvgFieldType = std::conditional_t, std::conditional_t, Decimal256, Decimal128>, NearestFieldType>; @@ -239,7 +237,7 @@ public: void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final { - this->data(place).numerator += static_cast &>(*columns[0]).getData()[row_num]; + this->data(place).numerator += static_cast &>(*columns[0]).getData()[row_num]; ++this->data(place).denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h index 68d48803718..6189ae92ce4 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h @@ -8,7 +8,7 @@ namespace DB struct Settings; template -using AvgWeightedFieldType = std::conditional_t, +using AvgWeightedFieldType = std::conditional_t, std::conditional_t, Decimal256, Decimal128>, std::conditional_t, Float64, // no way to do UInt128 * UInt128, better cast to Float64 @@ -34,10 +34,10 @@ public: void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override { - const auto& weights = static_cast &>(*columns[1]); + const auto& weights = static_cast &>(*columns[1]); this->data(place).numerator += static_cast( - static_cast &>(*columns[0]).getData()[row_num]) * + static_cast &>(*columns[0]).getData()[row_num]) * static_cast(weights.getData()[row_num]); this->data(place).denominator += static_cast(weights.getData()[row_num]); diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp index e4eff6be901..3e0f1849545 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp @@ -25,14 +25,14 @@ namespace template struct MovingSum { - using Data = MovingSumData, Decimal128, NearestFieldType>>; + using Data = MovingSumData, Decimal128, NearestFieldType>>; using Function = MovingImpl; }; template struct MovingAvg { - using Data = MovingAvgData, Decimal128, Float64>>; + using Data = MovingAvgData, Decimal128, Float64>>; using Function = MovingImpl; }; diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h index daaa7a69baf..a3b7dc841b6 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h @@ -87,18 +87,10 @@ class MovingImpl final public: using ResultT = typename Data::Accumulator; - using ColumnSource = std::conditional_t, - ColumnDecimal, - ColumnVector>; + using ColumnSource = ColumnVectorOrDecimal; /// Probably for overflow function in the future. - using ColumnResult = std::conditional_t, - ColumnDecimal, - ColumnVector>; - - using DataTypeResult = std::conditional_t, - DataTypeDecimal, - DataTypeNumber>; + using ColumnResult = ColumnVectorOrDecimal; explicit MovingImpl(const DataTypePtr & data_type_, UInt64 window_size_ = std::numeric_limits::max()) : IAggregateFunctionDataHelper>({data_type_}, {}) @@ -106,14 +98,7 @@ public: String getName() const override { return Data::name; } - DataTypePtr getReturnType() const override - { - if constexpr (IsDecimalNumber) - return std::make_shared(std::make_shared( - DataTypeResult::maxPrecision(), getDecimalScale(*this->argument_types.at(0)))); - else - return std::make_shared(std::make_shared()); - } + DataTypePtr getReturnType() const override { return std::make_shared(getReturnTypeElement()); } void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { @@ -196,6 +181,18 @@ public: { return true; } + +private: + auto getReturnTypeElement() const + { + if constexpr (!is_decimal) + return std::make_shared>(); + else + { + using Res = DataTypeDecimal; + return std::make_shared(Res::maxPrecision(), getDecimalScale(*this->argument_types.at(0))); + } + } }; #undef AGGREGATE_FUNCTION_MOVING_MAX_ARRAY_SIZE diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index e5471b8a727..8fa4b235c2c 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -44,7 +44,7 @@ struct SingleValueDataFixed { private: using Self = SingleValueDataFixed; - using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecType = ColumnVectorOrDecimal; bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf. T value; diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 65d1df9fe77..5ca52f6d73a 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -67,10 +67,10 @@ class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper> { private: - using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecType = ColumnVectorOrDecimal; static constexpr bool returns_float = !(std::is_same_v); - static_assert(!IsDecimalNumber || !returns_float); + static_assert(!is_decimal || !returns_float); QuantileLevels levels; diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.cpp b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp new file mode 100644 index 00000000000..7f1196173a7 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + + +namespace DB +{ + +struct Settings; + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +template