mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
Merge branch 'master' into rabbit-fix
This commit is contained in:
Kseniia Sumarokova
GitHub
commit
fbc048b2bf
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -246,3 +246,6 @@
|
||||
[submodule "contrib/bzip2"]
|
||||
path = contrib/bzip2
|
||||
url = https://github.com/ClickHouse-Extras/bzip2.git
|
||||
[submodule "contrib/magic_enum"]
|
||||
path = contrib/magic_enum
|
||||
url = https://github.com/Neargye/magic_enum
|
||||
|
||||
@ -152,6 +152,7 @@ if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD)
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
|
||||
endif ()
|
||||
|
||||
include (cmake/check_flags.cmake)
|
||||
include (cmake/add_warning.cmake)
|
||||
|
||||
if (NOT MSVC)
|
||||
@ -166,7 +167,8 @@ if (COMPILER_CLANG)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
|
||||
endif ()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0)
|
||||
if (HAS_USE_CTOR_HOMING)
|
||||
# For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
@ -192,7 +194,7 @@ endif ()
|
||||
# Make sure the final executable has symbols exported
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
|
||||
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
|
||||
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
|
||||
|
||||
if (NOT OBJCOPY_PATH AND OS_DARWIN)
|
||||
find_program (BREW_PATH NAMES "brew")
|
||||
@ -379,7 +381,7 @@ if (COMPILER_CLANG)
|
||||
endif ()
|
||||
|
||||
# Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
|
||||
|
||||
if (LLVM_AR_PATH)
|
||||
message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
|
||||
@ -388,7 +390,7 @@ if (COMPILER_CLANG)
|
||||
message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
|
||||
endif ()
|
||||
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
|
||||
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
|
||||
|
||||
if (LLVM_RANLIB_PATH)
|
||||
message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
|
||||
@ -629,9 +631,6 @@ include_directories(${ConfigIncludePath})
|
||||
# Add as many warnings as possible for our own code.
|
||||
include (cmake/warnings.cmake)
|
||||
|
||||
# Check if needed compiler flags are supported
|
||||
include (cmake/check_flags.cmake)
|
||||
|
||||
add_subdirectory (base)
|
||||
add_subdirectory (src)
|
||||
add_subdirectory (programs)
|
||||
|
||||
@ -85,6 +85,7 @@ target_link_libraries (common
|
||||
replxx
|
||||
cctz
|
||||
fmt
|
||||
magic_enum
|
||||
)
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
|
||||
157
base/common/Decimal.h
Normal file
157
base/common/Decimal.h
Normal file
@ -0,0 +1,157 @@
|
||||
#pragma once
|
||||
#include "common/extended_types.h"
|
||||
|
||||
#if !defined(NO_SANITIZE_UNDEFINED)
|
||||
#if defined(__clang__)
|
||||
#define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined")))
|
||||
#else
|
||||
#define NO_SANITIZE_UNDEFINED
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
template <class> struct Decimal;
|
||||
class DateTime64;
|
||||
|
||||
using Decimal32 = Decimal<Int32>;
|
||||
using Decimal64 = Decimal<Int64>;
|
||||
using Decimal128 = Decimal<Int128>;
|
||||
using Decimal256 = Decimal<Int256>;
|
||||
|
||||
template <class T>
|
||||
concept is_decimal =
|
||||
std::is_same_v<T, Decimal32>
|
||||
|| std::is_same_v<T, Decimal64>
|
||||
|| std::is_same_v<T, Decimal128>
|
||||
|| std::is_same_v<T, Decimal256>
|
||||
|| std::is_same_v<T, DateTime64>;
|
||||
|
||||
template <class T>
|
||||
concept is_over_big_int =
|
||||
std::is_same_v<T, Int128>
|
||||
|| std::is_same_v<T, UInt128>
|
||||
|| std::is_same_v<T, Int256>
|
||||
|| std::is_same_v<T, UInt256>
|
||||
|| std::is_same_v<T, Decimal128>
|
||||
|| std::is_same_v<T, Decimal256>;
|
||||
|
||||
template <class T> struct NativeTypeT { using Type = T; };
|
||||
template <is_decimal T> struct NativeTypeT<T> { using Type = typename T::NativeType; };
|
||||
template <class T> using NativeType = typename NativeTypeT<T>::Type;
|
||||
|
||||
/// Own FieldType for Decimal.
|
||||
/// It is only a "storage" for decimal.
|
||||
/// To perform operations, you also have to provide a scale (number of digits after point).
|
||||
template <typename T>
|
||||
struct Decimal
|
||||
{
|
||||
using NativeType = T;
|
||||
|
||||
constexpr Decimal() = default;
|
||||
constexpr Decimal(Decimal<T> &&) = default;
|
||||
constexpr Decimal(const Decimal<T> &) = default;
|
||||
|
||||
constexpr Decimal(const T & value_): value(value_) {}
|
||||
|
||||
template <typename U>
|
||||
constexpr Decimal(const Decimal<U> & x): value(x.value) {}
|
||||
|
||||
constexpr Decimal<T> & operator = (Decimal<T> &&) = default;
|
||||
constexpr Decimal<T> & operator = (const Decimal<T> &) = default;
|
||||
|
||||
constexpr operator T () const { return value; }
|
||||
|
||||
template <typename U>
|
||||
constexpr U convertTo() const
|
||||
{
|
||||
if constexpr (is_decimal<U>)
|
||||
return convertTo<typename U::NativeType>();
|
||||
else
|
||||
return static_cast<U>(value);
|
||||
}
|
||||
|
||||
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
|
||||
const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
|
||||
const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
|
||||
const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
|
||||
const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
|
||||
|
||||
template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
|
||||
template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
|
||||
template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
|
||||
template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
|
||||
template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
|
||||
|
||||
/// This is to avoid UB for sumWithOverflow()
|
||||
void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
|
||||
|
||||
T value;
|
||||
};
|
||||
|
||||
template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
|
||||
template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
|
||||
template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
|
||||
template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
|
||||
template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
|
||||
template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
|
||||
|
||||
template <typename T> inline Decimal<T> operator+ (const Decimal<T> & x, const Decimal<T> & y) { return x.value + y.value; }
|
||||
template <typename T> inline Decimal<T> operator- (const Decimal<T> & x, const Decimal<T> & y) { return x.value - y.value; }
|
||||
template <typename T> inline Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
|
||||
template <typename T> inline Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
|
||||
template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
|
||||
|
||||
/// Distinguishable type to allow function resolution/deduction based on value type,
|
||||
/// but also relatively easy to convert to/from Decimal64.
|
||||
class DateTime64 : public Decimal64
|
||||
{
|
||||
public:
|
||||
using Base = Decimal64;
|
||||
using Base::Base;
|
||||
using NativeType = Base::NativeType;
|
||||
|
||||
constexpr DateTime64(const Base & v): Base(v) {}
|
||||
};
|
||||
}
|
||||
|
||||
constexpr DB::UInt64 max_uint_mask = std::numeric_limits<DB::UInt64>::max();
|
||||
|
||||
namespace std
|
||||
{
|
||||
template <typename T>
|
||||
struct hash<DB::Decimal<T>>
|
||||
{
|
||||
size_t operator()(const DB::Decimal<T> & x) const { return hash<T>()(x.value); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct hash<DB::Decimal128>
|
||||
{
|
||||
size_t operator()(const DB::Decimal128 & x) const
|
||||
{
|
||||
return std::hash<DB::Int64>()(x.value >> 64)
|
||||
^ std::hash<DB::Int64>()(x.value & max_uint_mask);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct hash<DB::DateTime64>
|
||||
{
|
||||
size_t operator()(const DB::DateTime64 & x) const
|
||||
{
|
||||
return std::hash<DB::DateTime64::NativeType>()(x);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct hash<DB::Decimal256>
|
||||
{
|
||||
size_t operator()(const DB::Decimal256 & x) const
|
||||
{
|
||||
// FIXME temp solution
|
||||
return std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value >> 64 & max_uint_mask))
|
||||
^ std::hash<DB::Int64>()(static_cast<DB::Int64>(x.value & max_uint_mask));
|
||||
}
|
||||
};
|
||||
}
|
||||
38
base/common/EnumReflection.h
Normal file
38
base/common/EnumReflection.h
Normal file
@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
#include <magic_enum.hpp>
|
||||
#include <fmt/format.h>
|
||||
|
||||
template <class T> concept is_enum = std::is_enum_v<T>;
|
||||
|
||||
namespace detail
|
||||
{
|
||||
template <is_enum E, class F, size_t ...I>
|
||||
constexpr void static_for(F && f, std::index_sequence<I...>)
|
||||
{
|
||||
(std::forward<F>(f)(std::integral_constant<E, magic_enum::enum_value<E>(I)>()) , ...);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over enum values in compile-time (compile-time switch/case, loop unrolling).
|
||||
*
|
||||
* @example static_for<E>([](auto enum_value) { return template_func<enum_value>(); }
|
||||
* ^ enum_value can be used as a template parameter
|
||||
*/
|
||||
template <is_enum E, class F>
|
||||
constexpr void static_for(F && f)
|
||||
{
|
||||
constexpr size_t count = magic_enum::enum_count<E>();
|
||||
detail::static_for<E>(std::forward<F>(f), std::make_index_sequence<count>());
|
||||
}
|
||||
|
||||
/// Enable printing enum values as strings via fmt + magic_enum
|
||||
template <is_enum T>
|
||||
struct fmt::formatter<T> : fmt::formatter<std::string_view>
|
||||
{
|
||||
constexpr auto format(T value, auto& format_context)
|
||||
{
|
||||
return formatter<string_view>::format(magic_enum::enum_name(value), format_context);
|
||||
}
|
||||
};
|
||||
@ -16,6 +16,10 @@ extern "C"
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) && __clang_major__ >= 13
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
|
||||
@ -41,22 +41,14 @@ template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
|
||||
template <typename T>
|
||||
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
|
||||
|
||||
template <class T> concept is_integer =
|
||||
std::is_integral_v<T>
|
||||
|| std::is_same_v<T, Int128>
|
||||
|| std::is_same_v<T, UInt128>
|
||||
|| std::is_same_v<T, Int256>
|
||||
|| std::is_same_v<T, UInt256>;
|
||||
|
||||
/// TODO: is_integral includes char, char8_t and wchar_t.
|
||||
template <typename T>
|
||||
struct is_integer
|
||||
{
|
||||
static constexpr bool value = std::is_integral_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_integer<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_integer_v = is_integer<T>::value;
|
||||
|
||||
template <class T> concept is_floating_point = std::is_floating_point_v<T>;
|
||||
|
||||
template <typename T>
|
||||
struct is_arithmetic
|
||||
|
||||
@ -36,18 +36,7 @@
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
template <char s0>
|
||||
inline bool is_in(char x)
|
||||
{
|
||||
return x == s0;
|
||||
}
|
||||
|
||||
template <char s0, char s1, char... tail>
|
||||
inline bool is_in(char x)
|
||||
{
|
||||
return x == s0 || is_in<s1, tail...>(x);
|
||||
}
|
||||
template <char ...chars> constexpr bool is_in(char x) { return ((x == chars) || ...); }
|
||||
|
||||
#if defined(__SSE2__)
|
||||
template <char s0>
|
||||
@ -67,16 +56,10 @@ inline __m128i mm_is_in(__m128i bytes)
|
||||
#endif
|
||||
|
||||
template <bool positive>
|
||||
bool maybe_negate(bool x)
|
||||
{
|
||||
if constexpr (positive)
|
||||
return x;
|
||||
else
|
||||
return !x;
|
||||
}
|
||||
constexpr bool maybe_negate(bool x) { return x == positive; }
|
||||
|
||||
template <bool positive>
|
||||
uint16_t maybe_negate(uint16_t x)
|
||||
constexpr uint16_t maybe_negate(uint16_t x)
|
||||
{
|
||||
if constexpr (positive)
|
||||
return x;
|
||||
@ -149,12 +132,13 @@ template <bool positive, ReturnMode return_mode, size_t num_chars,
|
||||
char c05 = 0, char c06 = 0, char c07 = 0, char c08 = 0,
|
||||
char c09 = 0, char c10 = 0, char c11 = 0, char c12 = 0,
|
||||
char c13 = 0, char c14 = 0, char c15 = 0, char c16 = 0>
|
||||
inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end)
|
||||
inline const char * find_first_symbols_sse42(const char * const begin, const char * const end)
|
||||
{
|
||||
const char * pos = begin;
|
||||
|
||||
#if defined(__SSE4_2__)
|
||||
#define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT)
|
||||
constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT;
|
||||
|
||||
__m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16);
|
||||
|
||||
for (; pos + 15 < end; pos += 16)
|
||||
@ -163,16 +147,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons
|
||||
|
||||
if constexpr (positive)
|
||||
{
|
||||
if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE))
|
||||
return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE);
|
||||
if (_mm_cmpestrc(set, num_chars, bytes, 16, mode))
|
||||
return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY))
|
||||
return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY);
|
||||
if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY))
|
||||
return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY);
|
||||
}
|
||||
}
|
||||
#undef MODE
|
||||
#endif
|
||||
|
||||
for (; pos < end; ++pos)
|
||||
@ -197,20 +180,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons
|
||||
}
|
||||
|
||||
|
||||
template <bool positive, ReturnMode return_mode, char... symbols>
|
||||
inline const char * find_first_symbols_sse42(const char * begin, const char * end)
|
||||
{
|
||||
return find_first_symbols_sse42_impl<positive, return_mode, sizeof...(symbols), symbols...>(begin, end);
|
||||
}
|
||||
|
||||
/// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do.
|
||||
|
||||
template <bool positive, ReturnMode return_mode, char... symbols>
|
||||
inline const char * find_first_symbols_dispatch(const char * begin, const char * end)
|
||||
requires(0 <= sizeof...(symbols) && sizeof...(symbols) <= 16)
|
||||
{
|
||||
#if defined(__SSE4_2__)
|
||||
if (sizeof...(symbols) >= 5)
|
||||
return find_first_symbols_sse42<positive, return_mode, symbols...>(begin, end);
|
||||
return find_first_symbols_sse42<positive, return_mode, sizeof...(symbols), symbols...>(begin, end);
|
||||
else
|
||||
#endif
|
||||
return find_first_symbols_sse2<positive, return_mode, symbols...>(begin, end);
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
#if defined(__clang__) && __clang_major__ >= 13
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
/// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex.
|
||||
|
||||
#include <common/defines.h>
|
||||
|
||||
@ -15,15 +15,15 @@ private:
|
||||
public:
|
||||
using UnderlyingType = T;
|
||||
template <class Enable = typename std::is_copy_constructible<T>::type>
|
||||
explicit StrongTypedef(const T & t_) : t(t_) {}
|
||||
constexpr explicit StrongTypedef(const T & t_) : t(t_) {}
|
||||
template <class Enable = typename std::is_move_constructible<T>::type>
|
||||
explicit StrongTypedef(T && t_) : t(std::move(t_)) {}
|
||||
constexpr explicit StrongTypedef(T && t_) : t(std::move(t_)) {}
|
||||
|
||||
template <class Enable = typename std::is_default_constructible<T>::type>
|
||||
StrongTypedef(): t() {}
|
||||
constexpr StrongTypedef(): t() {}
|
||||
|
||||
StrongTypedef(const Self &) = default;
|
||||
StrongTypedef(Self &&) = default;
|
||||
constexpr StrongTypedef(const Self &) = default;
|
||||
constexpr StrongTypedef(Self &&) = default;
|
||||
|
||||
Self & operator=(const Self &) = default;
|
||||
Self & operator=(Self &&) = default;
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
|
||||
#if defined(__clang__) && __clang_major__ >= 13
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
constexpr size_t KiB = 1024;
|
||||
constexpr size_t MiB = 1024 * KiB;
|
||||
constexpr size_t GiB = 1024 * MiB;
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
#if defined(__clang__) && __clang_major__ >= 13
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <daemon/SentryWriter.h>
|
||||
|
||||
|
||||
@ -49,6 +49,8 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR)
|
||||
target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
target_no_warning(mysqlxx reserved-macro-identifier)
|
||||
|
||||
if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES)
|
||||
message(WARNING "Statically linking with system mysql/mariadb only works "
|
||||
"if mysql client libraries are built with same openssl version as "
|
||||
|
||||
@ -79,7 +79,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf
|
||||
std::lock_guard<std::mutex> lock(impl->mutex);
|
||||
if (auto entry = impl->pools.find(config_name); entry != impl->pools.end())
|
||||
{
|
||||
return *(entry->second.get());
|
||||
return *(entry->second);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -100,7 +100,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf
|
||||
impl->pools.insert_or_assign(config_name, pool);
|
||||
impl->pools_by_ids.insert_or_assign(entry_name, config_name);
|
||||
}
|
||||
return *(pool.get());
|
||||
return *pool;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -27,6 +27,10 @@
|
||||
#define _PATH_TTY "/dev/tty"
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) && __clang_major__ >= 13
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
#include <termios.h>
|
||||
#include <signal.h>
|
||||
#include <ctype.h>
|
||||
|
||||
@ -6,7 +6,7 @@ if (ENABLE_CLANG_TIDY)
|
||||
message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.")
|
||||
endif()
|
||||
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-13" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
|
||||
|
||||
if (CLANG_TIDY_PATH)
|
||||
message(STATUS
|
||||
|
||||
@ -4,3 +4,4 @@ include (CheckCCompilerFlag)
|
||||
check_cxx_compiler_flag("-Wsuggest-destructor-override" HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
|
||||
check_cxx_compiler_flag("-Wshadow" HAS_SHADOW)
|
||||
check_cxx_compiler_flag("-Wsuggest-override" HAS_SUGGEST_OVERRIDE)
|
||||
check_cxx_compiler_flag("-Xclang -fuse-ctor-homing" HAS_USE_CTOR_HOMING)
|
||||
|
||||
@ -192,4 +192,29 @@ elseif (COMPILER_GCC)
|
||||
# For some reason (bug in gcc?) macro 'GCC diagnostic ignored "-Wstringop-overflow"' doesn't help.
|
||||
add_cxx_compile_options(-Wno-stringop-overflow)
|
||||
endif()
|
||||
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11)
|
||||
# reinterpretAs.cpp:182:31: error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type
|
||||
# ‘using ToFieldType = using FieldType = using UUID = struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’
|
||||
# {aka ‘struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’} from an array of ‘const char8_t’
|
||||
add_cxx_compile_options(-Wno-error=class-memaccess)
|
||||
|
||||
# Maybe false positive...
|
||||
# In file included from /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:673,
|
||||
# In function ‘void std::__1::__libcpp_operator_delete(_Args ...) [with _Args = {void*, long unsigned int}]’,
|
||||
# inlined from ‘void std::__1::__do_deallocate_handle_size(void*, size_t, _Args ...) [with _Args = {}]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:271:34,
|
||||
# inlined from ‘void std::__1::__libcpp_deallocate(void*, size_t, size_t)’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:285:41,
|
||||
# inlined from ‘constexpr void std::__1::allocator<_Tp>::deallocate(_Tp*, size_t) [with _Tp = char]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:849:39,
|
||||
# inlined from ‘static constexpr void std::__1::allocator_traits<_Alloc>::deallocate(std::__1::allocator_traits<_Alloc>::allocator_type&, std::__1::allocator_traits<_Alloc>::pointer, std::__1::allocator_traits<_Alloc>::size_type) [with _Alloc = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/__memory/allocator_traits.h:476:24,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2219:35,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2213:1,
|
||||
# inlined from ‘DB::JSONBuilder::JSONMap::Pair::~Pair()’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:90:12,
|
||||
# inlined from ‘void DB::JSONBuilder::JSONMap::add(std::__1::string, DB::JSONBuilder::ItemPtr)’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:97:68,
|
||||
# inlined from ‘virtual void DB::ExpressionStep::describeActions(DB::JSONBuilder::JSONMap&) const’ at /home/jakalletti/ClickHouse/ClickHouse/src/Processors/QueryPlan/ExpressionStep.cpp:102:12:
|
||||
# /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:247:20: error: ‘void operator delete(void*, size_t)’ called on a pointer to an unallocated object ‘7598543875853023301’ [-Werror=free-nonheap-object]
|
||||
add_cxx_compile_options(-Wno-error=free-nonheap-object)
|
||||
|
||||
# AggregateFunctionAvg.h:203:100: error: ‘this’ pointer is null [-Werror=nonnull]
|
||||
add_cxx_compile_options(-Wno-error=nonnull)
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
1
contrib/CMakeLists.txt
vendored
1
contrib/CMakeLists.txt
vendored
@ -33,6 +33,7 @@ endif()
|
||||
set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1)
|
||||
|
||||
add_subdirectory (abseil-cpp-cmake)
|
||||
add_subdirectory (magic-enum-cmake)
|
||||
add_subdirectory (boost-cmake)
|
||||
add_subdirectory (cctz-cmake)
|
||||
add_subdirectory (consistent-hashing)
|
||||
|
||||
2
contrib/abseil-cpp
vendored
2
contrib/abseil-cpp
vendored
@ -1 +1 @@
|
||||
Subproject commit 4f3b686f86c3ebaba7e4e926e62a79cb1c659a54
|
||||
Subproject commit b004a8a02418b83de8b686caa0b0f6e39ac2191f
|
||||
2
contrib/fastops
vendored
2
contrib/fastops
vendored
@ -1 +1 @@
|
||||
Subproject commit 88752a5e03cf34639a4a37a4b41d8b463fffd2b5
|
||||
Subproject commit 012b777df9e2d145a24800a6c8c3d4a0249bb09e
|
||||
2
contrib/llvm
vendored
2
contrib/llvm
vendored
@ -1 +1 @@
|
||||
Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1
|
||||
Subproject commit f30bbecef78b75b527e257c1304d0be2f2f95975
|
||||
3
contrib/magic-enum-cmake/CMakeLists.txt
Normal file
3
contrib/magic-enum-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,3 @@
|
||||
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/magic_enum")
|
||||
add_library (magic_enum INTERFACE)
|
||||
target_include_directories(magic_enum INTERFACE ${LIBRARY_DIR}/include)
|
||||
1
contrib/magic_enum
vendored
Submodule
1
contrib/magic_enum
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 38f86e4d093cfc9034a140d37de2168e3951bef3
|
||||
2
contrib/rocksdb
vendored
2
contrib/rocksdb
vendored
@ -1 +1 @@
|
||||
Subproject commit b6480c69bf3ab6e298e0d019a07fd4f69029b26a
|
||||
Subproject commit 5ea892c8673e6c5a052887653673b967d44cc59b
|
||||
4
debian/rules
vendored
4
debian/rules
vendored
@ -36,8 +36,8 @@ endif
|
||||
|
||||
CMAKE_FLAGS += -DENABLE_UTILS=0
|
||||
|
||||
DEB_CC ?= $(shell which gcc-10 gcc-9 gcc | head -n1)
|
||||
DEB_CXX ?= $(shell which g++-10 g++-9 g++ | head -n1)
|
||||
DEB_CC ?= $(shell which gcc-11 gcc-10 gcc-9 gcc | head -n1)
|
||||
DEB_CXX ?= $(shell which g++-11 g++-10 g++-9 g++ | head -n1)
|
||||
|
||||
ifdef DEB_CXX
|
||||
DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ set -e
|
||||
#ccache -s # uncomment to display CCache statistics
|
||||
mkdir -p /server/build_docker
|
||||
cd /server/build_docker
|
||||
cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-12)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-12)"
|
||||
cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-13)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-13)"
|
||||
|
||||
# Set the number of build jobs to the half of number of virtual CPU cores (rounded up).
|
||||
# By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time.
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# docker build -t clickhouse/binary-builder .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
@ -41,18 +41,14 @@ RUN apt-get update \
|
||||
ccache \
|
||||
cmake \
|
||||
curl \
|
||||
g++-10 \
|
||||
gcc-10 \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
libicu-dev \
|
||||
libreadline-dev \
|
||||
clang-12 \
|
||||
clang-tidy-12 \
|
||||
lld-12 \
|
||||
llvm-12 \
|
||||
llvm-12-dev \
|
||||
clang-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
lld-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION}-dev \
|
||||
libicu-dev \
|
||||
libreadline-dev \
|
||||
moreutils \
|
||||
@ -104,15 +100,10 @@ RUN wget -nv "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.0
|
||||
# Download toolchain for FreeBSD 11.3
|
||||
RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz
|
||||
|
||||
# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable.
|
||||
# Current workaround is to use latest version proposed repo. Remove as soon as
|
||||
# gcc-10.2 appear in stable repo.
|
||||
RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install gcc-10 g++-10 --yes
|
||||
|
||||
RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update
|
||||
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
|
||||
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
||||
&& apt-get update \
|
||||
&& apt-get install gcc-11 g++-11 --yes
|
||||
|
||||
|
||||
COPY build.sh /
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# docker build -t clickhouse/deb-builder .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
@ -37,17 +37,17 @@ RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
alien \
|
||||
clang-12 \
|
||||
clang-tidy-12 \
|
||||
clang-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
cmake \
|
||||
debhelper \
|
||||
devscripts \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lld-12 \
|
||||
llvm-12 \
|
||||
llvm-12-dev \
|
||||
lld-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION}-dev \
|
||||
moreutils \
|
||||
ninja-build \
|
||||
perl \
|
||||
@ -57,15 +57,11 @@ RUN apt-get update \
|
||||
tzdata \
|
||||
--yes --no-install-recommends
|
||||
|
||||
# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable.
|
||||
# Current workaround is to use latest version proposed repo. Remove as soon as
|
||||
# gcc-10.2 appear in stable repo.
|
||||
RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list
|
||||
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
|
||||
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
||||
&& apt-get update \
|
||||
&& apt-get install gcc-11 g++-11 --yes
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install gcc-10 g++-10 --yes --no-install-recommends
|
||||
|
||||
RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
|
||||
|
||||
@ -205,7 +205,8 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--build-type", choices=("debug", ""), default="")
|
||||
parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
|
||||
"clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64",
|
||||
"clang-11-freebsd", "clang-12-freebsd", "gcc-10"), default="clang-12")
|
||||
"clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64",
|
||||
"clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13")
|
||||
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
|
||||
parser.add_argument("--unbundled", action="store_true")
|
||||
parser.add_argument("--split-binary", action="store_true")
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# docker build -t clickhouse/test-base .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libl
|
||||
# https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
|
||||
RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser
|
||||
|
||||
RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-12 -DCMAKE_C_COMPILER=clang-12 && make -j
|
||||
RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-13 -DCMAKE_C_COMPILER=clang-13 && make -j
|
||||
|
||||
ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator
|
||||
ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator
|
||||
@ -24,7 +24,7 @@ ENV SHA=nosha
|
||||
ENV DATA="data"
|
||||
|
||||
CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
|
||||
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-12 -DCMAKE_C_COMPILER=/usr/bin/clang-12 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
|
||||
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
|
||||
mkdir -p $HTML_RESULT_DIRECTORY && \
|
||||
$CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
|
||||
cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\
|
||||
|
||||
@ -80,7 +80,7 @@ LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAM
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
|
||||
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time --use-skip-list 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt
|
||||
|
||||
readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt")
|
||||
|
||||
@ -97,7 +97,7 @@ then
|
||||
|
||||
echo "Going to run again: ${FAILED_TESTS[*]}"
|
||||
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper --use-skip-list "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt
|
||||
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt
|
||||
else
|
||||
echo "No failed tests"
|
||||
fi
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# docker build -t clickhouse/fasttest .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
|
||||
|
||||
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT
|
||||
stage=${stage:-}
|
||||
|
||||
# Compiler version, normally set by Dockerfile
|
||||
export LLVM_VERSION=${LLVM_VERSION:-12}
|
||||
export LLVM_VERSION=${LLVM_VERSION:-13}
|
||||
|
||||
# A variable to pass additional flags to CMake.
|
||||
# Here we explicitly default it to nothing so that bash doesn't complain about
|
||||
@ -159,6 +159,7 @@ function clone_submodules
|
||||
cd "$FASTTEST_SOURCE"
|
||||
|
||||
SUBMODULES_TO_UPDATE=(
|
||||
contrib/magic_enum
|
||||
contrib/abseil-cpp
|
||||
contrib/boost
|
||||
contrib/zlib-ng
|
||||
@ -261,153 +262,8 @@ function run_tests
|
||||
|
||||
start_server
|
||||
|
||||
TESTS_TO_SKIP=(
|
||||
00105_shard_collations
|
||||
00109_shard_totals_after_having
|
||||
00110_external_sort
|
||||
00302_http_compression
|
||||
00417_kill_query
|
||||
00436_convert_charset
|
||||
00490_special_line_separators_and_characters_outside_of_bmp
|
||||
00652_replicated_mutations_zookeeper
|
||||
00682_empty_parts_merge
|
||||
00701_rollup
|
||||
00834_cancel_http_readonly_queries_on_client_close
|
||||
00911_tautological_compare
|
||||
|
||||
# Hyperscan
|
||||
00926_multimatch
|
||||
00929_multi_match_edit_distance
|
||||
01681_hyperscan_debug_assertion
|
||||
02004_max_hyperscan_regex_length
|
||||
|
||||
01176_mysql_client_interactive # requires mysql client
|
||||
01031_mutations_interpreter_and_context
|
||||
01053_ssd_dictionary # this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled
|
||||
01083_expressions_in_engine_arguments
|
||||
01092_memory_profiler
|
||||
01098_msgpack_format
|
||||
01098_temporary_and_external_tables
|
||||
01103_check_cpu_instructions_at_startup # avoid dependency on qemu -- invonvenient when running locally
|
||||
01193_metadata_loading
|
||||
01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently
|
||||
01251_dict_is_in_infinite_loop
|
||||
01259_dictionary_custom_settings_ddl
|
||||
01268_dictionary_direct_layout
|
||||
01280_ssd_complex_key_dictionary
|
||||
01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently
|
||||
01318_encrypt # Depends on OpenSSL
|
||||
01318_decrypt # Depends on OpenSSL
|
||||
01663_aes_msan # Depends on OpenSSL
|
||||
01667_aes_args_check # Depends on OpenSSL
|
||||
01683_codec_encrypted # Depends on OpenSSL
|
||||
01776_decrypt_aead_size_check # Depends on OpenSSL
|
||||
01811_filter_by_null # Depends on OpenSSL
|
||||
02012_sha512_fixedstring # Depends on OpenSSL
|
||||
01281_unsucceeded_insert_select_queries_counter
|
||||
01292_create_user
|
||||
01294_lazy_database_concurrent
|
||||
01305_replica_create_drop_zookeeper
|
||||
01354_order_by_tuple_collate_const
|
||||
01355_ilike
|
||||
01411_bayesian_ab_testing
|
||||
01798_uniq_theta_sketch
|
||||
01799_long_uniq_theta_sketch
|
||||
01890_stem # depends on libstemmer_c
|
||||
02003_compress_bz2 # depends on bzip2
|
||||
01059_storage_file_compression # depends on brotli and bzip2
|
||||
collate
|
||||
collation
|
||||
_orc_
|
||||
arrow
|
||||
avro
|
||||
base64
|
||||
brotli
|
||||
capnproto
|
||||
client
|
||||
ddl_dictionaries
|
||||
h3
|
||||
hashing
|
||||
hdfs
|
||||
java_hash
|
||||
json
|
||||
limit_memory
|
||||
live_view
|
||||
memory_leak
|
||||
memory_limit
|
||||
mysql
|
||||
odbc
|
||||
parallel_alter
|
||||
parquet
|
||||
protobuf
|
||||
secure
|
||||
sha256
|
||||
xz
|
||||
|
||||
# Not sure why these two fail even in sequential mode. Disabled for now
|
||||
# to make some progress.
|
||||
00646_url_engine
|
||||
00974_query_profiler
|
||||
|
||||
# In fasttest, ENABLE_LIBRARIES=0, so rocksdb engine is not enabled by default
|
||||
01504_rocksdb
|
||||
01686_rocksdb
|
||||
|
||||
# Look at DistributedFilesToInsert, so cannot run in parallel.
|
||||
01460_DistributedFilesToInsert
|
||||
|
||||
01541_max_memory_usage_for_user_long
|
||||
|
||||
# Require python libraries like scipy, pandas and numpy
|
||||
01322_ttest_scipy
|
||||
01561_mann_whitney_scipy
|
||||
|
||||
01545_system_errors
|
||||
# Checks system.errors
|
||||
01563_distributed_query_finish
|
||||
|
||||
# nc - command not found
|
||||
01601_proxy_protocol
|
||||
01622_defaults_for_url_engine
|
||||
|
||||
# JSON functions
|
||||
01666_blns
|
||||
|
||||
# Requires postgresql-client
|
||||
01802_test_postgresql_protocol_with_row_policy
|
||||
|
||||
# Depends on AWS
|
||||
01801_s3_cluster
|
||||
02012_settings_clause_for_s3
|
||||
|
||||
# needs psql
|
||||
01889_postgresql_protocol_null_fields
|
||||
|
||||
# needs pv
|
||||
01923_network_receive_time_metric_insert
|
||||
|
||||
01889_sqlite_read_write
|
||||
|
||||
# needs s2
|
||||
01849_geoToS2
|
||||
01851_s2_to_geo
|
||||
01852_s2_get_neighbours
|
||||
01853_s2_cells_intersect
|
||||
01854_s2_cap_contains
|
||||
01854_s2_cap_union
|
||||
|
||||
# needs s3
|
||||
01944_insert_partition_by
|
||||
|
||||
# depends on Go
|
||||
02013_zlib_read_after_eof
|
||||
|
||||
# Accesses CH via mysql table function (which is unavailable)
|
||||
01747_system_session_log_long
|
||||
)
|
||||
|
||||
time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \
|
||||
--no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" \
|
||||
time clickhouse-test --hung-check -j 8 --order=random \
|
||||
--fast-tests-only --no-long --testname --shard --zookeeper \
|
||||
-- "$FASTTEST_FOCUS" 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee "$FASTTEST_OUTPUT/test_log.txt"
|
||||
|
||||
@ -12,7 +12,7 @@ stage=${stage:-}
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
echo "$script_dir"
|
||||
repo_dir=ch
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-12_debug_none_bundled_unsplitted_disable_False_binary"}
|
||||
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"}
|
||||
|
||||
function clone
|
||||
{
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-12_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
|
||||
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
|
||||
|
||||
|
||||
|
||||
@ -38,7 +38,7 @@ RUN set -x \
|
||||
&& dpkg -i "${PKG_VERSION}.deb"
|
||||
|
||||
CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
|
||||
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-12 -DCMAKE_CXX_COMPILER=clang\+\+-12 \
|
||||
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
|
||||
&& ninja re2_st clickhouse_grpc_protos \
|
||||
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
|
||||
cp /repo_folder/pvs-studio.log /test_output; \
|
||||
|
||||
@ -108,7 +108,7 @@ function run_tests()
|
||||
ADDITIONAL_OPTIONS+=('--replicated-database')
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --use-skip-list --print-time "${ADDITIONAL_OPTIONS[@]}" \
|
||||
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \
|
||||
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
}
|
||||
|
||||
|
||||
@ -97,7 +97,7 @@ function run_tests()
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper --hung-check --print-time \
|
||||
--use-skip-list --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
--test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a test_output/test_result.txt
|
||||
}
|
||||
|
||||
@ -13,8 +13,4 @@ dpkg -i package_folder/clickhouse-test_*.deb
|
||||
|
||||
service clickhouse-server start && sleep 5
|
||||
|
||||
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
||||
SKIP_LIST_OPT="--use-skip-list"
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
clickhouse-test --testname --shard --zookeeper "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
@ -10,14 +10,6 @@ import logging
|
||||
import time
|
||||
|
||||
|
||||
def get_skip_list_cmd(path):
|
||||
with open(path, 'r') as f:
|
||||
for line in f:
|
||||
if '--use-skip-list' in line:
|
||||
return '--use-skip-list'
|
||||
return ''
|
||||
|
||||
|
||||
def get_options(i):
|
||||
options = []
|
||||
client_options = []
|
||||
@ -56,8 +48,6 @@ def get_options(i):
|
||||
|
||||
|
||||
def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit):
|
||||
skip_list_opt = get_skip_list_cmd(cmd)
|
||||
|
||||
global_time_limit_option = ''
|
||||
if global_time_limit:
|
||||
global_time_limit_option = "--global_time_limit={}".format(global_time_limit)
|
||||
@ -66,7 +56,7 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t
|
||||
pipes = []
|
||||
for i in range(0, len(output_paths)):
|
||||
f = open(output_paths[i], 'w')
|
||||
full_command = "{} {} {} {} {}".format(cmd, skip_list_opt, get_options(i), global_time_limit_option, skip_tests_option)
|
||||
full_command = "{} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option)
|
||||
logging.info("Run func tests '%s'", full_command)
|
||||
p = Popen(full_command, shell=True, stdout=f, stderr=f)
|
||||
pipes.append(p)
|
||||
|
||||
@ -76,7 +76,7 @@ cd ClickHouse
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
cmake --build . --config RelWithDebInfo
|
||||
cd ..
|
||||
```
|
||||
|
||||
@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build
|
||||
|
||||
Or cmake3 instead of cmake on older systems.
|
||||
|
||||
### Install clang-12 (recommended) {#install-clang-12}
|
||||
### Install clang-13 (recommended) {#install-clang-13}
|
||||
|
||||
On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
|
||||
|
||||
@ -33,11 +33,11 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
|
||||
|
||||
For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html).
|
||||
|
||||
#### Use clang-12 for Builds
|
||||
#### Use clang-13 for Builds
|
||||
|
||||
``` bash
|
||||
$ export CC=clang-12
|
||||
$ export CXX=clang++-12
|
||||
$ export CC=clang-13
|
||||
$ export CXX=clang++-13
|
||||
```
|
||||
|
||||
Gcc can also be used though it is discouraged.
|
||||
|
||||
@ -288,5 +288,7 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to
|
||||
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
|
||||
- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size)
|
||||
- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold)
|
||||
- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth)
|
||||
- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth)
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) <!--hide-->
|
||||
|
||||
@ -60,6 +60,7 @@ The supported formats are:
|
||||
| [LineAsString](#lineasstring) | ✔ | ✗ |
|
||||
| [Regexp](#data-format-regexp) | ✔ | ✗ |
|
||||
| [RawBLOB](#rawblob) | ✔ | ✔ |
|
||||
| [MsgPack](#msgpack) | ✔ | ✔ |
|
||||
|
||||
You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
|
||||
|
||||
@ -1551,4 +1552,31 @@ Result:
|
||||
f9725a22f9191e064120d718e26862a9 -
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/interfaces/formats/) <!--hide-->
|
||||
## MsgPack {#msgpack}
|
||||
|
||||
ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data files.
|
||||
|
||||
### Data Types Matching {#data-types-matching-msgpack}
|
||||
|
||||
| MsgPack data type | ClickHouse data type |
|
||||
|---------------------------------|----------------------------------------------------------------------------------|
|
||||
| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) |
|
||||
| `int N` | [IntN](../sql-reference/data-types/int-uint.md) |
|
||||
| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) |
|
||||
| `float 32` | [Float32](../sql-reference/data-types/float.md) |
|
||||
| `float 64` | [Float64](../sql-reference/data-types/float.md) |
|
||||
| `uint 16` | [Date](../sql-reference/data-types/date.md) |
|
||||
| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) |
|
||||
| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) |
|
||||
| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) |
|
||||
| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) |
|
||||
|
||||
Example:
|
||||
|
||||
Writing to a file ".msgpk":
|
||||
|
||||
```sql
|
||||
$ clickhouse-client --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE = Memory;"
|
||||
$ clickhouse-client --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])";
|
||||
$ clickhouse-client --query="SELECT * FROM msgpack FORMAT MsgPack" > tmp_msgpack.msgpk;
|
||||
```
|
||||
2
docs/en/interfaces/third-party/gui.md
vendored
2
docs/en/interfaces/third-party/gui.md
vendored
@ -84,7 +84,7 @@ Features:
|
||||
- Table data preview.
|
||||
- Full-text search.
|
||||
|
||||
By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set session_id to a random string (it uses the http connection under the hood). Then you can use any setting from the query window
|
||||
By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set `session_id` to a random string (it uses the http connection under the hood). Then you can use any setting from the query window.
|
||||
|
||||
### clickhouse-cli {#clickhouse-cli}
|
||||
|
||||
|
||||
@ -3,58 +3,58 @@ toc_priority: 66
|
||||
toc_title: ClickHouse Keeper
|
||||
---
|
||||
|
||||
# [pre-production] clickhouse-keeper
|
||||
# [pre-production] ClickHouse Keeper
|
||||
|
||||
ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
|
||||
ClickHouse server uses [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
|
||||
|
||||
!!! warning "Warning"
|
||||
This feature currently in pre-production stage. We test it in our CI and on small internal installations.
|
||||
This feature is currently in the pre-production stage. We test it in our CI and on small internal installations.
|
||||
|
||||
## Implementation details
|
||||
|
||||
ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages.
|
||||
ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper ClickHouse Keeper is written in C++ and uses [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages.
|
||||
|
||||
By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible.
|
||||
By default, ClickHouse Keeper provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with ClickHouse Keeper. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to ClickHouse Keeper snapshot. Interserver protocol in ClickHouse Keeper is also incompatible with ZooKeeper so mixed ZooKeeper / ClickHouse Keeper cluster is impossible.
|
||||
|
||||
## Configuration
|
||||
|
||||
`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is `<keeper_server>`. Keeper configuration has the following parameters:
|
||||
ClickHouse Keeper can be used as a standalone replacement for ZooKeeper or as an internal part of the ClickHouse server, but in both cases configuration is almost the same `.xml` file. The main ClickHouse Keeper configuration tag is `<keeper_server>`. Keeper configuration has the following parameters:
|
||||
|
||||
- `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`)
|
||||
- `tcp_port_secure` — the secure port for a client to connect
|
||||
- `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on)
|
||||
- `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper)
|
||||
- `snapshot_storage_path` — path to coordination snapshots
|
||||
- `tcp_port` — Port for a client to connect (default for ZooKeeper is `2181`).
|
||||
- `tcp_port_secure` — Secure port for a client to connect.
|
||||
- `server_id` — Unique server id, each participant of the ClickHouse Keeper cluster must have a unique number (1, 2, 3, and so on).
|
||||
- `log_storage_path` — Path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper).
|
||||
- `snapshot_storage_path` — Path to coordination snapshots.
|
||||
|
||||
Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on).
|
||||
Other common parameters are inherited from the ClickHouse server config (`listen_host`, `logger`, and so on).
|
||||
|
||||
Internal coordination settings are located in `<keeper_server>.<coordination_settings>` section:
|
||||
|
||||
- `operation_timeout_ms` — timeout for a single client operation (default: 10000)
|
||||
- `session_timeout_ms` — timeout for client session (default: 30000)
|
||||
- `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them (default: 500)
|
||||
- `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers (default: 500)
|
||||
- `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election (default: 1000)
|
||||
- `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election (default: 2000)
|
||||
- `rotate_log_storage_interval` — how many log records to store in a single file (default: 100000)
|
||||
- `reserved_log_items` — how many coordination log records to store before compaction (default: 100000)
|
||||
- `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of records in logs) (default: 100000)
|
||||
- `snapshots_to_keep` — how many snapshots to keep (default: 3)
|
||||
- `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs (default: 10000)
|
||||
- `fresh_log_gap` - when node became fresh (default: 200)
|
||||
- `max_requests_batch_size` - max size of batch in requests count before it will be sent to RAFT (default: 100)
|
||||
- `force_sync` — call `fsync` on each write to coordination log (default: true)
|
||||
- `quorum_reads` - execute read requests as writes through whole RAFT consesus with similar speed (default: false)
|
||||
- `raft_logs_level` — text logging level about coordination (trace, debug, and so on) (default: system default)
|
||||
- `auto_forwarding` - allow to forward write requests from followers to leader (default: true)
|
||||
- `shutdown_timeout` — wait to finish internal connections and shutdown (ms) (default: 5000)
|
||||
- `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000)
|
||||
- `operation_timeout_ms` — Timeout for a single client operation (ms) (default: 10000).
|
||||
- `session_timeout_ms` — Timeout for client session (ms) (default: 30000).
|
||||
- `dead_session_check_period_ms` — How often ClickHouse Keeper check dead sessions and remove them (ms) (default: 500).
|
||||
- `heart_beat_interval_ms` — How often a ClickHouse Keeper leader will send heartbeats to followers (ms) (default: 500).
|
||||
- `election_timeout_lower_bound_ms` — If the follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election (default: 1000).
|
||||
- `election_timeout_upper_bound_ms` — If the follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election (default: 2000).
|
||||
- `rotate_log_storage_interval` — How many log records to store in a single file (default: 100000).
|
||||
- `reserved_log_items` — How many coordination log records to store before compaction (default: 100000).
|
||||
- `snapshot_distance` — How often ClickHouse Keeper will create new snapshots (in the number of records in logs) (default: 100000).
|
||||
- `snapshots_to_keep` — How many snapshots to keep (default: 3).
|
||||
- `stale_log_gap` — Threshold when leader considers follower as stale and sends the snapshot to it instead of logs (default: 10000).
|
||||
- `fresh_log_gap` — When node became fresh (default: 200).
|
||||
- `max_requests_batch_size` - Max size of batch in requests count before it will be sent to RAFT (default: 100).
|
||||
- `force_sync` — Call `fsync` on each write to coordination log (default: true).
|
||||
- `quorum_reads` — Execute read requests as writes through whole RAFT consensus with similar speed (default: false).
|
||||
- `raft_logs_level` — Text logging level about coordination (trace, debug, and so on) (default: system default).
|
||||
- `auto_forwarding` — Allow to forward write requests from followers to the leader (default: true).
|
||||
- `shutdown_timeout` — Wait to finish internal connections and shutdown (ms) (default: 5000).
|
||||
- `startup_timeout` — If the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000).
|
||||
|
||||
Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `<server>` are:
|
||||
|
||||
- `id` — server_id in quorum
|
||||
- `hostname` — hostname where this server placed
|
||||
- `port` — port where this server listen for connections
|
||||
- `id` — Server identifier in a quorum.
|
||||
- `hostname` — Hostname where this server is placed.
|
||||
- `port` — Port where this server listens for connections.
|
||||
|
||||
|
||||
Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1:
|
||||
@ -94,7 +94,7 @@ Examples of configuration for quorum with three nodes can be found in [integrati
|
||||
|
||||
## How to run
|
||||
|
||||
`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `<keeper_server>` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with:
|
||||
ClickHouse Keeper is bundled into the ClickHouse server package, just add configuration of `<keeper_server>` and start ClickHouse server as always. If you want to run standalone ClickHouse Keeper you can start it in a similar way with:
|
||||
|
||||
```bash
|
||||
clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
|
||||
@ -102,17 +102,18 @@ clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
|
||||
|
||||
## [experimental] Migration from ZooKeeper
|
||||
|
||||
Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
|
||||
Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
|
||||
|
||||
1. Stop all ZooKeeper nodes.
|
||||
|
||||
2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot.
|
||||
2. Optional, but recommended: find ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create a consistent snapshot.
|
||||
|
||||
3. Run `clickhouse-keeper-converter` on leader, example
|
||||
3. Run `clickhouse-keeper-converter` on a leader, for example:
|
||||
|
||||
```bash
|
||||
clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots
|
||||
```
|
||||
|
||||
4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist on all nodes, otherwise empty nodes can be faster and one of them can becamse leader.
|
||||
4. Copy snapshot to ClickHouse server nodes with a configured `keeper` or start ClickHouse Keeper instead of ZooKeeper. The snapshot must persist on all nodes, otherwise, empty nodes can be faster and one of them can become a leader.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/clickhouse-keeper/) <!--hide-->
|
||||
|
||||
@ -69,29 +69,85 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression.
|
||||
</compression>
|
||||
```
|
||||
|
||||
<!--
|
||||
## encryption {#server-settings-encryption}
|
||||
|
||||
Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). The command, or a shell script, is expected to write a Base64-encoded key of any length to the stdout.
|
||||
Configures a command to obtain a key to be used by [encryption codecs](../../sql-reference/statements/create/table.md#create-query-encryption-codecs). Key (or keys) should be written in enviroment variables or be set in configuration file.
|
||||
|
||||
Keys can be hex or string. Their length must be equal to 16.
|
||||
|
||||
**Example**
|
||||
|
||||
For Linux with systemd:
|
||||
Load from config:
|
||||
|
||||
```xml
|
||||
<encryption>
|
||||
<key_command>/usr/bin/systemd-ask-password --id="clickhouse-server" --timeout=0 "Enter the ClickHouse encryption passphrase:" | base64</key_command>
|
||||
</encryption>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>12345567812345678</key>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
For other systems:
|
||||
!!! note "NOTE"
|
||||
Storing keys in configuration file is not recommended. It isn't secure. You can move the keys into a separate config file on a secure disk and put a symlink to that config file to `config.d/` folder.
|
||||
|
||||
Load from config, when key is in hex:
|
||||
|
||||
```xml
|
||||
<encryption>
|
||||
<key_command><![CDATA[IFS=; echo -n >/dev/tty "Enter the ClickHouse encryption passphrase: "; stty=`stty -F /dev/tty -g`; stty -F /dev/tty -echo; read k </dev/tty; stty -F /dev/tty "$stty"; echo -n $k | base64]]></key_command>
|
||||
</encryption>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key_hex>00112233445566778899aabbccddeeff</key_hex>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
-->
|
||||
|
||||
Load key from environment variable:
|
||||
|
||||
```xml
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key_hex from_env="KEY"></key_hex>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
Where current_key_id sets the current key for encryption, and all specified keys can be used for decryption.
|
||||
|
||||
All this methods can be applied for multiple keys:
|
||||
|
||||
```xml
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key_hex id="0">00112233445566778899aabbccddeeff</key_hex>
|
||||
<key_hex id="1" from_env=".."></key_hex>
|
||||
<current_key_id>1</current_key_id>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
Where `current_key_id` shows current key for encryption.
|
||||
|
||||
Also user can add nonce that must be 12 bytes long (by default encryption and decryption will use nonce consisting of zero bytes):
|
||||
|
||||
```xml
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<nonce>0123456789101</nonce>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
Or it can be set in hex:
|
||||
|
||||
```xml
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<nonce_hex>abcdefabcdef</nonce_hex>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
```
|
||||
|
||||
Everything above can be applied for `aes_256_gcm_siv` (but key must be 32 bytes length).
|
||||
|
||||
## custom_settings_prefixes {#custom_settings_prefixes}
|
||||
|
||||
List of prefixes for [custom settings](../../operations/settings/index.md#custom_settings). The prefixes must be separated with commas.
|
||||
|
||||
@ -181,6 +181,44 @@ Possible values:
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth}
|
||||
|
||||
Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches. This setting is applied to a particular table, unlike the [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server) setting, which is applied to the server.
|
||||
|
||||
You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_fetches_network_bandwidth_for_server` setting.
|
||||
|
||||
The setting isn't followed perfectly accurately.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
**Usage**
|
||||
|
||||
Could be used for throttling speed when replicating data to add or replace new nodes.
|
||||
|
||||
## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth}
|
||||
|
||||
Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends. This setting is applied to a particular table, unlike the [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server) setting, which is applied to the server.
|
||||
|
||||
You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_sends_network_bandwidth_for_server` setting.
|
||||
|
||||
The setting isn't followed perfectly accurately.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
**Usage**
|
||||
|
||||
Could be used for throttling speed when replicating data to add or replace new nodes.
|
||||
|
||||
## old_parts_lifetime {#old-parts-lifetime}
|
||||
|
||||
The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots.
|
||||
|
||||
@ -1140,6 +1140,40 @@ Possible values:
|
||||
|
||||
Default value: `5`.
|
||||
|
||||
## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server}
|
||||
|
||||
Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) setting.
|
||||
|
||||
The setting isn't followed perfectly accurately.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
**Usage**
|
||||
|
||||
Could be used for throttling speed when replicating the data to add or replace new nodes.
|
||||
|
||||
## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server}
|
||||
|
||||
Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) setting.
|
||||
|
||||
The setting isn't followed perfectly accurately.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited.
|
||||
|
||||
Default value: `0`.
|
||||
|
||||
**Usage**
|
||||
|
||||
Could be used for throttling speed when replicating the data to add or replace new nodes.
|
||||
|
||||
## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms}
|
||||
|
||||
The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the ‘shard’ and ‘replica’ sections are used in the cluster definition.
|
||||
|
||||
@ -25,14 +25,13 @@ Aliases:
|
||||
- `Int32` — `INT`, `INT4`, `INTEGER`.
|
||||
- `Int64` — `BIGINT`.
|
||||
|
||||
## Uint Ranges {#uint-ranges}
|
||||
## UInt Ranges {#uint-ranges}
|
||||
|
||||
- `UInt8` — \[0 : 255\]
|
||||
- `UInt16` — \[0 : 65535\]
|
||||
- `UInt32` — \[0 : 4294967295\]
|
||||
- `UInt64` — \[0 : 18446744073709551615\]
|
||||
- `UInt128` — \[0 : 340282366920938463463374607431768211455\]
|
||||
- `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\]
|
||||
|
||||
`UInt128` is not supported yet.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/data_types/int_uint/) <!--hide-->
|
||||
|
||||
@ -29,7 +29,7 @@ Returns the round number with largest absolute value that has an absolute value
|
||||
|
||||
Rounds a value to a specified number of decimal places.
|
||||
|
||||
The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker’s rounding for float number types and rounds away from zero for the other number types.
|
||||
The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function uses banker’s rounding for float number types and rounds away from zero for the other number types (Decimal).
|
||||
|
||||
``` sql
|
||||
round(expression [, decimal_places])
|
||||
@ -49,7 +49,7 @@ The rounded number of the same type as the input number.
|
||||
|
||||
### Examples {#examples}
|
||||
|
||||
**Example of use**
|
||||
**Example of use with Float**
|
||||
|
||||
``` sql
|
||||
SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3
|
||||
@ -63,6 +63,20 @@ SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3
|
||||
└─────┴──────────────────────────┘
|
||||
```
|
||||
|
||||
**Example of use with Decimal**
|
||||
|
||||
``` sql
|
||||
SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3
|
||||
```
|
||||
|
||||
``` text
|
||||
┌──────x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐
|
||||
│ 0.0000 │ 0.0000 │
|
||||
│ 0.5000 │ 1.0000 │
|
||||
│ 1.0000 │ 1.0000 │
|
||||
└────────┴──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Examples of rounding**
|
||||
|
||||
Rounding to the nearest number.
|
||||
|
||||
@ -74,7 +74,7 @@ $ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/
|
||||
$ rm -rf build
|
||||
$ mkdir build
|
||||
$ cd build
|
||||
$ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
|
||||
$ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
|
||||
$ cmake --build . --config RelWithDebInfo
|
||||
$ cd ..
|
||||
```
|
||||
|
||||
@ -15,7 +15,7 @@ toc_title: PostgreSQL
|
||||
|
||||
``` sql
|
||||
CREATE DATABASE test_database
|
||||
ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cache`]);
|
||||
ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `schema`, `use_table_cache`]);
|
||||
```
|
||||
|
||||
**Параметры движка**
|
||||
@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac
|
||||
- `database` — имя удаленной БД.
|
||||
- `user` — пользователь PostgreSQL.
|
||||
- `password` — пароль пользователя.
|
||||
- `schema` — схема PostgreSQL.
|
||||
- `use_table_cache` — определяет кеширование структуры таблиц БД. Необязательный параметр. Значение по умолчанию: `0`.
|
||||
|
||||
## Поддерживаемые типы данных {#data_types-support}
|
||||
|
||||
@ -253,4 +253,5 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
|
||||
- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size)
|
||||
- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size)
|
||||
- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold)
|
||||
|
||||
- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth)
|
||||
- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth)
|
||||
|
||||
@ -59,6 +59,7 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT
|
||||
| [LineAsString](#lineasstring) | ✔ | ✗ |
|
||||
| [Regexp](#data-format-regexp) | ✔ | ✗ |
|
||||
| [RawBLOB](#rawblob) | ✔ | ✔ |
|
||||
| [MsgPack](#msgpack) | ✔ | ✔ |
|
||||
|
||||
Вы можете регулировать некоторые параметры работы с форматами с помощью настроек ClickHouse. За дополнительной информацией обращайтесь к разделу [Настройки](../operations/settings/settings.md).
|
||||
|
||||
@ -1464,3 +1465,32 @@ $ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum
|
||||
``` text
|
||||
f9725a22f9191e064120d718e26862a9 -
|
||||
```
|
||||
|
||||
## MsgPack {#msgpack}
|
||||
|
||||
ClickHouse поддерживает запись и чтение из файлов в формате [MessagePack](https://msgpack.org/).
|
||||
|
||||
### Соответствие типов данных {#data-types-matching-msgpack}
|
||||
|
||||
| Тип данных MsgPack | Тип данных ClickHouse |
|
||||
|---------------------------------|------------------------------------------------------------------------------------|
|
||||
| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) |
|
||||
| `int N` | [IntN](../sql-reference/data-types/int-uint.md) |
|
||||
| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) |
|
||||
| `float 32` | [Float32](../sql-reference/data-types/float.md) |
|
||||
| `float 64` | [Float64](../sql-reference/data-types/float.md) |
|
||||
| `uint 16` | [Date](../sql-reference/data-types/date.md) |
|
||||
| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) |
|
||||
| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) |
|
||||
| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) |
|
||||
| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) |
|
||||
|
||||
Пример:
|
||||
|
||||
Запись в файл ".msgpk":
|
||||
|
||||
```sql
|
||||
$ clickhouse-client --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE = Memory;"
|
||||
$ clickhouse-client --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])";
|
||||
$ clickhouse-client --query="SELECT * FROM msgpack FORMAT MsgPack" > tmp_msgpack.msgpk;
|
||||
```
|
||||
8
docs/ru/interfaces/third-party/gui.md
vendored
8
docs/ru/interfaces/third-party/gui.md
vendored
@ -75,11 +75,13 @@ toc_title: "Визуальные интерфейсы от сторонних р
|
||||
|
||||
Основные возможности:
|
||||
|
||||
- Построение запросов с подсветкой синтаксиса;
|
||||
- Просмотр таблиц;
|
||||
- Автодополнение команд;
|
||||
- Построение запросов с подсветкой синтаксиса.
|
||||
- Просмотр таблиц.
|
||||
- Автодополнение команд.
|
||||
- Полнотекстовый поиск.
|
||||
|
||||
По умолчанию DBeaver не использует сессии при подключении (в отличие от CLI, например). Если вам нужна поддержка сессий (например, для установки настроек на сессию), измените настройки подключения драйвера и укажите для настройки `session_id` любое произвольное значение (драйвер использует подключение по http). После этого вы можете использовать любую настройку (setting) в окне запроса.
|
||||
|
||||
### clickhouse-cli {#clickhouse-cli}
|
||||
|
||||
[clickhouse-cli](https://github.com/hatarist/clickhouse-cli) - это альтернативный клиент командной строки для ClickHouse, написанный на Python 3.
|
||||
|
||||
119
docs/ru/operations/clickhouse-keeper.md
Normal file
119
docs/ru/operations/clickhouse-keeper.md
Normal file
@ -0,0 +1,119 @@
|
||||
---
|
||||
toc_priority: 66
|
||||
toc_title: ClickHouse Keeper
|
||||
---
|
||||
|
||||
# [пре-продакшн] ClickHouse Keeper
|
||||
|
||||
Сервер ClickHouse использует сервис координации [ZooKeeper](https://zookeeper.apache.org/) для [репликации](../engines/table-engines/mergetree-family/replication.md) данных и выполнения [распределенных DDL запросов](../sql-reference/distributed-ddl.md). ClickHouse Keeper — это альтернативный сервис координации, совместимый с ZooKeeper.
|
||||
|
||||
!!! warning "Предупреждение"
|
||||
ClickHouse Keeper находится в стадии пре-продакшн и тестируется в CI ClickHouse и на нескольких внутренних инсталляциях.
|
||||
|
||||
## Детали реализации
|
||||
|
||||
ZooKeeper — один из первых широко известных сервисов координации с открытым исходным кодом. Он реализован на языке программирования Java, имеет достаточно простую и мощную модель данных. Алгоритм координации Zookeeper называется ZAB (ZooKeeper Atomic Broadcast). Он не гарантирует линеаризуемость операций чтения, поскольку каждый узел ZooKeeper обслуживает чтения локально. В отличие от ZooKeeper, ClickHouse Keeper реализован на C++ и использует алгоритм [RAFT](https://raft.github.io/), [реализация](https://github.com/eBay/NuRaft). Этот алгоритм позволяет достичь линеаризуемости чтения и записи, имеет несколько реализаций с открытым исходным кодом на разных языках.
|
||||
|
||||
По умолчанию ClickHouse Keeper предоставляет те же гарантии, что и ZooKeeper (линеаризуемость записей, последовательная согласованность чтений). У него есть совместимый клиент-серверный протокол, поэтому любой стандартный клиент ZooKeeper может использоваться для взаимодействия с ClickHouse Keeper. Снэпшоты и журналы имеют несовместимый с ZooKeeper формат, однако можно конвертировать данные Zookeeper в снэпшот ClickHouse Keeper с помощью `clickhouse-keeper-converter`. Межсерверный протокол ClickHouse Keeper также несовместим с ZooKeeper, поэтому создание смешанного кластера ZooKeeper / ClickHouse Keeper невозможно.
|
||||
|
||||
## Конфигурация
|
||||
|
||||
ClickHouse Keeper может использоваться как равноценная замена ZooKeeper или как внутренняя часть сервера ClickHouse, но в обоих случаях конфигурация представлена файлом `.xml`. Главный тег конфигурации ClickHouse Keeper — это `<keeper_server>`. Параметры конфигурации:
|
||||
|
||||
- `tcp_port` — порт для подключения клиента (по умолчанию для ZooKeeper: `2181`).
|
||||
- `tcp_port_secure` — зашифрованный порт для подключения клиента.
|
||||
- `server_id` — уникальный идентификатор сервера, каждый участник кластера должен иметь уникальный номер (1, 2, 3 и т. д.).
|
||||
- `log_storage_path` — путь к журналам координации, лучше хранить их на незанятом устройстве (актуально и для ZooKeeper).
|
||||
- `snapshot_storage_path` — путь к снэпшотам координации.
|
||||
|
||||
Другие общие параметры наследуются из конфигурации сервера ClickHouse (`listen_host`, `logger`, и т. д.).
|
||||
|
||||
Настройки внутренней координации находятся в `<keeper_server>.<coordination_settings>`:
|
||||
|
||||
- `operation_timeout_ms` — максимальное время ожидания для одной клиентской операции в миллисекундах (по умолчанию: 10000).
|
||||
- `session_timeout_ms` — максимальное время ожидания для клиентской сессии в миллисекундах (по умолчанию: 30000).
|
||||
- `dead_session_check_period_ms` — частота, с которой ClickHouse Keeper проверяет мертвые сессии и удаляет их, в миллисекундах (по умолчанию: 500).
|
||||
- `heart_beat_interval_ms` — частота, с которой узел-лидер ClickHouse Keeper отправляет хартбиты узлам-последователям, в миллисекундах (по умолчанию: 500).
|
||||
- `election_timeout_lower_bound_ms` — время, после которого последователь может инициировать выборы лидера, если не получил от него сердцебиения (по умолчанию: 1000).
|
||||
- `election_timeout_upper_bound_ms` — время, после которого последователь должен инициировать выборы лидера, если не получил от него сердцебиения (по умолчанию: 2000).
|
||||
- `rotate_log_storage_interval` — количество записей в журнале координации для хранения в одном файле (по умолчанию: 100000).
|
||||
- `reserved_log_items` — минимальное количество записей в журнале координации которые нужно сохранять после снятия снепшота (по умолчанию: 100000).
|
||||
- `snapshot_distance` — частота, с которой ClickHouse Keeper делает новые снэпшоты (по количеству записей в журналах), в миллисекундах (по умолчанию: 100000).
|
||||
- `snapshots_to_keep` — количество снэпшотов для сохранения (по умолчанию: 3).
|
||||
- `stale_log_gap` — время, после которого лидер считает последователя устаревшим и отправляет ему снэпшот вместо журналов (по умолчанию: 10000).
|
||||
- `fresh_log_gap` — максимальное отставание от лидера в количестве записей журнала после которого последователь считает себя не отстающим (по умолчанию: 200).
|
||||
- `max_requests_batch_size` — количество запросов на запись, которые будут сгруппированы в один перед отправкой через RAFT (по умолчанию: 100).
|
||||
- `force_sync` — вызывать `fsync` при каждой записи в журнал координации (по умолчанию: true).
|
||||
- `quorum_reads` — выполнять запросы чтения аналогично запросам записи через весь консенсус RAFT с негативным эффектом на производительность и размер журналов (по умолчанию: false).
|
||||
- `raft_logs_level` — уровень логгирования сообщений в текстовый лог (trace, debug и т. д.) (по умолчанию: information).
|
||||
- `auto_forwarding` — разрешить пересылку запросов на запись от последователей лидеру (по умолчанию: true).
|
||||
- `shutdown_timeout` — время ожидания завершения внутренних подключений и выключения, в миллисекундах (по умолчанию: 5000).
|
||||
- `startup_timeout` — время отключения сервера, если он не подключается к другим участникам кворума, в миллисекундах (по умолчанию: 30000).
|
||||
|
||||
Конфигурация кворума находится в `<keeper_server>.<raft_configuration>` и содержит описание серверов. Единственный параметр для всего кворума — `secure`, который включает зашифрованное соединение для связи между участниками кворума. Параметры для каждого `<server>`:
|
||||
|
||||
- `id` — идентификатор сервера в кворуме.
|
||||
- `hostname` — имя хоста, на котором размещен сервер.
|
||||
- `port` — порт, на котором серверу доступны соединения для внутренней коммуникации.
|
||||
|
||||
|
||||
Примеры конфигурации кворума с тремя узлами можно найти в [интеграционных тестах](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) с префиксом `test_keeper_`. Пример конфигурации для сервера №1:
|
||||
|
||||
```xml
|
||||
<keeper_server>
|
||||
<tcp_port>2181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||
<session_timeout_ms>30000</session_timeout_ms>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>zoo1</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>zoo2</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>zoo3</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
```
|
||||
|
||||
## Как запустить
|
||||
|
||||
ClickHouse Keeper входит в пакет` clickhouse-server`, просто добавьте кофигурацию `<keeper_server>` и запустите сервер ClickHouse как обычно. Если вы хотите запустить ClickHouse Keeper автономно, сделайте это аналогичным способом:
|
||||
|
||||
```bash
|
||||
clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
|
||||
```
|
||||
|
||||
## [экспериментально] Переход с ZooKeeper
|
||||
|
||||
Плавный переход с ZooKeeper на ClickHouse Keeper невозможен, необходимо остановить кластер ZooKeeper, преобразовать данные и запустить ClickHouse Keeper. Утилита `clickhouse-keeper-converter` конвертирует журналы и снэпшоты ZooKeeper в снэпшот ClickHouse Keeper. Работа утилиты проверена только для версий ZooKeeper выше 3.4. Для миграции необходимо выполнить следующие шаги:
|
||||
|
||||
1. Остановите все узлы ZooKeeper.
|
||||
|
||||
2. Необязательно, но рекомендуется: найдите узел-лидер ZooKeeper, запустите и снова остановите его. Это заставит ZooKeeper создать консистентный снэпшот.
|
||||
|
||||
3. Запустите `clickhouse-keeper-converter` на лидере, например:
|
||||
|
||||
```bash
|
||||
clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots
|
||||
```
|
||||
|
||||
4. Скопируйте снэпшот на узлы сервера ClickHouse с настроенным `keeper` или запустите ClickHouse Keeper вместо ZooKeeper. Снэпшот должен сохраняться на всех узлах: в противном случае пустые узлы могут захватить лидерство и сконвертированные данные могут быть отброшены на старте.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/clickhouse-keeper/) <!--hide-->
|
||||
@ -201,6 +201,44 @@ Eсли суммарное число активных кусков во все
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth}
|
||||
|
||||
Ограничивает максимальную скорость скачивания данных в сети (в байтах в секунду) для синхронизаций между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server), которая применяется к серверу.
|
||||
|
||||
Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_fetches_network_bandwidth_for_server`.
|
||||
|
||||
Настройка соблюдается неточно.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Любое целое положительное число.
|
||||
- 0 — Скорость не ограничена.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Может быть использована для ограничения скорости передачи данных при репликации данных для добавления или замены новых узлов.
|
||||
|
||||
## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth}
|
||||
|
||||
Ограничивает максимальную скорость отправки данных по сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server), которая применяется к серверу.
|
||||
|
||||
Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_sends_network_bandwidth_for_server`.
|
||||
|
||||
Настройка следуется неточно.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Любое целое положительное число.
|
||||
- 0 — Скорость не ограничена.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов.
|
||||
|
||||
## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool}
|
||||
|
||||
Максимальный суммарный размер кусков (в байтах) в одном слиянии, если есть свободные ресурсы в фоновом пуле.
|
||||
|
||||
@ -1098,6 +1098,40 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test'
|
||||
|
||||
Значение по умолчанию: `5`.
|
||||
|
||||
## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server}
|
||||
|
||||
Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth).
|
||||
|
||||
Значение настройки соблюдается неточно.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Любое целое положительное число.
|
||||
- 0 — Скорость не ограничена.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов.
|
||||
|
||||
## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server}
|
||||
|
||||
Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для [репликационных](../../engines/table-engines/mergetree-family/replication.md) отправок. Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth).
|
||||
|
||||
Значение настройки соблюдается неточно.
|
||||
|
||||
Возможные значения:
|
||||
|
||||
- Любое целое положительное число.
|
||||
- 0 — Скорость не ограничена.
|
||||
|
||||
Значение по умолчанию: `0`.
|
||||
|
||||
**Использование**
|
||||
|
||||
Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов.
|
||||
|
||||
## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms}
|
||||
|
||||
Таймаут в миллисекундах на соединение с удалённым сервером, для движка таблиц Distributed, если используются секции shard и replica в описании кластера.
|
||||
|
||||
@ -78,10 +78,11 @@ active_replicas: 2
|
||||
|
||||
- `log_max_index` (`UInt64`) - максимальный номер записи в общем логе действий.
|
||||
- `log_pointer` (`UInt64`) - максимальный номер записи из общего лога действий, которую реплика скопировала в свою очередь для выполнения, плюс единица. Если log_pointer сильно меньше log_max_index, значит что-то не так.
|
||||
- `last_queue_update` (`DateTime`) - When the queue was updated last time.
|
||||
- `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has.
|
||||
- `last_queue_update` (`DateTime`) - время последнего обновления запроса.
|
||||
- `absolute_delay` (`UInt64`) - задержка (в секундах) для текущей реплики.
|
||||
- `total_replicas` (`UInt8`) - общее число известных реплик этой таблицы.
|
||||
- `active_replicas` (`UInt8`) - число реплик этой таблицы, имеющих сессию в ZK; то есть, число работающих реплик.
|
||||
- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — соответствие между именем реплики и признаком активности реплики.
|
||||
|
||||
Если запрашивать все столбцы, то таблица может работать слегка медленно, так как на каждую строчку делается несколько чтений из ZK.
|
||||
Если не запрашивать последние 4 столбца (log_max_index, log_pointer, total_replicas, active_replicas), то таблица работает быстро.
|
||||
|
||||
@ -27,7 +27,7 @@ N может быть отрицательным.
|
||||
|
||||
Округляет значение до указанного десятичного разряда.
|
||||
|
||||
Функция возвращает ближайшее значение указанного порядка. В случае, когда заданное число равноудалено от чисел необходимого порядка, функция возвращает то из них, которое имеет ближайшую чётную цифру (банковское округление).
|
||||
Функция возвращает ближайшее значение указанного порядка. В случае, когда заданное число равноудалено от чисел необходимого порядка, для типов с плавающей точкой (Float32/64) функция возвращает то из них, которое имеет ближайшую чётную цифру (банковское округление), для типов с фиксированной точкой (Decimal) функция использует округление в бо́льшую по модулю сторону (математическое округление).
|
||||
|
||||
``` sql
|
||||
round(expression [, decimal_places])
|
||||
@ -47,7 +47,7 @@ round(expression [, decimal_places])
|
||||
|
||||
### Примеры {#primery}
|
||||
|
||||
**Пример использования**
|
||||
**Пример использования с Float**
|
||||
|
||||
``` sql
|
||||
SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3
|
||||
@ -61,6 +61,21 @@ SELECT number / 2 AS x, round(x) FROM system.numbers LIMIT 3
|
||||
└─────┴──────────────────────────┘
|
||||
```
|
||||
|
||||
**Пример использования с Decimal**
|
||||
|
||||
|
||||
``` sql
|
||||
SELECT cast(number / 2 AS Decimal(10,4)) AS x, round(x) FROM system.numbers LIMIT 3
|
||||
```
|
||||
|
||||
``` text
|
||||
┌──────x─┬─round(CAST(divide(number, 2), 'Decimal(10, 4)'))─┐
|
||||
│ 0.0000 │ 0.0000 │
|
||||
│ 0.5000 │ 1.0000 │
|
||||
│ 1.0000 │ 1.0000 │
|
||||
└────────┴──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Примеры округления**
|
||||
|
||||
Округление до ближайшего числа.
|
||||
|
||||
@ -4,6 +4,7 @@ set (CLICKHOUSE_CLIENT_SOURCES
|
||||
QueryFuzzer.cpp
|
||||
Suggest.cpp
|
||||
TestHint.cpp
|
||||
TestTags.cpp
|
||||
)
|
||||
|
||||
set (CLICKHOUSE_CLIENT_LINK
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include "QueryFuzzer.h"
|
||||
#include "Suggest.h"
|
||||
#include "TestHint.h"
|
||||
#include "TestTags.h"
|
||||
|
||||
#if USE_REPLXX
|
||||
# include <common/ReplxxLineReader.h>
|
||||
@ -1078,12 +1079,17 @@ private:
|
||||
|
||||
bool echo_query = echo_queries;
|
||||
|
||||
/// Test tags are started with "--" so they are interpreted as comments anyway.
|
||||
/// But if the echo is enabled we have to remove the test tags from `all_queries_text`
|
||||
/// because we don't want test tags to be echoed.
|
||||
size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0;
|
||||
|
||||
/// Several queries separated by ';'.
|
||||
/// INSERT data is ended by the end of line, not ';'.
|
||||
/// An exception is VALUES format where we also support semicolon in
|
||||
/// addition to end of line.
|
||||
|
||||
const char * this_query_begin = all_queries_text.data();
|
||||
const char * this_query_begin = all_queries_text.data() + test_tags_length;
|
||||
const char * all_queries_end = all_queries_text.data() + all_queries_text.size();
|
||||
|
||||
while (this_query_begin < all_queries_end)
|
||||
@ -2033,8 +2039,21 @@ private:
|
||||
PullingAsyncPipelineExecutor executor(pipeline);
|
||||
|
||||
Block block;
|
||||
while (executor.pull(block))
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!executor.pull(block))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
e.addMessage(fmt::format("(in query: {})", full_query));
|
||||
throw;
|
||||
}
|
||||
|
||||
/// Check if server send Log packet
|
||||
receiveLogs();
|
||||
|
||||
|
||||
51
programs/client/TestTags.cpp
Normal file
51
programs/client/TestTags.cpp
Normal file
@ -0,0 +1,51 @@
|
||||
#include "TestTags.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
size_t getTestTagsLength(const String & multiline_query)
|
||||
{
|
||||
const String & text = multiline_query;
|
||||
size_t pos = 0;
|
||||
bool first_line = true;
|
||||
|
||||
while (true)
|
||||
{
|
||||
size_t line_start = pos;
|
||||
|
||||
/// Skip spaces.
|
||||
while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t'))
|
||||
++pos;
|
||||
|
||||
/// Skip comment "--".
|
||||
static constexpr const char comment[] = "--";
|
||||
if (text.compare(pos, strlen(comment), comment) != 0)
|
||||
return line_start;
|
||||
pos += strlen(comment);
|
||||
|
||||
/// Skip the prefix "Tags:" if it's the first line.
|
||||
if (first_line)
|
||||
{
|
||||
while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t'))
|
||||
++pos;
|
||||
|
||||
static constexpr const char tags_prefix[] = "Tags:";
|
||||
if (text.compare(pos, strlen(tags_prefix), tags_prefix) != 0)
|
||||
return 0;
|
||||
pos += strlen(tags_prefix);
|
||||
first_line = false;
|
||||
}
|
||||
|
||||
/// Skip end-of-line.
|
||||
size_t eol_pos = text.find_first_of("\r\n", pos);
|
||||
if (eol_pos == String::npos)
|
||||
return text.length();
|
||||
bool two_chars_eol = (eol_pos + 1 < text.length()) && ((text[eol_pos + 1] == '\r') || (text[eol_pos + 1] == '\n')) && (text[eol_pos + 1] != text[eol_pos]);
|
||||
size_t eol_length = two_chars_eol ? 2 : 1;
|
||||
pos = eol_pos + eol_length;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
18
programs/client/TestTags.h
Normal file
18
programs/client/TestTags.h
Normal file
@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/Types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Returns the length of a text looking like
|
||||
/// -- Tags: x, y, z
|
||||
/// -- Tag x: explanation of tag x
|
||||
/// -- Tag y: explanation of tag y
|
||||
/// -- Tag z: explanation of tag z
|
||||
///
|
||||
/// at the beginning of a multiline query.
|
||||
/// If there are no test tags in the multiline query the function returns 0.
|
||||
size_t getTestTagsLength(const String & multiline_query);
|
||||
|
||||
}
|
||||
@ -79,6 +79,7 @@
|
||||
#include <Server/ProtocolServerAdapter.h>
|
||||
#include <Server/HTTP/HTTPServer.h>
|
||||
#include <filesystem>
|
||||
#include <Compression/CompressionCodecEncrypted.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_core.h"
|
||||
@ -251,7 +252,6 @@ namespace ErrorCodes
|
||||
extern const int SUPPORT_IS_DISABLED;
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int INVALID_CONFIG_PARAMETER;
|
||||
extern const int SYSTEM_ERROR;
|
||||
extern const int FAILED_TO_GETPWUID;
|
||||
@ -456,40 +456,6 @@ void checkForUsersNotInMainConfig(
|
||||
}
|
||||
}
|
||||
|
||||
static void loadEncryptionKey(const std::string & key_command [[maybe_unused]], Poco::Logger * log)
|
||||
{
|
||||
#if USE_BASE64 && USE_SSL && USE_INTERNAL_SSL_LIBRARY
|
||||
|
||||
auto process = ShellCommand::execute(key_command);
|
||||
|
||||
std::string b64_key;
|
||||
readStringUntilEOF(b64_key, process->out);
|
||||
process->wait();
|
||||
|
||||
// turbob64 doesn't like whitespace characters in input. Strip
|
||||
// them before decoding.
|
||||
std::erase_if(b64_key, [](char c)
|
||||
{
|
||||
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
||||
});
|
||||
|
||||
std::vector<char> buf(b64_key.size());
|
||||
const size_t key_size = tb64dec(reinterpret_cast<const unsigned char *>(b64_key.data()), b64_key.size(),
|
||||
reinterpret_cast<unsigned char *>(buf.data()));
|
||||
if (!key_size)
|
||||
throw Exception("Failed to decode encryption key", ErrorCodes::INCORRECT_DATA);
|
||||
else if (key_size < 16)
|
||||
LOG_WARNING(log, "The encryption key should be at least 16 octets long.");
|
||||
|
||||
const std::string_view key = std::string_view(buf.data(), key_size);
|
||||
CompressionCodecEncrypted::setMasterKey(key);
|
||||
|
||||
#else
|
||||
LOG_WARNING(log, "Server was built without Base64 or SSL support. Encryption is disabled.");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
[[noreturn]] void forceShutdown()
|
||||
{
|
||||
#if defined(THREAD_SANITIZER) && defined(OS_LINUX)
|
||||
@ -904,6 +870,8 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
|
||||
global_context->updateStorageConfiguration(*config);
|
||||
global_context->updateInterserverCredentials(*config);
|
||||
|
||||
CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs");
|
||||
},
|
||||
/* already_loaded = */ false); /// Reload it right now (initial loading)
|
||||
|
||||
@ -976,9 +944,9 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
global_context->getMergeTreeSettings().sanityCheck(settings);
|
||||
global_context->getReplicatedMergeTreeSettings().sanityCheck(settings);
|
||||
|
||||
/// Set up encryption.
|
||||
if (config().has("encryption.key_command"))
|
||||
loadEncryptionKey(config().getString("encryption.key_command"), log);
|
||||
|
||||
/// try set up encryption. There are some errors in config, error will be printed and server wouldn't start.
|
||||
CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs");
|
||||
|
||||
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
|
||||
|
||||
|
||||
@ -1026,10 +1026,28 @@
|
||||
defined, or encryption codecs will be disabled otherwise. The
|
||||
command is executed through /bin/sh and is expected to write
|
||||
a Base64-encoded key to the stdout. -->
|
||||
<encryption>
|
||||
<!-- <key_command>/usr/bin/systemd-ask-password --id="clickhouse-server" --timeout=0 "Enter the ClickHouse encryption passphrase:" | base64</key_command> -->
|
||||
<!-- <key_command><![CDATA[IFS=; echo -n >/dev/tty "Enter the ClickHouse encryption passphrase: "; stty=`stty -F /dev/tty -g`; stty -F /dev/tty -echo; read k </dev/tty; stty -F /dev/tty "$stty"; echo -n $k | base64]]></key_command> -->
|
||||
</encryption>
|
||||
<encryption_codecs>
|
||||
<!-- aes_128_gcm_siv -->
|
||||
<!-- Example of getting hex key from env -->
|
||||
<!-- the code should use this key and throw an exception if its length is not 16 bytes -->
|
||||
<!--key_hex from_env="..."></key_hex -->
|
||||
|
||||
<!-- Example of multiple hex keys. They can be imported from env or be written down in config-->
|
||||
<!-- the code should use these keys and throw an exception if their length is not 16 bytes -->
|
||||
<!-- key_hex id="0">...</key_hex -->
|
||||
<!-- key_hex id="1" from_env=".."></key_hex -->
|
||||
<!-- key_hex id="2">...</key_hex -->
|
||||
<!-- current_key_id>2</current_key_id -->
|
||||
|
||||
<!-- Example of getting hex key from config -->
|
||||
<!-- the code should use this key and throw an exception if its length is not 16 bytes -->
|
||||
<!-- key>...</key -->
|
||||
|
||||
<!-- example of adding nonce -->
|
||||
<!-- nonce>...</nonce -->
|
||||
|
||||
<!-- /aes_128_gcm_siv -->
|
||||
</encryption_codecs>
|
||||
|
||||
<!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster.
|
||||
Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. -->
|
||||
|
||||
@ -15,8 +15,7 @@
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
#define UUID_PATTERN "[\\w]{8}-[\\w]{4}-[\\w]{4}-[\\w]{4}-[\\w]{12}"
|
||||
#define EXTRACT_UUID_PATTERN fmt::format(".*/({})/.*", UUID_PATTERN)
|
||||
#define EXTRACT_PATH_PATTERN ".*\\/store/(.*)"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -32,39 +31,102 @@ namespace ErrorCodes
|
||||
* If test-mode option is added, files will be put by given url via PUT request.
|
||||
*/
|
||||
|
||||
void processTableFiles(const fs::path & path, const String & files_prefix, String uuid,
|
||||
WriteBuffer & metadata_buf, std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf)
|
||||
void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf)
|
||||
{
|
||||
fs::directory_iterator dir_end;
|
||||
auto process_file = [&](const String & file_name, const String & file_path)
|
||||
String remote_path;
|
||||
RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path);
|
||||
bool is_directory = fs::is_directory(file_path);
|
||||
|
||||
writeText(file_path.filename().string(), metadata_buf);
|
||||
writeChar('\t', metadata_buf);
|
||||
writeBoolText(is_directory, metadata_buf);
|
||||
if (!is_directory)
|
||||
{
|
||||
auto remote_file_name = files_prefix + "-" + uuid + "-" + file_name;
|
||||
writeText(remote_file_name, metadata_buf);
|
||||
writeChar('\t', metadata_buf);
|
||||
writeIntText(fs::file_size(file_path), metadata_buf);
|
||||
writeChar('\n', metadata_buf);
|
||||
}
|
||||
writeChar('\n', metadata_buf);
|
||||
|
||||
auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
|
||||
auto dst_buf = create_dst_buf(remote_file_name);
|
||||
if (is_directory)
|
||||
return;
|
||||
|
||||
copyData(*src_buf, *dst_buf);
|
||||
dst_buf->next();
|
||||
dst_buf->finalize();
|
||||
};
|
||||
auto dst_file_path = fs::path(dst_path) / remote_path;
|
||||
|
||||
for (fs::directory_iterator dir_it(path); dir_it != dir_end; ++dir_it)
|
||||
auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path));
|
||||
std::shared_ptr<WriteBuffer> dst_buf;
|
||||
|
||||
/// test mode for integration tests.
|
||||
if (test_mode)
|
||||
dst_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT);
|
||||
else
|
||||
dst_buf = std::make_shared<WriteBufferFromFile>(dst_file_path);
|
||||
|
||||
copyData(*src_buf, *dst_buf);
|
||||
dst_buf->next();
|
||||
dst_buf->finalize();
|
||||
};
|
||||
|
||||
|
||||
void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode)
|
||||
{
|
||||
std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl;
|
||||
|
||||
String prefix;
|
||||
RE2::FullMatch(data_path.string(), EXTRACT_PATH_PATTERN, &prefix);
|
||||
|
||||
std::shared_ptr<WriteBuffer> root_meta;
|
||||
if (test_mode)
|
||||
{
|
||||
dst_path /= "store";
|
||||
auto files_root = dst_path / prefix;
|
||||
root_meta = std::make_shared<WriteBufferFromHTTP>(Poco::URI(files_root / ".index"), Poco::Net::HTTPRequest::HTTP_PUT);
|
||||
}
|
||||
else
|
||||
{
|
||||
dst_path = fs::canonical(dst_path);
|
||||
auto files_root = dst_path / prefix;
|
||||
fs::create_directories(files_root);
|
||||
root_meta = std::make_shared<WriteBufferFromFile>(files_root / ".index");
|
||||
}
|
||||
|
||||
fs::directory_iterator dir_end;
|
||||
for (fs::directory_iterator dir_it(data_path); dir_it != dir_end; ++dir_it)
|
||||
{
|
||||
if (dir_it->is_directory())
|
||||
{
|
||||
processFile(dir_it->path(), dst_path, test_mode, *root_meta);
|
||||
|
||||
String directory_prefix;
|
||||
RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix);
|
||||
|
||||
std::shared_ptr<WriteBuffer> directory_meta;
|
||||
if (test_mode)
|
||||
{
|
||||
auto files_root = dst_path / prefix;
|
||||
directory_meta = std::make_shared<WriteBufferFromHTTP>(Poco::URI(dst_path / directory_prefix / ".index"), Poco::Net::HTTPRequest::HTTP_PUT);
|
||||
}
|
||||
else
|
||||
{
|
||||
dst_path = fs::canonical(dst_path);
|
||||
auto files_root = dst_path / prefix;
|
||||
fs::create_directories(dst_path / directory_prefix);
|
||||
directory_meta = std::make_shared<WriteBufferFromFile>(dst_path / directory_prefix / ".index");
|
||||
}
|
||||
|
||||
fs::directory_iterator files_end;
|
||||
for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it)
|
||||
process_file(dir_it->path().filename().string() + "-" + file_it->path().filename().string(), file_it->path());
|
||||
processFile(file_it->path(), dst_path, test_mode, *directory_meta);
|
||||
|
||||
directory_meta->next();
|
||||
directory_meta->finalize();
|
||||
}
|
||||
else
|
||||
{
|
||||
process_file(dir_it->path().filename(), dir_it->path());
|
||||
processFile(dir_it->path(), dst_path, test_mode, *root_meta);
|
||||
}
|
||||
}
|
||||
root_meta->next();
|
||||
root_meta->finalize();
|
||||
}
|
||||
}
|
||||
|
||||
@ -80,8 +142,7 @@ try
|
||||
("metadata-path", po::value<std::string>(), "Metadata path (select data_paths from system.tables where name='table_name'")
|
||||
("test-mode", "Use test mode, which will put data on given url via PUT")
|
||||
("url", po::value<std::string>(), "Web server url for test mode")
|
||||
("output-dir", po::value<std::string>(), "Directory to put files in non-test mode")
|
||||
("files-prefix", po::value<std::string>(), "Prefix for stored files");
|
||||
("output-dir", po::value<std::string>(), "Directory to put files in non-test mode");
|
||||
|
||||
po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run();
|
||||
po::variables_map options;
|
||||
@ -94,18 +155,13 @@ try
|
||||
exit(0);
|
||||
}
|
||||
|
||||
String url, metadata_path, files_prefix;
|
||||
String metadata_path;
|
||||
|
||||
if (options.count("metadata-path"))
|
||||
metadata_path = options["metadata-path"].as<std::string>();
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No metadata-path option passed");
|
||||
|
||||
if (options.count("files-prefix"))
|
||||
files_prefix = options["files-prefix"].as<std::string>();
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No files-prefix option passed");
|
||||
|
||||
fs::path fs_path = fs::weakly_canonical(metadata_path);
|
||||
if (!fs::exists(fs_path))
|
||||
{
|
||||
@ -113,27 +169,14 @@ try
|
||||
return 1;
|
||||
}
|
||||
|
||||
String uuid;
|
||||
if (!RE2::Extract(metadata_path, EXTRACT_UUID_PATTERN, "\\1", &uuid))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot extract uuid for: {}", metadata_path);
|
||||
|
||||
std::shared_ptr<WriteBuffer> metadata_buf;
|
||||
std::function<std::shared_ptr<WriteBuffer>(const String &)> create_dst_buf;
|
||||
String root_path;
|
||||
|
||||
if (options.count("test-mode"))
|
||||
auto test_mode = options.contains("test-mode");
|
||||
if (test_mode)
|
||||
{
|
||||
if (options.count("url"))
|
||||
url = options["url"].as<std::string>();
|
||||
root_path = options["url"].as<std::string>();
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "No url option passed for test mode");
|
||||
|
||||
metadata_buf = std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / (".index-" + uuid)), Poco::Net::HTTPRequest::HTTP_PUT);
|
||||
|
||||
create_dst_buf = [&](const String & remote_file_name)
|
||||
{
|
||||
return std::make_shared<WriteBufferFromHTTP>(Poco::URI(fs::path(url) / remote_file_name), Poco::Net::HTTPRequest::HTTP_PUT);
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -141,17 +184,9 @@ try
|
||||
root_path = options["output-dir"].as<std::string>();
|
||||
else
|
||||
root_path = fs::current_path();
|
||||
|
||||
metadata_buf = std::make_shared<WriteBufferFromFile>(fs::path(root_path) / (".index-" + uuid));
|
||||
create_dst_buf = [&](const String & remote_file_name)
|
||||
{
|
||||
return std::make_shared<WriteBufferFromFile>(fs::path(root_path) / remote_file_name);
|
||||
};
|
||||
}
|
||||
|
||||
processTableFiles(fs_path, files_prefix, uuid, *metadata_buf, create_dst_buf);
|
||||
metadata_buf->next();
|
||||
metadata_buf->finalize();
|
||||
processTableFiles(fs_path, root_path, test_mode);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -21,11 +21,9 @@
|
||||
namespace DB
|
||||
{
|
||||
struct Settings;
|
||||
template <typename T>
|
||||
using DecimalOrVectorCol = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
|
||||
template <typename T> constexpr bool DecimalOrExtendedInt =
|
||||
IsDecimalNumber<T>
|
||||
is_decimal<T>
|
||||
|| std::is_same_v<T, Int128>
|
||||
|| std::is_same_v<T, Int256>
|
||||
|| std::is_same_v<T, UInt128>
|
||||
@ -44,7 +42,7 @@ struct AvgFraction
|
||||
/// Invoked only is either Numerator or Denominator are Decimal.
|
||||
Float64 NO_SANITIZE_UNDEFINED divideIfAnyDecimal(UInt32 num_scale, UInt32 denom_scale [[maybe_unused]]) const
|
||||
{
|
||||
if constexpr (IsDecimalNumber<Numerator> && IsDecimalNumber<Denominator>)
|
||||
if constexpr (is_decimal<Numerator> && is_decimal<Denominator>)
|
||||
{
|
||||
// According to the docs, num(S1) / denom(S2) would have scale S1
|
||||
|
||||
@ -60,7 +58,7 @@ struct AvgFraction
|
||||
/// Numerator is always casted to Float64 to divide correctly if the denominator is not Float64.
|
||||
Float64 num_converted;
|
||||
|
||||
if constexpr (IsDecimalNumber<Numerator>)
|
||||
if constexpr (is_decimal<Numerator>)
|
||||
num_converted = DecimalUtils::convertTo<Float64>(numerator, num_scale);
|
||||
else
|
||||
num_converted = static_cast<Float64>(numerator); /// all other types, including extended integral.
|
||||
@ -68,7 +66,7 @@ struct AvgFraction
|
||||
std::conditional_t<DecimalOrExtendedInt<Denominator>,
|
||||
Float64, Denominator> denom_converted;
|
||||
|
||||
if constexpr (IsDecimalNumber<Denominator>)
|
||||
if constexpr (is_decimal<Denominator>)
|
||||
denom_converted = DecimalUtils::convertTo<Float64>(denominator, denom_scale);
|
||||
else if constexpr (DecimalOrExtendedInt<Denominator>)
|
||||
/// no way to divide Float64 and extended integral type without an explicit cast.
|
||||
@ -139,7 +137,7 @@ public:
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
|
||||
{
|
||||
if constexpr (IsDecimalNumber<Numerator> || IsDecimalNumber<Denominator>)
|
||||
if constexpr (is_decimal<Numerator> || is_decimal<Denominator>)
|
||||
assert_cast<ColumnVector<Float64> &>(to).getData().push_back(
|
||||
this->data(place).divideIfAnyDecimal(num_scale, denom_scale));
|
||||
else
|
||||
@ -222,7 +220,7 @@ private:
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
using AvgFieldType = std::conditional_t<IsDecimalNumber<T>,
|
||||
using AvgFieldType = std::conditional_t<is_decimal<T>,
|
||||
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
|
||||
NearestFieldType<T>>;
|
||||
|
||||
@ -239,7 +237,7 @@ public:
|
||||
|
||||
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final
|
||||
{
|
||||
this->data(place).numerator += static_cast<const DecimalOrVectorCol<T> &>(*columns[0]).getData()[row_num];
|
||||
this->data(place).numerator += static_cast<const ColumnVectorOrDecimal<T> &>(*columns[0]).getData()[row_num];
|
||||
++this->data(place).denominator;
|
||||
}
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@ namespace DB
|
||||
struct Settings;
|
||||
|
||||
template <typename T>
|
||||
using AvgWeightedFieldType = std::conditional_t<IsDecimalNumber<T>,
|
||||
using AvgWeightedFieldType = std::conditional_t<is_decimal<T>,
|
||||
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
|
||||
std::conditional_t<DecimalOrExtendedInt<T>,
|
||||
Float64, // no way to do UInt128 * UInt128, better cast to Float64
|
||||
@ -34,10 +34,10 @@ public:
|
||||
|
||||
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
const auto& weights = static_cast<const DecimalOrVectorCol<Weight> &>(*columns[1]);
|
||||
const auto& weights = static_cast<const ColumnVectorOrDecimal<Weight> &>(*columns[1]);
|
||||
|
||||
this->data(place).numerator += static_cast<Numerator>(
|
||||
static_cast<const DecimalOrVectorCol<Value> &>(*columns[0]).getData()[row_num]) *
|
||||
static_cast<const ColumnVectorOrDecimal<Value> &>(*columns[0]).getData()[row_num]) *
|
||||
static_cast<Numerator>(weights.getData()[row_num]);
|
||||
|
||||
this->data(place).denominator += static_cast<Denominator>(weights.getData()[row_num]);
|
||||
|
||||
@ -25,14 +25,14 @@ namespace
|
||||
template <typename T, typename LimitNumberOfElements>
|
||||
struct MovingSum
|
||||
{
|
||||
using Data = MovingSumData<std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>>;
|
||||
using Data = MovingSumData<std::conditional_t<is_decimal<T>, Decimal128, NearestFieldType<T>>>;
|
||||
using Function = MovingImpl<T, LimitNumberOfElements, Data>;
|
||||
};
|
||||
|
||||
template <typename T, typename LimitNumberOfElements>
|
||||
struct MovingAvg
|
||||
{
|
||||
using Data = MovingAvgData<std::conditional_t<IsDecimalNumber<T>, Decimal128, Float64>>;
|
||||
using Data = MovingAvgData<std::conditional_t<is_decimal<T>, Decimal128, Float64>>;
|
||||
using Function = MovingImpl<T, LimitNumberOfElements, Data>;
|
||||
};
|
||||
|
||||
|
||||
@ -87,18 +87,10 @@ class MovingImpl final
|
||||
public:
|
||||
using ResultT = typename Data::Accumulator;
|
||||
|
||||
using ColumnSource = std::conditional_t<IsDecimalNumber<T>,
|
||||
ColumnDecimal<T>,
|
||||
ColumnVector<T>>;
|
||||
using ColumnSource = ColumnVectorOrDecimal<T>;
|
||||
|
||||
/// Probably for overflow function in the future.
|
||||
using ColumnResult = std::conditional_t<IsDecimalNumber<ResultT>,
|
||||
ColumnDecimal<ResultT>,
|
||||
ColumnVector<ResultT>>;
|
||||
|
||||
using DataTypeResult = std::conditional_t<IsDecimalNumber<ResultT>,
|
||||
DataTypeDecimal<ResultT>,
|
||||
DataTypeNumber<ResultT>>;
|
||||
using ColumnResult = ColumnVectorOrDecimal<ResultT>;
|
||||
|
||||
explicit MovingImpl(const DataTypePtr & data_type_, UInt64 window_size_ = std::numeric_limits<UInt64>::max())
|
||||
: IAggregateFunctionDataHelper<Data, MovingImpl<T, Tlimit_num_elems, Data>>({data_type_}, {})
|
||||
@ -106,14 +98,7 @@ public:
|
||||
|
||||
String getName() const override { return Data::name; }
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
if constexpr (IsDecimalNumber<ResultT>)
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeResult>(
|
||||
DataTypeResult::maxPrecision(), getDecimalScale(*this->argument_types.at(0))));
|
||||
else
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeResult>());
|
||||
}
|
||||
DataTypePtr getReturnType() const override { return std::make_shared<DataTypeArray>(getReturnTypeElement()); }
|
||||
|
||||
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
|
||||
{
|
||||
@ -196,6 +181,18 @@ public:
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
auto getReturnTypeElement() const
|
||||
{
|
||||
if constexpr (!is_decimal<ResultT>)
|
||||
return std::make_shared<DataTypeNumber<ResultT>>();
|
||||
else
|
||||
{
|
||||
using Res = DataTypeDecimal<ResultT>;
|
||||
return std::make_shared<Res>(Res::maxPrecision(), getDecimalScale(*this->argument_types.at(0)));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#undef AGGREGATE_FUNCTION_MOVING_MAX_ARRAY_SIZE
|
||||
|
||||
@ -44,7 +44,7 @@ struct SingleValueDataFixed
|
||||
{
|
||||
private:
|
||||
using Self = SingleValueDataFixed;
|
||||
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
using ColVecType = ColumnVectorOrDecimal<T>;
|
||||
|
||||
bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf.
|
||||
T value;
|
||||
|
||||
@ -67,10 +67,10 @@ class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper<Data
|
||||
AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>
|
||||
{
|
||||
private:
|
||||
using ColVecType = std::conditional_t<IsDecimalNumber<Value>, ColumnDecimal<Value>, ColumnVector<Value>>;
|
||||
using ColVecType = ColumnVectorOrDecimal<Value>;
|
||||
|
||||
static constexpr bool returns_float = !(std::is_same_v<FloatReturnType, void>);
|
||||
static_assert(!IsDecimalNumber<Value> || !returns_float);
|
||||
static_assert(!is_decimal<Value> || !returns_float);
|
||||
|
||||
QuantileLevels<Float64> levels;
|
||||
|
||||
|
||||
72
src/AggregateFunctions/AggregateFunctionSparkbar.cpp
Normal file
72
src/AggregateFunctions/AggregateFunctionSparkbar.cpp
Normal file
@ -0,0 +1,72 @@
|
||||
#include <AggregateFunctions/AggregateFunctionSparkbar.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <template <typename, typename> class AggregateFunctionTemplate, typename Data, typename ... TArgs>
|
||||
static IAggregateFunction * createWithUIntegerOrTimeType(const std::string & name, const IDataType & argument_type, TArgs && ... args)
|
||||
{
|
||||
WhichDataType which(argument_type);
|
||||
if (which.idx == TypeIndex::Date || which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::DateTime || which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt8) return new AggregateFunctionTemplate<UInt8, Data>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data>(std::forward<TArgs>(args)...);
|
||||
throw Exception("The first argument type must be UInt or Date or DateTime for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
template <typename ... TArgs>
|
||||
AggregateFunctionPtr createAggregateFunctionSparkbarImpl(const std::string & name, const IDataType & x_argument_type, const IDataType & y_argument_type, TArgs ... args)
|
||||
{
|
||||
WhichDataType which(y_argument_type);
|
||||
#define DISPATCH(TYPE) \
|
||||
if (which.idx == TypeIndex::TYPE) return AggregateFunctionPtr(createWithUIntegerOrTimeType<AggregateFunctionSparkbar, TYPE>(name, x_argument_type, std::forward<TArgs>(args)...));
|
||||
FOR_NUMERIC_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
|
||||
throw Exception("The second argument type must be numeric for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
|
||||
AggregateFunctionPtr createAggregateFunctionSparkbar(const std::string & name, const DataTypes & arguments, const Array & params, const Settings *)
|
||||
{
|
||||
assertBinary(name, arguments);
|
||||
|
||||
if (params.size() != 1 && params.size() != 3)
|
||||
throw Exception("The number of params does not match for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (params.size() == 3)
|
||||
{
|
||||
if (params.at(1).getType() != arguments[0]->getDefault().getType() || params.at(2).getType() != arguments[0]->getDefault().getType())
|
||||
{
|
||||
throw Exception("The second and third parameters are not the same type as the first arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
}
|
||||
return createAggregateFunctionSparkbarImpl(name, *arguments[0], *arguments[1], arguments, params);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionSparkbar(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("sparkbar", createAggregateFunctionSparkbar);
|
||||
}
|
||||
|
||||
}
|
||||
309
src/AggregateFunctions/AggregateFunctionSparkbar.h
Normal file
309
src/AggregateFunctions/AggregateFunctionSparkbar.h
Normal file
@ -0,0 +1,309 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <common/range.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template<typename X, typename Y>
|
||||
struct AggregateFunctionSparkbarData
|
||||
{
|
||||
|
||||
using Points = HashMap<X, Y>;
|
||||
Points points;
|
||||
|
||||
X min_x = std::numeric_limits<X>::max();
|
||||
X max_x = std::numeric_limits<X>::lowest();
|
||||
|
||||
Y min_y = std::numeric_limits<Y>::max();
|
||||
Y max_y = std::numeric_limits<Y>::lowest();
|
||||
|
||||
void insert(const X & x, const Y & y)
|
||||
{
|
||||
auto result = points.insert({x, y});
|
||||
if (!result.second)
|
||||
result.first->getMapped() += y;
|
||||
}
|
||||
|
||||
void add(X x, Y y)
|
||||
{
|
||||
insert(x, y);
|
||||
min_x = std::min(x, min_x);
|
||||
max_x = std::max(x, max_x);
|
||||
min_y = std::min(y, min_y);
|
||||
max_y = std::max(y, max_y);
|
||||
}
|
||||
|
||||
void merge(const AggregateFunctionSparkbarData & other)
|
||||
{
|
||||
if (other.points.empty())
|
||||
return;
|
||||
|
||||
for (auto & point : other.points)
|
||||
insert(point.getKey(), point.getMapped());
|
||||
|
||||
min_x = std::min(other.min_x, min_x);
|
||||
max_x = std::max(other.max_x, max_x);
|
||||
min_y = std::min(other.min_y, min_y);
|
||||
max_y = std::max(other.max_y, max_y);
|
||||
}
|
||||
|
||||
void serialize(WriteBuffer & buf) const
|
||||
{
|
||||
writeBinary(min_x, buf);
|
||||
writeBinary(max_x, buf);
|
||||
writeBinary(min_y, buf);
|
||||
writeBinary(max_y, buf);
|
||||
writeVarUInt(points.size(), buf);
|
||||
|
||||
for (const auto & elem : points)
|
||||
{
|
||||
writeBinary(elem.getKey(), buf);
|
||||
writeBinary(elem.getMapped(), buf);
|
||||
}
|
||||
}
|
||||
|
||||
void deserialize(ReadBuffer & buf)
|
||||
{
|
||||
readBinary(min_x, buf);
|
||||
readBinary(max_x, buf);
|
||||
readBinary(min_y, buf);
|
||||
readBinary(max_y, buf);
|
||||
size_t size;
|
||||
readVarUInt(size, buf);
|
||||
|
||||
/// TODO Protection against huge size
|
||||
X x;
|
||||
Y y;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
readBinary(x, buf);
|
||||
readBinary(y, buf);
|
||||
insert(x, y);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
template<typename X, typename Y>
|
||||
class AggregateFunctionSparkbar final
|
||||
: public IAggregateFunctionDataHelper<AggregateFunctionSparkbarData<X, Y>, AggregateFunctionSparkbar<X, Y>>
|
||||
{
|
||||
|
||||
private:
|
||||
size_t width;
|
||||
X min_x;
|
||||
X max_x;
|
||||
|
||||
String getBar(const UInt8 value) const
|
||||
{
|
||||
// ▁▂▃▄▅▆▇█
|
||||
switch (value)
|
||||
{
|
||||
case 1: return "▁";
|
||||
case 2: return "▂";
|
||||
case 3: return "▃";
|
||||
case 4: return "▄";
|
||||
case 5: return "▅";
|
||||
case 6: return "▆";
|
||||
case 7: return "▇";
|
||||
case 8: return "█";
|
||||
}
|
||||
return " ";
|
||||
}
|
||||
|
||||
/**
|
||||
* The minimum value of y is rendered as the lowest height "▁",
|
||||
* the maximum value of y is rendered as the highest height "█", and the middle value will be rendered proportionally.
|
||||
* If a bucket has no y value, it will be rendered as " ".
|
||||
* If the actual number of buckets is greater than the specified bucket, it will be compressed by width.
|
||||
* For example, there are actually 11 buckets, specify 10 buckets, and divide the 11 buckets as follows (11/10):
|
||||
* 0.0-1.1, 1.1-2.2, 2.2-3.3, 3.3-4.4, 4.4-5.5, 5.5-6.6, 6.6-7.7, 7.7-8.8, 8.8-9.9, 9.9-11.
|
||||
* The y value of the first bucket will be calculated as follows:
|
||||
* the actual y value of the first position + the actual second position y*0.1, and the remaining y*0.9 is reserved for the next bucket.
|
||||
* The next bucket will use the last y*0.9 + the actual third position y*0.2, and the remaining y*0.8 will be reserved for the next bucket. And so on.
|
||||
*/
|
||||
String render(const AggregateFunctionSparkbarData<X, Y> & data) const
|
||||
{
|
||||
String value;
|
||||
if (data.points.empty() || !width)
|
||||
return value;
|
||||
X local_min_x = data.min_x;
|
||||
X local_max_x = data.max_x;
|
||||
size_t diff_x = local_max_x - local_min_x;
|
||||
if ((diff_x + 1) <= width)
|
||||
{
|
||||
Y min_y = data.min_y;
|
||||
Y max_y = data.max_y;
|
||||
Float64 diff_y = max_y - min_y;
|
||||
|
||||
if (diff_y)
|
||||
{
|
||||
for (size_t i = 0; i <= diff_x; ++i)
|
||||
{
|
||||
auto it = data.points.find(local_min_x + i);
|
||||
bool found = it != data.points.end();
|
||||
value += getBar(found ? static_cast<UInt8>(std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1) : 0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i <= diff_x; ++i)
|
||||
value += getBar(data.points.has(local_min_x + i) ? 1 : 0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// begin reshapes to width buckets
|
||||
Float64 multiple_d = (diff_x + 1) / static_cast<Float64>(width);
|
||||
|
||||
std::optional<Float64> min_y;
|
||||
std::optional<Float64> max_y;
|
||||
|
||||
std::optional<Float64> new_y;
|
||||
std::vector<std::optional<Float64>> newPoints;
|
||||
newPoints.reserve(width);
|
||||
|
||||
std::pair<size_t, Float64> bound{0, 0.0};
|
||||
size_t cur_bucket_num = 0;
|
||||
// upper bound for bucket
|
||||
auto upperBound = [&](size_t bucket_num)
|
||||
{
|
||||
bound.second = (bucket_num + 1) * multiple_d;
|
||||
bound.first = std::floor(bound.second);
|
||||
};
|
||||
upperBound(cur_bucket_num);
|
||||
for (size_t i = 0; i <= (diff_x + 1); ++i)
|
||||
{
|
||||
if (i == bound.first) // is bound
|
||||
{
|
||||
Float64 proportion = bound.second - bound.first;
|
||||
auto it = data.points.find(local_min_x + i);
|
||||
bool found = (it != data.points.end());
|
||||
if (found)
|
||||
new_y = new_y.value_or(0) + it->getMapped() * proportion;
|
||||
|
||||
if (new_y)
|
||||
{
|
||||
Float64 avg_y = new_y.value() / multiple_d;
|
||||
|
||||
newPoints.emplace_back(avg_y);
|
||||
// If min_y has no value, or if the avg_y of the current bucket is less than min_y, update it.
|
||||
if (!min_y || avg_y < min_y)
|
||||
min_y = avg_y;
|
||||
if (!max_y || avg_y > max_y)
|
||||
max_y = avg_y;
|
||||
}
|
||||
else
|
||||
{
|
||||
newPoints.emplace_back();
|
||||
}
|
||||
|
||||
// next bucket
|
||||
new_y = found ? ((1 - proportion) * it->getMapped()) : std::optional<Float64>();
|
||||
upperBound(++cur_bucket_num);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto it = data.points.find(local_min_x + i);
|
||||
if (it != data.points.end())
|
||||
new_y = new_y.value_or(0) + it->getMapped();
|
||||
}
|
||||
}
|
||||
|
||||
if (!min_y || !max_y) // No value is set
|
||||
return {};
|
||||
|
||||
Float64 diff_y = max_y.value() - min_y.value();
|
||||
|
||||
auto getBars = [&] (const std::optional<Float64> & point_y)
|
||||
{
|
||||
value += getBar(point_y ? static_cast<UInt8>(std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1) : 0);
|
||||
};
|
||||
auto getBarsForConstant = [&] (const std::optional<Float64> & point_y)
|
||||
{
|
||||
value += getBar(point_y ? 1 : 0);
|
||||
};
|
||||
|
||||
if (diff_y)
|
||||
std::for_each(newPoints.begin(), newPoints.end(), getBars);
|
||||
else
|
||||
std::for_each(newPoints.begin(), newPoints.end(), getBarsForConstant);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
AggregateFunctionSparkbar(const DataTypes & arguments, const Array & params)
|
||||
: IAggregateFunctionDataHelper<AggregateFunctionSparkbarData<X, Y>, AggregateFunctionSparkbar>(
|
||||
arguments, params)
|
||||
{
|
||||
width = params.at(0).safeGet<UInt64>();
|
||||
if (params.size() == 3)
|
||||
{
|
||||
min_x = params.at(1).safeGet<X>();
|
||||
max_x = params.at(2).safeGet<X>();
|
||||
}
|
||||
else
|
||||
{
|
||||
min_x = std::numeric_limits<X>::min();
|
||||
max_x = std::numeric_limits<X>::max();
|
||||
}
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return "sparkbar";
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * /*arena*/) const override
|
||||
{
|
||||
X x = assert_cast<const ColumnVector<X> *>(columns[0])->getData()[row_num];
|
||||
if (min_x <= x && x <= max_x)
|
||||
{
|
||||
Y y = assert_cast<const ColumnVector<Y> *>(columns[1])->getData()[row_num];
|
||||
this->data(place).add(x, y);
|
||||
}
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * /*arena*/) const override
|
||||
{
|
||||
this->data(place).merge(this->data(rhs));
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).serialize(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).deserialize(buf);
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override { return false; }
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * /*arena*/) const override
|
||||
{
|
||||
auto & to_column = assert_cast<ColumnString &>(to);
|
||||
const auto & data = this->data(place);
|
||||
const String & value = render(data);
|
||||
to_column.insertData(value.data(), value.size());
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@ -49,7 +49,7 @@ struct StatFuncOneArg
|
||||
using Type1 = T;
|
||||
using Type2 = T;
|
||||
using ResultType = std::conditional_t<std::is_same_v<T, Float32>, Float32, Float64>;
|
||||
using Data = std::conditional_t<IsDecimalNumber<T>, VarMomentsDecimal<Decimal128, _level>, VarMoments<ResultType, _level>>;
|
||||
using Data = std::conditional_t<is_decimal<T>, VarMomentsDecimal<Decimal128, _level>, VarMoments<ResultType, _level>>;
|
||||
|
||||
static constexpr StatisticsFunctionKind kind = _kind;
|
||||
static constexpr UInt32 num_args = 1;
|
||||
@ -75,8 +75,8 @@ class AggregateFunctionVarianceSimple final
|
||||
public:
|
||||
using T1 = typename StatFunc::Type1;
|
||||
using T2 = typename StatFunc::Type2;
|
||||
using ColVecT1 = std::conditional_t<IsDecimalNumber<T1>, ColumnDecimal<T1>, ColumnVector<T1>>;
|
||||
using ColVecT2 = std::conditional_t<IsDecimalNumber<T2>, ColumnDecimal<T2>, ColumnVector<T2>>;
|
||||
using ColVecT1 = ColumnVectorOrDecimal<T1>;
|
||||
using ColVecT2 = ColumnVectorOrDecimal<T2>;
|
||||
using ResultType = typename StatFunc::ResultType;
|
||||
using ColVecResult = ColumnVector<ResultType>;
|
||||
|
||||
@ -132,7 +132,7 @@ public:
|
||||
static_cast<ResultType>(static_cast<const ColVecT2 &>(*columns[1]).getData()[row_num]));
|
||||
else
|
||||
{
|
||||
if constexpr (IsDecimalNumber<T1>)
|
||||
if constexpr (is_decimal<T1>)
|
||||
{
|
||||
this->data(place).add(static_cast<ResultType>(
|
||||
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value));
|
||||
@ -163,7 +163,7 @@ public:
|
||||
const auto & data = this->data(place);
|
||||
auto & dst = static_cast<ColVecResult &>(to).getData();
|
||||
|
||||
if constexpr (IsDecimalNumber<T1>)
|
||||
if constexpr (is_decimal<T1>)
|
||||
{
|
||||
if constexpr (StatFunc::kind == StatisticsFunctionKind::varPop)
|
||||
dst.push_back(data.getPopulation(src_scale * 2));
|
||||
|
||||
@ -20,10 +20,9 @@ template <typename T>
|
||||
struct SumSimple
|
||||
{
|
||||
/// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
|
||||
using ResultType = std::conditional_t<IsDecimalNumber<T>,
|
||||
using ResultType = std::conditional_t<is_decimal<T>,
|
||||
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
|
||||
NearestFieldType<T>>;
|
||||
// using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
|
||||
using AggregateDataType = AggregateFunctionSumData<ResultType>;
|
||||
using Function = AggregateFunctionSum<T, ResultType, AggregateDataType, AggregateFunctionTypeSum>;
|
||||
};
|
||||
@ -47,7 +46,7 @@ struct SumKahan
|
||||
template <typename T> using AggregateFunctionSumSimple = typename SumSimple<T>::Function;
|
||||
template <typename T> using AggregateFunctionSumWithOverflow = typename SumSameType<T>::Function;
|
||||
template <typename T> using AggregateFunctionSumKahan =
|
||||
std::conditional_t<IsDecimalNumber<T>, typename SumSimple<T>::Function, typename SumKahan<T>::Function>;
|
||||
std::conditional_t<is_decimal<T>, typename SumSimple<T>::Function, typename SumKahan<T>::Function>;
|
||||
|
||||
|
||||
template <template <typename> class Function>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <experimental/type_traits>
|
||||
#include <type_traits>
|
||||
@ -104,8 +105,8 @@ struct AggregateFunctionSumData
|
||||
const auto * end = ptr + count;
|
||||
|
||||
if constexpr (
|
||||
(is_integer_v<T> && !is_big_int_v<T>)
|
||||
|| (IsDecimalNumber<T> && !std::is_same_v<T, Decimal256> && !std::is_same_v<T, Decimal128>))
|
||||
(is_integer<T> && !is_big_int_v<T>)
|
||||
|| (is_decimal<T> && !std::is_same_v<T, Decimal256> && !std::is_same_v<T, Decimal128>))
|
||||
{
|
||||
/// For integers we can vectorize the operation if we replace the null check using a multiplication (by 0 for null, 1 for not null)
|
||||
/// https://quick-bench.com/q/MLTnfTvwC2qZFVeWHfOBR3U7a8I
|
||||
@ -123,6 +124,10 @@ struct AggregateFunctionSumData
|
||||
|
||||
if constexpr (std::is_floating_point_v<T>)
|
||||
{
|
||||
/// For floating point we use a similar trick as above, except that now we reinterpret the floating point number as an unsigned
|
||||
/// integer of the same size and use a mask instead (0 to discard, 0xFF..FF to keep)
|
||||
static_assert(sizeof(Value) == 4 || sizeof(Value) == 8);
|
||||
typedef typename std::conditional<sizeof(Value) == 4, UInt32, UInt64>::type equivalent_integer;
|
||||
constexpr size_t unroll_count = 128 / sizeof(T);
|
||||
T partial_sums[unroll_count]{};
|
||||
|
||||
@ -132,10 +137,12 @@ struct AggregateFunctionSumData
|
||||
{
|
||||
for (size_t i = 0; i < unroll_count; ++i)
|
||||
{
|
||||
if (!condition_map[i] == add_if_zero)
|
||||
{
|
||||
Impl::add(partial_sums[i], ptr[i]);
|
||||
}
|
||||
equivalent_integer value;
|
||||
std::memcpy(&value, &ptr[i], sizeof(Value));
|
||||
value &= (!condition_map[i] != add_if_zero) - 1;
|
||||
Value d;
|
||||
std::memcpy(&d, &value, sizeof(Value));
|
||||
Impl::add(partial_sums[i], d);
|
||||
}
|
||||
ptr += unroll_count;
|
||||
condition_map += unroll_count;
|
||||
@ -334,9 +341,7 @@ class AggregateFunctionSum final : public IAggregateFunctionDataHelper<Data, Agg
|
||||
public:
|
||||
static constexpr bool DateTime64Supported = false;
|
||||
|
||||
using ResultDataType = std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<TResult>, DataTypeNumber<TResult>>;
|
||||
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
||||
using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<TResult>, ColumnVector<TResult>>;
|
||||
using ColVecType = ColumnVectorOrDecimal<T>;
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
@ -361,10 +366,13 @@ public:
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
return std::make_shared<ResultDataType>(ResultDataType::maxPrecision(), scale);
|
||||
if constexpr (!is_decimal<T>)
|
||||
return std::make_shared<DataTypeNumber<TResult>>();
|
||||
else
|
||||
return std::make_shared<ResultDataType>();
|
||||
{
|
||||
using DataType = DataTypeDecimal<TResult>;
|
||||
return std::make_shared<DataType>(DataType::maxPrecision(), scale);
|
||||
}
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override { return false; }
|
||||
@ -431,8 +439,7 @@ public:
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
|
||||
{
|
||||
auto & column = assert_cast<ColVecResult &>(to);
|
||||
column.getData().push_back(this->data(place).get());
|
||||
castColumnToResult(to).getData().push_back(this->data(place).get());
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
@ -511,6 +518,14 @@ public:
|
||||
|
||||
private:
|
||||
UInt32 scale;
|
||||
|
||||
static constexpr auto & castColumnToResult(IColumn & to)
|
||||
{
|
||||
if constexpr (is_decimal<T>)
|
||||
return assert_cast<ColumnDecimal<TResult> &>(to);
|
||||
else
|
||||
return assert_cast<ColumnVector<TResult> &>(to);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -8,8 +8,6 @@
|
||||
namespace DB
|
||||
{
|
||||
template <typename T>
|
||||
using DecimalOrNumberDataType = std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<AvgFieldType<T>>, DataTypeNumber<AvgFieldType<T>>>;
|
||||
template <typename T>
|
||||
class AggregateFunctionSumCount final : public AggregateFunctionAvgBase<AvgFieldType<T>, UInt64, AggregateFunctionSumCount<T>>
|
||||
{
|
||||
public:
|
||||
@ -20,20 +18,13 @@ public:
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
DataTypes types;
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
types.emplace_back(std::make_shared<DecimalOrNumberDataType<T>>(DecimalOrNumberDataType<T>::maxPrecision(), scale));
|
||||
else
|
||||
types.emplace_back(std::make_shared<DecimalOrNumberDataType<T>>());
|
||||
|
||||
types.emplace_back(std::make_shared<DataTypeUInt64>());
|
||||
|
||||
return std::make_shared<DataTypeTuple>(types);
|
||||
auto second_elem = std::make_shared<DataTypeUInt64>();
|
||||
return std::make_shared<DataTypeTuple>(DataTypes{getReturnTypeFirstElement(), std::move(second_elem)});
|
||||
}
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const final
|
||||
{
|
||||
assert_cast<DecimalOrVectorCol<AvgFieldType<T>> &>((assert_cast<ColumnTuple &>(to)).getColumn(0)).getData().push_back(
|
||||
assert_cast<ColumnVectorOrDecimal<AvgFieldType<T>> &>((assert_cast<ColumnTuple &>(to)).getColumn(0)).getData().push_back(
|
||||
this->data(place).numerator);
|
||||
|
||||
assert_cast<ColumnUInt64 &>((assert_cast<ColumnTuple &>(to)).getColumn(1)).getData().push_back(
|
||||
@ -42,7 +33,7 @@ public:
|
||||
|
||||
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final
|
||||
{
|
||||
this->data(place).numerator += static_cast<const DecimalOrVectorCol<T> &>(*columns[0]).getData()[row_num];
|
||||
this->data(place).numerator += static_cast<const ColumnVectorOrDecimal<T> &>(*columns[0]).getData()[row_num];
|
||||
++this->data(place).denominator;
|
||||
}
|
||||
|
||||
@ -59,6 +50,19 @@ public:
|
||||
|
||||
private:
|
||||
UInt32 scale;
|
||||
|
||||
auto getReturnTypeFirstElement() const
|
||||
{
|
||||
using FieldType = AvgFieldType<T>;
|
||||
|
||||
if constexpr (!is_decimal<T>)
|
||||
return std::make_shared<DataTypeNumber<FieldType>>();
|
||||
else
|
||||
{
|
||||
using DataType = DataTypeDecimal<FieldType>;
|
||||
return std::make_shared<DataType>(DataType::maxPrecision(), scale);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -190,7 +190,7 @@ public:
|
||||
continue;
|
||||
|
||||
decltype(merged_maps.begin()) it;
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
if constexpr (is_decimal<T>)
|
||||
{
|
||||
// FIXME why is storing NearestFieldType not enough, and we
|
||||
// have to check for decimals again here?
|
||||
@ -217,7 +217,7 @@ public:
|
||||
new_values.resize(size);
|
||||
new_values[col] = value;
|
||||
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
if constexpr (is_decimal<T>)
|
||||
{
|
||||
UInt32 scale = static_cast<const ColumnDecimal<T> &>(key_column).getData().getScale();
|
||||
merged_maps.emplace(DecimalField<T>(key, scale), std::move(new_values));
|
||||
@ -280,7 +280,7 @@ public:
|
||||
for (size_t col = 0; col < values_types.size(); ++col)
|
||||
values_serializations[col]->deserializeBinary(values[col], buf);
|
||||
|
||||
if constexpr (IsDecimalNumber<T>)
|
||||
if constexpr (is_decimal<T>)
|
||||
merged_maps[key.get<DecimalField<T>>()] = values;
|
||||
else
|
||||
merged_maps[key.get<T>()] = values;
|
||||
@ -396,7 +396,7 @@ private:
|
||||
using Base = AggregateFunctionMapBase<T, Self, FieldVisitorSum, overflow, tuple_argument, true>;
|
||||
|
||||
/// ARCADIA_BUILD disallow unordered_set for big ints for some reason
|
||||
static constexpr const bool allow_hash = !OverBigInt<T>;
|
||||
static constexpr const bool allow_hash = !is_over_big_int<T>;
|
||||
using ContainerT = std::conditional_t<allow_hash, std::unordered_set<T>, std::set<T>>;
|
||||
|
||||
ContainerT keys_to_keep;
|
||||
|
||||
@ -30,7 +30,7 @@ struct QuantileExactWeighted
|
||||
};
|
||||
|
||||
using Weight = UInt64;
|
||||
using UnderlyingType = typename NativeType<Value>::Type;
|
||||
using UnderlyingType = NativeType<Value>;
|
||||
using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
|
||||
|
||||
/// When creating, the hash table must be small.
|
||||
|
||||
@ -121,7 +121,7 @@ public:
|
||||
{
|
||||
if (samples.empty())
|
||||
{
|
||||
if (DB::IsDecimalNumber<T>)
|
||||
if (DB::is_decimal<T>)
|
||||
return 0;
|
||||
return onEmpty<double>();
|
||||
}
|
||||
@ -134,7 +134,7 @@ public:
|
||||
size_t right_index = left_index + 1;
|
||||
if (right_index == samples.size())
|
||||
{
|
||||
if constexpr (DB::IsDecimalNumber<T>)
|
||||
if constexpr (DB::is_decimal<T>)
|
||||
return static_cast<double>(samples[left_index].value);
|
||||
else
|
||||
return static_cast<double>(samples[left_index]);
|
||||
@ -143,7 +143,7 @@ public:
|
||||
double left_coef = right_index - index;
|
||||
double right_coef = index - left_index;
|
||||
|
||||
if constexpr (DB::IsDecimalNumber<T>)
|
||||
if constexpr (DB::is_decimal<T>)
|
||||
return static_cast<double>(samples[left_index].value) * left_coef + static_cast<double>(samples[right_index].value) * right_coef;
|
||||
else
|
||||
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
|
||||
|
||||
@ -50,6 +50,7 @@ void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionSingleValueOrNull(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionSequenceNextNode(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionSparkbar(AggregateFunctionFactory &);
|
||||
|
||||
class AggregateFunctionCombinatorFactory;
|
||||
void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &);
|
||||
@ -119,6 +120,7 @@ void registerAggregateFunctions()
|
||||
registerWindowFunctions(factory);
|
||||
|
||||
registerAggregateFunctionIntervalLengthSum(factory);
|
||||
registerAggregateFunctionSparkbar(factory);
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@ -49,6 +49,7 @@ SRCS(
|
||||
AggregateFunctionSimpleLinearRegression.cpp
|
||||
AggregateFunctionSimpleState.cpp
|
||||
AggregateFunctionSingleValueOrNull.cpp
|
||||
AggregateFunctionSparkbar.cpp
|
||||
AggregateFunctionState.cpp
|
||||
AggregateFunctionStatistics.cpp
|
||||
AggregateFunctionStatisticsSimple.cpp
|
||||
|
||||
@ -83,9 +83,8 @@ namespace
|
||||
/// Replaces elements of types TEMPORARY_TABLE or ALL_TEMPORARY_TABLES with elements of type TABLE or DATABASE.
|
||||
void replaceTemporaryTablesWithTemporaryDatabase(Elements & elements)
|
||||
{
|
||||
for (size_t i = 0; i != elements.size(); ++i)
|
||||
for (auto & element : elements)
|
||||
{
|
||||
auto & element = elements[i];
|
||||
switch (element.type)
|
||||
{
|
||||
case ElementType::TEMPORARY_TABLE:
|
||||
|
||||
@ -27,8 +27,8 @@ HedgedConnectionsFactory::HedgedConnectionsFactory(
|
||||
: pool(pool_), settings(settings_), timeouts(timeouts_), table_to_check(table_to_check_), log(&Poco::Logger::get("HedgedConnectionsFactory"))
|
||||
{
|
||||
shuffled_pools = pool->getShuffledPools(settings);
|
||||
for (size_t i = 0; i != shuffled_pools.size(); ++i)
|
||||
replicas.emplace_back(ConnectionEstablisherAsync(shuffled_pools[i].pool, &timeouts, settings, log, table_to_check.get()));
|
||||
for (auto shuffled_pool : shuffled_pools)
|
||||
replicas.emplace_back(ConnectionEstablisherAsync(shuffled_pool.pool, &timeouts, settings, log, table_to_check.get()));
|
||||
|
||||
max_tries
|
||||
= (settings ? size_t{settings->connections_with_failover_max_tries} : size_t{DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES});
|
||||
|
||||
@ -506,8 +506,9 @@ static void pushBackAndCreateState(ColumnAggregateFunction::Container & data, Ar
|
||||
void ColumnAggregateFunction::insert(const Field & x)
|
||||
{
|
||||
if (x.getType() != Field::Types::AggregateFunctionState)
|
||||
throw Exception(String("Inserting field of type ") + x.getTypeName() + " into ColumnAggregateFunction. "
|
||||
"Expected " + Field::Types::toString(Field::Types::AggregateFunctionState), ErrorCodes::LOGICAL_ERROR);
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Inserting field of type {} into ColumnAggregateFunction. Expected {}",
|
||||
x.getTypeName(), Field::Types::AggregateFunctionState);
|
||||
|
||||
const auto & field_name = x.get<const AggregateFunctionStateData &>().name;
|
||||
if (type_string != field_name)
|
||||
|
||||
@ -57,9 +57,13 @@ public:
|
||||
*/
|
||||
static ColumnPtr wrap(ColumnPtr column)
|
||||
{
|
||||
/// The order of evaluation of function arguments is unspecified
|
||||
/// and could cause interacting with object in moved-from state
|
||||
const auto size = column->size();
|
||||
const auto bytes = column->allocatedBytes();
|
||||
return ColumnCompressed::create(
|
||||
column->size(),
|
||||
column->allocatedBytes(),
|
||||
size,
|
||||
bytes,
|
||||
[column = std::move(column)]{ return column; });
|
||||
}
|
||||
|
||||
@ -124,4 +128,3 @@ private:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -38,7 +38,7 @@ template class DecimalPaddedPODArray<Decimal128>;
|
||||
template class DecimalPaddedPODArray<Decimal256>;
|
||||
template class DecimalPaddedPODArray<DateTime64>;
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
|
||||
{
|
||||
auto & other = static_cast<const Self &>(rhs_);
|
||||
@ -50,7 +50,7 @@ int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) c
|
||||
return decimalLess<T>(b, a, other.scale, scale) ? 1 : (decimalLess<T>(a, b, scale, other.scale) ? -1 : 0);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::compareColumn(const IColumn & rhs, size_t rhs_row_num,
|
||||
PaddedPODArray<UInt64> * row_indexes, PaddedPODArray<Int8> & compare_results,
|
||||
int direction, int nan_direction_hint) const
|
||||
@ -59,13 +59,13 @@ void ColumnDecimal<T>::compareColumn(const IColumn & rhs, size_t rhs_row_num,
|
||||
compare_results, direction, nan_direction_hint);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
bool ColumnDecimal<T>::hasEqualValues() const
|
||||
{
|
||||
return this->template hasEqualValuesImpl<ColumnDecimal<T>>();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
StringRef ColumnDecimal<T>::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const
|
||||
{
|
||||
auto * pos = arena.allocContinue(sizeof(T), begin);
|
||||
@ -73,20 +73,20 @@ StringRef ColumnDecimal<T>::serializeValueIntoArena(size_t n, Arena & arena, cha
|
||||
return StringRef(pos, sizeof(T));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
const char * ColumnDecimal<T>::deserializeAndInsertFromArena(const char * pos)
|
||||
{
|
||||
data.push_back(unalignedLoad<T>(pos));
|
||||
return pos + sizeof(T);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
const char * ColumnDecimal<T>::skipSerializedInArena(const char * pos) const
|
||||
{
|
||||
return pos + sizeof(T);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
UInt64 ColumnDecimal<T>::get64([[maybe_unused]] size_t n) const
|
||||
{
|
||||
if constexpr (sizeof(T) > sizeof(UInt64))
|
||||
@ -95,13 +95,13 @@ UInt64 ColumnDecimal<T>::get64([[maybe_unused]] size_t n) const
|
||||
return static_cast<NativeT>(data[n]);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::updateHashWithValue(size_t n, SipHash & hash) const
|
||||
{
|
||||
hash.update(data[n].value);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash) const
|
||||
{
|
||||
auto s = data.size();
|
||||
@ -122,13 +122,13 @@ void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash) const
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::updateHashFast(SipHash & hash) const
|
||||
{
|
||||
hash.update(reinterpret_cast<const char *>(data.data()), size() * sizeof(data[0]));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::getPermutation(bool reverse, size_t limit, int , IColumn::Permutation & res) const
|
||||
{
|
||||
#if 1 /// TODO: perf test
|
||||
@ -147,7 +147,7 @@ void ColumnDecimal<T>::getPermutation(bool reverse, size_t limit, int , IColumn:
|
||||
permutation(reverse, limit, res);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_ranges) const
|
||||
{
|
||||
if (equal_ranges.empty())
|
||||
@ -228,7 +228,7 @@ void ColumnDecimal<T>::updatePermutation(bool reverse, size_t limit, int, IColum
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
ColumnPtr ColumnDecimal<T>::permute(const IColumn::Permutation & perm, size_t limit) const
|
||||
{
|
||||
size_t size = limit ? std::min(data.size(), limit) : data.size();
|
||||
@ -244,7 +244,7 @@ ColumnPtr ColumnDecimal<T>::permute(const IColumn::Permutation & perm, size_t li
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
MutableColumnPtr ColumnDecimal<T>::cloneResized(size_t size) const
|
||||
{
|
||||
auto res = this->create(0, scale);
|
||||
@ -268,7 +268,7 @@ MutableColumnPtr ColumnDecimal<T>::cloneResized(size_t size) const
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
|
||||
{
|
||||
T tmp;
|
||||
@ -276,7 +276,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
|
||||
data.emplace_back(tmp);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
|
||||
{
|
||||
const ColumnDecimal & src_vec = assert_cast<const ColumnDecimal &>(src);
|
||||
@ -292,7 +292,7 @@ void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
|
||||
memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
@ -321,19 +321,19 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::expand(const IColumn::Filter & mask, bool inverted)
|
||||
{
|
||||
expandDataByMask<T>(data, mask, inverted);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
ColumnPtr ColumnDecimal<T>::index(const IColumn & indexes, size_t limit) const
|
||||
{
|
||||
return selectIndexImpl(*this, indexes, limit);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
ColumnPtr ColumnDecimal<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
{
|
||||
size_t size = data.size();
|
||||
@ -360,13 +360,13 @@ ColumnPtr ColumnDecimal<T>::replicate(const IColumn::Offsets & offsets) const
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::gather(ColumnGathererStream & gatherer)
|
||||
{
|
||||
gatherer.gather(*this);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
ColumnPtr ColumnDecimal<T>::compress() const
|
||||
{
|
||||
size_t source_size = data.size() * sizeof(T);
|
||||
@ -390,7 +390,7 @@ ColumnPtr ColumnDecimal<T>::compress() const
|
||||
});
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
|
||||
{
|
||||
if (data.empty())
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
#include <Core/DecimalFunctions.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/sort.h>
|
||||
#include <Core/TypeId.h>
|
||||
#include <Core/TypeName.h>
|
||||
|
||||
#include <cmath>
|
||||
|
||||
@ -59,11 +61,9 @@ extern template class DecimalPaddedPODArray<Decimal256>;
|
||||
extern template class DecimalPaddedPODArray<DateTime64>;
|
||||
|
||||
/// A ColumnVector for Decimals
|
||||
template <typename T>
|
||||
template <is_decimal T>
|
||||
class ColumnDecimal final : public COWHelper<ColumnVectorHelper, ColumnDecimal<T>>
|
||||
{
|
||||
static_assert(IsDecimalNumber<T>);
|
||||
|
||||
private:
|
||||
using Self = ColumnDecimal;
|
||||
friend class COWHelper<ColumnVectorHelper, Self>;
|
||||
@ -210,7 +210,12 @@ protected:
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
template <class> class ColumnVector;
|
||||
template <class T> struct ColumnVectorOrDecimalT { using Col = ColumnVector<T>; };
|
||||
template <is_decimal T> struct ColumnVectorOrDecimalT<T> { using Col = ColumnDecimal<T>; };
|
||||
template <class T> using ColumnVectorOrDecimal = typename ColumnVectorOrDecimalT<T>::Col;
|
||||
|
||||
template <is_decimal T>
|
||||
template <typename Type>
|
||||
ColumnPtr ColumnDecimal<T>::indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const
|
||||
{
|
||||
|
||||
@ -276,7 +276,10 @@ bool ColumnMap::structureEquals(const IColumn & rhs) const
|
||||
ColumnPtr ColumnMap::compress() const
|
||||
{
|
||||
auto compressed = nested->compress();
|
||||
return ColumnCompressed::create(size(), compressed->byteSize(), [compressed = std::move(compressed)]
|
||||
const auto byte_size = compressed->byteSize();
|
||||
/// The order of evaluation of function arguments is unspecified
|
||||
/// and could cause interacting with object in moved-from state
|
||||
return ColumnCompressed::create(size(), byte_size, [compressed = std::move(compressed)]
|
||||
{
|
||||
return ColumnMap::create(compressed->decompress());
|
||||
});
|
||||
|
||||
@ -7,6 +7,8 @@
|
||||
#include <common/unaligned.h>
|
||||
#include <Core/Field.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Core/TypeId.h>
|
||||
#include <Core/TypeName.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -102,7 +104,7 @@ template <class U> struct CompareHelper<Float64, U> : public FloatCompareHelper<
|
||||
template <typename T>
|
||||
class ColumnVector final : public COWHelper<ColumnVectorHelper, ColumnVector<T>>
|
||||
{
|
||||
static_assert(!IsDecimalNumber<T>);
|
||||
static_assert(!is_decimal<T>);
|
||||
|
||||
private:
|
||||
using Self = ColumnVector;
|
||||
|
||||
@ -249,8 +249,8 @@ MaskInfo extractInvertedMask(
|
||||
|
||||
void inverseMask(PaddedPODArray<UInt8> & mask, MaskInfo & mask_info)
|
||||
{
|
||||
for (size_t i = 0; i != mask.size(); ++i)
|
||||
mask[i] = !mask[i];
|
||||
for (auto & byte : mask)
|
||||
byte = !byte;
|
||||
std::swap(mask_info.has_ones, mask_info.has_zeros);
|
||||
}
|
||||
|
||||
|
||||
@ -494,7 +494,6 @@
|
||||
M(523, UNKNOWN_ROW_POLICY) \
|
||||
M(524, ALTER_OF_COLUMN_IS_FORBIDDEN) \
|
||||
M(525, INCORRECT_DISK_INDEX) \
|
||||
M(526, UNKNOWN_VOLUME_TYPE) \
|
||||
M(527, NO_SUITABLE_FUNCTION_IMPLEMENTATION) \
|
||||
M(528, CASSANDRA_INTERNAL_ERROR) \
|
||||
M(529, NOT_A_LEADER) \
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
#include <Common/ExternalLoaderStatus.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
String toString(ExternalLoaderStatus status)
|
||||
{
|
||||
using Status = ExternalLoaderStatus;
|
||||
switch (status)
|
||||
{
|
||||
case Status::NOT_LOADED: return "NOT_LOADED";
|
||||
case Status::LOADED: return "LOADED";
|
||||
case Status::FAILED: return "FAILED";
|
||||
case Status::LOADING: return "LOADING";
|
||||
case Status::FAILED_AND_RELOADING: return "FAILED_AND_RELOADING";
|
||||
case Status::LOADED_AND_RELOADING: return "LOADED_AND_RELOADING";
|
||||
case Status::NOT_EXIST: return "NOT_EXIST";
|
||||
}
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
||||
std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues()
|
||||
{
|
||||
using Status = ExternalLoaderStatus;
|
||||
return std::vector<std::pair<String, Int8>>{
|
||||
{toString(Status::NOT_LOADED), static_cast<Int8>(Status::NOT_LOADED)},
|
||||
{toString(Status::LOADED), static_cast<Int8>(Status::LOADED)},
|
||||
{toString(Status::FAILED), static_cast<Int8>(Status::FAILED)},
|
||||
{toString(Status::LOADING), static_cast<Int8>(Status::LOADING)},
|
||||
{toString(Status::LOADED_AND_RELOADING), static_cast<Int8>(Status::LOADED_AND_RELOADING)},
|
||||
{toString(Status::FAILED_AND_RELOADING), static_cast<Int8>(Status::FAILED_AND_RELOADING)},
|
||||
{toString(Status::NOT_EXIST), static_cast<Int8>(Status::NOT_EXIST)},
|
||||
};
|
||||
}
|
||||
|
||||
std::ostream & operator<<(std::ostream & out, ExternalLoaderStatus status)
|
||||
{
|
||||
return out << toString(status);
|
||||
}
|
||||
}
|
||||
@ -1,13 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <utility>
|
||||
#include <ostream>
|
||||
#include <common/EnumReflection.h>
|
||||
#include <common/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
enum class ExternalLoaderStatus
|
||||
enum class ExternalLoaderStatus : Int8
|
||||
{
|
||||
NOT_LOADED, /// Object hasn't been tried to load. This is an initial state.
|
||||
LOADED, /// Object has been loaded successfully.
|
||||
@ -18,7 +17,14 @@ namespace DB
|
||||
NOT_EXIST, /// Object with this name wasn't found in the configuration.
|
||||
};
|
||||
|
||||
String toString(ExternalLoaderStatus status);
|
||||
std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues();
|
||||
std::ostream & operator<<(std::ostream & out, ExternalLoaderStatus status);
|
||||
inline std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues()
|
||||
{
|
||||
std::vector<std::pair<String, Int8>> out;
|
||||
out.reserve(magic_enum::enum_count<ExternalLoaderStatus>());
|
||||
|
||||
for (const auto & [value, str] : magic_enum::enum_entries<ExternalLoaderStatus>())
|
||||
out.emplace_back(std::string{str}, static_cast<Int8>(value));
|
||||
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user