mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
Merge branch 'ClickHouse:master' into opt_memcpy_small
This commit is contained in:
李扬
GitHub
commit
be68b21de4
@ -5,9 +5,7 @@
|
||||
# a) the new check is not controversial (this includes many checks in readability-* and google-*) or
|
||||
# b) too noisy (checks with > 100 new warnings are considered noisy, this includes e.g. cppcoreguidelines-*).
|
||||
|
||||
# TODO Let clang-tidy check headers in further directories
|
||||
# --> HeaderFilterRegex: '^.*/(src|base|programs|utils)/.*(h|hpp)$'
|
||||
HeaderFilterRegex: '^.*/(base|programs|utils)/.*(h|hpp)$'
|
||||
HeaderFilterRegex: '^.*/(base|src|programs|utils)/.*(h|hpp)$'
|
||||
|
||||
Checks: [
|
||||
'*',
|
||||
@ -22,6 +20,7 @@ Checks: [
|
||||
'-bugprone-branch-clone',
|
||||
'-bugprone-easily-swappable-parameters',
|
||||
'-bugprone-exception-escape',
|
||||
'-bugprone-forward-declaration-namespace',
|
||||
'-bugprone-implicit-widening-of-multiplication-result',
|
||||
'-bugprone-narrowing-conversions',
|
||||
'-bugprone-not-null-terminated-result',
|
||||
@ -37,6 +36,8 @@ Checks: [
|
||||
'-cert-oop54-cpp',
|
||||
'-cert-oop57-cpp',
|
||||
|
||||
'-clang-analyzer-optin.performance.Padding',
|
||||
|
||||
'-clang-analyzer-unix.Malloc',
|
||||
|
||||
'-cppcoreguidelines-*', # impractical in a codebase as large as ClickHouse, also slow
|
||||
|
||||
@ -61,11 +61,16 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
|
||||
# set CPU time limit to 1000 seconds
|
||||
set (RLIMIT_CPU 1000)
|
||||
|
||||
# -fsanitize=memory and address are too heavy
|
||||
# Sanitizers are too heavy
|
||||
if (SANITIZE OR SANITIZE_COVERAGE OR WITH_COVERAGE)
|
||||
set (RLIMIT_DATA 10000000000) # 10G
|
||||
endif()
|
||||
|
||||
# For some files currently building RISCV64 might be too slow. TODO: Improve compilation times per file
|
||||
if (ARCH_RISCV64)
|
||||
set (RLIMIT_CPU 1800)
|
||||
endif()
|
||||
|
||||
set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
|
||||
endif ()
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@ set (SRCS
|
||||
getPageSize.cpp
|
||||
getThreadId.cpp
|
||||
int8_to_string.cpp
|
||||
itoa.cpp
|
||||
JSON.cpp
|
||||
mremap.cpp
|
||||
phdr_cache.cpp
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/strong_typedef.h>
|
||||
#include <base/extended_types.h>
|
||||
#include <Common/formatIPv6.h>
|
||||
#include <base/strong_typedef.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
|
||||
namespace DB
|
||||
@ -62,7 +61,8 @@ namespace std
|
||||
{
|
||||
size_t operator()(const DB::IPv6 & x) const
|
||||
{
|
||||
return std::hash<std::string_view>{}(std::string_view(reinterpret_cast<const char*>(&x.toUnderType()), IPV6_BINARY_LENGTH));
|
||||
return std::hash<std::string_view>{}(
|
||||
std::string_view(reinterpret_cast<const char *>(&x.toUnderType()), sizeof(DB::IPv6::UnderlyingType)));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -50,9 +50,6 @@ std::optional<uint64_t> getCgroupsV2MemoryLimit()
|
||||
|
||||
}
|
||||
|
||||
/** Returns the size of physical memory (RAM) in bytes.
|
||||
* Returns 0 on unsupported platform
|
||||
*/
|
||||
uint64_t getMemoryAmountOrZero()
|
||||
{
|
||||
int64_t num_pages = sysconf(_SC_PHYS_PAGES);
|
||||
|
||||
@ -2,11 +2,10 @@
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
/** Returns the size of physical memory (RAM) in bytes.
|
||||
* Returns 0 on unsupported platform or if it cannot determine the size of physical memory.
|
||||
*/
|
||||
/// Returns the size in bytes of physical memory (RAM) available to the process. The value can
|
||||
/// be smaller than the total available RAM available to the system due to cgroups settings.
|
||||
/// Returns 0 on unsupported platform or if it cannot determine the size of physical memory.
|
||||
uint64_t getMemoryAmountOrZero();
|
||||
|
||||
/** Throws exception if it cannot determine the size of physical memory.
|
||||
*/
|
||||
/// Throws exception if it cannot determine the size of physical memory.
|
||||
uint64_t getMemoryAmount();
|
||||
|
||||
503
base/base/itoa.cpp
Normal file
503
base/base/itoa.cpp
Normal file
@ -0,0 +1,503 @@
|
||||
// Based on https://github.com/amdn/itoa and combined with our optimizations
|
||||
//
|
||||
//=== itoa.cpp - Fast integer to ascii conversion --*- C++ -*-//
|
||||
//
|
||||
// The MIT License (MIT)
|
||||
// Copyright (c) 2016 Arturo Martin-de-Nicolas
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included
|
||||
// in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
#include <base/defines.h>
|
||||
#include <base/extended_types.h>
|
||||
#include <base/itoa.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
template <typename T>
|
||||
ALWAYS_INLINE inline constexpr T pow10(size_t x)
|
||||
{
|
||||
return x ? 10 * pow10<T>(x - 1) : 1;
|
||||
}
|
||||
|
||||
// Division by a power of 10 is implemented using a multiplicative inverse.
|
||||
// This strength reduction is also done by optimizing compilers, but
|
||||
// presently the fastest results are produced by using the values
|
||||
// for the multiplication and the shift as given by the algorithm
|
||||
// described by Agner Fog in "Optimizing Subroutines in Assembly Language"
|
||||
//
|
||||
// http://www.agner.org/optimize/optimizing_assembly.pdf
|
||||
//
|
||||
// "Integer division by a constant (all processors)
|
||||
// A floating point number can be divided by a constant by multiplying
|
||||
// with the reciprocal. If we want to do the same with integers, we have
|
||||
// to scale the reciprocal by 2n and then shift the product to the right
|
||||
// by n. There are various algorithms for finding a suitable value of n
|
||||
// and compensating for rounding errors. The algorithm described below
|
||||
// was invented by Terje Mathisen, Norway, and not published elsewhere."
|
||||
|
||||
/// Division by constant is performed by:
|
||||
/// 1. Adding 1 if needed;
|
||||
/// 2. Multiplying by another constant;
|
||||
/// 3. Shifting right by another constant.
|
||||
template <typename UInt, bool add_, UInt multiplier_, unsigned shift_>
|
||||
struct Division
|
||||
{
|
||||
static constexpr bool add{add_};
|
||||
static constexpr UInt multiplier{multiplier_};
|
||||
static constexpr unsigned shift{shift_};
|
||||
};
|
||||
|
||||
/// Select a type with appropriate number of bytes from the list of types.
|
||||
/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes.
|
||||
/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t.
|
||||
template <size_t N, typename T, typename... Ts>
|
||||
struct SelectType
|
||||
{
|
||||
using Result = typename SelectType<N / 2, Ts...>::Result;
|
||||
};
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
struct SelectType<1, T, Ts...>
|
||||
{
|
||||
using Result = T;
|
||||
};
|
||||
|
||||
|
||||
/// Division by 10^N where N is the size of the type.
|
||||
template <size_t N>
|
||||
using DivisionBy10PowN = typename SelectType<
|
||||
N,
|
||||
Division<uint8_t, false, 205U, 11>, /// divide by 10
|
||||
Division<uint16_t, true, 41943U, 22>, /// divide by 100
|
||||
Division<uint32_t, false, 3518437209U, 45>, /// divide by 10000
|
||||
Division<uint64_t, false, 12379400392853802749ULL, 90> /// divide by 100000000
|
||||
>::Result;
|
||||
|
||||
template <size_t N>
|
||||
using UnsignedOfSize = typename SelectType<N, uint8_t, uint16_t, uint32_t, uint64_t, __uint128_t>::Result;
|
||||
|
||||
/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
|
||||
template <size_t N>
|
||||
struct QuotientAndRemainder
|
||||
{
|
||||
UnsignedOfSize<N> quotient; // quotient with fewer than 2*N decimal digits
|
||||
UnsignedOfSize<N / 2> remainder; // remainder with at most N decimal digits
|
||||
};
|
||||
|
||||
template <size_t N>
|
||||
QuotientAndRemainder<N> inline split(UnsignedOfSize<N> value)
|
||||
{
|
||||
constexpr DivisionBy10PowN<N> division;
|
||||
|
||||
UnsignedOfSize<N> quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift;
|
||||
UnsignedOfSize<N / 2> remainder = static_cast<UnsignedOfSize<N / 2>>(value - quotient * pow10<UnsignedOfSize<N / 2>>(N));
|
||||
|
||||
return {quotient, remainder};
|
||||
}
|
||||
|
||||
ALWAYS_INLINE inline char * outDigit(char * p, uint8_t value)
|
||||
{
|
||||
*p = '0' + value;
|
||||
++p;
|
||||
return p;
|
||||
}
|
||||
|
||||
// Using a lookup table to convert binary numbers from 0 to 99
|
||||
// into ascii characters as described by Andrei Alexandrescu in
|
||||
// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/
|
||||
|
||||
const char digits[201] = "00010203040506070809"
|
||||
"10111213141516171819"
|
||||
"20212223242526272829"
|
||||
"30313233343536373839"
|
||||
"40414243444546474849"
|
||||
"50515253545556575859"
|
||||
"60616263646566676869"
|
||||
"70717273747576777879"
|
||||
"80818283848586878889"
|
||||
"90919293949596979899";
|
||||
|
||||
ALWAYS_INLINE inline char * outTwoDigits(char * p, uint8_t value)
|
||||
{
|
||||
memcpy(p, &digits[value * 2], 2);
|
||||
p += 2;
|
||||
return p;
|
||||
}
|
||||
|
||||
namespace convert
|
||||
{
|
||||
template <typename UInt, size_t N = sizeof(UInt)>
|
||||
char * head(char * p, UInt u);
|
||||
template <typename UInt, size_t N = sizeof(UInt)>
|
||||
char * tail(char * p, UInt u);
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// head: find most significant digit, skip leading zeros
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// "x" contains quotient and remainder after division by 10^N
|
||||
// quotient is less than 10^N
|
||||
template <size_t N>
|
||||
ALWAYS_INLINE inline char * head(char * p, QuotientAndRemainder<N> x)
|
||||
{
|
||||
p = head(p, UnsignedOfSize<N / 2>(x.quotient));
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
// "u" is less than 10^2*N
|
||||
template <typename UInt, size_t N>
|
||||
ALWAYS_INLINE inline char * head(char * p, UInt u)
|
||||
{
|
||||
return u < pow10<UnsignedOfSize<N>>(N) ? head(p, UnsignedOfSize<N / 2>(u)) : head<N>(p, split<N>(u));
|
||||
}
|
||||
|
||||
// recursion base case, selected when "u" is one byte
|
||||
template <>
|
||||
ALWAYS_INLINE inline char * head<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
return u < 10 ? outDigit(p, u) : outTwoDigits(p, u);
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// tail: produce all digits including leading zeros
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// recursive step, "u" is less than 10^2*N
|
||||
template <typename UInt, size_t N>
|
||||
ALWAYS_INLINE inline char * tail(char * p, UInt u)
|
||||
{
|
||||
QuotientAndRemainder<N> x = split<N>(u);
|
||||
p = tail(p, UnsignedOfSize<N / 2>(x.quotient));
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
// recursion base case, selected when "u" is one byte
|
||||
template <>
|
||||
ALWAYS_INLINE inline char * tail<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
return outTwoDigits(p, u);
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// large values are >= 10^2*N
|
||||
// where x contains quotient and remainder after division by 10^N
|
||||
//===----------------------------------------------------------===//
|
||||
template <size_t N>
|
||||
ALWAYS_INLINE inline char * large(char * p, QuotientAndRemainder<N> x)
|
||||
{
|
||||
QuotientAndRemainder<N> y = split<N>(x.quotient);
|
||||
p = head(p, UnsignedOfSize<N / 2>(y.quotient));
|
||||
p = tail(p, y.remainder);
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// handle values of "u" that might be >= 10^2*N
|
||||
// where N is the size of "u" in bytes
|
||||
//===----------------------------------------------------------===//
|
||||
template <typename UInt, size_t N = sizeof(UInt)>
|
||||
ALWAYS_INLINE inline char * uitoa(char * p, UInt u)
|
||||
{
|
||||
if (u < pow10<UnsignedOfSize<N>>(N))
|
||||
return head(p, UnsignedOfSize<N / 2>(u));
|
||||
QuotientAndRemainder<N> x = split<N>(u);
|
||||
|
||||
return u < pow10<UnsignedOfSize<N>>(2 * N) ? head<N>(p, x) : large<N>(p, x);
|
||||
}
|
||||
|
||||
// selected when "u" is one byte
|
||||
template <>
|
||||
ALWAYS_INLINE inline char * uitoa<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
if (u < 10)
|
||||
return outDigit(p, u);
|
||||
else if (u < 100)
|
||||
return outTwoDigits(p, u);
|
||||
else
|
||||
{
|
||||
p = outDigit(p, u / 100);
|
||||
p = outTwoDigits(p, u % 100);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// handle unsigned and signed integral operands
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// itoa: handle unsigned integral operands (selected by SFINAE)
|
||||
template <typename U, std::enable_if_t<!std::is_signed_v<U> && std::is_integral_v<U>> * = nullptr>
|
||||
ALWAYS_INLINE inline char * itoa(U u, char * p)
|
||||
{
|
||||
return convert::uitoa(p, u);
|
||||
}
|
||||
|
||||
// itoa: handle signed integral operands (selected by SFINAE)
|
||||
template <typename I, size_t N = sizeof(I), std::enable_if_t<std::is_signed_v<I> && std::is_integral_v<I>> * = nullptr>
|
||||
ALWAYS_INLINE inline char * itoa(I i, char * p)
|
||||
{
|
||||
// Need "mask" to be filled with a copy of the sign bit.
|
||||
// If "i" is a negative value, then the result of "operator >>"
|
||||
// is implementation-defined, though usually it is an arithmetic
|
||||
// right shift that replicates the sign bit.
|
||||
// Use a conditional expression to be portable,
|
||||
// a good optimizing compiler generates an arithmetic right shift
|
||||
// and avoids the conditional branch.
|
||||
UnsignedOfSize<N> mask = i < 0 ? ~UnsignedOfSize<N>(0) : 0;
|
||||
// Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize<N>.
|
||||
// Cannot use std::abs() because the result is undefined
|
||||
// in 2's complement systems for the most-negative value.
|
||||
// Want to avoid conditional branch for performance reasons since
|
||||
// CPU branch prediction will be ineffective when negative values
|
||||
// occur randomly.
|
||||
// Let "u" be "i" cast to unsigned type UnsignedOfSize<N>.
|
||||
// Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative.
|
||||
// This yields the absolute value with the desired type without
|
||||
// using a conditional branch and without invoking undefined or
|
||||
// implementation defined behavior:
|
||||
UnsignedOfSize<N> u = ((2 * UnsignedOfSize<N>(i)) & ~mask) - UnsignedOfSize<N>(i);
|
||||
// Unconditionally store a minus sign when producing digits
|
||||
// in a forward direction and increment the pointer only if
|
||||
// the value is in fact negative.
|
||||
// This avoids a conditional branch and is safe because we will
|
||||
// always produce at least one digit and it will overwrite the
|
||||
// minus sign when the value is not negative.
|
||||
*p = '-';
|
||||
p += (mask & 1);
|
||||
p = convert::uitoa(p, u);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
const uint64_t max_multiple_of_hundred_that_fits_in_64_bits = 1'00'00'00'00'00'00'00'00'00ull;
|
||||
const int max_multiple_of_hundred_blocks = 9;
|
||||
static_assert(max_multiple_of_hundred_that_fits_in_64_bits % 100 == 0);
|
||||
|
||||
ALWAYS_INLINE inline char * writeUIntText(UInt128 _x, char * p)
|
||||
{
|
||||
/// If we the highest 64bit item is empty, we can print just the lowest item as u64
|
||||
if (_x.items[UInt128::_impl::little(1)] == 0)
|
||||
return convert::itoa(_x.items[UInt128::_impl::little(0)], p);
|
||||
|
||||
/// Doing operations using __int128 is faster and we already rely on this feature
|
||||
using T = unsigned __int128;
|
||||
T x = (T(_x.items[UInt128::_impl::little(1)]) << 64) + T(_x.items[UInt128::_impl::little(0)]);
|
||||
|
||||
/// We are going to accumulate blocks of 2 digits to print until the number is small enough to be printed as u64
|
||||
/// To do this we could do: x / 100, x % 100
|
||||
/// But these would mean doing many iterations with long integers, so instead we divide by a much longer integer
|
||||
/// multiple of 100 (100^9) and then get the blocks out of it (as u64)
|
||||
/// Once we reach u64::max we can stop and use the fast method to print that in the front
|
||||
static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits;
|
||||
static const T largest_uint64 = std::numeric_limits<uint64_t>::max();
|
||||
uint8_t two_values[20] = {0}; // 39 Max characters / 2
|
||||
|
||||
int current_block = 0;
|
||||
while (x > largest_uint64)
|
||||
{
|
||||
uint64_t u64_remainder = uint64_t(x % large_divisor);
|
||||
x /= large_divisor;
|
||||
|
||||
int pos = current_block;
|
||||
while (u64_remainder)
|
||||
{
|
||||
two_values[pos] = uint8_t(u64_remainder % 100);
|
||||
pos++;
|
||||
u64_remainder /= 100;
|
||||
}
|
||||
current_block += max_multiple_of_hundred_blocks;
|
||||
}
|
||||
|
||||
char * highest_part_print = convert::itoa(uint64_t(x), p);
|
||||
for (int i = 0; i < current_block; i++)
|
||||
{
|
||||
outTwoDigits(highest_part_print, two_values[current_block - 1 - i]);
|
||||
highest_part_print += 2;
|
||||
}
|
||||
|
||||
return highest_part_print;
|
||||
}
|
||||
|
||||
ALWAYS_INLINE inline char * writeUIntText(UInt256 _x, char * p)
|
||||
{
|
||||
/// If possible, treat it as a smaller integer as they are much faster to print
|
||||
if (_x.items[UInt256::_impl::little(3)] == 0 && _x.items[UInt256::_impl::little(2)] == 0)
|
||||
return writeUIntText(UInt128{_x.items[UInt256::_impl::little(0)], _x.items[UInt256::_impl::little(1)]}, p);
|
||||
|
||||
/// If available (x86) we transform from our custom class to _BitInt(256) which has better support in the compiler
|
||||
/// and produces better code
|
||||
using T =
|
||||
#if defined(__x86_64__)
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wbit-int-extension"
|
||||
unsigned _BitInt(256)
|
||||
# pragma clang diagnostic pop
|
||||
#else
|
||||
UInt256
|
||||
#endif
|
||||
;
|
||||
|
||||
#if defined(__x86_64__)
|
||||
T x = (T(_x.items[UInt256::_impl::little(3)]) << 192) + (T(_x.items[UInt256::_impl::little(2)]) << 128)
|
||||
+ (T(_x.items[UInt256::_impl::little(1)]) << 64) + T(_x.items[UInt256::_impl::little(0)]);
|
||||
#else
|
||||
T x = _x;
|
||||
#endif
|
||||
|
||||
/// Similar to writeUIntText(UInt128) only that in this case we will stop as soon as we reach the largest u128
|
||||
/// and switch to that function
|
||||
uint8_t two_values[39] = {0}; // 78 Max characters / 2
|
||||
int current_pos = 0;
|
||||
|
||||
static const T large_divisor = max_multiple_of_hundred_that_fits_in_64_bits;
|
||||
static const T largest_uint128 = T(std::numeric_limits<uint64_t>::max()) << 64 | T(std::numeric_limits<uint64_t>::max());
|
||||
|
||||
while (x > largest_uint128)
|
||||
{
|
||||
uint64_t u64_remainder = uint64_t(x % large_divisor);
|
||||
x /= large_divisor;
|
||||
|
||||
int pos = current_pos;
|
||||
while (u64_remainder)
|
||||
{
|
||||
two_values[pos] = uint8_t(u64_remainder % 100);
|
||||
pos++;
|
||||
u64_remainder /= 100;
|
||||
}
|
||||
current_pos += max_multiple_of_hundred_blocks;
|
||||
}
|
||||
|
||||
#if defined(__x86_64__)
|
||||
UInt128 pending{uint64_t(x), uint64_t(x >> 64)};
|
||||
#else
|
||||
UInt128 pending{x.items[UInt256::_impl::little(0)], x.items[UInt256::_impl::little(1)]};
|
||||
#endif
|
||||
|
||||
char * highest_part_print = writeUIntText(pending, p);
|
||||
for (int i = 0; i < current_pos; i++)
|
||||
{
|
||||
outTwoDigits(highest_part_print, two_values[current_pos - 1 - i]);
|
||||
highest_part_print += 2;
|
||||
}
|
||||
|
||||
return highest_part_print;
|
||||
}
|
||||
|
||||
ALWAYS_INLINE inline char * writeLeadingMinus(char * pos)
|
||||
{
|
||||
*pos = '-';
|
||||
return pos + 1;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
ALWAYS_INLINE inline char * writeSIntText(T x, char * pos)
|
||||
{
|
||||
static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
|
||||
|
||||
using UnsignedT = make_unsigned_t<T>;
|
||||
static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
|
||||
|
||||
if (unlikely(x == min_int))
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Int128>)
|
||||
{
|
||||
const char * res = "-170141183460469231731687303715884105728";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
else if constexpr (std::is_same_v<T, Int256>)
|
||||
{
|
||||
const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
}
|
||||
|
||||
if (x < 0)
|
||||
{
|
||||
x = -x;
|
||||
pos = writeLeadingMinus(pos);
|
||||
}
|
||||
return writeUIntText(UnsignedT(x), pos);
|
||||
}
|
||||
}
|
||||
|
||||
char * itoa(UInt8 i, char * p)
|
||||
{
|
||||
return convert::itoa(uint8_t(i), p);
|
||||
}
|
||||
|
||||
char * itoa(Int8 i, char * p)
|
||||
{
|
||||
return convert::itoa(int8_t(i), p);
|
||||
}
|
||||
|
||||
char * itoa(UInt128 i, char * p)
|
||||
{
|
||||
return writeUIntText(i, p);
|
||||
}
|
||||
|
||||
char * itoa(Int128 i, char * p)
|
||||
{
|
||||
return writeSIntText(i, p);
|
||||
}
|
||||
|
||||
char * itoa(UInt256 i, char * p)
|
||||
{
|
||||
return writeUIntText(i, p);
|
||||
}
|
||||
|
||||
char * itoa(Int256 i, char * p)
|
||||
{
|
||||
return writeSIntText(i, p);
|
||||
}
|
||||
|
||||
#define DEFAULT_ITOA(T) \
|
||||
char * itoa(T i, char * p) \
|
||||
{ \
|
||||
return convert::itoa(i, p); \
|
||||
}
|
||||
|
||||
#define FOR_MISSING_INTEGER_TYPES(M) \
|
||||
M(uint8_t) \
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(int8_t) \
|
||||
M(Int16) \
|
||||
M(Int32) \
|
||||
M(Int64)
|
||||
|
||||
FOR_MISSING_INTEGER_TYPES(DEFAULT_ITOA)
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
DEFAULT_ITOA(unsigned long)
|
||||
DEFAULT_ITOA(long)
|
||||
#endif
|
||||
|
||||
#undef FOR_MISSING_INTEGER_TYPES
|
||||
#undef DEFAULT_ITOA
|
||||
462
base/base/itoa.h
462
base/base/itoa.h
@ -1,446 +1,30 @@
|
||||
#pragma once
|
||||
|
||||
// Based on https://github.com/amdn/itoa and combined with our optimizations
|
||||
//
|
||||
//=== itoa.h - Fast integer to ascii conversion --*- C++ -*-//
|
||||
//
|
||||
// The MIT License (MIT)
|
||||
// Copyright (c) 2016 Arturo Martin-de-Nicolas
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included
|
||||
// in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
#include <base/extended_types.h>
|
||||
|
||||
#define FOR_INTEGER_TYPES(M) \
|
||||
M(uint8_t) \
|
||||
M(UInt8) \
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(UInt128) \
|
||||
M(UInt256) \
|
||||
M(int8_t) \
|
||||
M(Int8) \
|
||||
M(Int16) \
|
||||
M(Int32) \
|
||||
M(Int64) \
|
||||
M(Int128) \
|
||||
M(Int256)
|
||||
|
||||
template <typename T>
|
||||
inline int digits10(T x)
|
||||
{
|
||||
if (x < 10ULL)
|
||||
return 1;
|
||||
if (x < 100ULL)
|
||||
return 2;
|
||||
if (x < 1000ULL)
|
||||
return 3;
|
||||
#define INSTANTIATION(T) char * itoa(T i, char * p);
|
||||
FOR_INTEGER_TYPES(INSTANTIATION)
|
||||
|
||||
if (x < 1000000000000ULL)
|
||||
{
|
||||
if (x < 100000000ULL)
|
||||
{
|
||||
if (x < 1000000ULL)
|
||||
{
|
||||
if (x < 10000ULL)
|
||||
return 4;
|
||||
else
|
||||
return 5 + (x >= 100000ULL);
|
||||
}
|
||||
#if defined(OS_DARWIN)
|
||||
INSTANTIATION(unsigned long)
|
||||
INSTANTIATION(long)
|
||||
#endif
|
||||
|
||||
return 7 + (x >= 10000000ULL);
|
||||
}
|
||||
|
||||
if (x < 10000000000ULL)
|
||||
return 9 + (x >= 1000000000ULL);
|
||||
|
||||
return 11 + (x >= 100000000000ULL);
|
||||
}
|
||||
|
||||
return 12 + digits10(x / 1000000000000ULL);
|
||||
}
|
||||
|
||||
|
||||
namespace impl
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
static constexpr T pow10(size_t x)
|
||||
{
|
||||
return x ? 10 * pow10<T>(x - 1) : 1;
|
||||
}
|
||||
|
||||
// Division by a power of 10 is implemented using a multiplicative inverse.
|
||||
// This strength reduction is also done by optimizing compilers, but
|
||||
// presently the fastest results are produced by using the values
|
||||
// for the multiplication and the shift as given by the algorithm
|
||||
// described by Agner Fog in "Optimizing Subroutines in Assembly Language"
|
||||
//
|
||||
// http://www.agner.org/optimize/optimizing_assembly.pdf
|
||||
//
|
||||
// "Integer division by a constant (all processors)
|
||||
// A floating point number can be divided by a constant by multiplying
|
||||
// with the reciprocal. If we want to do the same with integers, we have
|
||||
// to scale the reciprocal by 2n and then shift the product to the right
|
||||
// by n. There are various algorithms for finding a suitable value of n
|
||||
// and compensating for rounding errors. The algorithm described below
|
||||
// was invented by Terje Mathisen, Norway, and not published elsewhere."
|
||||
|
||||
/// Division by constant is performed by:
|
||||
/// 1. Adding 1 if needed;
|
||||
/// 2. Multiplying by another constant;
|
||||
/// 3. Shifting right by another constant.
|
||||
template <typename UInt, bool add_, UInt multiplier_, unsigned shift_>
|
||||
struct Division
|
||||
{
|
||||
static constexpr bool add{add_};
|
||||
static constexpr UInt multiplier{multiplier_};
|
||||
static constexpr unsigned shift{shift_};
|
||||
};
|
||||
|
||||
/// Select a type with appropriate number of bytes from the list of types.
|
||||
/// First parameter is the number of bytes requested. Then goes a list of types with 1, 2, 4, ... number of bytes.
|
||||
/// Example: SelectType<4, uint8_t, uint16_t, uint32_t, uint64_t> will select uint32_t.
|
||||
template <size_t N, typename T, typename... Ts>
|
||||
struct SelectType
|
||||
{
|
||||
using Result = typename SelectType<N / 2, Ts...>::Result;
|
||||
};
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
struct SelectType<1, T, Ts...>
|
||||
{
|
||||
using Result = T;
|
||||
};
|
||||
|
||||
|
||||
/// Division by 10^N where N is the size of the type.
|
||||
template <size_t N>
|
||||
using DivisionBy10PowN = typename SelectType
|
||||
<
|
||||
N,
|
||||
Division<uint8_t, false, 205U, 11>, /// divide by 10
|
||||
Division<uint16_t, true, 41943U, 22>, /// divide by 100
|
||||
Division<uint32_t, false, 3518437209U, 45>, /// divide by 10000
|
||||
Division<uint64_t, false, 12379400392853802749ULL, 90> /// divide by 100000000
|
||||
>::Result;
|
||||
|
||||
template <size_t N>
|
||||
using UnsignedOfSize = typename SelectType
|
||||
<
|
||||
N,
|
||||
uint8_t,
|
||||
uint16_t,
|
||||
uint32_t,
|
||||
uint64_t,
|
||||
__uint128_t
|
||||
>::Result;
|
||||
|
||||
/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
|
||||
template <size_t N>
|
||||
struct QuotientAndRemainder
|
||||
{
|
||||
UnsignedOfSize<N> quotient; // quotient with fewer than 2*N decimal digits
|
||||
UnsignedOfSize<N / 2> remainder; // remainder with at most N decimal digits
|
||||
};
|
||||
|
||||
template <size_t N>
|
||||
QuotientAndRemainder<N> static inline split(UnsignedOfSize<N> value)
|
||||
{
|
||||
constexpr DivisionBy10PowN<N> division;
|
||||
|
||||
UnsignedOfSize<N> quotient = (division.multiplier * (UnsignedOfSize<2 * N>(value) + division.add)) >> division.shift;
|
||||
UnsignedOfSize<N / 2> remainder = static_cast<UnsignedOfSize<N / 2>>(value - quotient * pow10<UnsignedOfSize<N / 2>>(N));
|
||||
|
||||
return {quotient, remainder};
|
||||
}
|
||||
|
||||
|
||||
static inline char * outDigit(char * p, uint8_t value)
|
||||
{
|
||||
*p = '0' + value;
|
||||
++p;
|
||||
return p;
|
||||
}
|
||||
|
||||
// Using a lookup table to convert binary numbers from 0 to 99
|
||||
// into ascii characters as described by Andrei Alexandrescu in
|
||||
// https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920/
|
||||
|
||||
static const char digits[201] = "00010203040506070809"
|
||||
"10111213141516171819"
|
||||
"20212223242526272829"
|
||||
"30313233343536373839"
|
||||
"40414243444546474849"
|
||||
"50515253545556575859"
|
||||
"60616263646566676869"
|
||||
"70717273747576777879"
|
||||
"80818283848586878889"
|
||||
"90919293949596979899";
|
||||
|
||||
static inline char * outTwoDigits(char * p, uint8_t value)
|
||||
{
|
||||
memcpy(p, &digits[value * 2], 2);
|
||||
p += 2;
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
namespace convert
|
||||
{
|
||||
template <typename UInt, size_t N = sizeof(UInt)> static char * head(char * p, UInt u);
|
||||
template <typename UInt, size_t N = sizeof(UInt)> static char * tail(char * p, UInt u);
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// head: find most significant digit, skip leading zeros
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// "x" contains quotient and remainder after division by 10^N
|
||||
// quotient is less than 10^N
|
||||
template <size_t N>
|
||||
static inline char * head(char * p, QuotientAndRemainder<N> x)
|
||||
{
|
||||
p = head(p, UnsignedOfSize<N / 2>(x.quotient));
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
// "u" is less than 10^2*N
|
||||
template <typename UInt, size_t N>
|
||||
static inline char * head(char * p, UInt u)
|
||||
{
|
||||
return u < pow10<UnsignedOfSize<N>>(N)
|
||||
? head(p, UnsignedOfSize<N / 2>(u))
|
||||
: head<N>(p, split<N>(u));
|
||||
}
|
||||
|
||||
// recursion base case, selected when "u" is one byte
|
||||
template <>
|
||||
inline char * head<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
return u < 10
|
||||
? outDigit(p, u)
|
||||
: outTwoDigits(p, u);
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// tail: produce all digits including leading zeros
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// recursive step, "u" is less than 10^2*N
|
||||
template <typename UInt, size_t N>
|
||||
static inline char * tail(char * p, UInt u)
|
||||
{
|
||||
QuotientAndRemainder<N> x = split<N>(u);
|
||||
p = tail(p, UnsignedOfSize<N / 2>(x.quotient));
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
// recursion base case, selected when "u" is one byte
|
||||
template <>
|
||||
inline char * tail<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
return outTwoDigits(p, u);
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// large values are >= 10^2*N
|
||||
// where x contains quotient and remainder after division by 10^N
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
template <size_t N>
|
||||
static inline char * large(char * p, QuotientAndRemainder<N> x)
|
||||
{
|
||||
QuotientAndRemainder<N> y = split<N>(x.quotient);
|
||||
p = head(p, UnsignedOfSize<N / 2>(y.quotient));
|
||||
p = tail(p, y.remainder);
|
||||
p = tail(p, x.remainder);
|
||||
return p;
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// handle values of "u" that might be >= 10^2*N
|
||||
// where N is the size of "u" in bytes
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
template <typename UInt, size_t N = sizeof(UInt)>
|
||||
static inline char * uitoa(char * p, UInt u)
|
||||
{
|
||||
if (u < pow10<UnsignedOfSize<N>>(N))
|
||||
return head(p, UnsignedOfSize<N / 2>(u));
|
||||
QuotientAndRemainder<N> x = split<N>(u);
|
||||
|
||||
return u < pow10<UnsignedOfSize<N>>(2 * N)
|
||||
? head<N>(p, x)
|
||||
: large<N>(p, x);
|
||||
}
|
||||
|
||||
// selected when "u" is one byte
|
||||
template <>
|
||||
inline char * uitoa<UnsignedOfSize<1>, 1>(char * p, UnsignedOfSize<1> u)
|
||||
{
|
||||
if (u < 10)
|
||||
return outDigit(p, u);
|
||||
else if (u < 100)
|
||||
return outTwoDigits(p, u);
|
||||
else
|
||||
{
|
||||
p = outDigit(p, u / 100);
|
||||
p = outTwoDigits(p, u % 100);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
//===----------------------------------------------------------===//
|
||||
// handle unsigned and signed integral operands
|
||||
//===----------------------------------------------------------===//
|
||||
|
||||
// itoa: handle unsigned integral operands (selected by SFINAE)
|
||||
template <typename U, std::enable_if_t<!std::is_signed_v<U> && std::is_integral_v<U>> * = nullptr>
|
||||
static inline char * itoa(U u, char * p)
|
||||
{
|
||||
return convert::uitoa(p, u);
|
||||
}
|
||||
|
||||
// itoa: handle signed integral operands (selected by SFINAE)
|
||||
template <typename I, size_t N = sizeof(I), std::enable_if_t<std::is_signed_v<I> && std::is_integral_v<I>> * = nullptr>
|
||||
static inline char * itoa(I i, char * p)
|
||||
{
|
||||
// Need "mask" to be filled with a copy of the sign bit.
|
||||
// If "i" is a negative value, then the result of "operator >>"
|
||||
// is implementation-defined, though usually it is an arithmetic
|
||||
// right shift that replicates the sign bit.
|
||||
// Use a conditional expression to be portable,
|
||||
// a good optimizing compiler generates an arithmetic right shift
|
||||
// and avoids the conditional branch.
|
||||
UnsignedOfSize<N> mask = i < 0 ? ~UnsignedOfSize<N>(0) : 0;
|
||||
// Now get the absolute value of "i" and cast to unsigned type UnsignedOfSize<N>.
|
||||
// Cannot use std::abs() because the result is undefined
|
||||
// in 2's complement systems for the most-negative value.
|
||||
// Want to avoid conditional branch for performance reasons since
|
||||
// CPU branch prediction will be ineffective when negative values
|
||||
// occur randomly.
|
||||
// Let "u" be "i" cast to unsigned type UnsignedOfSize<N>.
|
||||
// Subtract "u" from 2*u if "i" is positive or 0 if "i" is negative.
|
||||
// This yields the absolute value with the desired type without
|
||||
// using a conditional branch and without invoking undefined or
|
||||
// implementation defined behavior:
|
||||
UnsignedOfSize<N> u = ((2 * UnsignedOfSize<N>(i)) & ~mask) - UnsignedOfSize<N>(i);
|
||||
// Unconditionally store a minus sign when producing digits
|
||||
// in a forward direction and increment the pointer only if
|
||||
// the value is in fact negative.
|
||||
// This avoids a conditional branch and is safe because we will
|
||||
// always produce at least one digit and it will overwrite the
|
||||
// minus sign when the value is not negative.
|
||||
*p = '-';
|
||||
p += (mask & 1);
|
||||
p = convert::uitoa(p, u);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
static inline char * writeUIntText(T x, char * p)
|
||||
{
|
||||
static_assert(is_unsigned_v<T>);
|
||||
|
||||
int len = digits10(x);
|
||||
auto * pp = p + len;
|
||||
while (x >= 100)
|
||||
{
|
||||
const auto i = x % 100;
|
||||
x /= 100;
|
||||
pp -= 2;
|
||||
outTwoDigits(pp, i);
|
||||
}
|
||||
if (x < 10)
|
||||
*p = '0' + x;
|
||||
else
|
||||
outTwoDigits(p, x);
|
||||
return p + len;
|
||||
}
|
||||
|
||||
static inline char * writeLeadingMinus(char * pos)
|
||||
{
|
||||
*pos = '-';
|
||||
return pos + 1;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline char * writeSIntText(T x, char * pos)
|
||||
{
|
||||
static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
|
||||
|
||||
using UnsignedT = make_unsigned_t<T>;
|
||||
static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
|
||||
|
||||
if (unlikely(x == min_int))
|
||||
{
|
||||
if constexpr (std::is_same_v<T, Int128>)
|
||||
{
|
||||
const char * res = "-170141183460469231731687303715884105728";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
else if constexpr (std::is_same_v<T, Int256>)
|
||||
{
|
||||
const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
}
|
||||
|
||||
if (x < 0)
|
||||
{
|
||||
x = -x;
|
||||
pos = writeLeadingMinus(pos);
|
||||
}
|
||||
return writeUIntText(UnsignedT(x), pos);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
char * itoa(I i, char * p)
|
||||
{
|
||||
return impl::convert::itoa(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(char8_t i, char * p)
|
||||
{
|
||||
return impl::convert::itoa(uint8_t(i), p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(UInt128 i, char * p)
|
||||
{
|
||||
return impl::writeUIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(Int128 i, char * p)
|
||||
{
|
||||
return impl::writeSIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(UInt256 i, char * p)
|
||||
{
|
||||
return impl::writeUIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(Int256 i, char * p)
|
||||
{
|
||||
return impl::writeSIntText(i, p);
|
||||
}
|
||||
#undef FOR_INTEGER_TYPES
|
||||
#undef INSTANTIATION
|
||||
|
||||
75
base/poco/Foundation/include/Poco/FPEnvironment_SUN.h
Normal file
75
base/poco/Foundation/include/Poco/FPEnvironment_SUN.h
Normal file
@ -0,0 +1,75 @@
|
||||
//
|
||||
// FPEnvironment_SUN.h
|
||||
//
|
||||
// Library: Foundation
|
||||
// Package: Core
|
||||
// Module: FPEnvironment
|
||||
//
|
||||
// Definitions of class FPEnvironmentImpl for Solaris.
|
||||
//
|
||||
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
|
||||
// and Contributors.
|
||||
//
|
||||
// SPDX-License-Identifier: BSL-1.0
|
||||
//
|
||||
|
||||
|
||||
#ifndef Foundation_FPEnvironment_SUN_INCLUDED
|
||||
#define Foundation_FPEnvironment_SUN_INCLUDED
|
||||
|
||||
|
||||
#include <ieeefp.h>
|
||||
#include "Poco/Foundation.h"
|
||||
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
|
||||
|
||||
class FPEnvironmentImpl
|
||||
{
|
||||
protected:
|
||||
enum RoundingModeImpl
|
||||
{
|
||||
FP_ROUND_DOWNWARD_IMPL = FP_RM,
|
||||
FP_ROUND_UPWARD_IMPL = FP_RP,
|
||||
FP_ROUND_TONEAREST_IMPL = FP_RN,
|
||||
FP_ROUND_TOWARDZERO_IMPL = FP_RZ
|
||||
};
|
||||
enum FlagImpl
|
||||
{
|
||||
FP_DIVIDE_BY_ZERO_IMPL = FP_X_DZ,
|
||||
FP_INEXACT_IMPL = FP_X_IMP,
|
||||
FP_OVERFLOW_IMPL = FP_X_OFL,
|
||||
FP_UNDERFLOW_IMPL = FP_X_UFL,
|
||||
FP_INVALID_IMPL = FP_X_INV
|
||||
};
|
||||
FPEnvironmentImpl();
|
||||
FPEnvironmentImpl(const FPEnvironmentImpl & env);
|
||||
~FPEnvironmentImpl();
|
||||
FPEnvironmentImpl & operator=(const FPEnvironmentImpl & env);
|
||||
void keepCurrentImpl();
|
||||
static void clearFlagsImpl();
|
||||
static bool isFlagImpl(FlagImpl flag);
|
||||
static void setRoundingModeImpl(RoundingModeImpl mode);
|
||||
static RoundingModeImpl getRoundingModeImpl();
|
||||
static bool isInfiniteImpl(float value);
|
||||
static bool isInfiniteImpl(double value);
|
||||
static bool isInfiniteImpl(long double value);
|
||||
static bool isNaNImpl(float value);
|
||||
static bool isNaNImpl(double value);
|
||||
static bool isNaNImpl(long double value);
|
||||
static float copySignImpl(float target, float source);
|
||||
static double copySignImpl(double target, double source);
|
||||
static long double copySignImpl(long double target, long double source);
|
||||
|
||||
private:
|
||||
fp_rnd _rnd;
|
||||
fp_except _exc;
|
||||
};
|
||||
|
||||
|
||||
} // namespace Poco
|
||||
|
||||
|
||||
#endif // Foundation_FPEnvironment_SUN_INCLUDED
|
||||
139
base/poco/Foundation/src/FPEnvironment_SUN.cpp
Normal file
139
base/poco/Foundation/src/FPEnvironment_SUN.cpp
Normal file
@ -0,0 +1,139 @@
|
||||
//
|
||||
// FPEnvironment_SUN.cpp
|
||||
//
|
||||
// Library: Foundation
|
||||
// Package: Core
|
||||
// Module: FPEnvironment
|
||||
//
|
||||
// Copyright (c) 2005-2006, Applied Informatics Software Engineering GmbH.
|
||||
// and Contributors.
|
||||
//
|
||||
// SPDX-License-Identifier: BSL-1.0
|
||||
//
|
||||
|
||||
|
||||
#include <math.h>
|
||||
#include "Poco/FPEnvironment_SUN.h"
|
||||
|
||||
|
||||
namespace Poco {
|
||||
|
||||
|
||||
FPEnvironmentImpl::FPEnvironmentImpl()
|
||||
{
|
||||
_rnd = fpgetround();
|
||||
_exc = fpgetmask();
|
||||
}
|
||||
|
||||
|
||||
FPEnvironmentImpl::FPEnvironmentImpl(const FPEnvironmentImpl& env)
|
||||
{
|
||||
_rnd = env._rnd;
|
||||
_exc = env._exc;
|
||||
}
|
||||
|
||||
|
||||
FPEnvironmentImpl::~FPEnvironmentImpl()
|
||||
{
|
||||
fpsetround(_rnd);
|
||||
fpsetmask(_exc);
|
||||
}
|
||||
|
||||
|
||||
FPEnvironmentImpl& FPEnvironmentImpl::operator = (const FPEnvironmentImpl& env)
|
||||
{
|
||||
_rnd = env._rnd;
|
||||
_exc = env._exc;
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isInfiniteImpl(float value)
|
||||
{
|
||||
int cls = fpclass(value);
|
||||
return cls == FP_PINF || cls == FP_NINF;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isInfiniteImpl(double value)
|
||||
{
|
||||
int cls = fpclass(value);
|
||||
return cls == FP_PINF || cls == FP_NINF;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isInfiniteImpl(long double value)
|
||||
{
|
||||
int cls = fpclass(value);
|
||||
return cls == FP_PINF || cls == FP_NINF;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isNaNImpl(float value)
|
||||
{
|
||||
return isnanf(value) != 0;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isNaNImpl(double value)
|
||||
{
|
||||
return isnan(value) != 0;
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isNaNImpl(long double value)
|
||||
{
|
||||
return isnan((double) value) != 0;
|
||||
}
|
||||
|
||||
|
||||
float FPEnvironmentImpl::copySignImpl(float target, float source)
|
||||
{
|
||||
return (float) copysign(target, source);
|
||||
}
|
||||
|
||||
|
||||
double FPEnvironmentImpl::copySignImpl(double target, double source)
|
||||
{
|
||||
return (float) copysign(target, source);
|
||||
}
|
||||
|
||||
|
||||
long double FPEnvironmentImpl::copySignImpl(long double target, long double source)
|
||||
{
|
||||
return (source > 0 && target > 0) || (source < 0 && target < 0) ? target : -target;
|
||||
}
|
||||
|
||||
|
||||
void FPEnvironmentImpl::keepCurrentImpl()
|
||||
{
|
||||
fpsetround(_rnd);
|
||||
fpsetmask(_exc);
|
||||
}
|
||||
|
||||
|
||||
void FPEnvironmentImpl::clearFlagsImpl()
|
||||
{
|
||||
fpsetsticky(0);
|
||||
}
|
||||
|
||||
|
||||
bool FPEnvironmentImpl::isFlagImpl(FlagImpl flag)
|
||||
{
|
||||
return (fpgetsticky() & flag) != 0;
|
||||
}
|
||||
|
||||
|
||||
void FPEnvironmentImpl::setRoundingModeImpl(RoundingModeImpl mode)
|
||||
{
|
||||
fpsetround((fp_rnd) mode);
|
||||
}
|
||||
|
||||
|
||||
FPEnvironmentImpl::RoundingModeImpl FPEnvironmentImpl::getRoundingModeImpl()
|
||||
{
|
||||
return (FPEnvironmentImpl::RoundingModeImpl) fpgetround();
|
||||
}
|
||||
|
||||
|
||||
} // namespace Poco
|
||||
@ -30,7 +30,6 @@ namespace Net
|
||||
|
||||
|
||||
class HTTPServerRequest;
|
||||
class HTTPServerResponse;
|
||||
class HTTPRequestHandler;
|
||||
|
||||
|
||||
|
||||
@ -26,13 +26,13 @@ const uint8_t MetroHash64::test_seed_1[8] = { 0x3B, 0x0D, 0x48, 0x1C, 0xF4, 0x
|
||||
|
||||
|
||||
|
||||
MetroHash64::MetroHash64(const uint64_t seed)
|
||||
MetroHash64::MetroHash64(uint64_t seed)
|
||||
{
|
||||
Initialize(seed);
|
||||
}
|
||||
|
||||
|
||||
void MetroHash64::Initialize(const uint64_t seed)
|
||||
void MetroHash64::Initialize(uint64_t seed)
|
||||
{
|
||||
vseed = (static_cast<uint64_t>(seed) + k2) * k0;
|
||||
|
||||
@ -47,7 +47,7 @@ void MetroHash64::Initialize(const uint64_t seed)
|
||||
}
|
||||
|
||||
|
||||
void MetroHash64::Update(const uint8_t * const buffer, const uint64_t length)
|
||||
void MetroHash64::Update(const uint8_t * const buffer, uint64_t length)
|
||||
{
|
||||
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
|
||||
const uint8_t * const end = ptr + length;
|
||||
@ -62,7 +62,7 @@ void MetroHash64::Update(const uint8_t * const buffer, const uint64_t length)
|
||||
memcpy(input.b + (bytes % 32), ptr, static_cast<size_t>(fill));
|
||||
ptr += fill;
|
||||
bytes += fill;
|
||||
|
||||
|
||||
// input buffer is still partially filled
|
||||
if ((bytes % 32) != 0) return;
|
||||
|
||||
@ -72,7 +72,7 @@ void MetroHash64::Update(const uint8_t * const buffer, const uint64_t length)
|
||||
state.v[2] += read_u64(&input.b[16]) * k2; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
|
||||
state.v[3] += read_u64(&input.b[24]) * k3; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
|
||||
}
|
||||
|
||||
|
||||
// bulk update
|
||||
bytes += static_cast<uint64_t>(end - ptr);
|
||||
while (ptr <= (end - 32))
|
||||
@ -83,14 +83,14 @@ void MetroHash64::Update(const uint8_t * const buffer, const uint64_t length)
|
||||
state.v[2] += read_u64(ptr) * k2; ptr += 8; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
|
||||
state.v[3] += read_u64(ptr) * k3; ptr += 8; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
|
||||
}
|
||||
|
||||
|
||||
// store remaining bytes in input buffer
|
||||
if (ptr < end)
|
||||
memcpy(input.b, ptr, static_cast<size_t>(end - ptr));
|
||||
}
|
||||
|
||||
|
||||
void MetroHash64::Finalize(uint8_t * const hash)
|
||||
void MetroHash64::Finalize(uint8_t * hash)
|
||||
{
|
||||
// finalize bulk loop, if used
|
||||
if (bytes >= 32)
|
||||
@ -102,11 +102,11 @@ void MetroHash64::Finalize(uint8_t * const hash)
|
||||
|
||||
state.v[0] = vseed + (state.v[0] ^ state.v[1]);
|
||||
}
|
||||
|
||||
|
||||
// process any bytes remaining in the input buffer
|
||||
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(input.b);
|
||||
const uint8_t * const end = ptr + (bytes % 32);
|
||||
|
||||
|
||||
if ((end - ptr) >= 16)
|
||||
{
|
||||
state.v[1] = state.v[0] + (read_u64(ptr) * k2); ptr += 8; state.v[1] = rotate_right(state.v[1],29) * k3;
|
||||
@ -139,7 +139,7 @@ void MetroHash64::Finalize(uint8_t * const hash)
|
||||
state.v[0] += read_u8 (ptr) * k3;
|
||||
state.v[0] ^= rotate_right(state.v[0], 37) * k1;
|
||||
}
|
||||
|
||||
|
||||
state.v[0] ^= rotate_right(state.v[0], 28);
|
||||
state.v[0] *= k0;
|
||||
state.v[0] ^= rotate_right(state.v[0], 29);
|
||||
@ -152,7 +152,7 @@ void MetroHash64::Finalize(uint8_t * const hash)
|
||||
}
|
||||
|
||||
|
||||
void MetroHash64::Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed)
|
||||
void MetroHash64::Hash(const uint8_t * buffer, uint64_t length, uint8_t * const hash, uint64_t seed)
|
||||
{
|
||||
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
|
||||
const uint8_t * const end = ptr + length;
|
||||
@ -238,7 +238,7 @@ bool MetroHash64::ImplementationVerified()
|
||||
|
||||
// verify incremental implementation
|
||||
MetroHash64 metro;
|
||||
|
||||
|
||||
metro.Initialize(0);
|
||||
metro.Update(reinterpret_cast<const uint8_t *>(MetroHash64::test_string), strlen(MetroHash64::test_string));
|
||||
metro.Finalize(hash);
|
||||
@ -262,9 +262,9 @@ void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
|
||||
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
|
||||
const uint8_t * const end = ptr + len;
|
||||
|
||||
|
||||
uint64_t hash = ((static_cast<uint64_t>(seed) + k2) * k0) + len;
|
||||
|
||||
|
||||
if (len >= 32)
|
||||
{
|
||||
uint64_t v[4];
|
||||
@ -272,7 +272,7 @@ void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v[1] = hash;
|
||||
v[2] = hash;
|
||||
v[3] = hash;
|
||||
|
||||
|
||||
do
|
||||
{
|
||||
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
|
||||
@ -288,7 +288,7 @@ void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 33) * k0;
|
||||
hash += v[0] ^ v[1];
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 16)
|
||||
{
|
||||
uint64_t v0 = hash + (read_u64(ptr) * k0); ptr += 8; v0 = rotate_right(v0,33) * k1;
|
||||
@ -297,32 +297,32 @@ void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v1 ^= rotate_right(v1 * k3, 35) + v0;
|
||||
hash += v1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 8)
|
||||
{
|
||||
hash += read_u64(ptr) * k3; ptr += 8;
|
||||
hash ^= rotate_right(hash, 33) * k1;
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 4)
|
||||
{
|
||||
hash += read_u32(ptr) * k3; ptr += 4;
|
||||
hash ^= rotate_right(hash, 15) * k1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 2)
|
||||
{
|
||||
hash += read_u16(ptr) * k3; ptr += 2;
|
||||
hash ^= rotate_right(hash, 13) * k1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 1)
|
||||
{
|
||||
hash += read_u8 (ptr) * k3;
|
||||
hash ^= rotate_right(hash, 25) * k1;
|
||||
}
|
||||
|
||||
|
||||
hash ^= rotate_right(hash, 33);
|
||||
hash *= k0;
|
||||
hash ^= rotate_right(hash, 33);
|
||||
@ -336,13 +336,13 @@ void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
static const uint64_t k0 = 0xD6D018F5;
|
||||
static const uint64_t k1 = 0xA2AA033B;
|
||||
static const uint64_t k2 = 0x62992FC1;
|
||||
static const uint64_t k3 = 0x30BC5B29;
|
||||
static const uint64_t k3 = 0x30BC5B29;
|
||||
|
||||
const uint8_t * ptr = reinterpret_cast<const uint8_t*>(key);
|
||||
const uint8_t * const end = ptr + len;
|
||||
|
||||
|
||||
uint64_t hash = ((static_cast<uint64_t>(seed) + k2) * k0) + len;
|
||||
|
||||
|
||||
if (len >= 32)
|
||||
{
|
||||
uint64_t v[4];
|
||||
@ -350,7 +350,7 @@ void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v[1] = hash;
|
||||
v[2] = hash;
|
||||
v[3] = hash;
|
||||
|
||||
|
||||
do
|
||||
{
|
||||
v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
|
||||
@ -366,7 +366,7 @@ void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 30) * k0;
|
||||
hash += v[0] ^ v[1];
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 16)
|
||||
{
|
||||
uint64_t v0 = hash + (read_u64(ptr) * k2); ptr += 8; v0 = rotate_right(v0,29) * k3;
|
||||
@ -375,31 +375,31 @@ void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * o
|
||||
v1 ^= rotate_right(v1 * k3, 34) + v0;
|
||||
hash += v1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 8)
|
||||
{
|
||||
hash += read_u64(ptr) * k3; ptr += 8;
|
||||
hash ^= rotate_right(hash, 36) * k1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 4)
|
||||
{
|
||||
hash += read_u32(ptr) * k3; ptr += 4;
|
||||
hash ^= rotate_right(hash, 15) * k1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 2)
|
||||
{
|
||||
hash += read_u16(ptr) * k3; ptr += 2;
|
||||
hash ^= rotate_right(hash, 15) * k1;
|
||||
}
|
||||
|
||||
|
||||
if ((end - ptr) >= 1)
|
||||
{
|
||||
hash += read_u8 (ptr) * k3;
|
||||
hash ^= rotate_right(hash, 23) * k1;
|
||||
}
|
||||
|
||||
|
||||
hash ^= rotate_right(hash, 28);
|
||||
hash *= k0;
|
||||
hash ^= rotate_right(hash, 29);
|
||||
|
||||
@ -25,24 +25,24 @@ public:
|
||||
static const uint32_t bits = 64;
|
||||
|
||||
// Constructor initializes the same as Initialize()
|
||||
explicit MetroHash64(const uint64_t seed=0);
|
||||
explicit MetroHash64(uint64_t seed=0);
|
||||
|
||||
// Initializes internal state for new hash with optional seed
|
||||
void Initialize(const uint64_t seed=0);
|
||||
void Initialize(uint64_t seed=0);
|
||||
|
||||
// Update the hash state with a string of bytes. If the length
|
||||
// is sufficiently long, the implementation switches to a bulk
|
||||
// hashing algorithm directly on the argument buffer for speed.
|
||||
void Update(const uint8_t * buffer, const uint64_t length);
|
||||
void Update(const uint8_t * buffer, uint64_t length);
|
||||
|
||||
// Constructs the final hash and writes it to the argument buffer.
|
||||
// After a hash is finalized, this instance must be Initialized()-ed
|
||||
// again or the behavior of Update() and Finalize() is undefined.
|
||||
void Finalize(uint8_t * const hash);
|
||||
void Finalize(uint8_t * hash);
|
||||
|
||||
// A non-incremental function implementation. This can be significantly
|
||||
// faster than the incremental implementation for some usage patterns.
|
||||
static void Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed=0);
|
||||
static void Hash(const uint8_t * buffer, uint64_t length, uint8_t * hash, uint64_t seed=0);
|
||||
|
||||
// Does implementation correctly execute test vectors?
|
||||
static bool ImplementationVerified();
|
||||
|
||||
@ -173,9 +173,15 @@ function fuzz
|
||||
|
||||
mkdir -p /var/run/clickhouse-server
|
||||
|
||||
# NOTE: we use process substitution here to preserve keep $! as a pid of clickhouse-server
|
||||
clickhouse-server --config-file db/config.xml --pid-file /var/run/clickhouse-server/clickhouse-server.pid -- --path db > server.log 2>&1 &
|
||||
server_pid=$!
|
||||
# server.log -> All server logs, including sanitizer
|
||||
# stderr.log -> Process logs (sanitizer) only
|
||||
clickhouse-server \
|
||||
--config-file db/config.xml \
|
||||
--pid-file /var/run/clickhouse-server/clickhouse-server.pid \
|
||||
-- --path db \
|
||||
--logger.console=0 \
|
||||
--logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 &
|
||||
server_pid=$(pidof clickhouse-server)
|
||||
|
||||
kill -0 $server_pid
|
||||
|
||||
@ -427,6 +433,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
|
||||
<a href="run.log">run.log</a>
|
||||
<a href="fuzzer.log.zst">fuzzer.log.zst</a>
|
||||
<a href="server.log.zst">server.log.zst</a>
|
||||
<a href="stderr.log">stderr.log</a>
|
||||
<a href="main.log">main.log</a>
|
||||
<a href="dmesg.log">dmesg.log</a>
|
||||
${CORE_LINK}
|
||||
|
||||
@ -126,7 +126,6 @@ RUN set -x \
|
||||
|
||||
COPY modprobe.sh /usr/local/bin/modprobe
|
||||
COPY dockerd-entrypoint.sh /usr/local/bin/
|
||||
COPY compose/ /compose/
|
||||
COPY misc/ /misc/
|
||||
|
||||
|
||||
|
||||
@ -51,22 +51,22 @@ fi
|
||||
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
|
||||
|
||||
if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; then
|
||||
sudo cat /etc/clickhouse-server/config.d/zookeeper.xml \
|
||||
| sed "/<use_compression>1<\/use_compression>/d" \
|
||||
> /etc/clickhouse-server/config.d/zookeeper.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/zookeeper.xml.tmp /etc/clickhouse-server/config.d/zookeeper.xml
|
||||
sudo sed -i "/<use_compression>1<\/use_compression>/d" /etc/clickhouse-server/config.d/zookeeper.xml
|
||||
|
||||
# it contains some new settings, but we can safely remove it
|
||||
rm /etc/clickhouse-server/config.d/handlers.yaml
|
||||
rm /etc/clickhouse-server/users.d/s3_cache_new.xml
|
||||
rm /etc/clickhouse-server/config.d/zero_copy_destructive_operations.xml
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
|
||||
|
||||
function remove_keeper_config()
|
||||
{
|
||||
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
|
||||
| sed "/<$1>$2<\/$1>/d" \
|
||||
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
}
|
||||
# commit_logs_cache_size_threshold setting doesn't exist on some older versions
|
||||
remove_keeper_config "commit_logs_cache_size_threshold" "[[:digit:]]\+"
|
||||
@ -77,7 +77,7 @@ fi
|
||||
if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
||||
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
||||
export THREAD_FUZZER_SLEEP_TIME_US=100000
|
||||
export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
|
||||
@ -88,10 +88,10 @@ if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
|
||||
|
||||
mkdir -p /var/run/clickhouse-server
|
||||
# simplest way to forward env variables to server
|
||||
@ -101,25 +101,13 @@ else
|
||||
fi
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
|
||||
| sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
|
||||
> /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
|
||||
mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
|
||||
sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
|
||||
|
||||
sudo cat /etc/clickhouse-server2/config.d/filesystem_caches_path.xml \
|
||||
| sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" \
|
||||
> /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp
|
||||
mv /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
|
||||
sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
|
||||
|
||||
sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
|
||||
| sed "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_1/</custom_cached_disks_base_directory>|" \
|
||||
> /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
|
||||
mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
|
||||
sudo sed -i "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_1/</custom_cached_disks_base_directory>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
|
||||
|
||||
sudo cat /etc/clickhouse-server2/config.d/filesystem_caches_path.xml \
|
||||
| sed "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_2/</custom_cached_disks_base_directory>|" \
|
||||
> /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp
|
||||
mv /etc/clickhouse-server2/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
|
||||
sudo sed -i "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_2/</custom_cached_disks_base_directory>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
|
||||
|
||||
mkdir -p /var/run/clickhouse-server1
|
||||
sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
|
||||
|
||||
@ -215,7 +215,7 @@ function check_server_start()
|
||||
function check_logs_for_critical_errors()
|
||||
{
|
||||
# Sanitizer asserts
|
||||
sed -n '/WARNING:.*anitizer/,/^$/p' >> /test_output/tmp
|
||||
sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv
|
||||
|
||||
@ -27,7 +27,7 @@ install_packages package_folder
|
||||
# and find more potential issues.
|
||||
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
||||
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
||||
export THREAD_FUZZER_SLEEP_TIME_US=100000
|
||||
export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_MIGRATE_PROBABILITY=1
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_MIGRATE_PROBABILITY=1
|
||||
@ -38,11 +38,11 @@ export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_PROBABILITY=0.001
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US_MAX=10000
|
||||
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US_MAX=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US_MAX=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
|
||||
|
||||
export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
|
||||
export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01
|
||||
|
||||
@ -67,10 +67,7 @@ configure
|
||||
|
||||
function remove_keeper_config()
|
||||
{
|
||||
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
|
||||
| sed "/<$1>$2<\/$1>/d" \
|
||||
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
sudo sed -i "/<$1>$2<\/$1>/d" /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
}
|
||||
|
||||
# async_replication setting doesn't exist on some older versions
|
||||
@ -80,16 +77,10 @@ remove_keeper_config "async_replication" "1"
|
||||
remove_keeper_config "create_if_not_exists" "[01]"
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
|
||||
| sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
|
||||
> /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
|
||||
sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
|
||||
| sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
|
||||
> /etc/clickhouse-server/config.d/storage_conf.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
|
||||
sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
|
||||
|
||||
# latest_logs_cache_size_threshold setting doesn't exist on some older versions
|
||||
remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
|
||||
@ -120,22 +111,13 @@ export ZOOKEEPER_FAULT_INJECTION=0
|
||||
configure
|
||||
|
||||
# force_sync=false doesn't work correctly on some older versions
|
||||
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
|
||||
| sed "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" \
|
||||
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
sudo sed -i "s|<force_sync>false</force_sync>|<force_sync>true</force_sync>|" /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo cat /etc/clickhouse-server/config.d/azure_storage_conf.xml \
|
||||
| sed "s|<object_storage_type>azure|<object_storage_type>azure_blob_storage|" \
|
||||
> /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/azure_storage_conf.xml.tmp /etc/clickhouse-server/config.d/azure_storage_conf.xml
|
||||
sudo sed -i "s|<object_storage_type>azure<|<object_storage_type>azure_blob_storage<|" /etc/clickhouse-server/config.d/azure_storage_conf.xml
|
||||
|
||||
#todo: remove these after 24.3 released.
|
||||
sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \
|
||||
| sed "s|<object_storage_type>local|<object_storage_type>local_blob_storage|" \
|
||||
> /etc/clickhouse-server/config.d/storage_conf.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml
|
||||
sudo sed -i "s|<object_storage_type>local<|<object_storage_type>local_blob_storage<|" /etc/clickhouse-server/config.d/storage_conf.xml
|
||||
|
||||
# async_replication setting doesn't exist on some older versions
|
||||
remove_keeper_config "async_replication" "1"
|
||||
@ -150,10 +132,7 @@ remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
|
||||
remove_keeper_config "commit_logs_cache_size_threshold" "[[:digit:]]\+"
|
||||
|
||||
# But we still need default disk because some tables loaded only into it
|
||||
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
|
||||
| sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
|
||||
> /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
|
||||
mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
sudo sed -i "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
|
||||
|
||||
@ -256,10 +235,7 @@ then
|
||||
fi
|
||||
|
||||
# Just in case previous version left some garbage in zk
|
||||
sudo cat /etc/clickhouse-server/config.d/lost_forever_check.xml \
|
||||
| sed "s|>1<|>0<|g" \
|
||||
> /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp
|
||||
sudo mv /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp /etc/clickhouse-server/config.d/lost_forever_check.xml
|
||||
sudo sed -i "s|>1<|>0<|g" /etc/clickhouse-server/config.d/lost_forever_check.xml \
|
||||
rm /etc/clickhouse-server/config.d/filesystem_caches_path.xml
|
||||
|
||||
start 500
|
||||
|
||||
293
docs/en/getting-started/example-datasets/tw-weather.md
Normal file
293
docs/en/getting-started/example-datasets/tw-weather.md
Normal file
@ -0,0 +1,293 @@
|
||||
---
|
||||
slug: /en/getting-started/example-datasets/tw-weather
|
||||
sidebar_label: Taiwan Historical Weather Datasets
|
||||
sidebar_position: 1
|
||||
description: 131 million rows of weather observation data for the last 128 yrs
|
||||
---
|
||||
|
||||
# Taiwan Historical Weather Datasets
|
||||
|
||||
This dataset contains historical meteorological observations measurements for the last 128 years. Each row is a measurement for a point in date time and weather station.
|
||||
|
||||
The origin of this dataset is available [here](https://github.com/Raingel/historical_weather) and the list of weather station numbers can be found [here](https://github.com/Raingel/weather_station_list).
|
||||
|
||||
> The sources of meteorological datasets include the meteorological stations that are established by the Central Weather Administration (station code is beginning with C0, C1, and 4) and the agricultural meteorological stations belonging to the Council of Agriculture (station code other than those mentioned above):
|
||||
|
||||
- StationId
|
||||
- MeasuredDate, the observation time
|
||||
- StnPres, the station air pressure
|
||||
- SeaPres, the sea level pressure
|
||||
- Td, the dew point temperature
|
||||
- RH, the relative humidity
|
||||
- Other elements where available
|
||||
|
||||
## Downloading the data
|
||||
|
||||
- A [pre-processed version](#pre-processed-data) of the data for the ClickHouse, which has been cleaned, re-structured, and enriched. This dataset covers the years from 1896 to 2023.
|
||||
- [Download the original raw data](#original-raw-data) and convert to the format required by ClickHouse. Users wanting to add their own columns may wish to explore or complete their approaches.
|
||||
|
||||
### Pre-processed data
|
||||
|
||||
The dataset has also been re-structured from a measurement per line to a row per weather station id and measured date, i.e.
|
||||
|
||||
```csv
|
||||
StationId,MeasuredDate,StnPres,Tx,RH,WS,WD,WSGust,WDGust,Precp,GloblRad,TxSoil0cm,TxSoil5cm,TxSoil20cm,TxSoil50cm,TxSoil100cm,SeaPres,Td,PrecpHour,SunShine,TxSoil10cm,EvapA,Visb,UVI,Cloud Amount,TxSoil30cm,TxSoil200cm,TxSoil300cm,TxSoil500cm,VaporPressure
|
||||
C0X100,2016-01-01 01:00:00,1022.1,16.1,72,1.1,8.0,,,,,,,,,,,,,,,,,,,,,,,
|
||||
C0X100,2016-01-01 02:00:00,1021.6,16.0,73,1.2,358.0,,,,,,,,,,,,,,,,,,,,,,,
|
||||
C0X100,2016-01-01 03:00:00,1021.3,15.8,74,1.5,353.0,,,,,,,,,,,,,,,,,,,,,,,
|
||||
C0X100,2016-01-01 04:00:00,1021.2,15.8,74,1.7,8.0,,,,,,,,,,,,,,,,,,,,,,,
|
||||
```
|
||||
|
||||
It is easy to query and ensure that the resulting table has less sparse and some elements are null because they're not available to be measured in this weather station.
|
||||
|
||||
This dataset is available in the following Google CloudStorage location. Either download the dataset to your local filesystem (and insert them with the ClickHouse client) or insert them directly into the ClickHouse (see [Inserting from URL](#inserting-from-url)).
|
||||
|
||||
To download:
|
||||
|
||||
```bash
|
||||
wget https://storage.googleapis.com/taiwan-weather-observaiton-datasets/preprocessed_weather_daily_1896_2023.tar.gz
|
||||
|
||||
# Option: Validate the checksum
|
||||
md5sum preprocessed_weather_daily_1896_2023.tar.gz
|
||||
# Checksum should be equal to: 11b484f5bd9ddafec5cfb131eb2dd008
|
||||
|
||||
tar -xzvf preprocessed_weather_daily_1896_2023.tar.gz
|
||||
daily_weather_preprocessed_1896_2023.csv
|
||||
|
||||
# Option: Validate the checksum
|
||||
md5sum daily_weather_preprocessed_1896_2023.csv
|
||||
# Checksum should be equal to: 1132248c78195c43d93f843753881754
|
||||
```
|
||||
|
||||
### Original raw data
|
||||
|
||||
The following details are about the steps to download the original raw data to transform and convert as you want.
|
||||
|
||||
#### Download
|
||||
|
||||
To download the original raw data:
|
||||
|
||||
```bash
|
||||
mkdir tw_raw_weather_data && cd tw_raw_weather_data
|
||||
|
||||
wget https://storage.googleapis.com/taiwan-weather-observaiton-datasets/raw_data_weather_daily_1896_2023.tar.gz
|
||||
|
||||
# Option: Validate the checksum
|
||||
md5sum raw_data_weather_daily_1896_2023.tar.gz
|
||||
# Checksum should be equal to: b66b9f137217454d655e3004d7d1b51a
|
||||
|
||||
tar -xzvf raw_data_weather_daily_1896_2023.tar.gz
|
||||
466920_1928.csv
|
||||
466920_1929.csv
|
||||
466920_1930.csv
|
||||
466920_1931.csv
|
||||
...
|
||||
|
||||
# Option: Validate the checksum
|
||||
cat *.csv | md5sum
|
||||
# Checksum should be equal to: b26db404bf84d4063fac42e576464ce1
|
||||
```
|
||||
|
||||
#### Retrieve the Taiwan weather stations
|
||||
|
||||
```bash
|
||||
wget -O weather_sta_list.csv https://github.com/Raingel/weather_station_list/raw/main/data/weather_sta_list.csv
|
||||
|
||||
# Option: Convert the UTF-8-BOM to UTF-8 encoding
|
||||
sed -i '1s/^\xEF\xBB\xBF//' weather_sta_list.csv
|
||||
```
|
||||
|
||||
## Create table schema
|
||||
|
||||
Create the MergeTree table in ClickHouse (from the ClickHouse client).
|
||||
|
||||
```bash
|
||||
CREATE TABLE tw_weather_data (
|
||||
StationId String null,
|
||||
MeasuredDate DateTime64,
|
||||
StnPres Float64 null,
|
||||
SeaPres Float64 null,
|
||||
Tx Float64 null,
|
||||
Td Float64 null,
|
||||
RH Float64 null,
|
||||
WS Float64 null,
|
||||
WD Float64 null,
|
||||
WSGust Float64 null,
|
||||
WDGust Float64 null,
|
||||
Precp Float64 null,
|
||||
PrecpHour Float64 null,
|
||||
SunShine Float64 null,
|
||||
GloblRad Float64 null,
|
||||
TxSoil0cm Float64 null,
|
||||
TxSoil5cm Float64 null,
|
||||
TxSoil10cm Float64 null,
|
||||
TxSoil20cm Float64 null,
|
||||
TxSoil50cm Float64 null,
|
||||
TxSoil100cm Float64 null,
|
||||
TxSoil30cm Float64 null,
|
||||
TxSoil200cm Float64 null,
|
||||
TxSoil300cm Float64 null,
|
||||
TxSoil500cm Float64 null,
|
||||
VaporPressure Float64 null,
|
||||
UVI Float64 null,
|
||||
"Cloud Amount" Float64 null,
|
||||
EvapA Float64 null,
|
||||
Visb Float64 null
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (MeasuredDate);
|
||||
```
|
||||
|
||||
## Inserting into ClickHouse
|
||||
|
||||
### Inserting from local file
|
||||
|
||||
Data can be inserted from a local file as follows (from the ClickHouse client):
|
||||
|
||||
```sql
|
||||
INSERT INTO tw_weather_data FROM INFILE '/path/to/daily_weather_preprocessed_1896_2023.csv'
|
||||
```
|
||||
|
||||
where `/path/to` represents the specific user path to the local file on the disk.
|
||||
|
||||
And the sample response output is as follows after inserting data into the ClickHouse:
|
||||
|
||||
```response
|
||||
Query id: 90e4b524-6e14-4855-817c-7e6f98fbeabb
|
||||
|
||||
Ok.
|
||||
131985329 rows in set. Elapsed: 71.770 sec. Processed 131.99 million rows, 10.06 GB (1.84 million rows/s., 140.14 MB/s.)
|
||||
Peak memory usage: 583.23 MiB.
|
||||
```
|
||||
|
||||
### Inserting from URL
|
||||
|
||||
```sql
|
||||
INSERT INTO tw_weather_data SELECT *
|
||||
FROM url('https://storage.googleapis.com/taiwan-weather-observaiton-datasets/daily_weather_preprocessed_1896_2023.csv', 'CSVWithNames')
|
||||
|
||||
```
|
||||
To know how to speed this up, please see our blog post on [tuning large data loads](https://clickhouse.com/blog/supercharge-your-clickhouse-data-loads-part2).
|
||||
|
||||
## Check data rows and sizes
|
||||
|
||||
1. Let's see how many rows are inserted:
|
||||
|
||||
```sql
|
||||
SELECT formatReadableQuantity(count())
|
||||
FROM tw_weather_data;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─formatReadableQuantity(count())─┐
|
||||
│ 131.99 million │
|
||||
└─────────────────────────────────┘
|
||||
```
|
||||
|
||||
2. Let's see how much disk space are used for this table:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
formatReadableSize(sum(bytes)) AS disk_size,
|
||||
formatReadableSize(sum(data_uncompressed_bytes)) AS uncompressed_size
|
||||
FROM system.parts
|
||||
WHERE (`table` = 'tw_weather_data') AND active
|
||||
```
|
||||
|
||||
```response
|
||||
┌─disk_size─┬─uncompressed_size─┐
|
||||
│ 2.13 GiB │ 32.94 GiB │
|
||||
└───────────┴───────────────────┘
|
||||
```
|
||||
|
||||
## Sample queries
|
||||
|
||||
### Q1: Retrieve the highest dew point temperature for each weather station in the specific year
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
StationId,
|
||||
max(Td) AS max_td
|
||||
FROM tw_weather_data
|
||||
WHERE (year(MeasuredDate) = 2023) AND (Td IS NOT NULL)
|
||||
GROUP BY StationId
|
||||
|
||||
┌─StationId─┬─max_td─┐
|
||||
│ 466940 │ 1 │
|
||||
│ 467300 │ 1 │
|
||||
│ 467540 │ 1 │
|
||||
│ 467490 │ 1 │
|
||||
│ 467080 │ 1 │
|
||||
│ 466910 │ 1 │
|
||||
│ 467660 │ 1 │
|
||||
│ 467270 │ 1 │
|
||||
│ 467350 │ 1 │
|
||||
│ 467571 │ 1 │
|
||||
│ 466920 │ 1 │
|
||||
│ 467650 │ 1 │
|
||||
│ 467550 │ 1 │
|
||||
│ 467480 │ 1 │
|
||||
│ 467610 │ 1 │
|
||||
│ 467050 │ 1 │
|
||||
│ 467590 │ 1 │
|
||||
│ 466990 │ 1 │
|
||||
│ 467060 │ 1 │
|
||||
│ 466950 │ 1 │
|
||||
│ 467620 │ 1 │
|
||||
│ 467990 │ 1 │
|
||||
│ 466930 │ 1 │
|
||||
│ 467110 │ 1 │
|
||||
│ 466881 │ 1 │
|
||||
│ 467410 │ 1 │
|
||||
│ 467441 │ 1 │
|
||||
│ 467420 │ 1 │
|
||||
│ 467530 │ 1 │
|
||||
│ 466900 │ 1 │
|
||||
└───────────┴────────┘
|
||||
|
||||
30 rows in set. Elapsed: 0.045 sec. Processed 6.41 million rows, 187.33 MB (143.92 million rows/s., 4.21 GB/s.)
|
||||
```
|
||||
|
||||
### Q2: Raw data fetching with the specific duration time range, fields and weather station
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
StnPres,
|
||||
SeaPres,
|
||||
Tx,
|
||||
Td,
|
||||
RH,
|
||||
WS,
|
||||
WD,
|
||||
WSGust,
|
||||
WDGust,
|
||||
Precp,
|
||||
PrecpHour
|
||||
FROM tw_weather_data
|
||||
WHERE (StationId = 'C0UB10') AND (MeasuredDate >= '2023-12-23') AND (MeasuredDate < '2023-12-24')
|
||||
ORDER BY MeasuredDate ASC
|
||||
LIMIT 10
|
||||
```
|
||||
|
||||
```response
|
||||
┌─StnPres─┬─SeaPres─┬───Tx─┬───Td─┬─RH─┬──WS─┬──WD─┬─WSGust─┬─WDGust─┬─Precp─┬─PrecpHour─┐
|
||||
│ 1029.5 │ ᴺᵁᴸᴸ │ 11.8 │ ᴺᵁᴸᴸ │ 78 │ 2.7 │ 271 │ 5.5 │ 275 │ -99.8 │ -99.8 │
|
||||
│ 1029.8 │ ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 78 │ 2.7 │ 289 │ 5.5 │ 308 │ -99.8 │ -99.8 │
|
||||
│ 1028.6 │ ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 79 │ 2.3 │ 251 │ 6.1 │ 289 │ -99.8 │ -99.8 │
|
||||
│ 1028.2 │ ᴺᵁᴸᴸ │ 13 │ ᴺᵁᴸᴸ │ 75 │ 4.3 │ 312 │ 7.5 │ 316 │ -99.8 │ -99.8 │
|
||||
│ 1027.8 │ ᴺᵁᴸᴸ │ 11.1 │ ᴺᵁᴸᴸ │ 89 │ 7.1 │ 310 │ 11.6 │ 322 │ -99.8 │ -99.8 │
|
||||
│ 1027.8 │ ᴺᵁᴸᴸ │ 11.6 │ ᴺᵁᴸᴸ │ 90 │ 3.1 │ 269 │ 10.7 │ 295 │ -99.8 │ -99.8 │
|
||||
│ 1027.9 │ ᴺᵁᴸᴸ │ 12.3 │ ᴺᵁᴸᴸ │ 89 │ 4.7 │ 296 │ 8.1 │ 310 │ -99.8 │ -99.8 │
|
||||
│ 1028.2 │ ᴺᵁᴸᴸ │ 12.2 │ ᴺᵁᴸᴸ │ 94 │ 2.5 │ 246 │ 7.1 │ 283 │ -99.8 │ -99.8 │
|
||||
│ 1028.4 │ ᴺᵁᴸᴸ │ 12.5 │ ᴺᵁᴸᴸ │ 94 │ 3.1 │ 265 │ 4.8 │ 297 │ -99.8 │ -99.8 │
|
||||
│ 1028.3 │ ᴺᵁᴸᴸ │ 13.6 │ ᴺᵁᴸᴸ │ 91 │ 1.2 │ 273 │ 4.4 │ 256 │ -99.8 │ -99.8 │
|
||||
└─────────┴─────────┴──────┴──────┴────┴─────┴─────┴────────┴────────┴───────┴───────────┘
|
||||
|
||||
10 rows in set. Elapsed: 0.009 sec. Processed 91.70 thousand rows, 2.33 MB (9.67 million rows/s., 245.31 MB/s.)
|
||||
```
|
||||
|
||||
## Credits
|
||||
|
||||
We would like to acknowledge the efforts of the Central Weather Administration and Agricultural Meteorological Observation Network (Station) of the Council of Agriculture for preparing, cleaning, and distributing this dataset. We appreciate your efforts.
|
||||
|
||||
Ou, J.-H., Kuo, C.-H., Wu, Y.-F., Lin, G.-C., Lee, M.-H., Chen, R.-K., Chou, H.-P., Wu, H.-Y., Chu, S.-C., Lai, Q.-J., Tsai, Y.-C., Lin, C.-C., Kuo, C.-C., Liao, C.-T., Chen, Y.-N., Chu, Y.-W., Chen, C.-Y., 2023. Application-oriented deep learning model for early warning of rice blast in Taiwan. Ecological Informatics 73, 101950. https://doi.org/10.1016/j.ecoinf.2022.101950 [13/12/2022]
|
||||
@ -178,7 +178,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--password` – The password. Default value: empty string.
|
||||
- `--ask-password` - Prompt the user to enter a password.
|
||||
- `--query, -q` – The query to process when using non-interactive mode. `--query` can be specified multiple times, e.g. `--query "SELECT 1" --query "SELECT 2"`. Cannot be used simultaneously with `--queries-file`.
|
||||
- `--queries-file` – file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--query queries1.sql --query queries2.sql`. Cannot be used simultaneously with `--query`.
|
||||
- `--queries-file` – file path with queries to execute. `--queries-file` can be specified multiple times, e.g. `--queries-file queries1.sql --queries-file queries2.sql`. Cannot be used simultaneously with `--query`.
|
||||
- `--multiquery, -n` – If specified, multiple queries separated by semicolons can be listed after the `--query` option. For convenience, it is also possible to omit `--query` and pass the queries directly after `--multiquery`.
|
||||
- `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter).
|
||||
- `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ by default).
|
||||
|
||||
@ -933,9 +933,9 @@ Hard limit is configured via system tools
|
||||
|
||||
## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec}
|
||||
|
||||
The delay before a table data is dropped in seconds. If the `DROP TABLE` query has a `SYNC` modifier, this setting is ignored.
|
||||
Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored.
|
||||
|
||||
Default value: `480` (8 minutes).
|
||||
Default value: `480` (8 minute).
|
||||
|
||||
## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec}
|
||||
|
||||
|
||||
@ -4,6 +4,67 @@ sidebar_label: Polygons
|
||||
title: "Functions for Working with Polygons"
|
||||
---
|
||||
|
||||
## WKT
|
||||
|
||||
Returns a WKT (Well Known Text) geometric object from various [Geo Data Types](../../data-types/geo.md). Supported WKT objects are:
|
||||
|
||||
- POINT
|
||||
- POLYGON
|
||||
- MULTIPOLYGON
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
WKT(geo_data)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
`geo_data` can be one of the following [Geo Data Types](../../data-types/geo.md) or their underlying primitive types:
|
||||
|
||||
- [Point](../../data-types/geo.md#point)
|
||||
- [Ring](../../data-types/geo.md#ring)
|
||||
- [Polygon](../../data-types/geo.md#polygon)
|
||||
- [MultiPolygon](../../data-types/geo.md#multipolygon)
|
||||
|
||||
**Returned value**
|
||||
|
||||
- WKT geometric object `POINT` is returned for a Point.
|
||||
- WKT geometric object `POLYGON` is returned for a Polygon
|
||||
- WKT geometric object `MULTIPOLYGON` is returned for a MultiPolygon.
|
||||
|
||||
**Examples**
|
||||
|
||||
POINT from tuple:
|
||||
|
||||
```sql
|
||||
SELECT wkt((0., 0.));
|
||||
```
|
||||
|
||||
```response
|
||||
POINT(0 0)
|
||||
```
|
||||
|
||||
POLYGON from an array of tuples or an array of tuple arrays:
|
||||
|
||||
```sql
|
||||
SELECT wkt([(0., 0.), (10., 0.), (10., 10.), (0., 10.)]);
|
||||
```
|
||||
|
||||
```response
|
||||
POLYGON((0 0,10 0,10 10,0 10))
|
||||
```
|
||||
|
||||
MULTIPOLYGON from an array of multi-dimensional tuple arrays:
|
||||
|
||||
```sql
|
||||
SELECT wkt([[[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]], [[(-10., -10.), (-10., -9.), (-9., 10.)]]]);
|
||||
```
|
||||
|
||||
```response
|
||||
MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))
|
||||
```
|
||||
|
||||
## readWKTMultiPolygon
|
||||
|
||||
Converts a WKT (Well Known Text) MultiPolygon into a MultiPolygon type.
|
||||
|
||||
@ -350,6 +350,7 @@ ALTER TABLE mt DELETE IN PARTITION ID '2' WHERE p = 2;
|
||||
You can specify the partition expression in `ALTER ... PARTITION` queries in different ways:
|
||||
|
||||
- As a value from the `partition` column of the `system.parts` table. For example, `ALTER TABLE visits DETACH PARTITION 201901`.
|
||||
- Using the keyword `ALL`. It can be used only with DROP/DETACH/ATTACH. For example, `ALTER TABLE visits ATTACH PARTITION ALL`.
|
||||
- As a tuple of expressions or constants that matches (in types) the table partitioning keys tuple. In the case of a single element partitioning key, the expression should be wrapped in the `tuple (...)` function. For example, `ALTER TABLE visits DETACH PARTITION tuple(toYYYYMM(toDate('2019-01-25')))`.
|
||||
- Using the partition ID. Partition ID is a string identifier of the partition (human-readable, if possible) that is used as the names of partitions in the file system and in ZooKeeper. The partition ID must be specified in the `PARTITION ID` clause, in a single quotes. For example, `ALTER TABLE visits DETACH PARTITION ID '201901'`.
|
||||
- In the [ALTER ATTACH PART](#alter_attach-partition) and [DROP DETACHED PART](#alter_drop-detached) query, to specify the name of a part, use string literal with a value from the `name` column of the [system.detached_parts](/docs/en/operations/system-tables/detached_parts.md/#system_tables-detached_parts) table. For example, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`.
|
||||
|
||||
@ -13,6 +13,13 @@ a system table called `system.dropped_tables`.
|
||||
|
||||
If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view.
|
||||
|
||||
:::note
|
||||
UNDROP TABLE is experimental. To use it add this setting:
|
||||
```sql
|
||||
set allow_experimental_undrop_table_query = 1;
|
||||
```
|
||||
:::
|
||||
|
||||
:::tip
|
||||
Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md)
|
||||
:::
|
||||
@ -25,53 +32,60 @@ UNDROP TABLE [db.]name [UUID '<uuid>'] [ON CLUSTER cluster]
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
set allow_experimental_undrop_table_query = 1;
|
||||
```
|
||||
|
||||
```sql
|
||||
CREATE TABLE tab
|
||||
CREATE TABLE undropMe
|
||||
(
|
||||
`id` UInt8
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY id;
|
||||
|
||||
DROP TABLE tab;
|
||||
|
||||
SELECT *
|
||||
FROM system.dropped_tables
|
||||
FORMAT Vertical;
|
||||
ORDER BY id
|
||||
```
|
||||
|
||||
```sql
|
||||
DROP TABLE undropMe
|
||||
```
|
||||
```sql
|
||||
SELECT *
|
||||
FROM system.dropped_tables
|
||||
FORMAT Vertical
|
||||
```
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
index: 0
|
||||
database: default
|
||||
table: tab
|
||||
table: undropMe
|
||||
uuid: aa696a1a-1d70-4e60-a841-4c80827706cc
|
||||
engine: MergeTree
|
||||
metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql
|
||||
metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql
|
||||
table_dropped_time: 2023-04-05 14:12:12
|
||||
|
||||
1 row in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
```sql
|
||||
UNDROP TABLE tab;
|
||||
|
||||
UNDROP TABLE undropMe
|
||||
```
|
||||
```response
|
||||
Ok.
|
||||
```
|
||||
```sql
|
||||
SELECT *
|
||||
FROM system.dropped_tables
|
||||
FORMAT Vertical;
|
||||
|
||||
FORMAT Vertical
|
||||
```
|
||||
```response
|
||||
Ok.
|
||||
|
||||
0 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
```sql
|
||||
DESCRIBE TABLE tab
|
||||
FORMAT Vertical;
|
||||
DESCRIBE TABLE undropMe
|
||||
FORMAT Vertical
|
||||
```
|
||||
|
||||
```response
|
||||
Row 1:
|
||||
──────
|
||||
|
||||
@ -34,6 +34,7 @@
|
||||
#include <Common/StudentTTest.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/ErrorCodes.h>
|
||||
#include <Core/BaseSettingsProgramOptions.h>
|
||||
|
||||
|
||||
/** A tool for evaluating ClickHouse performance.
|
||||
@ -623,7 +624,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
;
|
||||
|
||||
Settings settings;
|
||||
settings.addProgramOptions(desc);
|
||||
addProgramOptions(settings, desc);
|
||||
|
||||
boost::program_options::variables_map options;
|
||||
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
#include <Common/ErrorCodes.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/TerminalSize.h>
|
||||
#include <Core/BaseSettingsProgramOptions.h>
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
@ -102,7 +103,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
|
||||
{
|
||||
std::string_view name = field.getName();
|
||||
if (name == "max_parser_depth" || name == "max_query_size")
|
||||
cmd_settings.addProgramOption(desc, name, field);
|
||||
addProgramOption(cmd_settings, desc, name, field);
|
||||
}
|
||||
|
||||
boost::program_options::variables_map options;
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include "Commands.h"
|
||||
#include <queue>
|
||||
#include "KeeperClient.h"
|
||||
#include "Parsers/CommonParsers.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -106,13 +107,13 @@ bool CreateCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> &
|
||||
|
||||
int mode = zkutil::CreateMode::Persistent;
|
||||
|
||||
if (ParserKeyword{"PERSISTENT"}.ignore(pos, expected))
|
||||
if (ParserKeyword(Keyword::PERSISTENT).ignore(pos, expected))
|
||||
mode = zkutil::CreateMode::Persistent;
|
||||
else if (ParserKeyword{"EPHEMERAL"}.ignore(pos, expected))
|
||||
else if (ParserKeyword(Keyword::EPHEMERAL).ignore(pos, expected))
|
||||
mode = zkutil::CreateMode::Ephemeral;
|
||||
else if (ParserKeyword{"EPHEMERAL SEQUENTIAL"}.ignore(pos, expected))
|
||||
else if (ParserKeyword(Keyword::EPHEMERAL_SEQUENTIAL).ignore(pos, expected))
|
||||
mode = zkutil::CreateMode::EphemeralSequential;
|
||||
else if (ParserKeyword{"PERSISTENT SEQUENTIAL"}.ignore(pos, expected))
|
||||
else if (ParserKeyword(Keyword::PERSISTENT_SEQUENTIAL).ignore(pos, expected))
|
||||
mode = zkutil::CreateMode::PersistentSequential;
|
||||
|
||||
node->args.push_back(std::move(mode));
|
||||
@ -382,12 +383,16 @@ void RMRCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) co
|
||||
|
||||
bool ReconfigCommand::parse(IParser::Pos & pos, std::shared_ptr<ASTKeeperQuery> & node, DB::Expected & expected) const
|
||||
{
|
||||
ParserKeyword s_add(Keyword::ADD);
|
||||
ParserKeyword s_remove(Keyword::REMOVE);
|
||||
ParserKeyword s_set(Keyword::SET);
|
||||
|
||||
ReconfigCommand::Operation operation;
|
||||
if (ParserKeyword{"ADD"}.ignore(pos, expected))
|
||||
if (s_add.ignore(pos, expected))
|
||||
operation = ReconfigCommand::Operation::ADD;
|
||||
else if (ParserKeyword{"REMOVE"}.ignore(pos, expected))
|
||||
else if (s_remove.ignore(pos, expected))
|
||||
operation = ReconfigCommand::Operation::REMOVE;
|
||||
else if (ParserKeyword{"SET"}.ignore(pos, expected))
|
||||
else if (s_set.ignore(pos, expected))
|
||||
operation = ReconfigCommand::Operation::SET;
|
||||
else
|
||||
return false;
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <IO/UseSSL.h>
|
||||
#include <Core/ServerUUID.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/CgroupsMemoryUsageObserver.h>
|
||||
#include <Common/ErrorHandlers.h>
|
||||
#include <Common/assertProcessUserMatchesDataOwner.h>
|
||||
#include <Common/makeSocketAddress.h>
|
||||
@ -623,6 +624,25 @@ try
|
||||
buildLoggers(config(), logger());
|
||||
main_config_reloader->start();
|
||||
|
||||
std::optional<CgroupsMemoryUsageObserver> cgroups_memory_usage_observer;
|
||||
try
|
||||
{
|
||||
auto wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15);
|
||||
if (wait_time != 0)
|
||||
{
|
||||
cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time));
|
||||
/// Not calling cgroups_memory_usage_observer->setLimits() here (as for the normal ClickHouse server) because Keeper controls
|
||||
/// its memory usage by other means (via setting 'max_memory_usage_soft_limit').
|
||||
cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); });
|
||||
cgroups_memory_usage_observer->startThread();
|
||||
}
|
||||
}
|
||||
catch (Exception &)
|
||||
{
|
||||
tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization");
|
||||
}
|
||||
|
||||
|
||||
LOG_INFO(log, "Ready for connections.");
|
||||
|
||||
waitForTerminationRequest();
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
#include <Common/getNumberOfPhysicalCPUCores.h>
|
||||
#include <Common/getExecutablePath.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <Common/Scheduler/IResourceManager.h>
|
||||
#include <Common/ThreadProfileEvents.h>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/getMappedArea.h>
|
||||
@ -1296,7 +1297,7 @@ try
|
||||
std::optional<CgroupsMemoryUsageObserver> cgroups_memory_usage_observer;
|
||||
try
|
||||
{
|
||||
UInt64 wait_time = server_settings.cgroups_memory_usage_observer_wait_time;
|
||||
auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time;
|
||||
if (wait_time != 0)
|
||||
cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time));
|
||||
}
|
||||
@ -1362,7 +1363,7 @@ try
|
||||
{
|
||||
double hard_limit_ratio = new_server_settings.cgroup_memory_watcher_hard_limit_ratio;
|
||||
double soft_limit_ratio = new_server_settings.cgroup_memory_watcher_soft_limit_ratio;
|
||||
cgroups_memory_usage_observer->setLimits(
|
||||
cgroups_memory_usage_observer->setMemoryUsageLimits(
|
||||
static_cast<uint64_t>(max_server_memory_usage * hard_limit_ratio),
|
||||
static_cast<uint64_t>(max_server_memory_usage * soft_limit_ratio));
|
||||
}
|
||||
@ -1720,6 +1721,12 @@ try
|
||||
throw;
|
||||
}
|
||||
|
||||
if (cgroups_memory_usage_observer)
|
||||
{
|
||||
cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); });
|
||||
cgroups_memory_usage_observer->startThread();
|
||||
}
|
||||
|
||||
/// Reload config in SYSTEM RELOAD CONFIG query.
|
||||
global_context->setConfigReloadCallback([&]()
|
||||
{
|
||||
|
||||
@ -297,7 +297,7 @@ namespace
|
||||
|
||||
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> access_entities,
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
|
||||
const String & data_path_in_backup,
|
||||
size_t counter,
|
||||
const AccessControl & access_control)
|
||||
@ -326,7 +326,7 @@ void AccessRestorerFromBackup::addDataPath(const String & data_path)
|
||||
return;
|
||||
|
||||
fs::path data_path_in_backup_fs = data_path;
|
||||
Strings filenames = backup->listFiles(data_path);
|
||||
Strings filenames = backup->listFiles(data_path, /*recursive*/ false);
|
||||
if (filenames.empty())
|
||||
return;
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ struct RestoreSettings;
|
||||
|
||||
/// Makes a backup of access entities of a specified type.
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> access_entities,
|
||||
const std::vector<std::pair<UUID, AccessEntityPtr>> & access_entities,
|
||||
const String & data_path_in_backup,
|
||||
size_t counter,
|
||||
const AccessControl & access_control);
|
||||
|
||||
@ -13,63 +13,63 @@ namespace ErrorCodes
|
||||
|
||||
const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType type_)
|
||||
{
|
||||
static constexpr auto make_info = [](const char * raw_name_, bool is_password_ = false)
|
||||
static constexpr auto make_info = [](Keyword keyword_, bool is_password_ = false)
|
||||
{
|
||||
String init_name = raw_name_;
|
||||
String init_name = String(toStringView(keyword_));
|
||||
boost::to_lower(init_name);
|
||||
return AuthenticationTypeInfo{raw_name_, std::move(init_name), is_password_};
|
||||
return AuthenticationTypeInfo{keyword_, std::move(init_name), is_password_};
|
||||
};
|
||||
|
||||
switch (type_)
|
||||
{
|
||||
case AuthenticationType::NO_PASSWORD:
|
||||
{
|
||||
static const auto info = make_info("NO_PASSWORD");
|
||||
static const auto info = make_info(Keyword::NO_PASSWORD);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
||||
{
|
||||
static const auto info = make_info("PLAINTEXT_PASSWORD", true);
|
||||
static const auto info = make_info(Keyword::PLAINTEXT_PASSWORD, true);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::SHA256_PASSWORD:
|
||||
{
|
||||
static const auto info = make_info("SHA256_PASSWORD", true);
|
||||
static const auto info = make_info(Keyword::SHA256_PASSWORD, true);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
||||
{
|
||||
static const auto info = make_info("DOUBLE_SHA1_PASSWORD", true);
|
||||
static const auto info = make_info(Keyword::DOUBLE_SHA1_PASSWORD, true);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::LDAP:
|
||||
{
|
||||
static const auto info = make_info("LDAP");
|
||||
static const auto info = make_info(Keyword::LDAP);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::KERBEROS:
|
||||
{
|
||||
static const auto info = make_info("KERBEROS");
|
||||
static const auto info = make_info(Keyword::KERBEROS);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::SSL_CERTIFICATE:
|
||||
{
|
||||
static const auto info = make_info("SSL_CERTIFICATE");
|
||||
static const auto info = make_info(Keyword::SSL_CERTIFICATE);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::BCRYPT_PASSWORD:
|
||||
{
|
||||
static const auto info = make_info("BCRYPT_PASSWORD", true);
|
||||
static const auto info = make_info(Keyword::BCRYPT_PASSWORD, true);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::SSH_KEY:
|
||||
{
|
||||
static const auto info = make_info("SSH_KEY");
|
||||
static const auto info = make_info(Keyword::SSH_KEY);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::HTTP:
|
||||
{
|
||||
static const auto info = make_info("HTTP");
|
||||
static const auto info = make_info(Keyword::HTTP);
|
||||
return info;
|
||||
}
|
||||
case AuthenticationType::MAX:
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/types.h>
|
||||
#include <Parsers/CommonParsers.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -45,7 +46,7 @@ enum class AuthenticationType
|
||||
|
||||
struct AuthenticationTypeInfo
|
||||
{
|
||||
const char * const raw_name;
|
||||
Keyword keyword; // Keyword used in parser
|
||||
const String name; /// Lowercased with underscores, e.g. "sha256_password".
|
||||
bool is_password;
|
||||
static const AuthenticationTypeInfo & get(AuthenticationType type_);
|
||||
@ -53,7 +54,7 @@ struct AuthenticationTypeInfo
|
||||
|
||||
inline String toString(AuthenticationType type_)
|
||||
{
|
||||
return AuthenticationTypeInfo::get(type_).raw_name;
|
||||
return String(toStringView(AuthenticationTypeInfo::get(type_).keyword));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <base/range.h>
|
||||
|
||||
#include <boost/algorithm/string/case_conv.hpp>
|
||||
#include <boost/algorithm/string/classification.hpp>
|
||||
|
||||
@ -7,7 +7,7 @@ namespace DB
|
||||
{
|
||||
|
||||
ContextAccessParams::ContextAccessParams(
|
||||
const std::optional<UUID> user_id_,
|
||||
std::optional<UUID> user_id_,
|
||||
bool full_access_,
|
||||
bool use_default_roles_,
|
||||
const std::shared_ptr<const std::vector<UUID>> & current_roles_,
|
||||
|
||||
@ -15,7 +15,7 @@ class ContextAccessParams
|
||||
{
|
||||
public:
|
||||
ContextAccessParams(
|
||||
const std::optional<UUID> user_id_,
|
||||
std::optional<UUID> user_id_,
|
||||
bool full_access_,
|
||||
bool use_default_roles_,
|
||||
const std::shared_ptr<const std::vector<UUID>> & current_roles_,
|
||||
|
||||
@ -34,7 +34,7 @@ public:
|
||||
};
|
||||
|
||||
explicit GSSAcceptorContext(const Params & params_);
|
||||
virtual ~GSSAcceptorContext() override;
|
||||
~GSSAcceptorContext() override;
|
||||
|
||||
GSSAcceptorContext(const GSSAcceptorContext &) = delete;
|
||||
GSSAcceptorContext(GSSAcceptorContext &&) = delete;
|
||||
|
||||
@ -13,6 +13,8 @@
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
|
||||
|
||||
namespace Poco { class Logger; }
|
||||
namespace Poco::Net { class IPAddress; }
|
||||
|
||||
@ -204,7 +204,7 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchR
|
||||
}
|
||||
|
||||
|
||||
void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, const std::size_t external_roles_hash) const
|
||||
void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, std::size_t external_roles_hash) const
|
||||
{
|
||||
const auto & user_name = user.getName();
|
||||
auto & granted_roles = user.granted_roles;
|
||||
|
||||
@ -33,29 +33,29 @@ public:
|
||||
static constexpr char STORAGE_TYPE[] = "ldap";
|
||||
|
||||
explicit LDAPAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
virtual ~LDAPAccessStorage() override = default;
|
||||
~LDAPAccessStorage() override = default;
|
||||
|
||||
String getLDAPServerName() const;
|
||||
|
||||
// IAccessStorage implementations.
|
||||
virtual const char * getStorageType() const override;
|
||||
virtual String getStorageParamsJSON() const override;
|
||||
virtual bool isReadOnly() const override { return true; }
|
||||
virtual bool exists(const UUID & id) const override;
|
||||
const char * getStorageType() const override;
|
||||
String getStorageParamsJSON() const override;
|
||||
bool isReadOnly() const override { return true; }
|
||||
bool exists(const UUID & id) const override;
|
||||
|
||||
private: // IAccessStorage implementations.
|
||||
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
virtual std::optional<AuthResult> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
|
||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||
std::optional<AuthResult> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
|
||||
|
||||
void setConfiguration(const Poco::Util::AbstractConfiguration & config, const String & prefix);
|
||||
void processRoleChange(const UUID & id, const AccessEntityPtr & entity);
|
||||
|
||||
void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name);
|
||||
void assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles) const;
|
||||
void assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, const std::size_t external_roles_hash) const;
|
||||
void assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, std::size_t external_roles_hash) const;
|
||||
void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const;
|
||||
std::set<String> mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const;
|
||||
bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <Access/User.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Core/Protocol.h>
|
||||
#include <base/insertAtEnd.h>
|
||||
|
||||
|
||||
@ -252,7 +252,6 @@ void dumpFlameGraph(
|
||||
fillColumn(chars, offsets, out.str());
|
||||
}
|
||||
|
||||
// NOLINTBEGIN(clang-analyzer-optin.performance.Padding)
|
||||
struct AggregateFunctionFlameGraphData
|
||||
{
|
||||
struct Entry
|
||||
@ -469,7 +468,6 @@ struct AggregateFunctionFlameGraphData
|
||||
DB::dumpFlameGraph(tree.dump(max_depth, min_bytes), chars, offsets);
|
||||
}
|
||||
};
|
||||
// NOLINTEND(clang-analyzer-optin.performance.Padding)
|
||||
|
||||
/// Aggregate function which builds a flamegraph using the list of stacktraces.
|
||||
/// The output is an array of strings which can be used by flamegraph.pl util.
|
||||
|
||||
@ -157,7 +157,7 @@ public:
|
||||
|
||||
void update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient) override;
|
||||
|
||||
virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
|
||||
void write(WriteBuffer & buf) const override;
|
||||
|
||||
@ -189,7 +189,7 @@ public:
|
||||
|
||||
void update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient) override;
|
||||
|
||||
virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
|
||||
void write(WriteBuffer & buf) const override;
|
||||
|
||||
@ -226,7 +226,7 @@ public:
|
||||
|
||||
void update(UInt64 batch_size, std::vector<Float64> & weights, Float64 & bias, Float64 learning_rate, const std::vector<Float64> & batch_gradient) override;
|
||||
|
||||
virtual void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
void merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) override;
|
||||
|
||||
void write(WriteBuffer & buf) const override;
|
||||
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
|
||||
#include <unordered_set>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <bitset>
|
||||
#include <base/range.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <bitset>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -10,6 +10,8 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <base/range.h>
|
||||
|
||||
#include <bitset>
|
||||
#include <stack>
|
||||
|
||||
|
||||
@ -234,9 +234,6 @@ namespace ErrorCodes
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <template <typename> typename FunctionTemplate, StatisticsFunctionKind kind>
|
||||
AggregateFunctionPtr createAggregateFunctionStatisticsUnary(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *)
|
||||
@ -273,5 +270,3 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -42,9 +42,6 @@ struct UniqCombinedHashTableGrower : public HashTableGrowerWithPrecalculation<>
|
||||
void increaseSize() { increaseSizeDegree(1); }
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <typename T, UInt8 K, typename HashValueType>
|
||||
struct AggregateFunctionUniqCombinedData
|
||||
{
|
||||
@ -268,8 +265,6 @@ AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_typ
|
||||
return std::make_shared<typename WithK<K, HashValueType>::template AggregateFunctionVariadic<false, false>>(argument_types, params);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <UInt8 K>
|
||||
AggregateFunctionPtr createAggregateFunctionWithHashType(bool use_64_bit_hash, const DataTypes & argument_types, const Array & params)
|
||||
{
|
||||
|
||||
@ -521,7 +521,7 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
chassert(nullable_filters.size() > 0);
|
||||
chassert(!nullable_filters.empty());
|
||||
bool found_one = false;
|
||||
if (nullable_filters.size() == 1)
|
||||
{
|
||||
|
||||
@ -38,7 +38,7 @@ public:
|
||||
offset = other->offset;
|
||||
}
|
||||
|
||||
int length()
|
||||
int length() const
|
||||
{
|
||||
return static_cast<int>(bins.size());
|
||||
}
|
||||
|
||||
@ -150,7 +150,7 @@ struct QuantileExactExclusive : public QuantileExact<Value>
|
||||
return static_cast<Float64>(*std::min_element(array.begin(), array.end()));
|
||||
|
||||
::nth_element(array.begin(), array.begin() + n - 1, array.end());
|
||||
auto nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
auto * nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
|
||||
return static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_elem - array[n - 1]);
|
||||
}
|
||||
@ -179,7 +179,7 @@ struct QuantileExactExclusive : public QuantileExact<Value>
|
||||
else
|
||||
{
|
||||
::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end());
|
||||
auto nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
auto * nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
|
||||
result[indices[i]] = static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_elem - array[n - 1]);
|
||||
prev_n = n - 1;
|
||||
@ -214,7 +214,7 @@ struct QuantileExactInclusive : public QuantileExact<Value>
|
||||
else if (n < 1)
|
||||
return static_cast<Float64>(*std::min_element(array.begin(), array.end()));
|
||||
::nth_element(array.begin(), array.begin() + n - 1, array.end());
|
||||
auto nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
auto * nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
|
||||
return static_cast<Float64>(array[n - 1]) + (h - n) * static_cast<Float64>(*nth_elem - array[n - 1]);
|
||||
}
|
||||
@ -241,7 +241,7 @@ struct QuantileExactInclusive : public QuantileExact<Value>
|
||||
else
|
||||
{
|
||||
::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end());
|
||||
auto nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
auto * nth_elem = std::min_element(array.begin() + n, array.end());
|
||||
|
||||
result[indices[i]] = static_cast<Float64>(array[n - 1]) + (h - n) * (static_cast<Float64>(*nth_elem) - array[n - 1]);
|
||||
prev_n = n - 1;
|
||||
|
||||
@ -191,7 +191,7 @@ public:
|
||||
/// TODO: After implementation of "versioning aggregate function state",
|
||||
/// change the serialization format.
|
||||
Element elem;
|
||||
memset(&elem, 0, sizeof(elem));
|
||||
memset(&elem, 0, sizeof(elem)); /// NOLINT(bugprone-undefined-memory-manipulation)
|
||||
elem = samples[i];
|
||||
|
||||
DB::transformEndianness<std::endian::little>(elem);
|
||||
|
||||
@ -23,7 +23,7 @@ struct SingleValueDataBase
|
||||
/// For example argMin holds 1 of these (for the result), while keeping a template for the value
|
||||
static constexpr UInt32 MAX_STORAGE_SIZE = 64;
|
||||
|
||||
virtual ~SingleValueDataBase() { }
|
||||
virtual ~SingleValueDataBase() = default;
|
||||
virtual bool has() const = 0;
|
||||
virtual void insertResultInto(IColumn &) const = 0;
|
||||
virtual void write(WriteBuffer &, const ISerialization &) const = 0;
|
||||
|
||||
@ -39,8 +39,8 @@ public:
|
||||
/// This method will convert all the SingleLevelSet to TwoLevelSet in parallel if the hashsets are not all singlelevel or not all twolevel.
|
||||
static void parallelizeMergePrepare(const std::vector<UniqExactSet *> & data_vec, ThreadPool & thread_pool)
|
||||
{
|
||||
unsigned long single_level_set_num = 0;
|
||||
unsigned long all_single_hash_size = 0;
|
||||
UInt64 single_level_set_num = 0;
|
||||
UInt64 all_single_hash_size = 0;
|
||||
|
||||
for (auto ele : data_vec)
|
||||
{
|
||||
|
||||
@ -48,6 +48,7 @@
|
||||
#include <Interpreters/SelectQueryOptions.h>
|
||||
#include <Interpreters/Set.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Interpreters/ExternalDictionariesLoader.h>
|
||||
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
|
||||
|
||||
@ -776,6 +777,7 @@ struct IdentifierResolveScope
|
||||
std::unordered_map<QueryTreeNodePtr, TableExpressionData> table_expression_node_to_data;
|
||||
|
||||
QueryTreeNodePtrWithHashSet nullable_group_by_keys;
|
||||
QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> nullable_join_columns;
|
||||
|
||||
/// Use identifier lookup to result cache
|
||||
bool use_identifier_lookup_to_result_cache = true;
|
||||
@ -1276,7 +1278,11 @@ private:
|
||||
return {};
|
||||
}
|
||||
|
||||
static void convertJoinedColumnTypeToNullIfNeeded(QueryTreeNodePtr & resolved_identifier, const JoinKind & join_kind, std::optional<JoinTableSide> resolved_side)
|
||||
static QueryTreeNodePtr convertJoinedColumnTypeToNullIfNeeded(
|
||||
const QueryTreeNodePtr & resolved_identifier,
|
||||
const JoinKind & join_kind,
|
||||
std::optional<JoinTableSide> resolved_side,
|
||||
IdentifierResolveScope & scope)
|
||||
{
|
||||
if (resolved_identifier->getNodeType() == QueryTreeNodeType::COLUMN &&
|
||||
JoinCommon::canBecomeNullable(resolved_identifier->getResultType()) &&
|
||||
@ -1284,9 +1290,20 @@ private:
|
||||
(isLeft(join_kind) && resolved_side && *resolved_side == JoinTableSide::Right) ||
|
||||
(isRight(join_kind) && resolved_side && *resolved_side == JoinTableSide::Left)))
|
||||
{
|
||||
auto & resolved_column = resolved_identifier->as<ColumnNode &>();
|
||||
resolved_column.setColumnType(makeNullableOrLowCardinalityNullable(resolved_column.getColumnType()));
|
||||
auto nullable_resolved_identifier = resolved_identifier->clone();
|
||||
auto & resolved_column = nullable_resolved_identifier->as<ColumnNode &>();
|
||||
auto new_result_type = makeNullableOrLowCardinalityNullable(resolved_column.getColumnType());
|
||||
resolved_column.setColumnType(new_result_type);
|
||||
if (resolved_column.hasExpression())
|
||||
{
|
||||
auto & resolved_expression = resolved_column.getExpression();
|
||||
if (!resolved_expression->getResultType()->equals(*new_result_type))
|
||||
resolved_expression = buildCastFunction(resolved_expression, new_result_type, scope.context, true);
|
||||
}
|
||||
scope.nullable_join_columns[nullable_resolved_identifier] = resolved_identifier;
|
||||
return nullable_resolved_identifier;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/// Resolve identifier functions
|
||||
@ -3258,6 +3275,32 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromTableExpression(const Id
|
||||
return {};
|
||||
}
|
||||
|
||||
QueryTreeNodePtr checkIsMissedObjectJSONSubcolumn(const QueryTreeNodePtr & left_resolved_identifier,
|
||||
const QueryTreeNodePtr & right_resolved_identifier)
|
||||
{
|
||||
if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT
|
||||
&& right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
|
||||
auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
|
||||
if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return left_resolved_identifier;
|
||||
}
|
||||
else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
|
||||
if (left_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return left_resolved_identifier;
|
||||
}
|
||||
else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
|
||||
if (right_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return right_resolved_identifier;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLookup & identifier_lookup,
|
||||
const QueryTreeNodePtr & table_expression_node,
|
||||
IdentifierResolveScope & scope)
|
||||
@ -3358,28 +3401,8 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
|
||||
|
||||
/// If columns from left or right table were missed Object(Nullable('json')) subcolumns, they will be replaced
|
||||
/// to ConstantNode(NULL), which can't be cast to ColumnNode, so we resolve it here.
|
||||
if (left_resolved_identifier && right_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT
|
||||
&& right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
|
||||
auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
|
||||
if (left_resolved_column.getValueStringRepresentation() == "NULL" && right_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return left_resolved_identifier;
|
||||
}
|
||||
else if (left_resolved_identifier && left_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
resolved_side = JoinTableSide::Left;
|
||||
auto & left_resolved_column = left_resolved_identifier->as<ConstantNode &>();
|
||||
if (left_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return left_resolved_identifier;
|
||||
}
|
||||
else if (right_resolved_identifier && right_resolved_identifier->getNodeType() == QueryTreeNodeType::CONSTANT)
|
||||
{
|
||||
resolved_side = JoinTableSide::Right;
|
||||
auto & right_resolved_column = right_resolved_identifier->as<ConstantNode &>();
|
||||
if (right_resolved_column.getValueStringRepresentation() == "NULL")
|
||||
return right_resolved_identifier;
|
||||
}
|
||||
if (auto missed_subcolumn_identifier = checkIsMissedObjectJSONSubcolumn(left_resolved_identifier, right_resolved_identifier))
|
||||
return missed_subcolumn_identifier;
|
||||
|
||||
if (left_resolved_identifier && right_resolved_identifier)
|
||||
{
|
||||
@ -3521,8 +3544,9 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromJoin(const IdentifierLoo
|
||||
|
||||
if (scope.join_use_nulls)
|
||||
{
|
||||
resolved_identifier = resolved_identifier->clone();
|
||||
convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side);
|
||||
auto nullable_resolved_identifier = convertJoinedColumnTypeToNullIfNeeded(resolved_identifier, join_kind, resolved_side, scope);
|
||||
if (nullable_resolved_identifier)
|
||||
resolved_identifier = nullable_resolved_identifier;
|
||||
}
|
||||
|
||||
return resolved_identifier;
|
||||
@ -4402,6 +4426,28 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(
|
||||
const auto & join_using_column_nodes_list = join_using_column_node.getExpressionOrThrow()->as<ListNode &>();
|
||||
const auto & join_using_column_nodes = join_using_column_nodes_list.getNodes();
|
||||
|
||||
/** If column doesn't exists in the table, then do not match column from USING clause.
|
||||
* Example: SELECT a + 1 AS id, * FROM (SELECT 1 AS a) AS t1 JOIN (SELECT 2 AS id) AS t2 USING (id);
|
||||
* In this case `id` is not present in the left table expression,
|
||||
* so asterisk should return `id` from the right table expression.
|
||||
*/
|
||||
auto is_column_from_parent_scope = [&scope](const QueryTreeNodePtr & using_node_from_table)
|
||||
{
|
||||
const auto & using_column_from_table = using_node_from_table->as<ColumnNode &>();
|
||||
auto table_expression_data_it = scope.table_expression_node_to_data.find(using_column_from_table.getColumnSource());
|
||||
if (table_expression_data_it != scope.table_expression_node_to_data.end())
|
||||
{
|
||||
const auto & table_expression_data = table_expression_data_it->second;
|
||||
const auto & column_name = using_column_from_table.getColumnName();
|
||||
return !table_expression_data.column_name_to_column_node.contains(column_name);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
if (is_column_from_parent_scope(join_using_column_nodes.at(0)) ||
|
||||
is_column_from_parent_scope(join_using_column_nodes.at(1)))
|
||||
continue;
|
||||
|
||||
QueryTreeNodePtr matched_column_node;
|
||||
|
||||
if (isRight(join_node->getKind()))
|
||||
@ -4523,7 +4569,15 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I
|
||||
for (auto & [node, node_name] : matched_expression_nodes_with_names)
|
||||
{
|
||||
auto join_identifier_side = getColumnSideFromJoinTree(node, *nearest_scope_join_node);
|
||||
convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side);
|
||||
auto projection_name_it = node_to_projection_name.find(node);
|
||||
auto nullable_node = convertJoinedColumnTypeToNullIfNeeded(node, nearest_scope_join_node->getKind(), join_identifier_side, scope);
|
||||
if (nullable_node)
|
||||
{
|
||||
node = nullable_node;
|
||||
/// Set the same projection name for new nullable node
|
||||
if (projection_name_it != node_to_projection_name.end())
|
||||
node_to_projection_name.emplace(node, projection_name_it->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -7304,8 +7358,64 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS
|
||||
|
||||
join_using_identifiers.insert(identifier_full_name);
|
||||
|
||||
const auto & settings = scope.context->getSettingsRef();
|
||||
|
||||
/** While resolving JOIN USING identifier, try to resolve identifier from parent subquery projection.
|
||||
* Example: SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b
|
||||
* In this case `b` is not in the left table expression, but it is in the parent subquery projection.
|
||||
*/
|
||||
auto try_resolve_identifier_from_query_projection = [this](const String & identifier_full_name_,
|
||||
const QueryTreeNodePtr & left_table_expression,
|
||||
const IdentifierResolveScope & scope_) -> QueryTreeNodePtr
|
||||
{
|
||||
const QueryNode * query_node = scope_.scope_node ? scope_.scope_node->as<QueryNode>() : nullptr;
|
||||
if (!query_node)
|
||||
return nullptr;
|
||||
|
||||
const auto & projection_list = query_node->getProjection();
|
||||
for (const auto & projection_node : projection_list.getNodes())
|
||||
{
|
||||
if (projection_node->hasAlias() && identifier_full_name_ == projection_node->getAlias())
|
||||
{
|
||||
auto left_subquery = std::make_shared<QueryNode>(query_node->getMutableContext());
|
||||
left_subquery->getProjection().getNodes().push_back(projection_node->clone());
|
||||
left_subquery->getJoinTree() = left_table_expression;
|
||||
|
||||
IdentifierResolveScope left_subquery_scope(left_subquery, nullptr /*parent_scope*/);
|
||||
resolveQuery(left_subquery, left_subquery_scope);
|
||||
|
||||
const auto & resolved_nodes = left_subquery->getProjection().getNodes();
|
||||
if (resolved_nodes.size() == 1)
|
||||
{
|
||||
/// Create ColumnNode with expression from parent projection
|
||||
return std::make_shared<ColumnNode>(
|
||||
NameAndTypePair{identifier_full_name_, resolved_nodes.front()->getResultType()},
|
||||
resolved_nodes.front(), left_table_expression);
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
};
|
||||
|
||||
QueryTreeNodePtr result_left_table_expression = nullptr;
|
||||
/** With `analyzer_compatibility_join_using_top_level_identifier` alias in projection has higher priority than column from left table.
|
||||
* But if aliased expression cannot be resolved from left table, we get UNKNOW_IDENTIFIER error,
|
||||
* despite the fact that column from USING could be resolved from left table.
|
||||
* It's compatibility with a default behavior for old analyzer.
|
||||
*/
|
||||
if (settings.analyzer_compatibility_join_using_top_level_identifier)
|
||||
result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope);
|
||||
|
||||
IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION};
|
||||
auto result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
|
||||
if (!result_left_table_expression)
|
||||
result_left_table_expression = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_node_typed.getLeftTableExpression(), scope);
|
||||
|
||||
/// Here we may try to resolve identifier from projection in case it's not resolved from left table expression
|
||||
/// and analyzer_compatibility_join_using_top_level_identifier is disabled.
|
||||
/// For now we do not do this, because not all corner cases are clear.
|
||||
/// if (!settings.analyzer_compatibility_join_using_top_level_identifier && !result_left_table_expression)
|
||||
/// result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope);
|
||||
|
||||
if (!result_left_table_expression)
|
||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
|
||||
"JOIN {} using identifier '{}' cannot be resolved from left table expression. In scope {}",
|
||||
@ -7448,6 +7558,29 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node,
|
||||
scope.table_expressions_in_resolve_process.erase(join_tree_node.get());
|
||||
}
|
||||
|
||||
class ReplaceColumnsVisitor : public InDepthQueryTreeVisitor<ReplaceColumnsVisitor>
|
||||
{
|
||||
public:
|
||||
explicit ReplaceColumnsVisitor(const QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> & replacement_map_, const ContextPtr & context_)
|
||||
: replacement_map(replacement_map_)
|
||||
, context(context_)
|
||||
{}
|
||||
|
||||
void visitImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
if (auto it = replacement_map.find(node); it != replacement_map.end())
|
||||
node = it->second;
|
||||
if (auto * function_node = node->as<FunctionNode>())
|
||||
rerunFunctionResolve(function_node, context);
|
||||
}
|
||||
|
||||
bool shouldTraverseTopToBottom() const { return false; }
|
||||
|
||||
private:
|
||||
const QueryTreeNodePtrWithHashMap<QueryTreeNodePtr> & replacement_map;
|
||||
const ContextPtr & context;
|
||||
};
|
||||
|
||||
/** Resolve query.
|
||||
* This function modifies query node during resolve. It is caller responsibility to clone query node before resolve
|
||||
* if it is needed for later use.
|
||||
@ -7635,21 +7768,23 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
scope.scope_node->formatASTForErrorMessage());
|
||||
}
|
||||
|
||||
if (query_node_typed.getPrewhere())
|
||||
if (auto & prewhere_node = query_node_typed.getPrewhere())
|
||||
{
|
||||
/** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
|
||||
* Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE a = 1
|
||||
* Column `a` should be resolved from table and should not change its type to Nullable.
|
||||
*/
|
||||
bool join_use_nulls = scope.join_use_nulls;
|
||||
bool use_identifier_lookup_to_result_cache = scope.use_identifier_lookup_to_result_cache;
|
||||
scope.join_use_nulls = false;
|
||||
scope.use_identifier_lookup_to_result_cache = false;
|
||||
resolveExpressionNode(prewhere_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
|
||||
resolveExpressionNode(query_node_typed.getPrewhere(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
|
||||
scope.join_use_nulls = join_use_nulls;
|
||||
scope.use_identifier_lookup_to_result_cache = use_identifier_lookup_to_result_cache;
|
||||
if (scope.join_use_nulls)
|
||||
{
|
||||
/** Expression in PREWHERE with JOIN should not be modified by join_use_nulls.
|
||||
* Example: SELECT * FROM t1 JOIN t2 USING (id) PREWHERE b = 1
|
||||
* Column `a` should be resolved from table and should not change its type to Nullable.
|
||||
* More complicated example when column is somewhere inside an expression:
|
||||
* SELECT a + 1 as b FROM t1 JOIN t2 USING (id) PREWHERE b = 1
|
||||
* expression `a + 1 as b` in projection and in PREWHERE should have different `a`.
|
||||
*/
|
||||
prewhere_node = prewhere_node->clone();
|
||||
ReplaceColumnsVisitor replace_visitor(scope.nullable_join_columns, scope.context);
|
||||
replace_visitor.visit(prewhere_node);
|
||||
}
|
||||
}
|
||||
|
||||
if (query_node_typed.getWhere())
|
||||
|
||||
@ -77,7 +77,7 @@ public:
|
||||
* Available expression columns are extracted from table expression.
|
||||
* Table expression node must have query, union, table, table function type.
|
||||
*/
|
||||
QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_ = false);
|
||||
explicit QueryAnalysisPass(QueryTreeNodePtr table_expression_, bool only_analyze_ = false);
|
||||
|
||||
String getName() override
|
||||
{
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <Backups/BackupCoordinationRemote.h>
|
||||
|
||||
#include <base/hex.h>
|
||||
#include <boost/algorithm/string/split.hpp>
|
||||
|
||||
#include <Access/Common/AccessEntityType.h>
|
||||
#include <Backups/BackupCoordinationReplicatedAccess.h>
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Backups/DDLAdjustingForBackupVisitor.h>
|
||||
#include <Databases/IDatabase.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Storages/IStorage.h>
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Backups/BackupIO.h>
|
||||
#include <Common/Logger.h>
|
||||
#include <IO/ReadSettings.h>
|
||||
#include <IO/WriteSettings.h>
|
||||
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <Backups/BackupIO_Default.h>
|
||||
#include <Common/Logger.h>
|
||||
#include <Disks/DiskType.h>
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
|
||||
#if USE_AWS_S3
|
||||
#include <Backups/BackupIO_Default.h>
|
||||
#include <Common/Logger.h>
|
||||
#include <Disks/DiskType.h>
|
||||
#include <IO/S3Common.h>
|
||||
#include <Storages/StorageS3Settings.h>
|
||||
|
||||
@ -26,7 +26,7 @@ struct BackupSettings
|
||||
String password;
|
||||
|
||||
/// S3 storage class.
|
||||
String s3_storage_class = "";
|
||||
String s3_storage_class;
|
||||
|
||||
/// If this is set to true then only create queries will be written to backup,
|
||||
/// without the data of tables.
|
||||
|
||||
@ -327,7 +327,7 @@ public:
|
||||
metric_active_threads = CurrentMetrics::RestoreThreadsActive;
|
||||
metric_active_threads = CurrentMetrics::RestoreThreadsScheduled;
|
||||
max_threads = num_restore_threads;
|
||||
use_queue = (thread_pool_id != ThreadPoolId::RESTORE);
|
||||
use_queue = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -80,7 +80,7 @@ public:
|
||||
|
||||
/// Returns names of entries stored in a specified directory in the backup.
|
||||
/// If `directory` is empty or '/' the functions returns entries in the backup's root.
|
||||
virtual Strings listFiles(const String & directory, bool recursive = false) const = 0;
|
||||
virtual Strings listFiles(const String & directory, bool recursive) const = 0;
|
||||
|
||||
/// Checks if a specified directory contains any files.
|
||||
/// The function returns the same as `!listFiles(directory).empty()`.
|
||||
@ -108,11 +108,9 @@ public:
|
||||
virtual std::unique_ptr<SeekableReadBuffer> readFile(const SizeAndChecksum & size_and_checksum) const = 0;
|
||||
|
||||
/// Copies a file from the backup to a specified destination disk. Returns the number of bytes written.
|
||||
virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path,
|
||||
WriteMode write_mode = WriteMode::Rewrite) const = 0;
|
||||
virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const = 0;
|
||||
|
||||
virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path,
|
||||
WriteMode write_mode = WriteMode::Rewrite) const = 0;
|
||||
virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const = 0;
|
||||
|
||||
/// Puts a new entry to the backup.
|
||||
virtual void writeFile(const BackupFileInfo & file_info, BackupEntryPtr entry) = 0;
|
||||
|
||||
@ -273,7 +273,7 @@ void RestorerFromBackup::findRootPathsInBackup()
|
||||
root_paths_in_backup.push_back(root_path);
|
||||
|
||||
/// Add shard-related part to the root path.
|
||||
Strings shards_in_backup = backup->listFiles(root_path / "shards");
|
||||
Strings shards_in_backup = backup->listFiles(root_path / "shards", /*recursive*/ false);
|
||||
if (shards_in_backup.empty())
|
||||
{
|
||||
if (restore_settings.shard_num_in_backup > 1)
|
||||
@ -295,7 +295,7 @@ void RestorerFromBackup::findRootPathsInBackup()
|
||||
}
|
||||
|
||||
/// Add replica-related part to the root path.
|
||||
Strings replicas_in_backup = backup->listFiles(root_path / "replicas");
|
||||
Strings replicas_in_backup = backup->listFiles(root_path / "replicas", /*recursive*/ false);
|
||||
if (replicas_in_backup.empty())
|
||||
{
|
||||
if (restore_settings.replica_num_in_backup > 1)
|
||||
@ -514,7 +514,7 @@ void RestorerFromBackup::findDatabaseInBackupImpl(const String & database_name_i
|
||||
if (!metadata_path && !try_metadata_path.empty() && backup->fileExists(try_metadata_path))
|
||||
metadata_path = try_metadata_path;
|
||||
|
||||
Strings file_names = backup->listFiles(try_tables_metadata_path);
|
||||
Strings file_names = backup->listFiles(try_tables_metadata_path, /*recursive*/ false);
|
||||
for (const String & file_name : file_names)
|
||||
{
|
||||
if (!file_name.ends_with(".sql"))
|
||||
@ -575,7 +575,7 @@ void RestorerFromBackup::findEverythingInBackup(const std::set<String> & except_
|
||||
|
||||
for (const auto & root_path_in_backup : root_paths_in_backup)
|
||||
{
|
||||
Strings file_names = backup->listFiles(root_path_in_backup / "metadata");
|
||||
Strings file_names = backup->listFiles(root_path_in_backup / "metadata", /*recursive*/ false);
|
||||
for (String & file_name : file_names)
|
||||
{
|
||||
if (file_name.ends_with(".sql"))
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <base/safeExit.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Core/BaseSettingsProgramOptions.h>
|
||||
#include <Core/Protocol.h>
|
||||
#include <Common/DateLUT.h>
|
||||
#include <Common/MemoryTracker.h>
|
||||
@ -2714,9 +2715,9 @@ private:
|
||||
void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
|
||||
{
|
||||
if (allow_repeated_settings)
|
||||
cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value());
|
||||
addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
|
||||
else
|
||||
cmd_settings.addProgramOptions(options_description.main_description.value());
|
||||
addProgramOptions(cmd_settings, options_description.main_description.value());
|
||||
|
||||
if (allow_merge_tree_settings)
|
||||
{
|
||||
@ -2737,9 +2738,9 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description,
|
||||
return;
|
||||
|
||||
if (allow_repeated_settings)
|
||||
cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, name, setting);
|
||||
addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
|
||||
else
|
||||
cmd_merge_tree_settings.addProgramOption(main_options, name, setting);
|
||||
addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
|
||||
};
|
||||
|
||||
const auto & setting_name = setting.getName();
|
||||
|
||||
@ -159,7 +159,7 @@ private:
|
||||
size_t requested_connections_count = 0;
|
||||
|
||||
const size_t max_parallel_replicas = 0;
|
||||
const bool skip_unavailable_shards = 0;
|
||||
const bool skip_unavailable_shards = false;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
#include "LocalConnection.h"
|
||||
#include <Core/Protocol.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <Interpreters/executeQuery.h>
|
||||
#include <Processors/Executors/CompletedPipelineExecutor.h>
|
||||
#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
|
||||
#include <Processors/Executors/PushingPipelineExecutor.h>
|
||||
#include <Processors/Executors/PushingAsyncPipelineExecutor.h>
|
||||
#include <Processors/Executors/PushingPipelineExecutor.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
#include <Core/Protocol.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -77,12 +77,12 @@ private:
|
||||
{
|
||||
if (actual_server_error && std::find(server_errors.begin(), server_errors.end(), actual_server_error) == server_errors.end())
|
||||
return false;
|
||||
if (!actual_server_error && server_errors.size())
|
||||
if (!actual_server_error && !server_errors.empty())
|
||||
return false;
|
||||
|
||||
if (actual_client_error && std::find(client_errors.begin(), client_errors.end(), actual_client_error) == client_errors.end())
|
||||
return false;
|
||||
if (!actual_client_error && client_errors.size())
|
||||
if (!actual_client_error && !client_errors.empty())
|
||||
return false;
|
||||
|
||||
return true;
|
||||
@ -90,7 +90,7 @@ private:
|
||||
|
||||
bool lostExpectedError(int actual_server_error, int actual_client_error) const
|
||||
{
|
||||
return (server_errors.size() && !actual_server_error) || (client_errors.size() && !actual_client_error);
|
||||
return (!server_errors.empty() && !actual_server_error) || (!client_errors.empty() && !actual_client_error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -671,7 +671,7 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
|
||||
size_t max_dictionary_size)
|
||||
{
|
||||
auto overflowed_keys = column_holder->cloneEmpty();
|
||||
auto overflowed_keys_ptr = typeid_cast<ColumnType *>(overflowed_keys.get());
|
||||
auto * overflowed_keys_ptr = typeid_cast<ColumnType *>(overflowed_keys.get());
|
||||
if (!overflowed_keys_ptr)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid keys type for ColumnUnique.");
|
||||
|
||||
|
||||
@ -86,7 +86,7 @@ struct ReverseIndexHashTableCell
|
||||
{
|
||||
/// Careful: apparently this uses SFINAE to redefine isZero for all types
|
||||
/// except the IndexType, for which the default ZeroTraits::isZero is used.
|
||||
static_assert(!std::is_same_v<typename std::decay<T>::type, typename std::decay<IndexType>::type>);
|
||||
static_assert(!std::is_same_v<typename std::decay_t<T>, typename std::decay_t<IndexType>>);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -26,10 +26,10 @@ struct AllocatorWithMemoryTracking
|
||||
|
||||
[[nodiscard]] T * allocate(size_t n)
|
||||
{
|
||||
if (n > std::numeric_limits<size_t>::max() / sizeof(T))
|
||||
if (n > std::numeric_limits<size_t>::max() / sizeof(T)) /// NOLINT(bugprone-sizeof-expression)
|
||||
throw std::bad_alloc();
|
||||
|
||||
size_t bytes = n * sizeof(T);
|
||||
size_t bytes = n * sizeof(T); /// NOLINT(bugprone-sizeof-expression)
|
||||
auto trace = CurrentMemoryTracker::alloc(bytes);
|
||||
|
||||
T * p = static_cast<T *>(malloc(bytes));
|
||||
@ -43,7 +43,7 @@ struct AllocatorWithMemoryTracking
|
||||
|
||||
void deallocate(T * p, size_t n) noexcept
|
||||
{
|
||||
size_t bytes = n * sizeof(T);
|
||||
size_t bytes = n * sizeof(T); /// NOLINT(bugprone-sizeof-expression)
|
||||
|
||||
free(p);
|
||||
auto trace = CurrentMemoryTracker::free(bytes);
|
||||
|
||||
@ -49,7 +49,7 @@ private:
|
||||
|
||||
MemoryChunk() = default;
|
||||
|
||||
void swap(MemoryChunk & other)
|
||||
void swap(MemoryChunk & other) noexcept
|
||||
{
|
||||
std::swap(begin, other.begin);
|
||||
std::swap(pos, other.pos);
|
||||
@ -57,12 +57,12 @@ private:
|
||||
prev.swap(other.prev);
|
||||
}
|
||||
|
||||
MemoryChunk(MemoryChunk && other)
|
||||
MemoryChunk(MemoryChunk && other) noexcept
|
||||
{
|
||||
*this = std::move(other);
|
||||
}
|
||||
|
||||
MemoryChunk & operator=(MemoryChunk && other)
|
||||
MemoryChunk & operator=(MemoryChunk && other) noexcept
|
||||
{
|
||||
swap(other);
|
||||
return *this;
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <base/cgroupsv2.h>
|
||||
#include <base/getMemoryAmount.h>
|
||||
#include <base/sleep.h>
|
||||
|
||||
#include <filesystem>
|
||||
@ -36,7 +37,7 @@ namespace ErrorCodes
|
||||
CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_)
|
||||
: log(getLogger("CgroupsMemoryUsageObserver"))
|
||||
, wait_time(wait_time_)
|
||||
, file(log)
|
||||
, memory_usage_file(log)
|
||||
{
|
||||
LOG_INFO(log, "Initialized cgroups memory limit observer, wait time is {} sec", wait_time.count());
|
||||
}
|
||||
@ -46,13 +47,13 @@ CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver()
|
||||
stopThread();
|
||||
}
|
||||
|
||||
void CgroupsMemoryUsageObserver::setLimits(uint64_t hard_limit_, uint64_t soft_limit_)
|
||||
void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_)
|
||||
{
|
||||
std::lock_guard<std::mutex> limit_lock(limit_mutex);
|
||||
|
||||
if (hard_limit_ == hard_limit && soft_limit_ == soft_limit)
|
||||
return;
|
||||
|
||||
stopThread();
|
||||
|
||||
hard_limit = hard_limit_;
|
||||
soft_limit = soft_limit_;
|
||||
|
||||
@ -83,10 +84,10 @@ void CgroupsMemoryUsageObserver::setLimits(uint64_t hard_limit_, uint64_t soft_l
|
||||
mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
|
||||
#endif
|
||||
/// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them.
|
||||
uint64_t current_usage = readMemoryUsage();
|
||||
MemoryTracker::setRSS(current_usage, 0);
|
||||
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
|
||||
MemoryTracker::setRSS(memory_usage, 0);
|
||||
|
||||
LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(current_usage));
|
||||
LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -94,14 +95,13 @@ void CgroupsMemoryUsageObserver::setLimits(uint64_t hard_limit_, uint64_t soft_l
|
||||
}
|
||||
};
|
||||
|
||||
startThread();
|
||||
|
||||
LOG_INFO(log, "Set new limits, soft limit: {}, hard limit: {}", ReadableSize(soft_limit_), ReadableSize(hard_limit_));
|
||||
}
|
||||
|
||||
uint64_t CgroupsMemoryUsageObserver::readMemoryUsage() const
|
||||
void CgroupsMemoryUsageObserver::setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_)
|
||||
{
|
||||
return file.readMemoryUsage();
|
||||
std::lock_guard<std::mutex> memory_amount_available_changed_lock(memory_amount_available_changed_mutex);
|
||||
on_memory_amount_available_changed = on_memory_amount_available_changed_;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -163,7 +163,7 @@ std::pair<std::string, CgroupsMemoryUsageObserver::CgroupsVersion> getCgroupsFil
|
||||
|
||||
}
|
||||
|
||||
CgroupsMemoryUsageObserver::File::File(LoggerPtr log_)
|
||||
CgroupsMemoryUsageObserver::MemoryUsageFile::MemoryUsageFile(LoggerPtr log_)
|
||||
: log(log_)
|
||||
{
|
||||
std::tie(file_name, version) = getCgroupsFileName();
|
||||
@ -177,7 +177,7 @@ CgroupsMemoryUsageObserver::File::File(LoggerPtr log_)
|
||||
file_name, "Cannot open file '{}'", file_name);
|
||||
}
|
||||
|
||||
CgroupsMemoryUsageObserver::File::~File()
|
||||
CgroupsMemoryUsageObserver::MemoryUsageFile::~MemoryUsageFile()
|
||||
{
|
||||
assert(fd != -1);
|
||||
if (::close(fd) != 0)
|
||||
@ -195,7 +195,7 @@ CgroupsMemoryUsageObserver::File::~File()
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t CgroupsMemoryUsageObserver::File::readMemoryUsage() const
|
||||
uint64_t CgroupsMemoryUsageObserver::MemoryUsageFile::readMemoryUsage() const
|
||||
{
|
||||
/// File read is probably not read is thread-safe, just to be sure
|
||||
std::lock_guard lock(mutex);
|
||||
@ -278,6 +278,9 @@ void CgroupsMemoryUsageObserver::runThread()
|
||||
{
|
||||
setThreadName("CgrpMemUsgObsr");
|
||||
|
||||
last_available_memory_amount = getMemoryAmount();
|
||||
LOG_INFO(log, "Memory amount initially available to the process is {}", ReadableSize(last_available_memory_amount));
|
||||
|
||||
std::unique_lock lock(thread_mutex);
|
||||
while (true)
|
||||
{
|
||||
@ -286,8 +289,42 @@ void CgroupsMemoryUsageObserver::runThread()
|
||||
|
||||
try
|
||||
{
|
||||
uint64_t memory_usage = file.readMemoryUsage();
|
||||
processMemoryUsage(memory_usage);
|
||||
uint64_t available_memory_amount = getMemoryAmount();
|
||||
if (available_memory_amount != last_available_memory_amount)
|
||||
{
|
||||
LOG_INFO(log, "Memory amount available to the process changed from {} to {}", ReadableSize(last_available_memory_amount), ReadableSize(available_memory_amount));
|
||||
last_available_memory_amount = available_memory_amount;
|
||||
std::lock_guard<std::mutex> memory_amount_available_changed_lock(memory_amount_available_changed_mutex);
|
||||
on_memory_amount_available_changed();
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> limit_lock(limit_mutex);
|
||||
if (soft_limit > 0 && hard_limit > 0)
|
||||
{
|
||||
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
|
||||
if (memory_usage > hard_limit)
|
||||
{
|
||||
if (last_memory_usage <= hard_limit)
|
||||
on_hard_limit(true);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (last_memory_usage > hard_limit)
|
||||
on_hard_limit(false);
|
||||
}
|
||||
|
||||
if (memory_usage > soft_limit)
|
||||
{
|
||||
if (last_memory_usage <= soft_limit)
|
||||
on_soft_limit(true);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (last_memory_usage > soft_limit)
|
||||
on_soft_limit(false);
|
||||
}
|
||||
last_memory_usage = memory_usage;
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -296,33 +333,6 @@ void CgroupsMemoryUsageObserver::runThread()
|
||||
}
|
||||
}
|
||||
|
||||
void CgroupsMemoryUsageObserver::processMemoryUsage(uint64_t current_usage)
|
||||
{
|
||||
if (current_usage > hard_limit)
|
||||
{
|
||||
if (last_usage <= hard_limit)
|
||||
on_hard_limit(true);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (last_usage > hard_limit)
|
||||
on_hard_limit(false);
|
||||
}
|
||||
|
||||
if (current_usage > soft_limit)
|
||||
{
|
||||
if (last_usage <= soft_limit)
|
||||
on_soft_limit(true);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (last_usage > soft_limit)
|
||||
on_soft_limit(false);
|
||||
}
|
||||
|
||||
last_usage = current_usage;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -2,57 +2,71 @@
|
||||
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <mutex>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Periodically reads the current memory usage from Linux cgroups.
|
||||
/// You can specify soft or hard memory limits:
|
||||
/// - When the soft memory limit is hit, drop jemalloc cache.
|
||||
/// - When the hard memory limit is hit, update MemoryTracking metric to throw memory exceptions faster.
|
||||
/// Does two things:
|
||||
/// 1. Periodically reads the memory usage of the process from Linux cgroups.
|
||||
/// You can specify soft or hard memory limits:
|
||||
/// - When the soft memory limit is hit, drop jemalloc cache.
|
||||
/// - When the hard memory limit is hit, update MemoryTracking metric to throw memory exceptions faster.
|
||||
/// The goal of this is to avoid that the process hits the maximum allowed memory limit at which there is a good
|
||||
/// chance that the Limux OOM killer terminates it. All of this is done is because internal memory tracking in
|
||||
/// ClickHouse can unfortunately under-estimate the actually used memory.
|
||||
/// 2. Periodically reads the the maximum memory available to the process (which can change due to cgroups settings).
|
||||
/// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server
|
||||
/// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit'
|
||||
/// (Keeper) from which various other internal limits are calculated, including the soft and hard limits for (1.).
|
||||
/// The goal of this is to provide elasticity when the container is scaled-up/scaled-down. The mechanism (polling
|
||||
/// cgroups) is quite implicit, unfortunately there is currently no better way to communicate memory threshold changes
|
||||
/// to the database.
|
||||
#if defined(OS_LINUX)
|
||||
class CgroupsMemoryUsageObserver
|
||||
{
|
||||
public:
|
||||
using OnMemoryLimitFn = std::function<void(bool)>;
|
||||
using OnMemoryAmountAvailableChangedFn = std::function<void()>;
|
||||
|
||||
enum class CgroupsVersion
|
||||
{
|
||||
V1,
|
||||
V2
|
||||
|
||||
};
|
||||
|
||||
explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_);
|
||||
~CgroupsMemoryUsageObserver();
|
||||
|
||||
void setLimits(uint64_t hard_limit_, uint64_t soft_limit_);
|
||||
void setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_);
|
||||
void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_);
|
||||
|
||||
size_t getHardLimit() const { return hard_limit; }
|
||||
size_t getSoftLimit() const { return soft_limit; }
|
||||
|
||||
uint64_t readMemoryUsage() const;
|
||||
void startThread();
|
||||
|
||||
private:
|
||||
LoggerPtr log;
|
||||
|
||||
std::atomic<size_t> hard_limit = 0;
|
||||
std::atomic<size_t> soft_limit = 0;
|
||||
|
||||
const std::chrono::seconds wait_time;
|
||||
|
||||
using CallbackFn = std::function<void(bool)>;
|
||||
CallbackFn on_hard_limit;
|
||||
CallbackFn on_soft_limit;
|
||||
std::mutex limit_mutex;
|
||||
size_t hard_limit TSA_GUARDED_BY(limit_mutex) = 0;
|
||||
size_t soft_limit TSA_GUARDED_BY(limit_mutex) = 0;
|
||||
OnMemoryLimitFn on_hard_limit TSA_GUARDED_BY(limit_mutex);
|
||||
OnMemoryLimitFn on_soft_limit TSA_GUARDED_BY(limit_mutex);
|
||||
|
||||
uint64_t last_usage = 0;
|
||||
std::mutex memory_amount_available_changed_mutex;
|
||||
OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed TSA_GUARDED_BY(memory_amount_available_changed_mutex);
|
||||
|
||||
uint64_t last_memory_usage = 0; /// how much memory does the process use
|
||||
uint64_t last_available_memory_amount; /// how much memory can the process use
|
||||
|
||||
/// Represents the cgroup virtual file that shows the memory consumption of the process's cgroup.
|
||||
struct File
|
||||
struct MemoryUsageFile
|
||||
{
|
||||
public:
|
||||
explicit File(LoggerPtr log_);
|
||||
~File();
|
||||
explicit MemoryUsageFile(LoggerPtr log_);
|
||||
~MemoryUsageFile();
|
||||
uint64_t readMemoryUsage() const;
|
||||
private:
|
||||
LoggerPtr log;
|
||||
@ -62,13 +76,11 @@ private:
|
||||
std::string file_name;
|
||||
};
|
||||
|
||||
File file;
|
||||
MemoryUsageFile memory_usage_file;
|
||||
|
||||
void startThread();
|
||||
void stopThread();
|
||||
|
||||
void runThread();
|
||||
void processMemoryUsage(uint64_t usage);
|
||||
|
||||
std::mutex thread_mutex;
|
||||
std::condition_variable cond;
|
||||
@ -79,13 +91,13 @@ private:
|
||||
#else
|
||||
class CgroupsMemoryUsageObserver
|
||||
{
|
||||
using OnMemoryAmountAvailableChangedFn = std::function<void()>;
|
||||
public:
|
||||
explicit CgroupsMemoryUsageObserver(std::chrono::seconds) {}
|
||||
|
||||
void setLimits(uint64_t, uint64_t) {}
|
||||
size_t readMemoryUsage() { return 0; }
|
||||
size_t getHardLimit() { return 0; }
|
||||
size_t getSoftLimit() { return 0; }
|
||||
void setMemoryUsageLimits(uint64_t, uint64_t) {}
|
||||
void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn) {}
|
||||
void startThread() {}
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
@ -47,7 +47,6 @@ public:
|
||||
HashMap<UInt16, Float64> map;
|
||||
};
|
||||
|
||||
public:
|
||||
using Map = HashMap<StringRef, Float64>;
|
||||
using Container = std::vector<Language>;
|
||||
|
||||
|
||||
@ -10,14 +10,15 @@
|
||||
#include <Common/MemoryTrackerSwitcher.h>
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
#include <Poco/Net/HTTPClientSession.h>
|
||||
#include <Poco/Net/HTTPStream.h>
|
||||
#include <Poco/Net/HTTPFixedLengthStream.h>
|
||||
#include <Poco/Net/HTTPChunkedStream.h>
|
||||
#include <Poco/Net/HTTPClientSession.h>
|
||||
#include <Poco/Net/HTTPFixedLengthStream.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Poco/Net/HTTPResponse.h>
|
||||
#include <Poco/Net/HTTPStream.h>
|
||||
#include <Poco/Timespan.h>
|
||||
|
||||
#include <Poco/Net/HTTPResponse.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <queue>
|
||||
|
||||
#include "config.h"
|
||||
|
||||
|
||||
@ -44,11 +44,12 @@ public:
|
||||
virtual const Metrics & getMetrics() const = 0;
|
||||
virtual ~IHTTPConnectionPoolForEndpoint() = default;
|
||||
|
||||
IHTTPConnectionPoolForEndpoint(const IHTTPConnectionPoolForEndpoint &) = delete;
|
||||
IHTTPConnectionPoolForEndpoint & operator=(const IHTTPConnectionPoolForEndpoint &) = delete;
|
||||
|
||||
protected:
|
||||
IHTTPConnectionPoolForEndpoint() = default;
|
||||
|
||||
IHTTPConnectionPoolForEndpoint(const IHTTPConnectionPoolForEndpoint &) = delete;
|
||||
IHTTPConnectionPoolForEndpoint & operator=(const IHTTPConnectionPoolForEndpoint &) = delete;
|
||||
};
|
||||
|
||||
enum class HTTPConnectionGroupType
|
||||
@ -70,11 +71,12 @@ public:
|
||||
static constexpr size_t warning_step = 100;
|
||||
};
|
||||
|
||||
private:
|
||||
HTTPConnectionPools();
|
||||
HTTPConnectionPools(const HTTPConnectionPools &) = delete;
|
||||
HTTPConnectionPools & operator=(const HTTPConnectionPools &) = delete;
|
||||
|
||||
private:
|
||||
HTTPConnectionPools();
|
||||
|
||||
public:
|
||||
static HTTPConnectionPools & instance();
|
||||
|
||||
|
||||
@ -109,6 +109,9 @@ public:
|
||||
|
||||
using Base::Base;
|
||||
|
||||
FixedHashMap() = default;
|
||||
FixedHashMap(size_t ) {} /// NOLINT
|
||||
|
||||
template <typename Func, bool>
|
||||
void ALWAYS_INLINE mergeToViaEmplace(Self & that, Func && func)
|
||||
{
|
||||
|
||||
@ -92,7 +92,8 @@ inline bool bitEquals(T && a, T && b)
|
||||
using RealT = std::decay_t<T>;
|
||||
|
||||
if constexpr (std::is_floating_point_v<RealT>)
|
||||
return 0 == memcmp(&a, &b, sizeof(RealT)); /// Note that memcmp with constant size is compiler builtin.
|
||||
/// Note that memcmp with constant size is compiler builtin.
|
||||
return 0 == memcmp(&a, &b, sizeof(RealT)); /// NOLINT
|
||||
else
|
||||
return a == b;
|
||||
}
|
||||
@ -644,7 +645,7 @@ protected:
|
||||
|
||||
/// Copy to a new location and zero the old one.
|
||||
x.setHash(hash_value);
|
||||
memcpy(static_cast<void*>(&buf[place_value]), &x, sizeof(x));
|
||||
memcpy(static_cast<void*>(&buf[place_value]), &x, sizeof(x)); /// NOLINT(bugprone-undefined-memory-manipulation)
|
||||
x.setZero();
|
||||
|
||||
/// Then the elements that previously were in collision with this can move to the old place.
|
||||
|
||||
@ -12,7 +12,7 @@ struct StringHashMapCell : public HashMapCell<Key, TMapped, StringHashTableHash,
|
||||
using Base::Base;
|
||||
static constexpr bool need_zero_value_storage = false;
|
||||
// external
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringView(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const Key & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
@ -32,7 +32,7 @@ struct StringHashMapCell<StringKey16, TMapped> : public HashMapCell<StringKey16,
|
||||
void setZero() { this->value.first.items[1] = 0; }
|
||||
|
||||
// external
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringView(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const StringKey16 & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
@ -53,7 +53,7 @@ struct StringHashMapCell<StringKey24, TMapped> : public HashMapCell<StringKey24,
|
||||
void setZero() { this->value.first.c = 0; }
|
||||
|
||||
// external
|
||||
StringRef getKey() const { return toStringRef(this->value.first); } /// NOLINT
|
||||
StringRef getKey() const { return toStringView(this->value.first); } /// NOLINT
|
||||
// internal
|
||||
static const StringKey24 & getKey(const value_type & value_) { return value_.first; }
|
||||
};
|
||||
|
||||
@ -19,7 +19,7 @@ struct StringKey24
|
||||
bool operator==(const StringKey24 rhs) const { return a == rhs.a && b == rhs.b && c == rhs.c; }
|
||||
};
|
||||
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
|
||||
inline StringRef ALWAYS_INLINE toStringView(const StringKey8 & n)
|
||||
{
|
||||
assert(n != 0);
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
@ -28,7 +28,7 @@ inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
|
||||
return {reinterpret_cast<const char *>(&n), 8ul - (std::countl_zero(n) >> 3)};
|
||||
#endif
|
||||
}
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
|
||||
inline StringRef ALWAYS_INLINE toStringView(const StringKey16 & n)
|
||||
{
|
||||
assert(n.items[1] != 0);
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
@ -37,7 +37,7 @@ inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
|
||||
return {reinterpret_cast<const char *>(&n), 16ul - (std::countl_zero(n.items[1]) >> 3)};
|
||||
#endif
|
||||
}
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n)
|
||||
inline StringRef ALWAYS_INLINE toStringView(const StringKey24 & n)
|
||||
{
|
||||
assert(n.c != 0);
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
|
||||
@ -38,6 +38,7 @@ public:
|
||||
Impl impls[NUM_BUCKETS];
|
||||
|
||||
TwoLevelStringHashTable() = default;
|
||||
TwoLevelStringHashTable(size_t ) {} /// NOLINT
|
||||
|
||||
template <typename Source>
|
||||
explicit TwoLevelStringHashTable(const Source & src)
|
||||
|
||||
@ -67,8 +67,8 @@ public:
|
||||
class Entry
|
||||
{
|
||||
public:
|
||||
explicit Entry(Entry && entry) = default;
|
||||
explicit Entry(Entry & entry) = delete;
|
||||
Entry(Entry && entry) = default;
|
||||
Entry(Entry & entry) = delete;
|
||||
|
||||
// no access as r-value
|
||||
const String * operator->() && = delete;
|
||||
@ -89,7 +89,7 @@ public:
|
||||
|
||||
Entry(HostResolver & pool_, Poco::Net::IPAddress address_)
|
||||
: pool(pool_.getWeakFromThis())
|
||||
, address(std::move(address_))
|
||||
, address(address_)
|
||||
, resolved_host(address.toString())
|
||||
{ }
|
||||
|
||||
@ -126,14 +126,14 @@ protected:
|
||||
struct Record
|
||||
{
|
||||
Record(Poco::Net::IPAddress address_, Poco::Timestamp resolve_time_)
|
||||
: address(std::move(address_))
|
||||
: address(address_)
|
||||
, resolve_time(resolve_time_)
|
||||
{}
|
||||
|
||||
explicit Record(Record && rec) = default;
|
||||
Record(Record && rec) = default;
|
||||
Record& operator=(Record && s) = default;
|
||||
|
||||
explicit Record(const Record & rec) = default;
|
||||
Record(const Record & rec) = default;
|
||||
Record& operator=(const Record & s) = default;
|
||||
|
||||
Poco::Net::IPAddress address;
|
||||
@ -198,10 +198,11 @@ class HostResolversPool
|
||||
{
|
||||
private:
|
||||
HostResolversPool() = default;
|
||||
|
||||
public:
|
||||
HostResolversPool(const HostResolversPool &) = delete;
|
||||
HostResolversPool & operator=(const HostResolversPool &) = delete;
|
||||
|
||||
public:
|
||||
static HostResolversPool & instance();
|
||||
|
||||
void dropCache();
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#include <Common/IntervalKind.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <base/EnumReflection.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -10,6 +12,11 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
std::string_view IntervalKind::toString() const
|
||||
{
|
||||
return magic_enum::enum_name(kind);
|
||||
}
|
||||
|
||||
Int64 IntervalKind::toAvgNanoseconds() const
|
||||
{
|
||||
static constexpr Int64 NANOSECONDS_PER_MICROSECOND = 1000;
|
||||
@ -240,7 +247,7 @@ const char * IntervalKind::toNameOfFunctionExtractTimePart() const
|
||||
return "toDayOfMonth";
|
||||
case IntervalKind::Kind::Week:
|
||||
// TODO: SELECT toRelativeWeekNum(toDate('2017-06-15')) - toRelativeWeekNum(toStartOfYear(toDate('2017-06-15')))
|
||||
// else if (ParserKeyword("WEEK").ignore(pos, expected))
|
||||
// else if (ParserKeyword(Keyword::WEEK).ignore(pos, expected))
|
||||
// function_name = "toRelativeWeekNum";
|
||||
throw Exception(ErrorCodes::SYNTAX_ERROR, "The syntax 'EXTRACT(WEEK FROM date)' is not supported, cannot extract the number of a week");
|
||||
case IntervalKind::Kind::Month:
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/types.h>
|
||||
#include <base/EnumReflection.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -27,7 +26,7 @@ struct IntervalKind
|
||||
IntervalKind(Kind kind_ = Kind::Second) : kind(kind_) {} /// NOLINT
|
||||
operator Kind() const { return kind; } /// NOLINT
|
||||
|
||||
constexpr std::string_view toString() const { return magic_enum::enum_name(kind); }
|
||||
std::string_view toString() const;
|
||||
|
||||
/// Returns number of nanoseconds in one interval.
|
||||
/// For `Month`, `Quarter` and `Year` the function returns an average number of nanoseconds.
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
*/
|
||||
|
||||
|
||||
namespace
|
||||
namespace impl
|
||||
{
|
||||
/// After the most significant bit 1, set all subsequent less significant bits to 1 as well.
|
||||
inline UInt64 toMask(UInt64 n)
|
||||
@ -85,7 +85,7 @@ void intervalBinaryPartition(UInt64 first, UInt64 last, F && callback)
|
||||
/// split = 15: 00001111
|
||||
|
||||
UInt64 diff = first ^ last;
|
||||
UInt64 mask = toMask(diff) >> 1;
|
||||
UInt64 mask = impl::toMask(diff) >> 1;
|
||||
|
||||
/// The current interval represents a whole range with fixed prefix.
|
||||
if ((first & mask) == 0 && (last & mask) == mask)
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
#include <Common/NamedCollections/NamedCollections.h>
|
||||
#include <Common/NamedCollections/NamedCollectionConfiguration.h>
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ public:
|
||||
}
|
||||
|
||||
NetException * clone() const override { return new NetException(*this); }
|
||||
void rethrow() const override { throw *this; }
|
||||
void rethrow() const override { throw *this; } /// NOLINT(cert-err60-cpp)
|
||||
|
||||
private:
|
||||
const char * name() const noexcept override { return "DB::NetException"; }
|
||||
|
||||
@ -301,6 +301,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
/// NOLINTBEGIN(bugprone-sizeof-expression)
|
||||
|
||||
template <typename T, size_t initial_bytes, typename TAllocator, size_t pad_right_, size_t pad_left_>
|
||||
class PODArray : public PODArrayBase<sizeof(T), initial_bytes, TAllocator, pad_right_, pad_left_>
|
||||
{
|
||||
@ -755,6 +757,8 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
/// NOLINTEND(bugprone-sizeof-expression)
|
||||
|
||||
template <typename T, size_t initial_bytes, typename TAllocator, size_t pad_right_, size_t pad_left_>
|
||||
void swap(PODArray<T, initial_bytes, TAllocator, pad_right_, pad_left_> & lhs, PODArray<T, initial_bytes, TAllocator, pad_right_, pad_left_> & rhs) /// NOLINT
|
||||
{
|
||||
|
||||
@ -50,7 +50,7 @@ struct ProxyConfiguration
|
||||
bool tunneling = false;
|
||||
Protocol original_request_protocol = Protocol::HTTP;
|
||||
|
||||
bool isEmpty() const { return host.size() == 0; }
|
||||
bool isEmpty() const { return host.empty(); }
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -12,6 +12,9 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ISchedulerNode;
|
||||
using SchedulerNodePtr = std::shared_ptr<ISchedulerNode>;
|
||||
|
||||
/*
|
||||
* Instance of derived class holds everything required for resource consumption,
|
||||
* including resources currently registered at `SchedulerRoot`. This is required to avoid
|
||||
|
||||
@ -85,7 +85,6 @@ private:
|
||||
StatePtr state; // hold state to avoid ResourceLink invalidation due to resource deregistration from SchedulerRoot
|
||||
};
|
||||
|
||||
private:
|
||||
SchedulerRoot scheduler;
|
||||
std::mutex mutex;
|
||||
StatePtr state;
|
||||
|
||||
@ -248,7 +248,6 @@ private:
|
||||
parent->activateChild(this);
|
||||
}
|
||||
|
||||
private:
|
||||
/// Beginning of `items` vector is heap of active children: [0; `heap_size`).
|
||||
/// Next go inactive children in unsorted order.
|
||||
/// NOTE: we have to track vruntime of inactive children for max-min fairness.
|
||||
|
||||
25
src/Common/Scheduler/ResouceLink.cpp
Normal file
25
src/Common/Scheduler/ResouceLink.cpp
Normal file
@ -0,0 +1,25 @@
|
||||
#include <Common/Scheduler/ISchedulerQueue.h>
|
||||
#include <Common/Scheduler/ResourceLink.h>
|
||||
#include <Common/Scheduler/ResourceRequest.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
void ResourceLink::adjust(ResourceCost estimated_cost, ResourceCost real_cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->adjustBudget(estimated_cost, real_cost);
|
||||
}
|
||||
|
||||
void ResourceLink::consumed(ResourceCost cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->consumeBudget(cost);
|
||||
}
|
||||
|
||||
void ResourceLink::accumulate(DB::ResourceCost cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->accumulateBudget(cost);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,9 +2,10 @@
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
#include <Common/Scheduler/ISchedulerConstraint.h>
|
||||
#include <Common/Scheduler/ISchedulerQueue.h>
|
||||
#include <Common/Scheduler/ResourceRequest.h>
|
||||
#include <Common/Scheduler/ResourceLink.h>
|
||||
#include <Common/Scheduler/ISchedulerConstraint.h>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
|
||||
@ -2,12 +2,10 @@
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
#include <Common/Scheduler/ResourceRequest.h>
|
||||
#include <Common/Scheduler/ISchedulerQueue.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class ISchedulerQueue;
|
||||
using ResourceCost = Int64;
|
||||
|
||||
/*
|
||||
* Everything required for resource consumption. Connection to a specific resource queue.
|
||||
@ -17,23 +15,11 @@ struct ResourceLink
|
||||
ISchedulerQueue * queue = nullptr;
|
||||
bool operator==(const ResourceLink &) const = default;
|
||||
|
||||
void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->adjustBudget(estimated_cost, real_cost);
|
||||
}
|
||||
void adjust(ResourceCost estimated_cost, ResourceCost real_cost) const;
|
||||
|
||||
void consumed(ResourceCost cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->consumeBudget(cost);
|
||||
}
|
||||
void consumed(ResourceCost cost) const;
|
||||
|
||||
void accumulate(ResourceCost cost) const
|
||||
{
|
||||
if (queue)
|
||||
queue->accumulateBudget(cost);
|
||||
}
|
||||
void accumulate(ResourceCost cost) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -231,7 +231,6 @@ private:
|
||||
value->next = nullptr;
|
||||
}
|
||||
|
||||
private:
|
||||
void schedulerThread()
|
||||
{
|
||||
while (!stop_flag.load())
|
||||
@ -253,7 +252,6 @@ private:
|
||||
request->execute();
|
||||
}
|
||||
|
||||
private:
|
||||
TResource * current = nullptr; // round-robin pointer
|
||||
std::unordered_map<ISchedulerNode *, TResource> children; // resources by pointer
|
||||
std::atomic<bool> stop_flag = false;
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
|
||||
#include <Common/SymbolIndex.h>
|
||||
#include <Common/MemorySanitizer.h>
|
||||
#include <base/hex.h>
|
||||
#include <base/sort.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
@ -11,8 +13,6 @@
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
#include <base/sort.h>
|
||||
|
||||
/**
|
||||
|
||||
ELF object can contain three different places with symbol names and addresses:
|
||||
|
||||
@ -86,12 +86,12 @@ static std::atomic<int> num_cpus = 0;
|
||||
static std::atomic<double> NAME##_before_yield_probability = 0; \
|
||||
static std::atomic<double> NAME##_before_migrate_probability = 0; \
|
||||
static std::atomic<double> NAME##_before_sleep_probability = 0; \
|
||||
static std::atomic<double> NAME##_before_sleep_time_us = 0; \
|
||||
static std::atomic<double> NAME##_before_sleep_time_us_max = 0; \
|
||||
\
|
||||
static std::atomic<double> NAME##_after_yield_probability = 0; \
|
||||
static std::atomic<double> NAME##_after_migrate_probability = 0; \
|
||||
static std::atomic<double> NAME##_after_sleep_probability = 0; \
|
||||
static std::atomic<double> NAME##_after_sleep_time_us = 0;
|
||||
static std::atomic<double> NAME##_after_sleep_time_us_max = 0;
|
||||
|
||||
FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
|
||||
|
||||
@ -110,7 +110,7 @@ void ThreadFuzzer::initConfiguration()
|
||||
initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY");
|
||||
initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY");
|
||||
initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY");
|
||||
initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US");
|
||||
initFromEnv(sleep_time_us_max, "THREAD_FUZZER_SLEEP_TIME_US_MAX");
|
||||
initFromEnv(explicit_sleep_probability, "THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY");
|
||||
initFromEnv(explicit_memory_exception_probability, "THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY");
|
||||
|
||||
@ -119,13 +119,12 @@ void ThreadFuzzer::initConfiguration()
|
||||
initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
|
||||
initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
|
||||
initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
|
||||
initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \
|
||||
initFromEnv(NAME##_before_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US_MAX"); \
|
||||
\
|
||||
initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
|
||||
initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
|
||||
initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
|
||||
initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
|
||||
|
||||
initFromEnv(NAME##_after_sleep_time_us_max, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US_MAX");
|
||||
FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
|
||||
|
||||
# undef INIT_WRAPPER_PARAMS
|
||||
@ -146,7 +145,7 @@ bool ThreadFuzzer::isEffective() const
|
||||
return true; \
|
||||
if (NAME##_before_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
|
||||
return true; \
|
||||
if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
|
||||
if (NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
|
||||
return true; \
|
||||
\
|
||||
if (NAME##_after_yield_probability.load(std::memory_order_relaxed) > 0.0) \
|
||||
@ -155,7 +154,7 @@ bool ThreadFuzzer::isEffective() const
|
||||
return true; \
|
||||
if (NAME##_after_sleep_probability.load(std::memory_order_relaxed) > 0.0) \
|
||||
return true; \
|
||||
if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed) > 0.0) \
|
||||
if (NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed) > 0.0) \
|
||||
return true;
|
||||
|
||||
FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
|
||||
@ -166,7 +165,7 @@ bool ThreadFuzzer::isEffective() const
|
||||
return cpu_time_period_us != 0
|
||||
&& (yield_probability > 0
|
||||
|| migrate_probability > 0
|
||||
|| (sleep_probability > 0 && sleep_time_us > 0));
|
||||
|| (sleep_probability > 0 && sleep_time_us_max > 0));
|
||||
}
|
||||
|
||||
void ThreadFuzzer::stop()
|
||||
@ -190,7 +189,7 @@ static void injectionImpl(
|
||||
double yield_probability,
|
||||
double migrate_probability,
|
||||
double sleep_probability,
|
||||
double sleep_time_us)
|
||||
double sleep_time_us_max)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
if (!ThreadFuzzer::isStarted())
|
||||
@ -221,10 +220,10 @@ static void injectionImpl(
|
||||
#endif
|
||||
|
||||
if (sleep_probability > 0
|
||||
&& sleep_time_us > 0
|
||||
&& sleep_time_us_max > 0
|
||||
&& std::bernoulli_distribution(sleep_probability)(thread_local_rng))
|
||||
{
|
||||
sleepForNanoseconds(static_cast<uint64_t>(sleep_time_us * 1000));
|
||||
sleepForNanoseconds((thread_local_rng() % static_cast<uint64_t>(sleep_time_us_max)) * 1000); /*may sleep(0)*/
|
||||
}
|
||||
}
|
||||
|
||||
@ -232,19 +231,19 @@ static ALWAYS_INLINE void injection(
|
||||
double yield_probability,
|
||||
double migrate_probability,
|
||||
double sleep_probability,
|
||||
double sleep_time_us)
|
||||
double sleep_time_us_max)
|
||||
{
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
if (!ThreadFuzzer::isStarted())
|
||||
return;
|
||||
|
||||
injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us);
|
||||
injectionImpl(yield_probability, migrate_probability, sleep_probability, sleep_time_us_max);
|
||||
}
|
||||
|
||||
void ThreadFuzzer::maybeInjectSleep()
|
||||
{
|
||||
auto & fuzzer = ThreadFuzzer::instance();
|
||||
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us);
|
||||
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.explicit_sleep_probability, fuzzer.sleep_time_us_max);
|
||||
}
|
||||
|
||||
/// Sometimes maybeInjectSleep() is not enough and we need to inject an exception.
|
||||
@ -265,7 +264,7 @@ void ThreadFuzzer::signalHandler(int)
|
||||
DENY_ALLOCATIONS_IN_SCOPE;
|
||||
auto saved_errno = errno;
|
||||
auto & fuzzer = ThreadFuzzer::instance();
|
||||
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us);
|
||||
injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us_max);
|
||||
errno = saved_errno;
|
||||
}
|
||||
|
||||
@ -309,13 +308,13 @@ void ThreadFuzzer::setup() const
|
||||
NAME##_before_yield_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_before_migrate_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_before_sleep_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_before_sleep_time_us.load(std::memory_order_relaxed));
|
||||
NAME##_before_sleep_time_us_max.load(std::memory_order_relaxed));
|
||||
#define INJECTION_AFTER(NAME) \
|
||||
injectionImpl( \
|
||||
NAME##_after_yield_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_after_migrate_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_after_sleep_probability.load(std::memory_order_relaxed), \
|
||||
NAME##_after_sleep_time_us.load(std::memory_order_relaxed));
|
||||
NAME##_after_sleep_time_us_max.load(std::memory_order_relaxed));
|
||||
|
||||
/// ThreadFuzzer intercepts pthread_mutex_lock()/pthread_mutex_unlock().
|
||||
///
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user