mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 15:42:02 +00:00
Merge branch 'master' into add-more-s3-tests
This commit is contained in:
mergify[bot]
GitHub
commit
89c36a4eca
292
.clang-tidy
292
.clang-tidy
@ -1,172 +1,148 @@
|
||||
Checks: '-*,
|
||||
misc-misplaced-const,
|
||||
misc-redundant-expression,
|
||||
misc-static-assert,
|
||||
misc-throw-by-value-catch-by-reference,
|
||||
misc-unconventional-assign-operator,
|
||||
misc-uniqueptr-reset-release,
|
||||
misc-unused-alias-decls,
|
||||
misc-unused-parameters,
|
||||
misc-unused-using-decls,
|
||||
Checks: '*,
|
||||
-abseil-*,
|
||||
|
||||
modernize-avoid-bind,
|
||||
modernize-loop-convert,
|
||||
modernize-make-shared,
|
||||
modernize-make-unique,
|
||||
modernize-raw-string-literal,
|
||||
modernize-redundant-void-arg,
|
||||
modernize-replace-random-shuffle,
|
||||
modernize-use-bool-literals,
|
||||
modernize-use-equals-default,
|
||||
modernize-use-equals-delete,
|
||||
modernize-use-nullptr,
|
||||
modernize-use-using,
|
||||
-altera-*,
|
||||
|
||||
performance-faster-string-find,
|
||||
performance-for-range-copy,
|
||||
performance-implicit-conversion-in-loop,
|
||||
performance-inefficient-algorithm,
|
||||
performance-inefficient-vector-operation,
|
||||
performance-move-const-arg,
|
||||
performance-move-constructor-init,
|
||||
performance-no-automatic-move,
|
||||
performance-noexcept-move-constructor,
|
||||
performance-trivially-destructible,
|
||||
performance-unnecessary-copy-initialization,
|
||||
-android-*,
|
||||
|
||||
readability-avoid-const-params-in-decls,
|
||||
readability-const-return-type,
|
||||
readability-container-contains,
|
||||
readability-container-size-empty,
|
||||
readability-convert-member-functions-to-static,
|
||||
readability-delete-null-pointer,
|
||||
readability-deleted-default,
|
||||
readability-identifier-naming,
|
||||
readability-inconsistent-declaration-parameter-name,
|
||||
readability-make-member-function-const,
|
||||
readability-misplaced-array-index,
|
||||
readability-non-const-parameter,
|
||||
readability-qualified-auto,
|
||||
readability-redundant-access-specifiers,
|
||||
readability-redundant-control-flow,
|
||||
readability-redundant-function-ptr-dereference,
|
||||
readability-redundant-member-init,
|
||||
readability-redundant-smartptr-get,
|
||||
readability-redundant-string-cstr,
|
||||
readability-redundant-string-init,
|
||||
readability-simplify-boolean-expr,
|
||||
readability-simplify-subscript-expr,
|
||||
readability-static-definition-in-anonymous-namespace,
|
||||
readability-string-compare,
|
||||
readability-uniqueptr-delete-release,
|
||||
-bugprone-assert-side-effect,
|
||||
-bugprone-branch-clone,
|
||||
-bugprone-dynamic-static-initializers,
|
||||
-bugprone-easily-swappable-parameters,
|
||||
-bugprone-exception-escape,
|
||||
-bugprone-forwarding-reference-overload,
|
||||
-bugprone-implicit-widening-of-multiplication-result,
|
||||
-bugprone-lambda-function-name,
|
||||
-bugprone-misplaced-widening-cast,
|
||||
-bugprone-narrowing-conversions,
|
||||
-bugprone-no-escape,
|
||||
-bugprone-not-null-terminated-result,
|
||||
-bugprone-signal-handler,
|
||||
-bugprone-spuriously-wake-up-functions,
|
||||
-bugprone-suspicious-semicolon,
|
||||
-bugprone-unhandled-exception-at-new,
|
||||
-bugprone-unhandled-self-assignment,
|
||||
|
||||
bugprone-argument-comment,
|
||||
bugprone-bad-signal-to-kill-thread,
|
||||
bugprone-bool-pointer-implicit-conversion,
|
||||
bugprone-copy-constructor-init,
|
||||
bugprone-dangling-handle,
|
||||
bugprone-fold-init-type,
|
||||
bugprone-forward-declaration-namespace,
|
||||
bugprone-inaccurate-erase,
|
||||
bugprone-incorrect-roundings,
|
||||
bugprone-infinite-loop,
|
||||
bugprone-integer-division,
|
||||
bugprone-macro-parentheses,
|
||||
bugprone-macro-repeated-side-effects,
|
||||
bugprone-misplaced-operator-in-strlen-in-alloc,
|
||||
bugprone-misplaced-pointer-artithmetic-in-alloc,
|
||||
bugprone-misplaced-widening-cast,
|
||||
bugprone-move-forwarding-reference,
|
||||
bugprone-multiple-statement-macro,
|
||||
bugprone-parent-virtual-call,
|
||||
bugprone-posix-return,
|
||||
bugprone-reserved-identifier,
|
||||
bugprone-signed-char-misuse,
|
||||
bugprone-sizeof-container,
|
||||
bugprone-sizeof-expression,
|
||||
bugprone-string-constructor,
|
||||
bugprone-string-integer-assignment,
|
||||
bugprone-string-literal-with-embedded-nul,
|
||||
bugprone-suspicious-enum-usage,
|
||||
bugprone-suspicious-include,
|
||||
bugprone-suspicious-memset-usage,
|
||||
bugprone-suspicious-missing-comma,
|
||||
bugprone-suspicious-string-compare,
|
||||
bugprone-swapped-arguments,
|
||||
bugprone-terminating-continue,
|
||||
bugprone-throw-keyword-missing,
|
||||
bugprone-too-small-loop-variable,
|
||||
bugprone-undefined-memory-manipulation,
|
||||
bugprone-undelegated-constructor,
|
||||
bugprone-unhandled-self-assignment,
|
||||
bugprone-unused-raii,
|
||||
bugprone-unused-return-value,
|
||||
bugprone-use-after-move,
|
||||
bugprone-virtual-near-miss,
|
||||
-cert-dcl16-c,
|
||||
-cert-dcl37-c,
|
||||
-cert-dcl51-cpp,
|
||||
-cert-dcl58-cpp,
|
||||
-cert-err58-cpp,
|
||||
-cert-err60-cpp,
|
||||
-cert-msc32-c,
|
||||
-cert-msc51-cpp,
|
||||
-cert-oop54-cpp,
|
||||
-cert-oop57-cpp,
|
||||
-cert-oop58-cpp,
|
||||
|
||||
cert-dcl21-cpp,
|
||||
cert-dcl50-cpp,
|
||||
cert-env33-c,
|
||||
cert-err34-c,
|
||||
cert-err52-cpp,
|
||||
cert-flp30-c,
|
||||
cert-mem57-cpp,
|
||||
cert-msc50-cpp,
|
||||
cert-oop58-cpp,
|
||||
-clang-analyzer-core.DynamicTypePropagation,
|
||||
-clang-analyzer-core.uninitialized.CapturedBlockVariable,
|
||||
|
||||
google-build-explicit-make-pair,
|
||||
google-build-namespaces,
|
||||
google-default-arguments,
|
||||
google-explicit-constructor,
|
||||
google-readability-avoid-underscore-in-googletest-name,
|
||||
google-readability-casting,
|
||||
google-runtime-int,
|
||||
google-runtime-operator,
|
||||
-clang-analyzer-optin.performance.Padding,
|
||||
-clang-analyzer-optin.portability.UnixAPI,
|
||||
|
||||
hicpp-exception-baseclass,
|
||||
-clang-analyzer-security.insecureAPI.bzero,
|
||||
-clang-analyzer-security.insecureAPI.strcpy,
|
||||
|
||||
clang-analyzer-core.CallAndMessage,
|
||||
clang-analyzer-core.DivideZero,
|
||||
clang-analyzer-core.NonNullParamChecker,
|
||||
clang-analyzer-core.NullDereference,
|
||||
clang-analyzer-core.StackAddressEscape,
|
||||
clang-analyzer-core.UndefinedBinaryOperatorResult,
|
||||
clang-analyzer-core.VLASize,
|
||||
clang-analyzer-core.uninitialized.ArraySubscript,
|
||||
clang-analyzer-core.uninitialized.Assign,
|
||||
clang-analyzer-core.uninitialized.Branch,
|
||||
clang-analyzer-core.uninitialized.CapturedBlockVariable,
|
||||
clang-analyzer-core.uninitialized.UndefReturn,
|
||||
clang-analyzer-cplusplus.InnerPointer,
|
||||
clang-analyzer-cplusplus.Move,
|
||||
clang-analyzer-cplusplus.NewDelete,
|
||||
clang-analyzer-cplusplus.NewDeleteLeaks,
|
||||
clang-analyzer-cplusplus.PlacementNewChecker,
|
||||
clang-analyzer-cplusplus.SelfAssignment,
|
||||
clang-analyzer-deadcode.DeadStores,
|
||||
clang-analyzer-optin.cplusplus.UninitializedObject,
|
||||
clang-analyzer-optin.cplusplus.VirtualCall,
|
||||
clang-analyzer-security.insecureAPI.UncheckedReturn,
|
||||
clang-analyzer-security.insecureAPI.bcmp,
|
||||
clang-analyzer-security.insecureAPI.bcopy,
|
||||
clang-analyzer-security.insecureAPI.bzero,
|
||||
clang-analyzer-security.insecureAPI.getpw,
|
||||
clang-analyzer-security.insecureAPI.gets,
|
||||
clang-analyzer-security.insecureAPI.mkstemp,
|
||||
clang-analyzer-security.insecureAPI.mktemp,
|
||||
clang-analyzer-security.insecureAPI.rand,
|
||||
clang-analyzer-security.insecureAPI.strcpy,
|
||||
clang-analyzer-unix.Malloc,
|
||||
clang-analyzer-unix.MallocSizeof,
|
||||
clang-analyzer-unix.MismatchedDeallocator,
|
||||
clang-analyzer-unix.Vfork,
|
||||
clang-analyzer-unix.cstring.BadSizeArg,
|
||||
clang-analyzer-unix.cstring.NullArg,
|
||||
-cppcoreguidelines-*,
|
||||
|
||||
boost-use-to-string,
|
||||
-concurrency-mt-unsafe,
|
||||
|
||||
alpha.security.cert.env.InvalidPtr,
|
||||
-darwin-*,
|
||||
|
||||
-fuchsia-*,
|
||||
|
||||
-google-build-using-namespace,
|
||||
-google-global-names-in-headers,
|
||||
-google-readability-braces-around-statements,
|
||||
-google-readability-function-size,
|
||||
-google-readability-namespace-comments,
|
||||
-google-readability-todo,
|
||||
-google-upgrade-googletest-case,
|
||||
|
||||
-hicpp-avoid-c-arrays,
|
||||
-hicpp-avoid-goto,
|
||||
-hicpp-braces-around-statements,
|
||||
-hicpp-deprecated-headers,
|
||||
-hicpp-explicit-conversions,
|
||||
-hicpp-function-size,
|
||||
-hicpp-invalid-access-moved,
|
||||
-hicpp-member-init,
|
||||
-hicpp-move-const-arg,
|
||||
-hicpp-multiway-paths-covered,
|
||||
-hicpp-named-parameter,
|
||||
-hicpp-no-array-decay,
|
||||
-hicpp-no-assembler,
|
||||
-hicpp-no-malloc,
|
||||
-hicpp-signed-bitwise,
|
||||
-hicpp-special-member-functions,
|
||||
-hicpp-uppercase-literal-suffix,
|
||||
-hicpp-use-auto,
|
||||
-hicpp-use-emplace,
|
||||
-hicpp-use-equals-default,
|
||||
-hicpp-use-noexcept,
|
||||
-hicpp-use-override,
|
||||
-hicpp-vararg,
|
||||
|
||||
-llvm-*,
|
||||
|
||||
-llvmlibc-*,
|
||||
|
||||
-openmp-*,
|
||||
|
||||
-misc-definitions-in-headers,
|
||||
-misc-new-delete-overloads,
|
||||
-misc-no-recursion,
|
||||
-misc-non-copyable-objects,
|
||||
-misc-non-private-member-variables-in-classes,
|
||||
-misc-static-assert,
|
||||
|
||||
-modernize-avoid-c-arrays,
|
||||
-modernize-concat-nested-namespaces,
|
||||
-modernize-deprecated-headers,
|
||||
-modernize-deprecated-ios-base-aliases,
|
||||
-modernize-pass-by-value,
|
||||
-modernize-replace-auto-ptr,
|
||||
-modernize-replace-disallow-copy-and-assign-macro,
|
||||
-modernize-return-braced-init-list,
|
||||
-modernize-unary-static-assert,
|
||||
-modernize-use-auto,
|
||||
-modernize-use-default-member-init,
|
||||
-modernize-use-emplace,
|
||||
-modernize-use-equals-default,
|
||||
-modernize-use-nodiscard,
|
||||
-modernize-use-noexcept,
|
||||
-modernize-use-override,
|
||||
-modernize-use-trailing-return-type,
|
||||
|
||||
-performance-inefficient-string-concatenation,
|
||||
-performance-no-int-to-ptr,
|
||||
-performance-type-promotion-in-math-fn,
|
||||
-performance-trivially-destructible,
|
||||
-performance-unnecessary-value-param,
|
||||
|
||||
-portability-simd-intrinsics,
|
||||
|
||||
-readability-convert-member-functions-to-static,
|
||||
-readability-braces-around-statements,
|
||||
-readability-else-after-return,
|
||||
-readability-function-cognitive-complexity,
|
||||
-readability-function-size,
|
||||
-readability-implicit-bool-conversion,
|
||||
-readability-isolate-declaration,
|
||||
-readability-magic-numbers,
|
||||
-readability-misleading-indentation,
|
||||
-readability-named-parameter,
|
||||
-readability-qualified-auto,
|
||||
-readability-redundant-declaration,
|
||||
-readability-static-accessed-through-instance,
|
||||
-readability-suspicious-call-argument,
|
||||
-readability-uppercase-literal-suffix,
|
||||
-readability-use-anyofallof,
|
||||
|
||||
-zirkon-*,
|
||||
'
|
||||
|
||||
WarningsAsErrors: '*'
|
||||
|
||||
CheckOptions:
|
||||
|
||||
1
.github/workflows/docs_check.yml
vendored
1
.github/workflows/docs_check.yml
vendored
@ -1,4 +1,3 @@
|
||||
---
|
||||
name: DocsCheck
|
||||
|
||||
env:
|
||||
|
||||
@ -25,7 +25,6 @@
|
||||
* Introduce format `ProtobufList` (all records as repeated messages in out Protobuf). Closes [#16436](https://github.com/ClickHouse/ClickHouse/issues/16436). [#35152](https://github.com/ClickHouse/ClickHouse/pull/35152) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Add `h3PointDistM`, `h3PointDistKm`, `h3PointDistRads`, `h3GetRes0Indexes`, `h3GetPentagonIndexes` functions. [#34568](https://github.com/ClickHouse/ClickHouse/pull/34568) ([Bharat Nallan](https://github.com/bharatnc)).
|
||||
* Add `toLastDayOfMonth` function which rounds up a date or date with time to the last day of the month. [#33501](https://github.com/ClickHouse/ClickHouse/issues/33501). [#34394](https://github.com/ClickHouse/ClickHouse/pull/34394) ([Habibullah Oladepo](https://github.com/holadepo)).
|
||||
* New aggregation function groupSortedArray to obtain an array of first N values. [#34055](https://github.com/ClickHouse/ClickHouse/pull/34055) ([palegre-tiny](https://github.com/palegre-tiny)).
|
||||
* Added load balancing setting for \[Zoo\]Keeper client. Closes [#29617](https://github.com/ClickHouse/ClickHouse/issues/29617). [#30325](https://github.com/ClickHouse/ClickHouse/pull/30325) ([小路](https://github.com/nicelulu)).
|
||||
* Add a new kind of row policies named `simple`. Before this PR we had two kinds or row policies: `permissive` and `restrictive`. A `simple` row policy adds a new filter on a table without any side-effects like it was for permissive and restrictive policies. [#35345](https://github.com/ClickHouse/ClickHouse/pull/35345) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Added an ability to specify cluster secret in replicated database. [#35333](https://github.com/ClickHouse/ClickHouse/pull/35333) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
@ -19,6 +19,33 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
|
||||
endif ()
|
||||
endif()
|
||||
|
||||
# Check if environment is polluted.
|
||||
if (NOT $ENV{CFLAGS} STREQUAL ""
|
||||
OR NOT $ENV{CXXFLAGS} STREQUAL ""
|
||||
OR NOT $ENV{LDFLAGS} STREQUAL ""
|
||||
OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_SHARED_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
|
||||
OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_SHARED_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
|
||||
|
||||
message("CFLAGS: $ENV{CFLAGS}")
|
||||
message("CXXFLAGS: $ENV{CXXFLAGS}")
|
||||
message("LDFLAGS: $ENV{LDFLAGS}")
|
||||
message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}")
|
||||
message("CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
|
||||
message("CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}")
|
||||
message("CMAKE_SHARED_LINKER_FLAGS: ${CMAKE_SHARED_LINKER_FLAGS}")
|
||||
message("CMAKE_MODULE_LINKER_FLAGS: ${CMAKE_MODULE_LINKER_FLAGS}")
|
||||
|
||||
message(FATAL_ERROR "
|
||||
Some of the variables like CFLAGS, CXXFLAGS, LDFLAGS are not empty.
|
||||
It is not possible to build ClickHouse with custom flags.
|
||||
These variables can be set up by previous invocation of some other build tools.
|
||||
You should cleanup these variables and start over again.
|
||||
|
||||
Run the `env` command to check the details.
|
||||
You will also need to remove the contents of the build directory.
|
||||
|
||||
Note: if you don't like this behavior, you can manually edit the cmake files, but please don't complain to developers.")
|
||||
endif()
|
||||
|
||||
# Default toolchain - this is needed to avoid dependency on OS files.
|
||||
execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
|
||||
|
||||
@ -3,9 +3,6 @@ if (USE_CLANG_TIDY)
|
||||
endif ()
|
||||
|
||||
add_subdirectory (base)
|
||||
add_subdirectory (daemon)
|
||||
add_subdirectory (loggers)
|
||||
add_subdirectory (pcg-random)
|
||||
add_subdirectory (widechar_width)
|
||||
add_subdirectory (readpassphrase)
|
||||
add_subdirectory (bridge)
|
||||
|
||||
@ -37,7 +37,8 @@ struct StringRef
|
||||
size_t size = 0;
|
||||
|
||||
/// Non-constexpr due to reinterpret_cast.
|
||||
template <typename CharT, typename = std::enable_if_t<sizeof(CharT) == 1>>
|
||||
template <typename CharT>
|
||||
requires (sizeof(CharT) == 1)
|
||||
StringRef(const CharT * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_)
|
||||
{
|
||||
/// Sanity check for overflowed values.
|
||||
|
||||
@ -21,10 +21,12 @@ public:
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr basic_scope_guard(basic_scope_guard<G> && src) : function{src.release()} {}
|
||||
|
||||
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr basic_scope_guard & operator=(basic_scope_guard<G> && src)
|
||||
{
|
||||
if (this != &src)
|
||||
@ -35,10 +37,12 @@ public:
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr basic_scope_guard(const G & function_) : function{function_} {}
|
||||
|
||||
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
constexpr basic_scope_guard(G && function_) : function{std::move(function_)} {}
|
||||
|
||||
~basic_scope_guard() { invoke(); }
|
||||
@ -64,7 +68,8 @@ public:
|
||||
return std::exchange(function, {});
|
||||
}
|
||||
|
||||
template <typename G, typename = std::enable_if_t<std::is_convertible_v<G, F>, void>>
|
||||
template <typename G>
|
||||
requires std::is_convertible_v<G, F>
|
||||
basic_scope_guard<F> & join(basic_scope_guard<G> && other)
|
||||
{
|
||||
if (other.function)
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
|
||||
/** Allows to make std::shared_ptr from T with protected constructor.
|
||||
*
|
||||
* Derive your T class from shared_ptr_helper<T> and add shared_ptr_helper<T> as a friend
|
||||
* and you will have static 'create' method in your class.
|
||||
*/
|
||||
template <typename T>
|
||||
struct shared_ptr_helper
|
||||
{
|
||||
template <typename... TArgs>
|
||||
static std::shared_ptr<T> create(TArgs &&... args)
|
||||
{
|
||||
return std::shared_ptr<T>(new T(std::forward<TArgs>(args)...));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename T>
|
||||
struct is_shared_ptr
|
||||
{
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
|
||||
template <typename T>
|
||||
struct is_shared_ptr<std::shared_ptr<T>>
|
||||
{
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_shared_ptr_v = is_shared_ptr<T>::value;
|
||||
@ -1,13 +0,0 @@
|
||||
add_library (bridge
|
||||
IBridge.cpp
|
||||
)
|
||||
|
||||
target_include_directories (daemon PUBLIC ..)
|
||||
target_link_libraries (bridge
|
||||
PRIVATE
|
||||
daemon
|
||||
dbms
|
||||
Poco::Data
|
||||
Poco::Data::ODBC
|
||||
)
|
||||
|
||||
@ -158,12 +158,8 @@ elseif (COMPILER_GCC)
|
||||
add_cxx_compile_options(-Wsizeof-array-argument)
|
||||
# Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof
|
||||
add_cxx_compile_options(-Wsizeof-pointer-memaccess)
|
||||
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9)
|
||||
# Warn about overriding virtual functions that are not marked with the override keyword
|
||||
add_cxx_compile_options(-Wsuggest-override)
|
||||
endif ()
|
||||
|
||||
# Warn about overriding virtual functions that are not marked with the override keyword
|
||||
add_cxx_compile_options(-Wsuggest-override)
|
||||
# Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type
|
||||
add_cxx_compile_options(-Wswitch-bool)
|
||||
# Warn if a self-comparison always evaluates to true or false
|
||||
@ -178,41 +174,36 @@ elseif (COMPILER_GCC)
|
||||
# Warn when a literal 0 is used as null pointer constant.
|
||||
add_cxx_compile_options(-Wzero-as-null-pointer-constant)
|
||||
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
|
||||
# XXX: gcc10 stuck with this option while compiling GatherUtils code
|
||||
# (anyway there are builds with clang, that will warn)
|
||||
add_cxx_compile_options(-Wno-sequence-point)
|
||||
# XXX: gcc10 false positive with this warning in MergeTreePartition.cpp
|
||||
# inlined from 'void writeHexByteLowercase(UInt8, void*)' at ../src/Common/hex.h:39:11,
|
||||
# inlined from 'DB::String DB::MergeTreePartition::getID(const DB::Block&) const' at ../src/Storages/MergeTree/MergeTreePartition.cpp:85:30:
|
||||
# ../contrib/libc-headers/x86_64-linux-gnu/bits/string_fortified.h:34:33: error: writing 2 bytes into a region of size 0 [-Werror=stringop-overflow=]
|
||||
# 34 | return __builtin___memcpy_chk (__dest, __src, __len, __bos0 (__dest));
|
||||
# For some reason (bug in gcc?) macro 'GCC diagnostic ignored "-Wstringop-overflow"' doesn't help.
|
||||
add_cxx_compile_options(-Wno-stringop-overflow)
|
||||
endif()
|
||||
# The following warnings are generally useful but had to be disabled because of compiler bugs with older GCCs.
|
||||
# XXX: We should try again on more recent GCCs (--> see CMake variable GCC_MINIMUM_VERSION).
|
||||
|
||||
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11)
|
||||
# reinterpretAs.cpp:182:31: error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type
|
||||
# ‘using ToFieldType = using FieldType = using UUID = struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’
|
||||
# {aka ‘struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’} from an array of ‘const char8_t’
|
||||
add_cxx_compile_options(-Wno-error=class-memaccess)
|
||||
|
||||
# Maybe false positive...
|
||||
# In file included from /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:673,
|
||||
# In function ‘void std::__1::__libcpp_operator_delete(_Args ...) [with _Args = {void*, long unsigned int}]’,
|
||||
# inlined from ‘void std::__1::__do_deallocate_handle_size(void*, size_t, _Args ...) [with _Args = {}]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:271:34,
|
||||
# inlined from ‘void std::__1::__libcpp_deallocate(void*, size_t, size_t)’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:285:41,
|
||||
# inlined from ‘constexpr void std::__1::allocator<_Tp>::deallocate(_Tp*, size_t) [with _Tp = char]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:849:39,
|
||||
# inlined from ‘static constexpr void std::__1::allocator_traits<_Alloc>::deallocate(std::__1::allocator_traits<_Alloc>::allocator_type&, std::__1::allocator_traits<_Alloc>::pointer, std::__1::allocator_traits<_Alloc>::size_type) [with _Alloc = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/__memory/allocator_traits.h:476:24,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2219:35,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2213:1,
|
||||
# inlined from ‘DB::JSONBuilder::JSONMap::Pair::~Pair()’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:90:12,
|
||||
# inlined from ‘void DB::JSONBuilder::JSONMap::add(std::__1::string, DB::JSONBuilder::ItemPtr)’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:97:68,
|
||||
# inlined from ‘virtual void DB::ExpressionStep::describeActions(DB::JSONBuilder::JSONMap&) const’ at /home/jakalletti/ClickHouse/ClickHouse/src/Processors/QueryPlan/ExpressionStep.cpp:102:12:
|
||||
# /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:247:20: error: ‘void operator delete(void*, size_t)’ called on a pointer to an unallocated object ‘7598543875853023301’ [-Werror=free-nonheap-object]
|
||||
add_cxx_compile_options(-Wno-error=free-nonheap-object)
|
||||
|
||||
# AggregateFunctionAvg.h:203:100: error: ‘this’ pointer is null [-Werror=nonnull]
|
||||
add_cxx_compile_options(-Wno-error=nonnull)
|
||||
endif()
|
||||
# gcc10 stuck with this option while compiling GatherUtils code, anyway there are builds with clang that will warn
|
||||
add_cxx_compile_options(-Wno-sequence-point)
|
||||
# gcc10 false positive with this warning in MergeTreePartition.cpp
|
||||
# inlined from 'void writeHexByteLowercase(UInt8, void*)' at ../src/Common/hex.h:39:11,
|
||||
# inlined from 'DB::String DB::MergeTreePartition::getID(const DB::Block&) const' at ../src/Storages/MergeTree/MergeTreePartition.cpp:85:30:
|
||||
# ../contrib/libc-headers/x86_64-linux-gnu/bits/string_fortified.h:34:33: error: writing 2 bytes into a region of size 0 [-Werror=stringop-overflow=]
|
||||
# 34 | return __builtin___memcpy_chk (__dest, __src, __len, __bos0 (__dest));
|
||||
# For some reason (bug in gcc?) macro 'GCC diagnostic ignored "-Wstringop-overflow"' doesn't help.
|
||||
add_cxx_compile_options(-Wno-stringop-overflow)
|
||||
# reinterpretAs.cpp:182:31: error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type
|
||||
# ‘using ToFieldType = using FieldType = using UUID = struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’
|
||||
# {aka ‘struct StrongTypedef<wide::integer<128, unsigned int>, DB::UUIDTag>’} from an array of ‘const char8_t’
|
||||
add_cxx_compile_options(-Wno-error=class-memaccess)
|
||||
# Maybe false positive...
|
||||
# In file included from /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:673,
|
||||
# In function ‘void std::__1::__libcpp_operator_delete(_Args ...) [with _Args = {void*, long unsigned int}]’,
|
||||
# inlined from ‘void std::__1::__do_deallocate_handle_size(void*, size_t, _Args ...) [with _Args = {}]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:271:34,
|
||||
# inlined from ‘void std::__1::__libcpp_deallocate(void*, size_t, size_t)’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:285:41,
|
||||
# inlined from ‘constexpr void std::__1::allocator<_Tp>::deallocate(_Tp*, size_t) [with _Tp = char]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:849:39,
|
||||
# inlined from ‘static constexpr void std::__1::allocator_traits<_Alloc>::deallocate(std::__1::allocator_traits<_Alloc>::allocator_type&, std::__1::allocator_traits<_Alloc>::pointer, std::__1::allocator_traits<_Alloc>::size_type) [with _Alloc = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/__memory/allocator_traits.h:476:24,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2219:35,
|
||||
# inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits<char>; _Allocator = std::__1::allocator<char>]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2213:1,
|
||||
# inlined from ‘DB::JSONBuilder::JSONMap::Pair::~Pair()’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:90:12,
|
||||
# inlined from ‘void DB::JSONBuilder::JSONMap::add(std::__1::string, DB::JSONBuilder::ItemPtr)’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:97:68,
|
||||
# inlined from ‘virtual void DB::ExpressionStep::describeActions(DB::JSONBuilder::JSONMap&) const’ at /home/jakalletti/ClickHouse/ClickHouse/src/Processors/QueryPlan/ExpressionStep.cpp:102:12:
|
||||
# /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:247:20: error: ‘void operator delete(void*, size_t)’ called on a pointer to an unallocated object ‘7598543875853023301’ [-Werror=free-nonheap-object]
|
||||
add_cxx_compile_options(-Wno-error=free-nonheap-object)
|
||||
# AggregateFunctionAvg.h:203:100: error: ‘this’ pointer is null [-Werror=nonnull]
|
||||
add_cxx_compile_options(-Wno-error=nonnull)
|
||||
endif ()
|
||||
|
||||
2
contrib/minizip-ng
vendored
2
contrib/minizip-ng
vendored
@ -1 +1 @@
|
||||
Subproject commit 6cffc951851620e0fac1993be75e4713c334de03
|
||||
Subproject commit f3d400e999056ca290998b3fd89cc5a74e4b8b58
|
||||
2
contrib/poco
vendored
2
contrib/poco
vendored
@ -1 +1 @@
|
||||
Subproject commit 008b16469471d55b176db181756c94e3f14dd2dc
|
||||
Subproject commit 6c1a233744d13414e8e8db396c75177b857b2c22
|
||||
@ -256,20 +256,6 @@ for conn_index, c in enumerate(all_connections):
|
||||
|
||||
reportStageEnd("settings")
|
||||
|
||||
# Check tables that should exist. If they don't exist, just skip this test.
|
||||
tables = [e.text for e in root.findall("preconditions/table_exists")]
|
||||
for t in tables:
|
||||
for c in all_connections:
|
||||
try:
|
||||
res = c.execute("select 1 from {} limit 1".format(t))
|
||||
except:
|
||||
exception_message = traceback.format_exception_only(*sys.exc_info()[:2])[-1]
|
||||
skipped_message = " ".join(exception_message.split("\n")[:2])
|
||||
print(f"skipped\t{tsv_escape(skipped_message)}")
|
||||
sys.exit(0)
|
||||
|
||||
reportStageEnd("preconditions")
|
||||
|
||||
if not args.use_existing_tables:
|
||||
# Run create and fill queries. We will run them simultaneously for both
|
||||
# servers, to save time. The weird XML search + filter is because we want to
|
||||
|
||||
@ -652,12 +652,16 @@ if args.report == "main":
|
||||
# Don't show mildly unstable queries, only the very unstable ones we
|
||||
# treat as errors.
|
||||
if very_unstable_queries:
|
||||
if very_unstable_queries > 5:
|
||||
error_tests += very_unstable_queries
|
||||
status = "failure"
|
||||
message_array.append(str(very_unstable_queries) + " unstable")
|
||||
# FIXME: uncomment the following lines when tests are stable and
|
||||
# reliable
|
||||
# if very_unstable_queries > 5:
|
||||
# error_tests += very_unstable_queries
|
||||
# status = "failure"
|
||||
#
|
||||
# error_tests += slow_average_tests
|
||||
# FIXME: until here
|
||||
|
||||
error_tests += slow_average_tests
|
||||
if error_tests:
|
||||
status = "failure"
|
||||
message_array.insert(0, str(error_tests) + " errors")
|
||||
|
||||
@ -131,6 +131,23 @@ ls -la /
|
||||
|
||||
clickhouse-client -q "system flush logs" ||:
|
||||
|
||||
# Stop server so we can safely read data with clickhouse-local.
|
||||
# Why do we read data with clickhouse-local?
|
||||
# Because it's the simplest way to read it when server has crashed.
|
||||
if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
clickhouse-client -q "system shutdown" ||:
|
||||
sleep 10
|
||||
else
|
||||
sudo clickhouse stop ||:
|
||||
fi
|
||||
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
clickhouse-client --port 19000 -q "system shutdown" ||:
|
||||
clickhouse-client --port 29000 -q "system shutdown" ||:
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz &
|
||||
|
||||
@ -143,18 +160,17 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous
|
||||
# for files >64MB, we want this files to be compressed explicitly
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log
|
||||
do
|
||||
clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz &
|
||||
clickhouse-local --path /var/lib/clickhouse/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.tsv.gz ||:
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz &
|
||||
clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz &
|
||||
clickhouse-local --path /var/lib/clickhouse1/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.1.tsv.gz ||:
|
||||
clickhouse-local --path /var/lib/clickhouse2/ -q "select * from system.$table format TSVWithNamesAndTypes" | pigz > /test_output/$table.2.tsv.gz ||:
|
||||
fi
|
||||
done
|
||||
wait ||:
|
||||
|
||||
# Also export trace log in flamegraph-friendly format.
|
||||
for trace_type in CPU Memory Real
|
||||
do
|
||||
clickhouse-client -q "
|
||||
clickhouse-local --path /var/lib/clickhouse/ -q "
|
||||
select
|
||||
arrayStringConcat((arrayMap(x -> concat(splitByChar('/', addressToLine(x))[-1], '#', demangle(addressToSymbol(x)) ), trace)), ';') AS stack,
|
||||
count(*) AS samples
|
||||
@ -164,10 +180,9 @@ do
|
||||
order by samples desc
|
||||
settings allow_introspection_functions = 1
|
||||
format TabSeparated" \
|
||||
| pigz > "/test_output/trace-log-$trace_type-flamegraph.tsv.gz" &
|
||||
| pigz > "/test_output/trace-log-$trace_type-flamegraph.tsv.gz" ||:
|
||||
done
|
||||
|
||||
wait ||:
|
||||
|
||||
# Compressed (FIXME: remove once only github actions will be left)
|
||||
rm /var/log/clickhouse-server/clickhouse-server.log
|
||||
|
||||
@ -167,6 +167,34 @@ Config is read from multiple files (in XML or YAML format) and merged into singl
|
||||
|
||||
For queries and subsystems other than `Server` config is accessible using `Context::getConfigRef()` method. Every subsystem that is capable of reloading it's config without server restart should register itself in reload callback in `Server::main()` method. Note that if newer config has an error, most subsystems will ignore new config, log warning messages and keep working with previously loaded config. Due to the nature of `AbstractConfiguration` it is not possible to pass reference to specific section, so `String config_prefix` is usually used instead.
|
||||
|
||||
## Threads and jobs {#threads-and-jobs}
|
||||
|
||||
To execute queries and do side activities ClickHouse allocates threads from one of thread pools to avoid frequent thread creation and destruction. There are a few thread pools, which are selected depending on a purpose and structure of a job:
|
||||
* Server pool for incoming client sessions.
|
||||
* Global thread pool for general purpose jobs, background activities and standalone threads.
|
||||
* IO thread pool for jobs that are mostly blocked on some IO and are not CPU-intensive.
|
||||
* Background pools for periodic tasks.
|
||||
* Pools for preemptable tasks that can be split into steps.
|
||||
|
||||
Server pool is a `Poco::ThreadPool` class instance defined in `Server::main()` method. It can have at most `max_connection` threads. Every thread is dedicated to a single active connection.
|
||||
|
||||
Global thread pool is `GlobalThreadPool` singleton class. To allocate thread from it `ThreadFromGlobalPool` is used. It has an interface similar to `std::thread`, but pulls thread from the global pool and does all necessary initializations. It is configured with the following settings:
|
||||
* `max_thread_pool_size` - limit on thread count in pool.
|
||||
* `max_thread_pool_free_size` - limit on idle thread count waiting for new jobs.
|
||||
* `thread_pool_queue_size` - limit on scheduled job count.
|
||||
|
||||
Global pool is universal and all pools described below are implemented on top of it. This can be thought of as a hierarchy of pools. Any specialized pool takes its threads from the global pool using `ThreadPool` class. So the main purpose of any specialized pool is to apply limit on the number of simultaneous jobs and do job scheduling. If there are more jobs scheduled than threads in a pool, `ThreadPool` accumulates jobs in a queue with priorities. Each job has an integer priority. Default priority is zero. All jobs with higher priority values are started before any job with lower priority value. But there is no difference between already executing jobs, thus priority matters only when the pool in overloaded.
|
||||
|
||||
IO thread pool is implemented as a plain `ThreadPool` accessible via `IOThreadPool::get()` method. It is configured in the same way as global pool with `max_io_thread_pool_size`, `max_io_thread_pool_free_size` and `io_thread_pool_queue_size` settings. The main purpose of IO thread pool is to avoid exhaustion of the global pool with IO jobs, which could prevent queries from fully utilizing CPU.
|
||||
|
||||
For periodic task execution there is `BackgroundSchedulePool` class. You can register tasks using `BackgroundSchedulePool::TaskHolder` objects and the pool ensures that no task runs two jobs at the same time. It also allows you to postpone task execution to a specific instant in the future or temporarily deactivate task. Global `Context` provides a few instances of this class for different purposes. For general purpose tasks `Context::getSchedulePool()` is used.
|
||||
|
||||
There are also specialized thread pools for preemptable tasks. Such `IExecutableTask` task can be split into ordered sequence of jobs, called steps. To schedule these tasks in a manner allowing short tasks to be prioritied over long ones `MergeTreeBackgroundExecutor` is used. As name suggests it is used for background MergeTree related operations such as merges, mutations, fetches and moves. Pool instances are available using `Context::getCommonExecutor()` and other similar methods.
|
||||
|
||||
No matter what pool is used for a job, at start `ThreadStatus` instance is created for this job. It encapsulates all per-thread information: thread id, query id, performance counters, resource consumption and many other useful data. Job can access it via thread local pointer by `CurrentThread::get()` call, so we do not need to pass it to every function.
|
||||
|
||||
If thread is related to query execution, then the most important thing attached to `ThreadStatus` is query context `ContextPtr`. Every query has its master thread in the server pool. Master thread does the attachment by holding an `ThreadStatus::QueryScope query_scope(query_context)` object. Master thread also creates a thread group represented with `ThreadGroupStatus` object. Every additional thread that is allocated during this query execution is attached to its thread group by `CurrentThread::attachTo(thread_group)` call. Thread groups are used to aggregate profile event counters and track memory consumption by all threads dedicated to a single task (see `MemoryTracker` and `ProfileEvents::Counters` classes for more information).
|
||||
|
||||
## Distributed Query Execution {#distributed-query-execution}
|
||||
|
||||
Servers in a cluster setup are mostly independent. You can create a `Distributed` table on one or all servers in a cluster. The `Distributed` table does not store data itself – it only provides a “view” to all local tables on multiple nodes of a cluster. When you SELECT from a `Distributed` table, it rewrites that query, chooses remote nodes according to load balancing settings, and sends the query to them. The `Distributed` table requests remote servers to process a query just up to a stage where intermediate results from different servers can be merged. Then it receives the intermediate results and merges them. The distributed table tries to distribute as much work as possible to remote servers and does not send much intermediate data over the network.
|
||||
|
||||
@ -83,7 +83,7 @@ $ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
|
||||
|
||||
Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/tests/performance-comparison` tool . See the readme file for invocation.
|
||||
|
||||
Each test run one or multiple queries (possibly with combinations of parameters) in a loop. Some tests can contain preconditions on preloaded test dataset.
|
||||
Each test run one or multiple queries (possibly with combinations of parameters) in a loop.
|
||||
|
||||
If you want to improve performance of ClickHouse in some scenario, and if improvements can be observed on simple queries, it is highly recommended to write a performance test. It always makes sense to use `perf top` or other perf tools during your tests.
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_
|
||||
|
||||
Engine parameters:
|
||||
|
||||
- `database` – Database name. Instead of the database name, you can use a constant expression that returns a string.
|
||||
- `database` – Database name. You can use `currentDatabase()` or another constant expression that returns a string.
|
||||
- `table` – Table to flush data to.
|
||||
- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16.
|
||||
- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer.
|
||||
|
||||
@ -9,66 +9,67 @@ ClickHouse can accept and return data in various formats. A format supported for
|
||||
results of a `SELECT`, and to perform `INSERT`s into a file-backed table.
|
||||
|
||||
The supported formats are:
|
||||
| Input | Output |
|
||||
|-------------------------------------------------------------------------------------------|-------|-------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TabSeparatedRawWithNames](#tabseparatedrawwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedRawWithNamesAndTypes](#tabseparatedrawwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
| [JSONAsString](#jsonasstring) | ✔ | ✗ |
|
||||
| [JSONStrings](#jsonstrings) | ✗ | ✔ |
|
||||
| [JSONCompact](#jsoncompact) | ✗ | ✔ |
|
||||
| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ |
|
||||
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
|
||||
| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ |
|
||||
| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNames](#jsoncompacteachrowwithnames) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRowWithNames](#jsoncompactstringseachrowwithnames) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TSKV](#tskv) | ✔ | ✔ |
|
||||
| [Pretty](#pretty) | ✗ | ✔ |
|
||||
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
|
||||
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
|
||||
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
|
||||
| [PrettySpace](#prettyspace) | ✗ | ✔ |
|
||||
| [Prometheus](#prometheus) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [ProtobufSingle](#protobufsingle) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Arrow](#data-format-arrow) | ✔ | ✔ |
|
||||
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
|
||||
| [ORC](#data-format-orc) | ✔ | ✔ |
|
||||
| [RowBinary](#rowbinary) | ✔ | ✔ |
|
||||
| [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [Native](#native) | ✔ | ✔ |
|
||||
| [Null](#null) | ✗ | ✔ |
|
||||
| [XML](#xml) | ✗ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✔ |
|
||||
| [LineAsString](#lineasstring) | ✔ | ✗ |
|
||||
| [Regexp](#data-format-regexp) | ✔ | ✗ |
|
||||
| [RawBLOB](#rawblob) | ✔ | ✔ |
|
||||
| [MsgPack](#msgpack) | ✔ | ✔ |
|
||||
| [MySQLDump](#mysqldump) | ✔ | ✗ |
|
||||
|
||||
| Format | Input | Output |
|
||||
|-----------------------------------------------------------------------------------------|-------|--------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TabSeparatedRawWithNames](#tabseparatedrawwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedRawWithNamesAndTypes](#tabseparatedrawwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CSVWithNamesAndTypes](#csvwithnamesandtypes) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNames](#customseparatedwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparatedWithNamesAndTypes](#customseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
| [JSONAsString](#jsonasstring) | ✔ | ✗ |
|
||||
| [JSONStrings](#jsonstrings) | ✗ | ✔ |
|
||||
| [JSONCompact](#jsoncompact) | ✗ | ✔ |
|
||||
| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ |
|
||||
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
|
||||
| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ |
|
||||
| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNames](#jsoncompacteachrowwithnames) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRowWithNames](#jsoncompactstringseachrowwithnames) | ✔ | ✔ |
|
||||
| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TSKV](#tskv) | ✔ | ✔ |
|
||||
| [Pretty](#pretty) | ✗ | ✔ |
|
||||
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
|
||||
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
|
||||
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
|
||||
| [PrettySpace](#prettyspace) | ✗ | ✔ |
|
||||
| [Prometheus](#prometheus) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [ProtobufSingle](#protobufsingle) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Arrow](#data-format-arrow) | ✔ | ✔ |
|
||||
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
|
||||
| [ORC](#data-format-orc) | ✔ | ✔ |
|
||||
| [RowBinary](#rowbinary) | ✔ | ✔ |
|
||||
| [RowBinaryWithNames](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [Native](#native) | ✔ | ✔ |
|
||||
| [Null](#null) | ✗ | ✔ |
|
||||
| [XML](#xml) | ✗ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✔ |
|
||||
| [LineAsString](#lineasstring) | ✔ | ✗ |
|
||||
| [Regexp](#data-format-regexp) | ✔ | ✗ |
|
||||
| [RawBLOB](#rawblob) | ✔ | ✔ |
|
||||
| [MsgPack](#msgpack) | ✔ | ✔ |
|
||||
|
||||
You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
|
||||
|
||||
@ -184,7 +185,7 @@ Differs from the `TabSeparated` format in that the column names are written in t
|
||||
During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness.
|
||||
|
||||
If setting [input_format_with_names_use_header](../operations/settings/settings.md#settings-input_format_with_names_use_header) is set to 1,
|
||||
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#settings-input_format_skip_unknown_fields) is set to 1.
|
||||
the columns from input data will be mapped to the columns from the table by their names, columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#settings-input-format-skip-unknown-fields) is set to 1.
|
||||
Otherwise, the first row will be skipped.
|
||||
|
||||
This format is also available under the name `TSVWithNames`.
|
||||
@ -1776,3 +1777,70 @@ $ clickhouse-client --query="CREATE TABLE msgpack (array Array(UInt8)) ENGINE =
|
||||
$ clickhouse-client --query="INSERT INTO msgpack VALUES ([0, 1, 2, 3, 42, 253, 254, 255]), ([255, 254, 253, 42, 3, 2, 1, 0])";
|
||||
$ clickhouse-client --query="SELECT * FROM msgpack FORMAT MsgPack" > tmp_msgpack.msgpk;
|
||||
```
|
||||
|
||||
## MySQLDump {#msgpack}
|
||||
|
||||
ClickHouse supports reading MySQL [dumps](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html).
|
||||
It reads all data from INSERT queries belonging to one table in dump. If there are more than one table, by default it reads data from the first one.
|
||||
You can specify the name of the table from which to read data from using [input_format_mysql_dump_table_name](../operations/settings/settings.md#settings-input-format-mysql-dump-table-name) settings.
|
||||
If setting [input_format_mysql_dump_map_columns](../operations/settings/settings.md#settings-input-format-mysql-dump-map-columns) is set to 1 and
|
||||
dump contains CREATE query for specified table or column names in INSERT query the columns from input data will be mapped to the columns from the table by their names,
|
||||
columns with unknown names will be skipped if setting [input_format_skip_unknown_fields](../operations/settings/settings.md#settings-input-format-skip-unknown-fields) is set to 1.
|
||||
This format supports schema inference: if the dump contains CREATE query for the specified table, the structure is extracted from it, otherwise schema is inferred from the data of INSERT queries.
|
||||
|
||||
Examples:
|
||||
|
||||
File dump.sql:
|
||||
```sql
|
||||
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
||||
/*!50503 SET character_set_client = utf8mb4 */;
|
||||
CREATE TABLE `test` (
|
||||
`x` int DEFAULT NULL,
|
||||
`y` int DEFAULT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
INSERT INTO `test` VALUES (1,NULL),(2,NULL),(3,NULL),(3,NULL),(4,NULL),(5,NULL),(6,7);
|
||||
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
||||
/*!50503 SET character_set_client = utf8mb4 */;
|
||||
CREATE TABLE `test 3` (
|
||||
`y` int DEFAULT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
INSERT INTO `test 3` VALUES (1);
|
||||
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
||||
/*!50503 SET character_set_client = utf8mb4 */;
|
||||
CREATE TABLE `test2` (
|
||||
`x` int DEFAULT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
INSERT INTO `test2` VALUES (1),(2),(3);
|
||||
```
|
||||
|
||||
Queries:
|
||||
|
||||
```sql
|
||||
:) desc file(dump.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'
|
||||
|
||||
DESCRIBE TABLE file(dump.sql, MySQLDump)
|
||||
SETTINGS input_format_mysql_dump_table_name = 'test2'
|
||||
|
||||
Query id: 25e66c89-e10a-42a8-9b42-1ee8bbbde5ef
|
||||
|
||||
┌─name─┬─type────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
|
||||
│ x │ Nullable(Int32) │ │ │ │ │ │
|
||||
└──────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
|
||||
|
||||
:) select * from file(dump.sql, MySQLDump) settings input_format_mysql_dump_table_name='test2'
|
||||
|
||||
SELECT *
|
||||
FROM file(dump.sql, MySQLDump)
|
||||
SETTINGS input_format_mysql_dump_table_name = 'test2'
|
||||
|
||||
Query id: 17d59664-ebce-4053-bb79-d46a516fb590
|
||||
|
||||
┌─x─┐
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
│ 3 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
@ -267,7 +267,7 @@ See also “[Executable User Defined Functions](../../sql-reference/functions/in
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<user_defined_executable_functions_config>*_dictionary.xml</user_defined_executable_functions_config>
|
||||
<user_defined_executable_functions_config>*_function.xml</user_defined_executable_functions_config>
|
||||
```
|
||||
|
||||
## dictionaries_lazy_load {#server_configuration_parameters-dictionaries_lazy_load}
|
||||
|
||||
@ -6,6 +6,29 @@ slug: /en/operations/settings/settings
|
||||
|
||||
# Settings {#settings}
|
||||
|
||||
## allow_nondeterministic_mutations {#allow_nondeterministic_mutations}
|
||||
|
||||
User-level setting that allows mutations on replicated tables to make use of non-deterministic functions such as `dictGet`.
|
||||
|
||||
Given that, for example, dictionaries, can be out of sync across nodes, mutations that pull values from them are disallowed on replicated tables by default. Enabling this setting allows this behavior, making it the user's responsibility to ensure that the data used is in sync across all nodes.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
**Example**
|
||||
|
||||
``` xml
|
||||
<profiles>
|
||||
<default>
|
||||
<allow_nondeterministic_mutations>1</allow_nondeterministic_mutations>
|
||||
|
||||
<!-- ... -->
|
||||
</default>
|
||||
|
||||
<!-- ... -->
|
||||
|
||||
</profiles>
|
||||
```
|
||||
|
||||
## distributed_product_mode {#distributed-product-mode}
|
||||
|
||||
Changes the behaviour of [distributed subqueries](../../sql-reference/operators/in.md).
|
||||
@ -4225,3 +4248,18 @@ Default value: 0.
|
||||
The waiting time in seconds for currently handled connections when shutdown server.
|
||||
|
||||
Default Value: 5.
|
||||
|
||||
## input_format_mysql_dump_table_name (#input-format-mysql-dump-table-name)
|
||||
|
||||
The name of the table from which to read data from in MySQLDump input format.
|
||||
|
||||
## input_format_mysql_dump_map_columns (#input-format-mysql-dump-map-columns)
|
||||
|
||||
Enables matching columns from table in MySQL dump and columns from ClickHouse table by names in MySQLDump input format.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Disabled.
|
||||
- 1 — Enabled.
|
||||
|
||||
Default value: 1.
|
||||
|
||||
@ -84,6 +84,7 @@ Result:
|
||||
Returns the inclusive lower bound of the corresponding tumbling window.
|
||||
|
||||
``` sql
|
||||
tumbleStart(bounds_tuple);
|
||||
tumbleStart(time_attr, interval [, timezone]);
|
||||
```
|
||||
|
||||
@ -92,6 +93,7 @@ tumbleStart(time_attr, interval [, timezone]);
|
||||
Returns the exclusive upper bound of the corresponding tumbling window.
|
||||
|
||||
``` sql
|
||||
tumbleEnd(bounds_tuple);
|
||||
tumbleEnd(time_attr, interval [, timezone]);
|
||||
```
|
||||
|
||||
@ -100,6 +102,7 @@ tumbleEnd(time_attr, interval [, timezone]);
|
||||
Returns the inclusive lower bound of the corresponding hopping window.
|
||||
|
||||
``` sql
|
||||
hopStart(bounds_tuple);
|
||||
hopStart(time_attr, hop_interval, window_interval [, timezone]);
|
||||
```
|
||||
|
||||
@ -108,5 +111,6 @@ hopStart(time_attr, hop_interval, window_interval [, timezone]);
|
||||
Returns the exclusive upper bound of the corresponding hopping window.
|
||||
|
||||
``` sql
|
||||
hopEnd(bounds_tuple);
|
||||
hopEnd(time_attr, hop_interval, window_interval [, timezone]);
|
||||
```
|
||||
```
|
||||
|
||||
@ -263,7 +263,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
|
||||
**Пример**
|
||||
|
||||
``` xml
|
||||
<user_defined_executable_functions_config>*_dictionary.xml</user_defined_executable_functions_config>
|
||||
<user_defined_executable_functions_config>*_function.xml</user_defined_executable_functions_config>
|
||||
```
|
||||
|
||||
## dictionaries_lazy_load {#server_configuration_parameters-dictionaries_lazy_load}
|
||||
|
||||
@ -22,12 +22,7 @@ It’ll take some effort to go through, but the result will be very close to pro
|
||||
|
||||
For the first time you’ll need to:
|
||||
|
||||
#### 1. Install CLI tools from npm
|
||||
|
||||
1. `sudo apt-get install npm` for Debian/Ubuntu or `brew install npm` on Mac OS X.
|
||||
2. `sudo npm install -g purify-css amphtml-validator`.
|
||||
|
||||
#### 2. Set up virtualenv
|
||||
#### 1. Set up virtualenv
|
||||
|
||||
``` bash
|
||||
$ cd ClickHouse/docs/tools
|
||||
@ -37,7 +32,7 @@ $ source venv/bin/activate
|
||||
$ pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
#### 3. Run build.py
|
||||
#### 2. Run build.py
|
||||
|
||||
When all prerequisites are installed, running `build.py` without args (there are some, check `build.py --help`) will generate `ClickHouse/docs/build` folder with complete static html website.
|
||||
|
||||
|
||||
@ -1,105 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import bs4
|
||||
import cssmin
|
||||
import jinja2
|
||||
import mkdocs.commands.build
|
||||
|
||||
import mdx_clickhouse
|
||||
import test
|
||||
import util
|
||||
import website
|
||||
|
||||
|
||||
def prepare_amp_html(lang, args, root, site_temp, main_site_dir):
|
||||
src_path = root
|
||||
src_index = os.path.join(src_path, "index.html")
|
||||
rel_path = os.path.relpath(src_path, site_temp)
|
||||
dst_path = os.path.join(main_site_dir, rel_path, "amp")
|
||||
dst_index = os.path.join(dst_path, "index.html")
|
||||
|
||||
logging.debug(f"Generating AMP version for {rel_path} ({lang})")
|
||||
os.makedirs(dst_path)
|
||||
with open(src_index, "r") as f:
|
||||
content = f.read()
|
||||
css_in = " ".join(website.get_css_in(args))
|
||||
command = f"purifycss --min {css_in} '{src_index}'"
|
||||
logging.debug(command)
|
||||
inline_css = subprocess.check_output(command, shell=True).decode("utf-8")
|
||||
inline_css = inline_css.replace("!important", "").replace("/*!", "/*")
|
||||
inline_css = cssmin.cssmin(inline_css)
|
||||
content = content.replace("CUSTOM_CSS_PLACEHOLDER", inline_css)
|
||||
|
||||
with open(dst_index, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
return dst_index
|
||||
|
||||
|
||||
def build_amp(lang, args, cfg):
|
||||
# AMP docs: https://amp.dev/documentation/
|
||||
logging.info(f"Building AMP version for {lang}")
|
||||
with util.temp_dir() as site_temp:
|
||||
extra = cfg.data["extra"]
|
||||
main_site_dir = cfg.data["site_dir"]
|
||||
extra["is_amp"] = True
|
||||
cfg.load_dict({"site_dir": site_temp, "extra": extra})
|
||||
|
||||
try:
|
||||
mkdocs.commands.build.build(cfg)
|
||||
except jinja2.exceptions.TemplateError:
|
||||
if not args.version_prefix:
|
||||
raise
|
||||
mdx_clickhouse.PatchedMacrosPlugin.disabled = True
|
||||
mkdocs.commands.build.build(cfg)
|
||||
|
||||
paths = []
|
||||
for root, _, filenames in os.walk(site_temp):
|
||||
if "index.html" in filenames:
|
||||
paths.append(
|
||||
prepare_amp_html(lang, args, root, site_temp, main_site_dir)
|
||||
)
|
||||
logging.info(f"Finished building AMP version for {lang}")
|
||||
|
||||
|
||||
def html_to_amp(content):
|
||||
soup = bs4.BeautifulSoup(content, features="html.parser")
|
||||
|
||||
for tag in soup.find_all():
|
||||
if tag.attrs.get("id") == "tostring":
|
||||
tag.attrs["id"] = "_tostring"
|
||||
if tag.name == "img":
|
||||
tag.name = "amp-img"
|
||||
tag.attrs["layout"] = "responsive"
|
||||
src = tag.attrs["src"]
|
||||
if not (src.startswith("/") or src.startswith("http")):
|
||||
tag.attrs["src"] = f"../{src}"
|
||||
if not tag.attrs.get("width"):
|
||||
tag.attrs["width"] = "640"
|
||||
if not tag.attrs.get("height"):
|
||||
tag.attrs["height"] = "320"
|
||||
if tag.name == "iframe":
|
||||
tag.name = "amp-iframe"
|
||||
tag.attrs["layout"] = "responsive"
|
||||
del tag.attrs["alt"]
|
||||
del tag.attrs["allowfullscreen"]
|
||||
if not tag.attrs.get("width"):
|
||||
tag.attrs["width"] = "640"
|
||||
if not tag.attrs.get("height"):
|
||||
tag.attrs["height"] = "320"
|
||||
elif tag.name == "a":
|
||||
href = tag.attrs.get("href")
|
||||
if href:
|
||||
if not (href.startswith("/") or href.startswith("http")):
|
||||
if "#" in href:
|
||||
href, anchor = href.split("#")
|
||||
else:
|
||||
anchor = None
|
||||
href = f"../{href}amp/"
|
||||
if anchor:
|
||||
href = f"{href}#{anchor}"
|
||||
tag.attrs["href"] = href
|
||||
content = str(soup)
|
||||
return website.minify_html(content)
|
||||
@ -74,7 +74,6 @@ def build_for_lang(lang, args):
|
||||
events=args.events,
|
||||
languages=languages,
|
||||
includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"),
|
||||
is_amp=False,
|
||||
is_blog=True,
|
||||
post_meta=post_meta,
|
||||
today=datetime.date.today().isoformat(),
|
||||
|
||||
@ -19,12 +19,9 @@ from mkdocs import config
|
||||
from mkdocs import exceptions
|
||||
import mkdocs.commands.build
|
||||
|
||||
import amp
|
||||
import blog
|
||||
import mdx_clickhouse
|
||||
import redirects
|
||||
import single_page
|
||||
import test
|
||||
import util
|
||||
import website
|
||||
|
||||
@ -49,7 +46,6 @@ markdown.extensions.ClickHouseMarkdown = ClickHouseMarkdown
|
||||
|
||||
def build_for_lang(lang, args):
|
||||
logging.info(f"Building {lang} docs")
|
||||
os.environ["SINGLE_PAGE"] = "0"
|
||||
|
||||
try:
|
||||
theme_cfg = {
|
||||
@ -104,7 +100,6 @@ def build_for_lang(lang, args):
|
||||
plugins=plugins,
|
||||
extra=dict(
|
||||
now=datetime.datetime.now().isoformat(),
|
||||
single_page=False,
|
||||
rev=args.rev,
|
||||
rev_short=args.rev_short,
|
||||
rev_url=args.rev_url,
|
||||
@ -112,14 +107,10 @@ def build_for_lang(lang, args):
|
||||
events=args.events,
|
||||
languages=languages,
|
||||
includes_dir=os.path.join(os.path.dirname(__file__), "..", "_includes"),
|
||||
is_amp=False,
|
||||
is_blog=False,
|
||||
),
|
||||
)
|
||||
|
||||
# Clean to be safe if last build finished abnormally
|
||||
single_page.remove_temporary_files(lang, args)
|
||||
|
||||
raw_config["nav"] = nav.build_docs_nav(lang, args)
|
||||
|
||||
cfg = config.load_config(**raw_config)
|
||||
@ -127,14 +118,6 @@ def build_for_lang(lang, args):
|
||||
if not args.skip_multi_page:
|
||||
mkdocs.commands.build.build(cfg)
|
||||
|
||||
if not args.skip_amp:
|
||||
amp.build_amp(lang, args, cfg)
|
||||
|
||||
if not args.skip_single_page:
|
||||
single_page.build_single_page_version(
|
||||
lang, args, raw_config.get("nav"), cfg
|
||||
)
|
||||
|
||||
mdx_clickhouse.PatchedMacrosPlugin.disabled = False
|
||||
|
||||
logging.info(f"Finished building {lang} docs")
|
||||
@ -174,7 +157,6 @@ def build(args):
|
||||
|
||||
if not args.skip_website:
|
||||
website.process_benchmark_results(args)
|
||||
website.minify_website(args)
|
||||
redirects.build_static_redirects(args)
|
||||
|
||||
|
||||
@ -197,22 +179,15 @@ if __name__ == "__main__":
|
||||
arg_parser.add_argument("--output-dir", default="build")
|
||||
arg_parser.add_argument("--nav-limit", type=int, default="0")
|
||||
arg_parser.add_argument("--skip-multi-page", action="store_true")
|
||||
arg_parser.add_argument("--skip-single-page", action="store_true")
|
||||
arg_parser.add_argument("--skip-amp", action="store_true")
|
||||
arg_parser.add_argument("--skip-website", action="store_true")
|
||||
arg_parser.add_argument("--skip-blog", action="store_true")
|
||||
arg_parser.add_argument("--skip-git-log", action="store_true")
|
||||
arg_parser.add_argument("--skip-docs", action="store_true")
|
||||
arg_parser.add_argument("--test-only", action="store_true")
|
||||
arg_parser.add_argument("--minify", action="store_true")
|
||||
arg_parser.add_argument("--htmlproofer", action="store_true")
|
||||
arg_parser.add_argument("--no-docs-macros", action="store_true")
|
||||
arg_parser.add_argument("--save-raw-single-page", type=str)
|
||||
arg_parser.add_argument("--livereload", type=int, default="0")
|
||||
arg_parser.add_argument("--verbose", action="store_true")
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
args.minify = False # TODO remove
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO, stream=sys.stderr
|
||||
@ -238,15 +213,6 @@ if __name__ == "__main__":
|
||||
args.rev_url = f"https://github.com/ClickHouse/ClickHouse/commit/{args.rev}"
|
||||
args.events = get_events(args)
|
||||
|
||||
if args.test_only:
|
||||
args.skip_multi_page = True
|
||||
args.skip_blog = True
|
||||
args.skip_website = True
|
||||
args.skip_amp = True
|
||||
|
||||
if args.skip_git_log or args.skip_amp:
|
||||
mdx_clickhouse.PatchedMacrosPlugin.skip_git_log = True
|
||||
|
||||
from build import build
|
||||
|
||||
build(args)
|
||||
|
||||
@ -39,7 +39,6 @@ MARKDOWN_EXTENSIONS = [
|
||||
|
||||
class ClickHouseLinkMixin(object):
|
||||
def handleMatch(self, m, data):
|
||||
single_page = os.environ.get("SINGLE_PAGE") == "1"
|
||||
try:
|
||||
el, start, end = super(ClickHouseLinkMixin, self).handleMatch(m, data)
|
||||
except IndexError:
|
||||
@ -51,13 +50,6 @@ class ClickHouseLinkMixin(object):
|
||||
if is_external:
|
||||
if not href.startswith("https://clickhouse.com"):
|
||||
el.set("rel", "external nofollow noreferrer")
|
||||
elif single_page:
|
||||
if "#" in href:
|
||||
el.set("href", "#" + href.split("#", 1)[1])
|
||||
else:
|
||||
el.set(
|
||||
"href", "#" + href.replace("/index.md", "/").replace(".md", "/")
|
||||
)
|
||||
return el, start, end
|
||||
|
||||
|
||||
@ -103,7 +95,6 @@ def get_translations(dirname, lang):
|
||||
|
||||
class PatchedMacrosPlugin(macros.plugin.MacrosPlugin):
|
||||
disabled = False
|
||||
skip_git_log = False
|
||||
|
||||
def on_config(self, config):
|
||||
super(PatchedMacrosPlugin, self).on_config(config)
|
||||
@ -141,16 +132,6 @@ class PatchedMacrosPlugin(macros.plugin.MacrosPlugin):
|
||||
lang = config.data["theme"]["language"]
|
||||
page.canonical_url = page.canonical_url.replace(f"/{lang}/", "/en/", 1)
|
||||
|
||||
if config.data["extra"].get("version_prefix") or config.data["extra"].get(
|
||||
"single_page"
|
||||
):
|
||||
return markdown
|
||||
if self.skip_git_log:
|
||||
return markdown
|
||||
src_path = page.file.abs_src_path
|
||||
|
||||
# There was a code that determined the minimum and maximum modification dates for a page.
|
||||
# It was removed due to being obnoxiously slow.
|
||||
return markdown
|
||||
|
||||
def render_impl(self, markdown):
|
||||
|
||||
7561
docs/tools/package-lock.json
generated
7561
docs/tools/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -1,31 +0,0 @@
|
||||
{
|
||||
"name": "build",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"build": "NODE_ENV=production webpack --config webpack.config.js",
|
||||
"watch": "NODE_ENV=development webpack --config webpack.config.js"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.10.2",
|
||||
"@babel/preset-env": "^7.10.2",
|
||||
"autoprefixer": "^9.8.0",
|
||||
"babel-loader": "^8.1.0",
|
||||
"cssnano": "^4.1.10",
|
||||
"file-loader": "^6.0.0",
|
||||
"node-sass-glob-importer": "^5.3.2",
|
||||
"postcss-loader": "^3.0.0",
|
||||
"sass": "^1.26.8",
|
||||
"sass-loader": "^8.0.2",
|
||||
"webpack": "^4.43.0",
|
||||
"webpack-cli": "^3.3.11"
|
||||
},
|
||||
"dependencies": {
|
||||
"amphtml-validator": "^1.0.35",
|
||||
"bootstrap": "^4.4.1",
|
||||
"purify-css": "^1.2.5"
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,7 @@ PUBLISH_DIR="${BASE_DIR}/../publish"
|
||||
BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.com}"
|
||||
GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}"
|
||||
GIT_PROD_URI="git@github.com:ClickHouse/clickhouse-website-content.git"
|
||||
EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---minify --verbose}"
|
||||
EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---verbose}"
|
||||
|
||||
if [[ -z "$1" ]]
|
||||
then
|
||||
@ -19,7 +19,7 @@ then
|
||||
# Will make a repository with website content as the only commit.
|
||||
git init
|
||||
git remote add origin "${GIT_TEST_URI}"
|
||||
git config user.email "robot-clickhouse@yandex-team.ru"
|
||||
git config user.email "robot-clickhouse@clickhouse.com"
|
||||
git config user.name "robot-clickhouse"
|
||||
|
||||
# Add files.
|
||||
|
||||
@ -1,234 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import yaml
|
||||
|
||||
import bs4
|
||||
import mkdocs.commands.build
|
||||
|
||||
import test
|
||||
import util
|
||||
import website
|
||||
|
||||
TEMPORARY_FILE_NAME = "single.md"
|
||||
|
||||
|
||||
def recursive_values(item):
|
||||
if isinstance(item, dict):
|
||||
for _, value in list(item.items()):
|
||||
yield from recursive_values(value)
|
||||
elif isinstance(item, list):
|
||||
for value in item:
|
||||
yield from recursive_values(value)
|
||||
elif isinstance(item, str):
|
||||
yield item
|
||||
|
||||
|
||||
anchor_not_allowed_chars = re.compile(r"[^\w\-]")
|
||||
|
||||
|
||||
def generate_anchor_from_path(path):
|
||||
return re.sub(anchor_not_allowed_chars, "-", path)
|
||||
|
||||
|
||||
absolute_link = re.compile(r"^https?://")
|
||||
|
||||
|
||||
def replace_link(match, path):
|
||||
title = match.group(1)
|
||||
link = match.group(2)
|
||||
|
||||
# Not a relative link
|
||||
if re.search(absolute_link, link):
|
||||
return match.group(0)
|
||||
|
||||
if link.endswith("/"):
|
||||
link = link[0:-1] + ".md"
|
||||
|
||||
return "{}(#{})".format(
|
||||
title,
|
||||
generate_anchor_from_path(
|
||||
os.path.normpath(os.path.join(os.path.dirname(path), link))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Concatenates Markdown files to a single file.
|
||||
def concatenate(lang, docs_path, single_page_file, nav):
|
||||
lang_path = os.path.join(docs_path, lang)
|
||||
|
||||
proj_config = f"{docs_path}/toc_{lang}.yml"
|
||||
if os.path.exists(proj_config):
|
||||
with open(proj_config) as cfg_file:
|
||||
nav = yaml.full_load(cfg_file.read())["nav"]
|
||||
|
||||
files_to_concatenate = list(recursive_values(nav))
|
||||
files_count = len(files_to_concatenate)
|
||||
logging.info(
|
||||
f"{files_count} files will be concatenated into single md-file for {lang}."
|
||||
)
|
||||
logging.debug("Concatenating: " + ", ".join(files_to_concatenate))
|
||||
assert files_count > 0, f"Empty single-page for {lang}"
|
||||
|
||||
link_regexp = re.compile(r"(\[[^\]]+\])\(([^)#]+)(?:#[^\)]+)?\)")
|
||||
|
||||
for path in files_to_concatenate:
|
||||
try:
|
||||
with open(os.path.join(lang_path, path)) as f:
|
||||
# Insert a horizontal ruler. Then insert an anchor that we will link to. Its name will be a path to the .md file.
|
||||
single_page_file.write(
|
||||
'\n______\n<a name="%s"></a>\n' % generate_anchor_from_path(path)
|
||||
)
|
||||
|
||||
in_metadata = False
|
||||
for line in f:
|
||||
# Skip YAML metadata.
|
||||
if line == "---\n":
|
||||
in_metadata = not in_metadata
|
||||
continue
|
||||
|
||||
if not in_metadata:
|
||||
# Increase the level of headers.
|
||||
if line.startswith("#"):
|
||||
line = "#" + line
|
||||
|
||||
# Replace links within the docs.
|
||||
|
||||
if re.search(link_regexp, line):
|
||||
line = re.sub(
|
||||
link_regexp,
|
||||
lambda match: replace_link(match, path),
|
||||
line,
|
||||
)
|
||||
|
||||
# If failed to replace the relative link, print to log
|
||||
# But with some exceptions:
|
||||
# - "../src/" -- for cmake-in-clickhouse.md (link to sources)
|
||||
# - "../usr/share" -- changelog entry that has "../usr/share/zoneinfo"
|
||||
if (
|
||||
"../" in line
|
||||
and (not "../usr/share" in line)
|
||||
and (not "../src/" in line)
|
||||
):
|
||||
logging.info("Failed to resolve relative link:")
|
||||
logging.info(path)
|
||||
logging.info(line)
|
||||
|
||||
single_page_file.write(line)
|
||||
|
||||
except IOError as e:
|
||||
logging.warning(str(e))
|
||||
|
||||
single_page_file.flush()
|
||||
|
||||
|
||||
def get_temporary_file_name(lang, args):
|
||||
return os.path.join(args.docs_dir, lang, TEMPORARY_FILE_NAME)
|
||||
|
||||
|
||||
def remove_temporary_files(lang, args):
|
||||
single_md_path = get_temporary_file_name(lang, args)
|
||||
if os.path.exists(single_md_path):
|
||||
os.unlink(single_md_path)
|
||||
|
||||
|
||||
def build_single_page_version(lang, args, nav, cfg):
|
||||
logging.info(f"Building single page version for {lang}")
|
||||
os.environ["SINGLE_PAGE"] = "1"
|
||||
extra = cfg.data["extra"]
|
||||
extra["single_page"] = True
|
||||
extra["is_amp"] = False
|
||||
|
||||
single_md_path = get_temporary_file_name(lang, args)
|
||||
with open(single_md_path, "w") as single_md:
|
||||
concatenate(lang, args.docs_dir, single_md, nav)
|
||||
|
||||
with util.temp_dir() as site_temp:
|
||||
with util.temp_dir() as docs_temp:
|
||||
docs_src_lang = os.path.join(args.docs_dir, lang)
|
||||
docs_temp_lang = os.path.join(docs_temp, lang)
|
||||
shutil.copytree(docs_src_lang, docs_temp_lang)
|
||||
for root, _, filenames in os.walk(docs_temp_lang):
|
||||
for filename in filenames:
|
||||
if filename != "single.md" and filename.endswith(".md"):
|
||||
os.unlink(os.path.join(root, filename))
|
||||
|
||||
cfg.load_dict(
|
||||
{
|
||||
"docs_dir": docs_temp_lang,
|
||||
"site_dir": site_temp,
|
||||
"extra": extra,
|
||||
"nav": [{cfg.data.get("site_name"): "single.md"}],
|
||||
}
|
||||
)
|
||||
|
||||
if not args.test_only:
|
||||
mkdocs.commands.build.build(cfg)
|
||||
|
||||
single_page_output_path = os.path.join(
|
||||
args.docs_dir, args.docs_output_dir, lang, "single"
|
||||
)
|
||||
|
||||
if os.path.exists(single_page_output_path):
|
||||
shutil.rmtree(single_page_output_path)
|
||||
|
||||
shutil.copytree(
|
||||
os.path.join(site_temp, "single"), single_page_output_path
|
||||
)
|
||||
|
||||
single_page_index_html = os.path.join(
|
||||
single_page_output_path, "index.html"
|
||||
)
|
||||
single_page_content_js = os.path.join(
|
||||
single_page_output_path, "content.js"
|
||||
)
|
||||
|
||||
with open(single_page_index_html, "r") as f:
|
||||
sp_prefix, sp_js, sp_suffix = f.read().split("<!-- BREAK -->")
|
||||
|
||||
with open(single_page_index_html, "w") as f:
|
||||
f.write(sp_prefix)
|
||||
f.write(sp_suffix)
|
||||
|
||||
with open(single_page_content_js, "w") as f:
|
||||
if args.minify:
|
||||
import jsmin
|
||||
|
||||
sp_js = jsmin.jsmin(sp_js)
|
||||
f.write(sp_js)
|
||||
|
||||
logging.info(f"Re-building single page for {lang} pdf/test")
|
||||
with util.temp_dir() as test_dir:
|
||||
extra["single_page"] = False
|
||||
cfg.load_dict(
|
||||
{
|
||||
"docs_dir": docs_temp_lang,
|
||||
"site_dir": test_dir,
|
||||
"extra": extra,
|
||||
"nav": [{cfg.data.get("site_name"): "single.md"}],
|
||||
}
|
||||
)
|
||||
mkdocs.commands.build.build(cfg)
|
||||
|
||||
css_in = " ".join(website.get_css_in(args))
|
||||
js_in = " ".join(website.get_js_in(args))
|
||||
subprocess.check_call(
|
||||
f"cat {css_in} > {test_dir}/css/base.css", shell=True
|
||||
)
|
||||
subprocess.check_call(
|
||||
f"cat {js_in} > {test_dir}/js/base.js", shell=True
|
||||
)
|
||||
|
||||
if args.save_raw_single_page:
|
||||
shutil.copytree(test_dir, args.save_raw_single_page)
|
||||
|
||||
logging.info(f"Running tests for {lang}")
|
||||
test.test_single_page(
|
||||
os.path.join(test_dir, "single", "index.html"), lang
|
||||
)
|
||||
|
||||
logging.info(f"Finished building single page version for {lang}")
|
||||
|
||||
remove_temporary_files(lang, args)
|
||||
@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import bs4
|
||||
import subprocess
|
||||
|
||||
|
||||
def test_single_page(input_path, lang):
|
||||
if not (lang == "en"):
|
||||
return
|
||||
|
||||
with open(input_path) as f:
|
||||
soup = bs4.BeautifulSoup(f, features="html.parser")
|
||||
|
||||
anchor_points = set()
|
||||
|
||||
duplicate_anchor_points = 0
|
||||
links_to_nowhere = 0
|
||||
|
||||
for tag in soup.find_all():
|
||||
for anchor_point in [tag.attrs.get("name"), tag.attrs.get("id")]:
|
||||
if anchor_point:
|
||||
anchor_points.add(anchor_point)
|
||||
|
||||
for tag in soup.find_all():
|
||||
href = tag.attrs.get("href")
|
||||
if href and href.startswith("#") and href != "#":
|
||||
if href[1:] not in anchor_points:
|
||||
links_to_nowhere += 1
|
||||
logging.info("Tag %s", tag)
|
||||
logging.info("Link to nowhere: %s" % href)
|
||||
|
||||
if links_to_nowhere:
|
||||
logging.error(f"Found {links_to_nowhere} links to nowhere in {lang}")
|
||||
sys.exit(1)
|
||||
|
||||
if len(anchor_points) <= 10:
|
||||
logging.error("Html parsing is probably broken")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
|
||||
test_single_page(sys.argv[1], sys.argv[2])
|
||||
@ -107,14 +107,12 @@ yaml.add_representer(collections.OrderedDict, represent_ordereddict)
|
||||
|
||||
|
||||
def init_jinja2_filters(env):
|
||||
import amp
|
||||
import website
|
||||
|
||||
chunk_size = 10240
|
||||
env.filters["chunks"] = lambda line: [
|
||||
line[i : i + chunk_size] for i in range(0, len(line), chunk_size)
|
||||
]
|
||||
env.filters["html_to_amp"] = amp.html_to_amp
|
||||
env.filters["adjust_markdown_html"] = website.adjust_markdown_html
|
||||
env.filters["to_rfc882"] = lambda d: datetime.datetime.strptime(
|
||||
d, "%Y-%m-%d"
|
||||
|
||||
@ -1,23 +1,14 @@
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import bs4
|
||||
import closure
|
||||
import cssmin
|
||||
import htmlmin
|
||||
import jsmin
|
||||
|
||||
import util
|
||||
|
||||
|
||||
def handle_iframe(iframe, soup):
|
||||
allowed_domains = ["https://www.youtube.com/", "https://datalens.yandex/"]
|
||||
allowed_domains = ["https://www.youtube.com/"]
|
||||
illegal_domain = True
|
||||
iframe_src = iframe.attrs["src"]
|
||||
for domain in allowed_domains:
|
||||
@ -127,10 +118,6 @@ def adjust_markdown_html(content):
|
||||
return str(soup)
|
||||
|
||||
|
||||
def minify_html(content):
|
||||
return htmlmin.minify(content)
|
||||
|
||||
|
||||
def build_website(args):
|
||||
logging.info("Building website")
|
||||
env = util.init_jinja2_env(args)
|
||||
@ -191,132 +178,6 @@ def build_website(args):
|
||||
f.write(content.encode("utf-8"))
|
||||
|
||||
|
||||
def get_css_in(args):
|
||||
return [
|
||||
f"'{args.website_dir}/css/bootstrap.css'",
|
||||
f"'{args.website_dir}/css/docsearch.css'",
|
||||
f"'{args.website_dir}/css/base.css'",
|
||||
f"'{args.website_dir}/css/blog.css'",
|
||||
f"'{args.website_dir}/css/docs.css'",
|
||||
f"'{args.website_dir}/css/highlight.css'",
|
||||
f"'{args.website_dir}/css/main.css'",
|
||||
]
|
||||
|
||||
|
||||
def get_js_in(args):
|
||||
return [
|
||||
f"'{args.website_dir}/js/jquery.js'",
|
||||
f"'{args.website_dir}/js/popper.js'",
|
||||
f"'{args.website_dir}/js/bootstrap.js'",
|
||||
f"'{args.website_dir}/js/sentry.js'",
|
||||
f"'{args.website_dir}/js/base.js'",
|
||||
f"'{args.website_dir}/js/index.js'",
|
||||
f"'{args.website_dir}/js/docsearch.js'",
|
||||
f"'{args.website_dir}/js/docs.js'",
|
||||
f"'{args.website_dir}/js/main.js'",
|
||||
]
|
||||
|
||||
|
||||
def minify_file(path, css_digest, js_digest):
|
||||
if not (path.endswith(".html") or path.endswith(".css")):
|
||||
return
|
||||
|
||||
logging.info("Minifying %s", path)
|
||||
with open(path, "rb") as f:
|
||||
content = f.read().decode("utf-8")
|
||||
if path.endswith(".html"):
|
||||
content = minify_html(content)
|
||||
content = content.replace("base.css?css_digest", f"base.css?{css_digest}")
|
||||
content = content.replace("base.js?js_digest", f"base.js?{js_digest}")
|
||||
# TODO: restore cssmin
|
||||
# elif path.endswith('.css'):
|
||||
# content = cssmin.cssmin(content)
|
||||
# TODO: restore jsmin
|
||||
# elif path.endswith('.js'):
|
||||
# content = jsmin.jsmin(content)
|
||||
with open(path, "wb") as f:
|
||||
f.write(content.encode("utf-8"))
|
||||
|
||||
|
||||
def minify_website(args):
|
||||
css_in = " ".join(get_css_in(args))
|
||||
css_out = f"{args.output_dir}/docs/css/base.css"
|
||||
os.makedirs(f"{args.output_dir}/docs/css")
|
||||
|
||||
if args.minify and False: # TODO: return closure
|
||||
command = (
|
||||
f"purifycss -w '*algolia*' --min {css_in} '{args.output_dir}/*.html' "
|
||||
f"'{args.output_dir}/docs/en/**/*.html' '{args.website_dir}/js/**/*.js' > {css_out}"
|
||||
)
|
||||
logging.info(css_in)
|
||||
logging.info(command)
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
logging.debug(output)
|
||||
|
||||
else:
|
||||
command = f"cat {css_in}"
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
with open(css_out, "wb+") as f:
|
||||
f.write(output)
|
||||
|
||||
with open(css_out, "rb") as f:
|
||||
css_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8]
|
||||
|
||||
js_in = " ".join(get_js_in(args))
|
||||
js_out = f"{args.output_dir}/docs/js/base.js"
|
||||
os.makedirs(f"{args.output_dir}/docs/js")
|
||||
|
||||
if args.minify and False: # TODO: return closure
|
||||
js_in = [js[1:-1] for js in js_in]
|
||||
closure_args = [
|
||||
"--js",
|
||||
*js_in,
|
||||
"--js_output_file",
|
||||
js_out,
|
||||
"--compilation_level",
|
||||
"SIMPLE",
|
||||
"--dependency_mode",
|
||||
"NONE",
|
||||
"--third_party",
|
||||
"--use_types_for_optimization",
|
||||
"--isolation_mode",
|
||||
"IIFE",
|
||||
]
|
||||
logging.info(closure_args)
|
||||
if closure.run(*closure_args):
|
||||
raise RuntimeError("failed to run closure compiler")
|
||||
with open(js_out, "r") as f:
|
||||
js_content = jsmin.jsmin(f.read())
|
||||
with open(js_out, "w") as f:
|
||||
f.write(js_content)
|
||||
|
||||
else:
|
||||
command = f"cat {js_in}"
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
with open(js_out, "wb+") as f:
|
||||
f.write(output)
|
||||
|
||||
with open(js_out, "rb") as f:
|
||||
js_digest = hashlib.sha3_224(f.read()).hexdigest()[0:8]
|
||||
logging.info(js_digest)
|
||||
|
||||
if args.minify:
|
||||
logging.info("Minifying website")
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
futures = []
|
||||
for root, _, filenames in os.walk(args.output_dir):
|
||||
for filename in filenames:
|
||||
path = os.path.join(root, filename)
|
||||
futures.append(
|
||||
executor.submit(minify_file, path, css_digest, js_digest)
|
||||
)
|
||||
for future in futures:
|
||||
exc = future.exception()
|
||||
if exc:
|
||||
logging.error(exc)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def process_benchmark_results(args):
|
||||
benchmark_root = os.path.join(args.website_dir, "benchmark")
|
||||
required_keys = {
|
||||
@ -329,7 +190,7 @@ def process_benchmark_results(args):
|
||||
results_root = os.path.join(benchmark_root, benchmark_kind, "results")
|
||||
for result in sorted(os.listdir(results_root)):
|
||||
result_file = os.path.join(results_root, result)
|
||||
logging.debug(f"Reading benchmark result from {result_file}")
|
||||
logging.info(f"Reading benchmark result from {result_file}")
|
||||
with open(result_file, "r") as f:
|
||||
result = json.loads(f.read())
|
||||
for item in result:
|
||||
|
||||
@ -5,38 +5,37 @@ sidebar_label: 体验平台
|
||||
|
||||
# ClickHouse Playground {#clickhouse-playground}
|
||||
|
||||
[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster.
|
||||
Several example datasets are available in Playground.
|
||||
无需搭建服务或集群，[ClickHouse Playground](https://play.clickhouse.com/play?user=play)允许人们通过执行查询语句立即体验ClickHouse，在Playground中我们提供了一些示例数据集。
|
||||
|
||||
You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md).
|
||||
你可以使用任意HTTP客户端向Playground提交查询语句，比如[curl](https://curl.haxx.se)或者[wget](https://www.gnu.org/software/wget/)，也可以通过[JDBC](../interfaces/jdbc.md)或者[ODBC](../interfaces/odbc.md)驱动建立连接，更多信息详见[客户端](../interfaces/index.md)。
|
||||
|
||||
## Credentials {#credentials}
|
||||
|
||||
| Parameter | Value |
|
||||
## 使用凭证 {#credentials}
|
||||
|
||||
| 参数 | 值 |
|
||||
|:--------------------|:-----------------------------------|
|
||||
| HTTPS endpoint | `https://play.clickhouse.com:443/` |
|
||||
| Native TCP endpoint | `play.clickhouse.com:9440` |
|
||||
| User | `explorer` or `play` |
|
||||
| Password | (empty) |
|
||||
| HTTPS连接地址 | `https://play.clickhouse.com:443/` |
|
||||
| 原生TCP连接地址 | `play.clickhouse.com:9440` |
|
||||
| 用户名 | `explorer`或者`play` |
|
||||
| 密码 | (空) |
|
||||
|
||||
## Limitations {#limitations}
|
||||
## 限制 {#limitations}
|
||||
|
||||
The queries are executed as a read-only user. It implies some limitations:
|
||||
所有查询语句都会视为一个具有只读权限用户的操作，这意味着存在如下一些限制：
|
||||
|
||||
- DDL queries are not allowed
|
||||
- INSERT queries are not allowed
|
||||
- 不允许执行DDL语句
|
||||
- 不允许执行INSERT语句
|
||||
|
||||
The service also have quotas on its usage.
|
||||
此外，Playground服务对资源使用也有限制。
|
||||
|
||||
## Examples {#examples}
|
||||
## 示例 {#examples}
|
||||
|
||||
HTTPS endpoint example with `curl`:
|
||||
使用`curl`命令：
|
||||
|
||||
``` bash
|
||||
curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'"
|
||||
```
|
||||
|
||||
TCP endpoint example with [CLI](../interfaces/cli.md):
|
||||
使用[命令行客户端](../interfaces/cli.md)：
|
||||
|
||||
``` bash
|
||||
clickhouse client --secure --host play.clickhouse.com --user explorer
|
||||
|
||||
@ -603,10 +603,23 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
{
|
||||
using boost::program_options::value;
|
||||
|
||||
/// Note: according to the standard, subsequent calls to getenv can mangle previous result.
|
||||
/// So we copy the results to std::string.
|
||||
std::optional<std::string> env_user_str;
|
||||
std::optional<std::string> env_password_str;
|
||||
|
||||
const char * env_user = getenv("CLICKHOUSE_USER");
|
||||
if (env_user != nullptr)
|
||||
env_user_str.emplace(std::string(env_user));
|
||||
|
||||
const char * env_password = getenv("CLICKHOUSE_PASSWORD");
|
||||
if (env_password != nullptr)
|
||||
env_password_str.emplace(std::string(env_password));
|
||||
|
||||
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
|
||||
desc.add_options()
|
||||
("help", "produce help message")
|
||||
("query", value<std::string>()->default_value(""), "query to execute")
|
||||
("query,q", value<std::string>()->default_value(""), "query to execute")
|
||||
("concurrency,c", value<unsigned>()->default_value(1), "number of parallel queries")
|
||||
("delay,d", value<double>()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)")
|
||||
("stage", value<std::string>()->default_value("complete"), "request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit")
|
||||
@ -615,12 +628,12 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
("randomize,r", value<bool>()->default_value(false), "randomize order of execution")
|
||||
("json", value<std::string>()->default_value(""), "write final report to specified file in JSON format")
|
||||
("host,h", value<Strings>()->multitoken(), "list of hosts")
|
||||
("port,p", value<Ports>()->multitoken(), "list of ports")
|
||||
("port", value<Ports>()->multitoken(), "list of ports")
|
||||
("roundrobin", "Instead of comparing queries for different --host/--port just pick one random --host/--port for every query and send query to it.")
|
||||
("cumulative", "prints cumulative data instead of data per interval")
|
||||
("secure,s", "Use TLS connection")
|
||||
("user", value<std::string>()->default_value("default"), "")
|
||||
("password", value<std::string>()->default_value(""), "")
|
||||
("user,u", value<std::string>()->default_value(env_user_str.value_or("default")), "")
|
||||
("password", value<std::string>()->default_value(env_password_str.value_or("")), "")
|
||||
("database", value<std::string>()->default_value("default"), "")
|
||||
("stacktrace", "print stack traces of exceptions")
|
||||
("confidence", value<size_t>()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)")
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <optional>
|
||||
#include <base/scope_guard_safe.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <boost/program_options.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <filesystem>
|
||||
@ -125,13 +125,9 @@ std::vector<String> Client::loadWarningMessages()
|
||||
continue;
|
||||
|
||||
case Protocol::Server::Progress:
|
||||
continue;
|
||||
case Protocol::Server::ProfileInfo:
|
||||
continue;
|
||||
case Protocol::Server::Totals:
|
||||
continue;
|
||||
case Protocol::Server::Extremes:
|
||||
continue;
|
||||
case Protocol::Server::Log:
|
||||
continue;
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
#include <Common/TerminalSize.h>
|
||||
#include <IO/ConnectionTimeoutsContext.h>
|
||||
#include <Formats/registerFormats.h>
|
||||
#include <base/scope_guard_safe.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <unistd.h>
|
||||
#include <filesystem>
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Poco/Util/ServerApplication.h>
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
|
||||
#include "ClusterCopier.h"
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
#include <Poco/SplitterChannel.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
#include <Poco/ConsoleChannel.h>
|
||||
#include <Poco/AutoPtr.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
|
||||
|
||||
@ -89,9 +89,9 @@ if (BUILD_STANDALONE_KEEPER)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperLock.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperNodeCache.cpp
|
||||
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/BaseDaemon.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/SentryWriter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../base/daemon/GraphiteWriter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/BaseDaemon.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/SentryWriter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/GraphiteWriter.cpp
|
||||
|
||||
Keeper.cpp
|
||||
TinyContext.cpp
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
#include <filesystem>
|
||||
#include <IO/UseSSL.h>
|
||||
#include <Core/ServerUUID.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <base/ErrorHandlers.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/ErrorHandlers.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <base/safeExit.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Server/IServer.h>
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
#include "TinyContext.h"
|
||||
|
||||
namespace Poco
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Server/HTTP/HTTPRequestHandlerFactory.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Server/HTTP/HTTPRequestHandler.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include "SharedLibraryHandler.h"
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <bridge/IBridge.h>
|
||||
#include <Bridge/IBridge.h>
|
||||
#include "HandlerFactory.h"
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include "LibraryInterface.h"
|
||||
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/SharedLibrary.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include "LibraryUtils.h"
|
||||
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include <Interpreters/loadMetadata.h>
|
||||
#include <Interpreters/DatabaseCatalog.h>
|
||||
#include <base/getFQDNOrHostName.h>
|
||||
#include <base/scope_guard_safe.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <Interpreters/UserDefinedSQLObjectsLoader.h>
|
||||
#include <Interpreters/Session.h>
|
||||
#include <Access/AccessControl.h>
|
||||
@ -23,14 +23,15 @@
|
||||
#include <Common/TLDListsHolder.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <loggers/Loggers.h>
|
||||
#include <Loggers/Loggers.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
#include <IO/UseSSL.h>
|
||||
#include <IO/IOThreadPool.h>
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <base/ErrorHandlers.h>
|
||||
#include <Common/ErrorHandlers.h>
|
||||
#include <Functions/registerFunctions.h>
|
||||
#include <AggregateFunctions/registerAggregateFunctions.h>
|
||||
#include <TableFunctions/registerTableFunctions.h>
|
||||
@ -105,6 +106,17 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
||||
auto loaded_config = config_processor.loadConfig();
|
||||
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
|
||||
}
|
||||
|
||||
GlobalThreadPool::initialize(
|
||||
config().getUInt("max_thread_pool_size", 10000),
|
||||
config().getUInt("max_thread_pool_free_size", 1000),
|
||||
config().getUInt("thread_pool_queue_size", 10000)
|
||||
);
|
||||
|
||||
IOThreadPool::initialize(
|
||||
config().getUInt("max_io_thread_pool_size", 100),
|
||||
config().getUInt("max_io_thread_pool_free_size", 0),
|
||||
config().getUInt("io_thread_pool_queue_size", 10000));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <Common/ProgressIndication.h>
|
||||
#include <Common/StatusFile.h>
|
||||
#include <Common/InterruptListener.h>
|
||||
#include <loggers/Loggers.h>
|
||||
#include <Loggers/Loggers.h>
|
||||
#include <Core/Settings.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
#include <Poco/Net/HTTPServerRequest.h>
|
||||
#include <Poco/Net/HTTPServerResponse.h>
|
||||
#include <Poco/NumberParser.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include "getIdentifierQuote.h"
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <Common/config.h>
|
||||
#include <Poco/URI.h>
|
||||
#include <Poco/Net/HTTPServerRequest.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Poco/Net/HTTPServerRequest.h>
|
||||
#include <Poco/Net/HTTPServerResponse.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include "getIdentifierQuote.h"
|
||||
#include "validateODBCConnectionString.h"
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#include <QueryPipeline/QueryPipeline.h>
|
||||
#include <Processors/Executors/CompletedPipelineExecutor.h>
|
||||
#include <Processors/Formats/IInputFormat.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Server/HTTP/HTMLForm.h>
|
||||
#include <Common/config.h>
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include "ODBCBlockOutputStream.h"
|
||||
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Processors/Formats/IOutputFormat.h>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <bridge/IBridge.h>
|
||||
#include <Bridge/IBridge.h>
|
||||
#include "HandlerFactory.h"
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <nanodbc/nanodbc.h>
|
||||
#include <mutex>
|
||||
#include <base/BorrowedObjectPool.h>
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Poco/Net/HTTPServerRequest.h>
|
||||
#include <Poco/Net/HTTPServerResponse.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include "validateODBCConnectionString.h"
|
||||
#include "ODBCConnectionFactory.h"
|
||||
#include <sql.h>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#if USE_ODBC
|
||||
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <sql.h>
|
||||
#include <sqlext.h>
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/setThreadName.h>
|
||||
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Poco/Util/LayeredConfiguration.h>
|
||||
|
||||
@ -14,11 +14,11 @@
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <base/scope_guard_safe.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <base/defines.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/phdr_cache.h>
|
||||
#include <base/ErrorHandlers.h>
|
||||
#include <Common/ErrorHandlers.h>
|
||||
#include <base/getMemoryAmount.h>
|
||||
#include <base/getAvailableMemoryAmount.h>
|
||||
#include <base/errnoToString.h>
|
||||
@ -528,16 +528,19 @@ static int readNumber(const String & path)
|
||||
|
||||
#endif
|
||||
|
||||
static void sanityChecks(Server * server)
|
||||
static void sanityChecks(Server & server)
|
||||
{
|
||||
std::string data_path = getCanonicalPath(server->config().getString("path", DBMS_DEFAULT_PATH));
|
||||
std::string logs_path = server->config().getString("logger.log", "");
|
||||
std::string data_path = getCanonicalPath(server.config().getString("path", DBMS_DEFAULT_PATH));
|
||||
std::string logs_path = server.config().getString("logger.log", "");
|
||||
|
||||
if (server.logger().is(Poco::Message::PRIO_TEST))
|
||||
server.context()->addWarningMessage("Server logging level is set to 'test' and performance is degraded. This cannot be used in production.");
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
try
|
||||
{
|
||||
if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos)
|
||||
server->context()->addWarningMessage("Linux is not using fast TSC clock source. Performance can be degraded.");
|
||||
server.context()->addWarningMessage("Linux is not using fast TSC clock source. Performance can be degraded.");
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -546,7 +549,7 @@ static void sanityChecks(Server * server)
|
||||
try
|
||||
{
|
||||
if (readNumber("/proc/sys/vm/overcommit_memory") == 2)
|
||||
server->context()->addWarningMessage("Linux memory overcommit is disabled.");
|
||||
server.context()->addWarningMessage("Linux memory overcommit is disabled.");
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -555,7 +558,7 @@ static void sanityChecks(Server * server)
|
||||
try
|
||||
{
|
||||
if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos)
|
||||
server->context()->addWarningMessage("Linux transparent hugepage are set to \"always\".");
|
||||
server.context()->addWarningMessage("Linux transparent hugepage are set to \"always\".");
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -564,7 +567,7 @@ static void sanityChecks(Server * server)
|
||||
try
|
||||
{
|
||||
if (readNumber("/proc/sys/kernel/pid_max") < 30000)
|
||||
server->context()->addWarningMessage("Linux max PID is too low.");
|
||||
server.context()->addWarningMessage("Linux max PID is too low.");
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -573,7 +576,7 @@ static void sanityChecks(Server * server)
|
||||
try
|
||||
{
|
||||
if (readNumber("/proc/sys/kernel/threads-max") < 30000)
|
||||
server->context()->addWarningMessage("Linux threads max count is too low.");
|
||||
server.context()->addWarningMessage("Linux threads max count is too low.");
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -581,21 +584,22 @@ static void sanityChecks(Server * server)
|
||||
|
||||
std::string dev_id = getBlockDeviceId(data_path);
|
||||
if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0)
|
||||
server->context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
|
||||
server.context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded.");
|
||||
#endif
|
||||
|
||||
try
|
||||
{
|
||||
if (getAvailableMemoryAmount() < (2l << 30))
|
||||
server->context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
|
||||
server.context()->addWarningMessage("Available memory at server startup is too low (2GiB).");
|
||||
|
||||
if (!enoughSpaceInDirectory(data_path, 1ull << 30))
|
||||
server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
|
||||
server.context()->addWarningMessage("Available disk space for data at server startup is too low (1GiB): " + String(data_path));
|
||||
|
||||
if (!logs_path.empty())
|
||||
{
|
||||
if (!enoughSpaceInDirectory(fs::path(logs_path).parent_path(), 1ull << 30))
|
||||
server->context()->addWarningMessage("Available disk space at server startup is too low (1GiB).");
|
||||
auto logs_parent = fs::path(logs_path).parent_path();
|
||||
if (!enoughSpaceInDirectory(logs_parent, 1ull << 30))
|
||||
server.context()->addWarningMessage("Available disk space for logs at server startup is too low (1GiB): " + String(logs_parent));
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
@ -643,7 +647,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
global_context->addWarningMessage("Server was built with sanitizer. It will work slowly.");
|
||||
#endif
|
||||
|
||||
sanityChecks(this);
|
||||
sanityChecks(*this);
|
||||
|
||||
// Initialize global thread pool. Do it before we fetch configs from zookeeper
|
||||
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
#include <Server/IServer.h>
|
||||
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
|
||||
/** Server provides three interfaces:
|
||||
* 1. HTTP - simple interface for any applications.
|
||||
|
||||
@ -236,8 +236,8 @@
|
||||
<openSSL>
|
||||
<server> <!-- Used for https server AND secure tcp port -->
|
||||
<!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
|
||||
<certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
|
||||
<privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
|
||||
<!-- <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
|
||||
<privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile> -->
|
||||
<!-- dhparams are optional. You can delete the <dhParamsFile> element.
|
||||
To generate dhparams, use the following command:
|
||||
openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
|
||||
@ -367,7 +367,7 @@
|
||||
|
||||
<!-- Path to temporary data for processing hard queries. -->
|
||||
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
||||
|
||||
|
||||
<!-- Disable AuthType plaintext_password and no_password for ACL. -->
|
||||
<!-- <allow_plaintext_password>0</allow_plaintext_password> -->
|
||||
<!-- <allow_no_password>0</allow_no_password> -->`
|
||||
@ -1297,8 +1297,8 @@
|
||||
-->
|
||||
|
||||
<!-- Uncomment if enable merge tree metadata cache -->
|
||||
<merge_tree_metadata_cache>
|
||||
<!--merge_tree_metadata_cache>
|
||||
<lru_cache_size>268435456</lru_cache_size>
|
||||
<continue_if_corrupted>true</continue_if_corrupted>
|
||||
</merge_tree_metadata_cache>
|
||||
</merge_tree_metadata_cache-->
|
||||
</clickhouse>
|
||||
|
||||
@ -177,8 +177,8 @@ openSSL:
|
||||
server:
|
||||
# Used for https server AND secure tcp port
|
||||
# openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt
|
||||
certificateFile: /etc/clickhouse-server/server.crt
|
||||
privateKeyFile: /etc/clickhouse-server/server.key
|
||||
# certificateFile: /etc/clickhouse-server/server.crt
|
||||
# privateKeyFile: /etc/clickhouse-server/server.key
|
||||
|
||||
# dhparams are optional. You can delete the dhParamsFile: element.
|
||||
# To generate dhparams, use the following command:
|
||||
|
||||
@ -233,7 +233,7 @@
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
td.right
|
||||
.right
|
||||
{
|
||||
text-align: right;
|
||||
}
|
||||
@ -272,6 +272,26 @@
|
||||
max-width: none;
|
||||
}
|
||||
|
||||
td.transposed
|
||||
{
|
||||
max-width: none;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
td.empty-result
|
||||
{
|
||||
text-align: center;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
div.empty-result
|
||||
{
|
||||
opacity: 10%;
|
||||
font-size: 7vw;
|
||||
font-family: Liberation Sans, DejaVu Sans, sans-serif;
|
||||
}
|
||||
|
||||
/* The style for SQL NULL */
|
||||
.null
|
||||
{
|
||||
@ -613,8 +633,108 @@
|
||||
}
|
||||
}
|
||||
|
||||
function renderCell(cell, col_idx, settings)
|
||||
{
|
||||
let td = document.createElement('td');
|
||||
|
||||
let is_null = (cell === null);
|
||||
let is_link = false;
|
||||
|
||||
/// Test: SELECT number, toString(number) AS str, number % 2 ? number : NULL AS nullable, range(number) AS arr, CAST((['hello', 'world'], [number, number % 2]) AS Map(String, UInt64)) AS map FROM numbers(10)
|
||||
let text;
|
||||
if (is_null) {
|
||||
text = 'ᴺᵁᴸᴸ';
|
||||
} else if (typeof(cell) === 'object') {
|
||||
text = JSON.stringify(cell);
|
||||
} else {
|
||||
text = cell;
|
||||
|
||||
/// If it looks like URL, create a link. This is for convenience.
|
||||
if (typeof(cell) == 'string' && cell.match(/^https?:\/\/\S+$/)) {
|
||||
is_link = true;
|
||||
}
|
||||
}
|
||||
|
||||
let node = document.createTextNode(text);
|
||||
if (is_link) {
|
||||
let link = document.createElement('a');
|
||||
link.appendChild(node);
|
||||
link.href = text;
|
||||
link.setAttribute('target', '_blank');
|
||||
node = link;
|
||||
}
|
||||
|
||||
if (settings.is_transposed) {
|
||||
td.className = 'left transposed';
|
||||
} else {
|
||||
td.className = settings.column_is_number[col_idx] ? 'right' : 'left';
|
||||
}
|
||||
if (is_null) {
|
||||
td.className += ' null';
|
||||
}
|
||||
|
||||
/// If it's a number, render bar in background.
|
||||
if (!settings.is_transposed && settings.column_need_render_bars[col_idx] && text > 0) {
|
||||
const ratio = 100 * text / settings.column_maximums[col_idx];
|
||||
|
||||
let div = document.createElement('div');
|
||||
|
||||
div.style.width = '100%';
|
||||
div.style.background = `linear-gradient(to right,
|
||||
var(--bar-color) 0%, var(--bar-color) ${ratio}%,
|
||||
transparent ${ratio}%, transparent 100%)`;
|
||||
|
||||
div.appendChild(node);
|
||||
node = div;
|
||||
}
|
||||
|
||||
td.appendChild(node);
|
||||
return td;
|
||||
}
|
||||
|
||||
function renderTableTransposed(response)
|
||||
{
|
||||
let tbody = document.createElement('tbody');
|
||||
for (let col_idx in response.meta) {
|
||||
let tr = document.createElement('tr');
|
||||
{
|
||||
let th = document.createElement('th');
|
||||
th.className = 'right';
|
||||
th.style.width = '0';
|
||||
th.appendChild(document.createTextNode(response.meta[col_idx].name));
|
||||
tr.appendChild(th);
|
||||
}
|
||||
for (let row_idx in response.data)
|
||||
{
|
||||
let cell = response.data[row_idx][col_idx];
|
||||
const td = renderCell(cell, col_idx, {is_transposed: true});
|
||||
tr.appendChild(td);
|
||||
}
|
||||
if (response.data.length == 0 && col_idx == 0)
|
||||
{
|
||||
/// If result is empty, show this fact with a style.
|
||||
let td = document.createElement('td');
|
||||
td.rowSpan = response.meta.length;
|
||||
td.className = 'empty-result';
|
||||
let div = document.createElement('div');
|
||||
div.appendChild(document.createTextNode("empty result"));
|
||||
div.className = 'empty-result';
|
||||
td.appendChild(div);
|
||||
tr.appendChild(td);
|
||||
}
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
let table = document.getElementById('data-table');
|
||||
table.appendChild(tbody);
|
||||
}
|
||||
|
||||
function renderTable(response)
|
||||
{
|
||||
if (response.data.length <= 1 && response.meta.length >= 5) {
|
||||
renderTableTransposed(response)
|
||||
return;
|
||||
}
|
||||
|
||||
let thead = document.createElement('thead');
|
||||
for (let idx in response.meta) {
|
||||
let th = document.createElement('th');
|
||||
@ -633,61 +753,20 @@
|
||||
const column_minimums = column_is_number.map((elem, idx) => elem ? Math.min(...response.data.map(row => Math.max(0, row[idx]))) : 0);
|
||||
const column_need_render_bars = column_is_number.map((elem, idx) => column_maximums[idx] > 0 && column_maximums[idx] > column_minimums[idx]);
|
||||
|
||||
const settings = {
|
||||
is_transposed: false,
|
||||
column_is_number: column_is_number,
|
||||
column_maximums: column_maximums,
|
||||
column_minimums: column_minimums,
|
||||
column_need_render_bars: column_need_render_bars,
|
||||
};
|
||||
|
||||
let tbody = document.createElement('tbody');
|
||||
for (let row_idx in response.data) {
|
||||
let tr = document.createElement('tr');
|
||||
for (let col_idx in response.data[row_idx]) {
|
||||
let td = document.createElement('td');
|
||||
let cell = response.data[row_idx][col_idx];
|
||||
|
||||
let is_null = (cell === null);
|
||||
let is_link = false;
|
||||
|
||||
/// Test: SELECT number, toString(number) AS str, number % 2 ? number : NULL AS nullable, range(number) AS arr, CAST((['hello', 'world'], [number, number % 2]) AS Map(String, UInt64)) AS map FROM numbers(10)
|
||||
let text;
|
||||
if (is_null) {
|
||||
text = 'ᴺᵁᴸᴸ';
|
||||
} else if (typeof(cell) === 'object') {
|
||||
text = JSON.stringify(cell);
|
||||
} else {
|
||||
text = cell;
|
||||
|
||||
/// If it looks like URL, create a link. This is for convenience.
|
||||
if (typeof(cell) == 'string' && cell.match(/^https?:\/\/\S+$/)) {
|
||||
is_link = true;
|
||||
}
|
||||
}
|
||||
|
||||
let node = document.createTextNode(text);
|
||||
if (is_link) {
|
||||
let link = document.createElement('a');
|
||||
link.appendChild(node);
|
||||
link.href = text;
|
||||
link.setAttribute('target', '_blank');
|
||||
node = link;
|
||||
}
|
||||
|
||||
td.className = column_is_number[col_idx] ? 'right' : 'left';
|
||||
if (is_null) {
|
||||
td.className += ' null';
|
||||
}
|
||||
|
||||
/// If it's a number, render bar in background.
|
||||
if (column_need_render_bars[col_idx] && text > 0) {
|
||||
const ratio = 100 * text / column_maximums[col_idx];
|
||||
|
||||
let div = document.createElement('div');
|
||||
|
||||
div.style.width = '100%';
|
||||
div.style.background = `linear-gradient(to right,
|
||||
var(--bar-color) 0%, var(--bar-color) ${ratio}%,
|
||||
transparent ${ratio}%, transparent 100%)`;
|
||||
|
||||
div.appendChild(node);
|
||||
node = div;
|
||||
}
|
||||
|
||||
td.appendChild(node);
|
||||
const td = renderCell(cell, col_idx, settings);
|
||||
tr.appendChild(td);
|
||||
}
|
||||
tbody.appendChild(tr);
|
||||
@ -787,10 +866,7 @@
|
||||
document.documentElement.setAttribute('data-theme', theme);
|
||||
}
|
||||
|
||||
/**
|
||||
* First we check if theme is set via the 'theme' GET parameter, if not, we check localStorage,
|
||||
* otherwise we check OS preference
|
||||
*/
|
||||
/// First we check if theme is set via the 'theme' GET parameter, if not, we check localStorage, otherwise we check OS preference.
|
||||
let theme = current_url.searchParams.get('theme');
|
||||
if (['dark', 'light'].indexOf(theme) === -1) {
|
||||
theme = window.localStorage.getItem('theme');
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <Access/AccessRights.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/sort.h>
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
|
||||
@ -158,6 +158,7 @@ enum class AccessType
|
||||
M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_RESTORE_REPLICA, "RESTORE REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_SYNC_DATABASE_REPLICA, "SYNC DATABASE REPLICA", DATABASE, SYSTEM) \
|
||||
M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH_LOGS, "FLUSH LOGS", GLOBAL, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include <Access/Common/AllowedClientHosts.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <Functions/likePatternToRegexp.h>
|
||||
#include <Poco/Net/SocketAddress.h>
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
#include <Core/Settings.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <boost/range/algorithm/set_algorithm.hpp>
|
||||
#include <assert.h>
|
||||
|
||||
@ -5,7 +5,6 @@
|
||||
#include <Interpreters/ClientInfo.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <base/shared_ptr_helper.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <Interpreters/Access/InterpreterCreateUserQuery.h>
|
||||
#include <Interpreters/Access/InterpreterShowGrantsQuery.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Poco/JSON/JSON.h>
|
||||
#include <Poco/JSON/Object.h>
|
||||
#include <Poco/JSON/Stringifier.h>
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <Access/Credentials.h>
|
||||
#include <Access/LDAPClient.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
#include <Poco/JSON/JSON.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#include <Access/LDAPClient.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <base/scope_guard.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
#include <Poco/Logger.h>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
#include <Poco/JSON/JSON.h>
|
||||
#include <Poco/JSON/Object.h>
|
||||
#include <Poco/JSON/Stringifier.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
#include <cstring>
|
||||
|
||||
@ -53,7 +53,7 @@ TEST(AccessRights, Union)
|
||||
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
|
||||
"SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
|
||||
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
|
||||
"SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
|
||||
"SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -225,26 +225,38 @@ public:
|
||||
}
|
||||
|
||||
void
|
||||
addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const final
|
||||
addBatchSinglePlace(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos) const final
|
||||
{
|
||||
AggregateFunctionSumData<Numerator> sum_data;
|
||||
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
sum_data.addManyConditional(column.getData().data(), flags.data(), batch_size);
|
||||
this->data(place).denominator += countBytesInFilter(flags.data(), batch_size);
|
||||
sum_data.addManyConditional(column.getData().data(), flags.data(), row_begin, row_end);
|
||||
this->data(place).denominator += countBytesInFilter(flags.data(), row_begin, row_end);
|
||||
}
|
||||
else
|
||||
{
|
||||
sum_data.addMany(column.getData().data(), batch_size);
|
||||
this->data(place).denominator += batch_size;
|
||||
sum_data.addMany(column.getData().data(), row_begin, row_end);
|
||||
this->data(place).denominator += (row_end - row_begin);
|
||||
}
|
||||
increment(place, sum_data.sum);
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena *, ssize_t if_argument_pos)
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos)
|
||||
const final
|
||||
{
|
||||
AggregateFunctionSumData<Numerator> sum_data;
|
||||
@ -253,22 +265,22 @@ public:
|
||||
{
|
||||
/// Merge the 2 sets of flags (null and if) into a single one. This allows us to use parallelizable sums when available
|
||||
const auto * if_flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData().data();
|
||||
auto final_flags = std::make_unique<UInt8[]>(batch_size);
|
||||
auto final_flags = std::make_unique<UInt8[]>(row_end);
|
||||
size_t used_value = 0;
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
UInt8 kept = (!null_map[i]) & !!if_flags[i];
|
||||
final_flags[i] = kept;
|
||||
used_value += kept;
|
||||
}
|
||||
|
||||
sum_data.addManyConditional(column.getData().data(), final_flags.get(), batch_size);
|
||||
sum_data.addManyConditional(column.getData().data(), final_flags.get(), row_begin, row_end);
|
||||
this->data(place).denominator += used_value;
|
||||
}
|
||||
else
|
||||
{
|
||||
sum_data.addManyNotNull(column.getData().data(), null_map, batch_size);
|
||||
this->data(place).denominator += batch_size - countBytesInFilter(null_map, batch_size);
|
||||
sum_data.addManyNotNull(column.getData().data(), null_map, row_begin, row_end);
|
||||
this->data(place).denominator += (row_end - row_begin) - countBytesInFilter(null_map, row_begin, row_end);
|
||||
}
|
||||
increment(place, sum_data.sum);
|
||||
}
|
||||
|
||||
@ -54,7 +54,12 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos) const override
|
||||
{
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
@ -63,12 +68,13 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
data(place).count += batch_size;
|
||||
data(place).count += row_end - row_begin;
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
@ -78,11 +84,12 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
data(place).count += countBytesInFilterWithNull(flags, null_map);
|
||||
data(place).count += countBytesInFilterWithNull(flags, null_map, row_begin, row_end);
|
||||
}
|
||||
else
|
||||
{
|
||||
data(place).count += batch_size - countBytesInFilter(null_map, batch_size);
|
||||
size_t rows = row_end - row_begin;
|
||||
data(place).count += rows - countBytesInFilter(null_map, row_begin, row_end);
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,17 +211,23 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos) const override
|
||||
{
|
||||
const auto & nc = assert_cast<const ColumnNullable &>(*columns[0]);
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
data(place).count += countBytesInFilterWithNull(flags, nc.getNullMapData().data());
|
||||
data(place).count += countBytesInFilterWithNull(flags, nc.getNullMapData().data(), row_begin, row_end);
|
||||
}
|
||||
else
|
||||
{
|
||||
data(place).count += batch_size - countBytesInFilter(nc.getNullMapData().data(), batch_size);
|
||||
size_t rows = row_end - row_begin;
|
||||
data(place).count += rows - countBytesInFilter(nc.getNullMapData().data(), row_begin, row_end);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -200,7 +200,7 @@ public:
|
||||
arguments_raw[i] = arguments[i].get();
|
||||
|
||||
assert(!arguments.empty());
|
||||
nested_func->addBatchSinglePlace(arguments[0]->size(), getNestedPlace(place), arguments_raw.data(), arena);
|
||||
nested_func->addBatchSinglePlace(0, arguments[0]->size(), getNestedPlace(place), arguments_raw.data(), arena);
|
||||
nested_func->insertResultInto(getNestedPlace(place), to, arena);
|
||||
}
|
||||
|
||||
|
||||
@ -37,18 +37,18 @@ inline TColumn readItem(const IColumn * column, Arena * arena, size_t row)
|
||||
|
||||
template <typename TColumn, typename TFilter = void>
|
||||
size_t
|
||||
getFirstNElements_low_threshold(const TColumn * data, int num_elements, int threshold, size_t * results, const TFilter * filter = nullptr)
|
||||
getFirstNElements_low_threshold(const TColumn * data, size_t row_begin, size_t row_end, size_t threshold, size_t * results, const TFilter * filter = nullptr)
|
||||
{
|
||||
for (int i = 0; i < threshold; i++)
|
||||
for (size_t i = 0; i < threshold; i++)
|
||||
{
|
||||
results[i] = 0;
|
||||
}
|
||||
|
||||
threshold = std::min(num_elements, threshold);
|
||||
int current_max = 0;
|
||||
int cur;
|
||||
int z;
|
||||
for (int i = 0; i < num_elements; i++)
|
||||
threshold = std::min(row_end - row_begin, threshold);
|
||||
size_t current_max = 0;
|
||||
size_t cur;
|
||||
size_t z;
|
||||
for (size_t i = row_begin; i < row_end; i++)
|
||||
{
|
||||
if constexpr (!std::is_same_v<TFilter, void>)
|
||||
{
|
||||
@ -90,12 +90,12 @@ struct SortableItem
|
||||
|
||||
template <typename TColumn, typename TFilter = void>
|
||||
size_t getFirstNElements_high_threshold(
|
||||
const TColumn * data, size_t num_elements, size_t threshold, size_t * results, const TFilter * filter = nullptr)
|
||||
const TColumn * data, size_t row_begin, size_t row_end, size_t threshold, size_t * results, const TFilter * filter = nullptr)
|
||||
{
|
||||
std::vector<SortableItem<TColumn>> dataIndexed(num_elements);
|
||||
std::vector<SortableItem<TColumn>> dataIndexed(row_end);
|
||||
size_t num_elements_filtered = 0;
|
||||
|
||||
for (size_t i = 0; i < num_elements; i++)
|
||||
for (size_t i = row_begin; i < row_end; i++)
|
||||
{
|
||||
if constexpr (!std::is_same_v<TFilter, void>)
|
||||
{
|
||||
@ -124,21 +124,21 @@ size_t getFirstNElements_high_threshold(
|
||||
static const size_t THRESHOLD_MAX_CUSTOM_FUNCTION = 1000;
|
||||
|
||||
template <typename TColumn>
|
||||
size_t getFirstNElements(const TColumn * data, size_t num_elements, size_t threshold, size_t * results, const UInt8 * filter = nullptr)
|
||||
size_t getFirstNElements(const TColumn * data, size_t row_begin, size_t row_end, size_t threshold, size_t * results, const UInt8 * filter = nullptr)
|
||||
{
|
||||
if (threshold < THRESHOLD_MAX_CUSTOM_FUNCTION)
|
||||
{
|
||||
if (filter != nullptr)
|
||||
return getFirstNElements_low_threshold(data, num_elements, threshold, results, filter);
|
||||
return getFirstNElements_low_threshold(data, row_begin, row_end, threshold, results, filter);
|
||||
else
|
||||
return getFirstNElements_low_threshold(data, num_elements, threshold, results);
|
||||
return getFirstNElements_low_threshold(data, row_begin, row_end, threshold, results);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (filter != nullptr)
|
||||
return getFirstNElements_high_threshold(data, num_elements, threshold, results, filter);
|
||||
return getFirstNElements_high_threshold(data, row_begin, row_end, threshold, results, filter);
|
||||
else
|
||||
return getFirstNElements_high_threshold(data, num_elements, threshold, results);
|
||||
return getFirstNElements_high_threshold(data, row_begin, row_end, threshold, results);
|
||||
}
|
||||
}
|
||||
|
||||
@ -203,7 +203,7 @@ public:
|
||||
|
||||
template <typename TColumn, bool is_plain, typename TFunc>
|
||||
void
|
||||
forFirstRows(size_t batch_size, const IColumn ** columns, size_t data_column, Arena * arena, ssize_t if_argument_pos, TFunc func) const
|
||||
forFirstRows(size_t row_begin, size_t row_end, const IColumn ** columns, size_t data_column, Arena * arena, ssize_t if_argument_pos, TFunc func) const
|
||||
{
|
||||
const TColumn * values = nullptr;
|
||||
std::unique_ptr<std::vector<TColumn>> values_vector;
|
||||
@ -211,8 +211,8 @@ public:
|
||||
|
||||
if constexpr (std::is_same_v<TColumn, StringRef>)
|
||||
{
|
||||
values_vector.reset(new std::vector<TColumn>(batch_size));
|
||||
for (size_t i = 0; i < batch_size; i++)
|
||||
values_vector.reset(new std::vector<TColumn>(row_end));
|
||||
for (size_t i = row_begin; i < row_end; i++)
|
||||
(*values_vector)[i] = readItem<TColumn, is_plain>(columns[data_column], arena, i);
|
||||
values = (*values_vector).data();
|
||||
}
|
||||
@ -231,7 +231,7 @@ public:
|
||||
filter = reinterpret_cast<const UInt8 *>(refFilter.data);
|
||||
}
|
||||
|
||||
size_t num_elements = getFirstNElements(values, batch_size, threshold, best_rows.data(), filter);
|
||||
size_t num_elements = getFirstNElements(values, row_begin, row_end, threshold, best_rows.data(), filter);
|
||||
for (size_t i = 0; i < num_elements; i++)
|
||||
{
|
||||
func(best_rows[i], values);
|
||||
@ -239,14 +239,19 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos) const override
|
||||
{
|
||||
State & data = this->data(place);
|
||||
|
||||
if constexpr (use_column_b)
|
||||
{
|
||||
forFirstRows<TColumnB, is_plain_b>(
|
||||
batch_size, columns, 1, arena, if_argument_pos, [columns, &arena, &data](size_t row, const TColumnB * values)
|
||||
row_begin, row_end, columns, 1, arena, if_argument_pos, [columns, &arena, &data](size_t row, const TColumnB * values)
|
||||
{
|
||||
data.add(readItem<TColumnA, is_plain_a>(columns[0], arena, row), values[row]);
|
||||
});
|
||||
@ -254,7 +259,7 @@ public:
|
||||
else
|
||||
{
|
||||
forFirstRows<TColumnA, is_plain_a>(
|
||||
batch_size, columns, 0, arena, if_argument_pos, [&data](size_t row, const TColumnA * values)
|
||||
row_begin, row_end, columns, 0, arena, if_argument_pos, [&data](size_t row, const TColumnA * values)
|
||||
{
|
||||
data.add(values[row]);
|
||||
});
|
||||
|
||||
@ -119,7 +119,13 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t) const override
|
||||
void addBatchSinglePlace(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t) const override
|
||||
{
|
||||
const ColumnNullable * column = assert_cast<const ColumnNullable *>(columns[0]);
|
||||
const UInt8 * null_map = column->getNullMapData().data();
|
||||
@ -142,25 +148,31 @@ public:
|
||||
/// Combine the 2 flag arrays so we can call a simplified version (one check vs 2)
|
||||
/// Note that now the null map will contain 0 if not null and not filtered, or 1 for null or filtered (or both)
|
||||
|
||||
auto final_nulls = std::make_unique<UInt8[]>(batch_size);
|
||||
auto final_nulls = std::make_unique<UInt8[]>(row_end);
|
||||
|
||||
if (filter_null_map)
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
final_nulls[i] = (!!null_map[i]) | (!filter_values[i]) | (!!filter_null_map[i]);
|
||||
else
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
final_nulls[i] = (!!null_map[i]) | (!filter_values[i]);
|
||||
|
||||
if constexpr (result_is_nullable)
|
||||
{
|
||||
if (!memoryIsByte(final_nulls.get(), batch_size, 1))
|
||||
if (!memoryIsByte(final_nulls.get(), row_begin, row_end, 1))
|
||||
this->setFlag(place);
|
||||
else
|
||||
return; /// No work to do.
|
||||
}
|
||||
|
||||
this->nested_function->addBatchSinglePlaceNotNull(
|
||||
batch_size, this->nestedPlace(place), columns_param, final_nulls.get(), arena, -1);
|
||||
row_begin,
|
||||
row_end,
|
||||
this->nestedPlace(place),
|
||||
columns_param,
|
||||
final_nulls.get(),
|
||||
arena,
|
||||
-1);
|
||||
}
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
|
||||
@ -98,31 +98,38 @@ public:
|
||||
}
|
||||
|
||||
void addBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t) const override
|
||||
{
|
||||
nested_func->addBatch(batch_size, places, place_offset, columns, arena, num_arguments - 1);
|
||||
nested_func->addBatch(row_begin, row_end, places, place_offset, columns, arena, num_arguments - 1);
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t) const override
|
||||
{
|
||||
nested_func->addBatchSinglePlace(batch_size, place, columns, arena, num_arguments - 1);
|
||||
nested_func->addBatchSinglePlace(row_begin, row_end, place, columns, arena, num_arguments - 1);
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
Arena * arena,
|
||||
ssize_t) const override
|
||||
{
|
||||
nested_func->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, num_arguments - 1);
|
||||
nested_func->addBatchSinglePlaceNotNull(row_begin, row_end, place, columns, null_map, arena, num_arguments - 1);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override
|
||||
@ -131,13 +138,14 @@ public:
|
||||
}
|
||||
|
||||
void mergeBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const AggregateDataPtr * rhs,
|
||||
Arena * arena) const override
|
||||
{
|
||||
nested_func->mergeBatch(batch_size, places, place_offset, rhs, arena);
|
||||
nested_func->mergeBatch(row_begin, row_end, places, place_offset, rhs, arena);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> version) const override
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/sort.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
@ -1159,7 +1159,12 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos) const override
|
||||
{
|
||||
if constexpr (is_any)
|
||||
if (this->data(place).has())
|
||||
@ -1167,7 +1172,7 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i])
|
||||
{
|
||||
@ -1179,7 +1184,7 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
this->data(place).changeIfBetter(*columns[0], i, arena);
|
||||
if constexpr (is_any)
|
||||
@ -1189,7 +1194,8 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
@ -1203,7 +1209,7 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (!null_map[i] && flags[i])
|
||||
{
|
||||
@ -1215,7 +1221,7 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (!null_map[i])
|
||||
{
|
||||
|
||||
@ -307,17 +307,22 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace( /// NOLINT
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1) const override
|
||||
{
|
||||
const ColumnNullable * column = assert_cast<const ColumnNullable *>(columns[0]);
|
||||
const IColumn * nested_column = &column->getNestedColumn();
|
||||
const UInt8 * null_map = column->getNullMapData().data();
|
||||
|
||||
this->nested_function->addBatchSinglePlaceNotNull(
|
||||
batch_size, this->nestedPlace(place), &nested_column, null_map, arena, if_argument_pos);
|
||||
row_begin, row_end, this->nestedPlace(place), &nested_column, null_map, arena, if_argument_pos);
|
||||
|
||||
if constexpr (result_is_nullable)
|
||||
if (!memoryIsByte(null_map, batch_size, 1))
|
||||
if (!memoryIsByte(null_map, row_begin, row_end, 1))
|
||||
this->setFlag(place);
|
||||
}
|
||||
|
||||
|
||||
@ -109,7 +109,8 @@ public:
|
||||
}
|
||||
|
||||
void addBatch( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
@ -119,7 +120,7 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i] && places[i])
|
||||
add(places[i] + place_offset, columns, i, arena);
|
||||
@ -127,21 +128,26 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
nested_function->addBatch(batch_size, places, place_offset, columns, arena, if_argument_pos);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
nested_function->addBatch(row_begin, row_end, places, place_offset, columns, arena, if_argument_pos);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
if (places[i])
|
||||
(places[i] + place_offset)[size_of_data] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlace( /// NOLINT
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1) const override
|
||||
{
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
nested_function->addBatchSinglePlace(batch_size, place, columns, arena, if_argument_pos);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
nested_function->addBatchSinglePlace(row_begin, row_end, place, columns, arena, if_argument_pos);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i])
|
||||
{
|
||||
@ -152,16 +158,17 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
if (batch_size)
|
||||
if (row_end != row_begin)
|
||||
{
|
||||
nested_function->addBatchSinglePlace(batch_size, place, columns, arena, if_argument_pos);
|
||||
nested_function->addBatchSinglePlace(row_begin, row_end, place, columns, arena, if_argument_pos);
|
||||
place[size_of_data] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
@ -171,8 +178,8 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, if_argument_pos);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
nested_function->addBatchSinglePlaceNotNull(row_begin, row_end, place, columns, null_map, arena, if_argument_pos);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i] && !null_map[i])
|
||||
{
|
||||
@ -183,10 +190,10 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
if (batch_size)
|
||||
if (row_end != row_begin)
|
||||
{
|
||||
nested_function->addBatchSinglePlaceNotNull(batch_size, place, columns, null_map, arena, if_argument_pos);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
nested_function->addBatchSinglePlaceNotNull(row_begin, row_end, place, columns, null_map, arena, if_argument_pos);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (!null_map[i])
|
||||
{
|
||||
@ -208,14 +215,15 @@ public:
|
||||
}
|
||||
|
||||
void mergeBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const AggregateDataPtr * rhs,
|
||||
Arena * arena) const override
|
||||
{
|
||||
nested_function->mergeBatch(batch_size, places, place_offset, rhs, arena);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
nested_function->mergeBatch(row_begin, row_end, places, place_offset, rhs, arena);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
(places[i] + place_offset)[size_of_data] |= rhs[i][size_of_data];
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
|
||||
|
||||
@ -52,6 +52,8 @@ public:
|
||||
return nested_func->getDefaultVersion();
|
||||
}
|
||||
|
||||
size_t getVersionFromRevision(size_t revision) const override { return nested_func->getVersionFromRevision(revision); }
|
||||
|
||||
void create(AggregateDataPtr __restrict place) const override
|
||||
{
|
||||
nested_func->create(place);
|
||||
|
||||
@ -59,9 +59,11 @@ struct AggregateFunctionSumData
|
||||
|
||||
/// Vectorized version
|
||||
template <typename Value>
|
||||
void NO_SANITIZE_UNDEFINED NO_INLINE addMany(const Value * __restrict ptr, size_t count)
|
||||
void NO_SANITIZE_UNDEFINED NO_INLINE addMany(const Value * __restrict ptr, size_t start, size_t end)
|
||||
{
|
||||
const auto * end = ptr + count;
|
||||
ptr += start;
|
||||
size_t count = end - start;
|
||||
const auto * end_ptr = ptr + count;
|
||||
|
||||
if constexpr (std::is_floating_point_v<T>)
|
||||
{
|
||||
@ -87,7 +89,7 @@ struct AggregateFunctionSumData
|
||||
|
||||
/// clang cannot vectorize the loop if accumulator is class member instead of local variable.
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
while (ptr < end_ptr)
|
||||
{
|
||||
Impl::add(local_sum, *ptr);
|
||||
++ptr;
|
||||
@ -97,9 +99,11 @@ struct AggregateFunctionSumData
|
||||
|
||||
template <typename Value, bool add_if_zero>
|
||||
void NO_SANITIZE_UNDEFINED NO_INLINE
|
||||
addManyConditionalInternal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t count)
|
||||
addManyConditionalInternal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t start, size_t end)
|
||||
{
|
||||
const auto * end = ptr + count;
|
||||
ptr += start;
|
||||
size_t count = end - start;
|
||||
const auto * end_ptr = ptr + count;
|
||||
|
||||
if constexpr (
|
||||
(is_integer<T> && !is_big_int_v<T>)
|
||||
@ -108,7 +112,7 @@ struct AggregateFunctionSumData
|
||||
/// For integers we can vectorize the operation if we replace the null check using a multiplication (by 0 for null, 1 for not null)
|
||||
/// https://quick-bench.com/q/MLTnfTvwC2qZFVeWHfOBR3U7a8I
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
while (ptr < end_ptr)
|
||||
{
|
||||
T multiplier = !*condition_map == add_if_zero;
|
||||
Impl::add(local_sum, *ptr * multiplier);
|
||||
@ -151,7 +155,7 @@ struct AggregateFunctionSumData
|
||||
}
|
||||
|
||||
T local_sum{};
|
||||
while (ptr < end)
|
||||
while (ptr < end_ptr)
|
||||
{
|
||||
if (!*condition_map == add_if_zero)
|
||||
Impl::add(local_sum, *ptr);
|
||||
@ -162,15 +166,15 @@ struct AggregateFunctionSumData
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
|
||||
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t start, size_t end)
|
||||
{
|
||||
return addManyConditionalInternal<Value, true>(ptr, null_map, count);
|
||||
return addManyConditionalInternal<Value, true>(ptr, null_map, start, end);
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t count)
|
||||
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t start, size_t end)
|
||||
{
|
||||
return addManyConditionalInternal<Value, false>(ptr, cond_map, count);
|
||||
return addManyConditionalInternal<Value, false>(ptr, cond_map, start, end);
|
||||
}
|
||||
|
||||
void NO_SANITIZE_UNDEFINED merge(const AggregateFunctionSumData & rhs)
|
||||
@ -220,7 +224,7 @@ struct AggregateFunctionSumKahanData
|
||||
|
||||
/// Vectorized version
|
||||
template <typename Value>
|
||||
void NO_INLINE addMany(const Value * __restrict ptr, size_t count)
|
||||
void NO_INLINE addMany(const Value * __restrict ptr, size_t start, size_t end)
|
||||
{
|
||||
/// Less than in ordinary sum, because the algorithm is more complicated and too large loop unrolling is questionable.
|
||||
/// But this is just a guess.
|
||||
@ -228,7 +232,10 @@ struct AggregateFunctionSumKahanData
|
||||
T partial_sums[unroll_count]{};
|
||||
T partial_compensations[unroll_count]{};
|
||||
|
||||
const auto * end = ptr + count;
|
||||
ptr += start;
|
||||
size_t count = end - start;
|
||||
|
||||
const auto * end_ptr = ptr + count;
|
||||
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
||||
|
||||
while (ptr < unrolled_end)
|
||||
@ -241,7 +248,7 @@ struct AggregateFunctionSumKahanData
|
||||
for (size_t i = 0; i < unroll_count; ++i)
|
||||
mergeImpl(sum, compensation, partial_sums[i], partial_compensations[i]);
|
||||
|
||||
while (ptr < end)
|
||||
while (ptr < end_ptr)
|
||||
{
|
||||
addImpl(*ptr, sum, compensation);
|
||||
++ptr;
|
||||
@ -249,13 +256,16 @@ struct AggregateFunctionSumKahanData
|
||||
}
|
||||
|
||||
template <typename Value, bool add_if_zero>
|
||||
void NO_INLINE addManyConditionalInternal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t count)
|
||||
void NO_INLINE addManyConditionalInternal(const Value * __restrict ptr, const UInt8 * __restrict condition_map, size_t start, size_t end)
|
||||
{
|
||||
constexpr size_t unroll_count = 4;
|
||||
T partial_sums[unroll_count]{};
|
||||
T partial_compensations[unroll_count]{};
|
||||
|
||||
const auto * end = ptr + count;
|
||||
ptr += start;
|
||||
size_t count = end - start;
|
||||
|
||||
const auto * end_ptr = ptr + count;
|
||||
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
||||
|
||||
while (ptr < unrolled_end)
|
||||
@ -270,7 +280,7 @@ struct AggregateFunctionSumKahanData
|
||||
for (size_t i = 0; i < unroll_count; ++i)
|
||||
mergeImpl(sum, compensation, partial_sums[i], partial_compensations[i]);
|
||||
|
||||
while (ptr < end)
|
||||
while (ptr < end_ptr)
|
||||
{
|
||||
if ((!*condition_map) == add_if_zero)
|
||||
addImpl(*ptr, sum, compensation);
|
||||
@ -280,15 +290,15 @@ struct AggregateFunctionSumKahanData
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t count)
|
||||
void ALWAYS_INLINE addManyNotNull(const Value * __restrict ptr, const UInt8 * __restrict null_map, size_t start, size_t end)
|
||||
{
|
||||
return addManyConditionalInternal<Value, true>(ptr, null_map, count);
|
||||
return addManyConditionalInternal<Value, true>(ptr, null_map, start, end);
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t count)
|
||||
void ALWAYS_INLINE addManyConditional(const Value * __restrict ptr, const UInt8 * __restrict cond_map, size_t start, size_t end)
|
||||
{
|
||||
return addManyConditionalInternal<Value, false>(ptr, cond_map, count);
|
||||
return addManyConditionalInternal<Value, false>(ptr, cond_map, start, end);
|
||||
}
|
||||
|
||||
void ALWAYS_INLINE mergeImpl(T & to_sum, T & to_compensation, T from_sum, T from_compensation)
|
||||
@ -385,22 +395,33 @@ public:
|
||||
}
|
||||
|
||||
void addBatchSinglePlace(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *, ssize_t if_argument_pos) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos) const override
|
||||
{
|
||||
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
this->data(place).addManyConditional(column.getData().data(), flags.data(), batch_size);
|
||||
this->data(place).addManyConditional(column.getData().data(), flags.data(), row_begin, row_end);
|
||||
}
|
||||
else
|
||||
{
|
||||
this->data(place).addMany(column.getData().data(), batch_size);
|
||||
this->data(place).addMany(column.getData().data(), row_begin, row_end);
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull(
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena *, ssize_t if_argument_pos)
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
Arena *,
|
||||
ssize_t if_argument_pos)
|
||||
const override
|
||||
{
|
||||
const auto & column = assert_cast<const ColVecType &>(*columns[0]);
|
||||
@ -408,15 +429,15 @@ public:
|
||||
{
|
||||
/// Merge the 2 sets of flags (null and if) into a single one. This allows us to use parallelizable sums when available
|
||||
const auto * if_flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData().data();
|
||||
auto final_flags = std::make_unique<UInt8[]>(batch_size);
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
auto final_flags = std::make_unique<UInt8[]>(row_end);
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
final_flags[i] = (!null_map[i]) & if_flags[i];
|
||||
|
||||
this->data(place).addManyConditional(column.getData().data(), final_flags.get(), batch_size);
|
||||
this->data(place).addManyConditional(column.getData().data(), final_flags.get(), row_begin, row_end);
|
||||
}
|
||||
else
|
||||
{
|
||||
this->data(place).addManyNotNull(column.getData().data(), null_map, batch_size);
|
||||
this->data(place).addManyNotNull(column.getData().data(), null_map, row_begin, row_end);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#include <Common/assert_cast.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <map>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
|
||||
|
||||
|
||||
@ -175,7 +175,8 @@ public:
|
||||
* and do a single call to "addBatch" for devirtualization and inlining.
|
||||
*/
|
||||
virtual void addBatch( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
@ -184,13 +185,16 @@ public:
|
||||
|
||||
/// The version of "addBatch", that handle sparse columns as arguments.
|
||||
virtual void addBatchSparse(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
Arena * arena) const = 0;
|
||||
|
||||
virtual void mergeBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const AggregateDataPtr * rhs,
|
||||
@ -199,17 +203,27 @@ public:
|
||||
/** The same for single place.
|
||||
*/
|
||||
virtual void addBatchSinglePlace( /// NOLINT
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const = 0;
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1) const = 0;
|
||||
|
||||
/// The version of "addBatchSinglePlace", that handle sparse columns as arguments.
|
||||
virtual void addBatchSparseSinglePlace(
|
||||
AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0;
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena) const = 0;
|
||||
|
||||
/** The same for single place when need to aggregate only filtered data.
|
||||
* Instead of using an if-column, the condition is combined inside the null_map
|
||||
*/
|
||||
virtual void addBatchSinglePlaceNotNull( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
@ -217,7 +231,12 @@ public:
|
||||
ssize_t if_argument_pos = -1) const = 0;
|
||||
|
||||
virtual void addBatchSinglePlaceFromInterval( /// NOLINT
|
||||
size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1)
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1)
|
||||
const = 0;
|
||||
|
||||
/** In addition to addBatch, this method collects multiple rows of arguments into array "places"
|
||||
@ -226,7 +245,8 @@ public:
|
||||
* "places" contains a large number of same values consecutively.
|
||||
*/
|
||||
virtual void addBatchArray(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
@ -237,7 +257,8 @@ public:
|
||||
* and pointers to aggregation states are stored in AggregateDataPtr[256] lookup table.
|
||||
*/
|
||||
virtual void addBatchLookupTable8(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
std::function<void(AggregateDataPtr &)> init,
|
||||
@ -251,7 +272,8 @@ public:
|
||||
* All places that were not inserted must be destroyed if there was exception during insert into result column.
|
||||
*/
|
||||
virtual void insertResultIntoBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
IColumn & to,
|
||||
@ -261,7 +283,8 @@ public:
|
||||
/** Destroy batch of aggregate places.
|
||||
*/
|
||||
virtual void destroyBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset) const noexcept = 0;
|
||||
|
||||
@ -355,7 +378,8 @@ public:
|
||||
AddFunc getAddressOfAddFunction() const override { return &addFree; }
|
||||
|
||||
void addBatch( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
@ -365,7 +389,7 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i] && places[i])
|
||||
static_cast<const Derived *>(this)->add(places[i] + place_offset, columns, i, arena);
|
||||
@ -373,13 +397,15 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
if (places[i])
|
||||
static_cast<const Derived *>(this)->add(places[i] + place_offset, columns, i, arena);
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSparse(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
@ -387,33 +413,42 @@ public:
|
||||
{
|
||||
const auto & column_sparse = assert_cast<const ColumnSparse &>(*columns[0]);
|
||||
const auto * values = &column_sparse.getValuesColumn();
|
||||
size_t batch_size = column_sparse.size();
|
||||
auto offset_it = column_sparse.begin();
|
||||
|
||||
for (size_t i = 0; i < batch_size; ++i, ++offset_it)
|
||||
/// FIXME: make it more optimal
|
||||
for (size_t i = 0; i < row_begin; ++i, ++offset_it)
|
||||
;
|
||||
|
||||
for (size_t i = 0; i < row_end; ++i, ++offset_it)
|
||||
static_cast<const Derived *>(this)->add(places[offset_it.getCurrentRow()] + place_offset,
|
||||
&values, offset_it.getValueIndex(), arena);
|
||||
}
|
||||
|
||||
void mergeBatch(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const AggregateDataPtr * rhs,
|
||||
Arena * arena) const override
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
if (places[i])
|
||||
static_cast<const Derived *>(this)->merge(places[i] + place_offset, rhs[i], arena);
|
||||
}
|
||||
|
||||
void addBatchSinglePlace( /// NOLINT
|
||||
size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1) const override
|
||||
{
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i])
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
@ -421,26 +456,34 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSparseSinglePlace(
|
||||
AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena) const override
|
||||
{
|
||||
/// TODO: add values and defaults separately if order of adding isn't important.
|
||||
const auto & column_sparse = assert_cast<const ColumnSparse &>(*columns[0]);
|
||||
const auto * values = &column_sparse.getValuesColumn();
|
||||
size_t batch_size = column_sparse.size();
|
||||
auto offset_it = column_sparse.begin();
|
||||
|
||||
for (size_t i = 0; i < batch_size; ++i, ++offset_it)
|
||||
/// FIXME: make it more optimal
|
||||
for (size_t i = 0; i < row_begin; ++i, ++offset_it)
|
||||
;
|
||||
|
||||
for (size_t i = 0; i < row_end; ++i, ++offset_it)
|
||||
static_cast<const Derived *>(this)->add(place, &values, offset_it.getValueIndex(), arena);
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceNotNull( /// NOLINT
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
const UInt8 * null_map,
|
||||
@ -450,26 +493,31 @@ public:
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
if (!null_map[i] && flags[i])
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
if (!null_map[i])
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchSinglePlaceFromInterval( /// NOLINT
|
||||
size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena, ssize_t if_argument_pos = -1)
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr place,
|
||||
const IColumn ** columns,
|
||||
Arena * arena,
|
||||
ssize_t if_argument_pos = -1)
|
||||
const override
|
||||
{
|
||||
if (if_argument_pos >= 0)
|
||||
{
|
||||
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
|
||||
for (size_t i = batch_begin; i < batch_end; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
if (flags[i])
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
@ -477,17 +525,23 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = batch_begin; i < batch_end; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
static_cast<const Derived *>(this)->add(place, columns, i, arena);
|
||||
}
|
||||
}
|
||||
|
||||
void addBatchArray(
|
||||
size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena)
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
const IColumn ** columns,
|
||||
const UInt64 * offsets,
|
||||
Arena * arena)
|
||||
const override
|
||||
{
|
||||
size_t current_offset = 0;
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
size_t next_offset = offsets[i];
|
||||
for (size_t j = current_offset; j < next_offset; ++j)
|
||||
@ -498,7 +552,8 @@ public:
|
||||
}
|
||||
|
||||
void addBatchLookupTable8(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * map,
|
||||
size_t place_offset,
|
||||
std::function<void(AggregateDataPtr &)> init,
|
||||
@ -508,10 +563,10 @@ public:
|
||||
{
|
||||
static constexpr size_t UNROLL_COUNT = 8;
|
||||
|
||||
size_t i = 0;
|
||||
size_t i = row_begin;
|
||||
|
||||
size_t batch_size_unrolled = batch_size / UNROLL_COUNT * UNROLL_COUNT;
|
||||
for (; i < batch_size_unrolled; i += UNROLL_COUNT)
|
||||
size_t size_unrolled = (row_end - row_begin) / UNROLL_COUNT * UNROLL_COUNT;
|
||||
for (; i < size_unrolled; i += UNROLL_COUNT)
|
||||
{
|
||||
AggregateDataPtr places[UNROLL_COUNT];
|
||||
for (size_t j = 0; j < UNROLL_COUNT; ++j)
|
||||
@ -527,7 +582,7 @@ public:
|
||||
static_cast<const Derived *>(this)->add(places[j] + place_offset, columns, i + j, arena);
|
||||
}
|
||||
|
||||
for (; i < batch_size; ++i)
|
||||
for (; i < row_end; ++i)
|
||||
{
|
||||
AggregateDataPtr & place = map[key[i]];
|
||||
if (unlikely(!place))
|
||||
@ -536,13 +591,20 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
void insertResultIntoBatch(size_t batch_size, AggregateDataPtr * places, size_t place_offset, IColumn & to, Arena * arena, bool destroy_place_after_insert) const override
|
||||
void insertResultIntoBatch(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset,
|
||||
IColumn & to,
|
||||
Arena * arena,
|
||||
bool destroy_place_after_insert) const override
|
||||
{
|
||||
size_t batch_index = 0;
|
||||
size_t batch_index = row_begin;
|
||||
|
||||
try
|
||||
{
|
||||
for (; batch_index < batch_size; ++batch_index)
|
||||
for (; batch_index < row_end; ++batch_index)
|
||||
{
|
||||
static_cast<const Derived *>(this)->insertResultInto(places[batch_index] + place_offset, to, arena);
|
||||
|
||||
@ -552,16 +614,20 @@ public:
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
for (size_t destroy_index = batch_index; destroy_index < batch_size; ++destroy_index)
|
||||
for (size_t destroy_index = batch_index; destroy_index < row_end; ++destroy_index)
|
||||
static_cast<const Derived *>(this)->destroy(places[destroy_index] + place_offset);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void destroyBatch(size_t batch_size, AggregateDataPtr * places, size_t place_offset) const noexcept override
|
||||
void destroyBatch(
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * places,
|
||||
size_t place_offset) const noexcept override
|
||||
{
|
||||
for (size_t i = 0; i < batch_size; ++i)
|
||||
for (size_t i = row_begin; i < row_end; ++i)
|
||||
{
|
||||
static_cast<const Derived *>(this)->destroy(places[i] + place_offset);
|
||||
}
|
||||
@ -612,7 +678,8 @@ public:
|
||||
}
|
||||
|
||||
void addBatchLookupTable8(
|
||||
size_t batch_size,
|
||||
size_t row_begin,
|
||||
size_t row_end,
|
||||
AggregateDataPtr * map,
|
||||
size_t place_offset,
|
||||
std::function<void(AggregateDataPtr &)> init,
|
||||
@ -626,7 +693,7 @@ public:
|
||||
|
||||
if (func.allocatesMemoryInArena() || sizeof(Data) > 16 || func.sizeOfData() != sizeof(Data))
|
||||
{
|
||||
IAggregateFunctionHelper<Derived>::addBatchLookupTable8(batch_size, map, place_offset, init, key, columns, arena);
|
||||
IAggregateFunctionHelper<Derived>::addBatchLookupTable8(row_begin, row_end, map, place_offset, init, key, columns, arena);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -637,12 +704,12 @@ public:
|
||||
std::unique_ptr<Data[]> places{new Data[256 * UNROLL_COUNT]};
|
||||
bool has_data[256 * UNROLL_COUNT]{}; /// Separate flags array to avoid heavy initialization.
|
||||
|
||||
size_t i = 0;
|
||||
size_t i = row_begin;
|
||||
|
||||
/// Aggregate data into different lookup tables.
|
||||
|
||||
size_t batch_size_unrolled = batch_size / UNROLL_COUNT * UNROLL_COUNT;
|
||||
for (; i < batch_size_unrolled; i += UNROLL_COUNT)
|
||||
size_t size_unrolled = (row_end - row_begin) / UNROLL_COUNT * UNROLL_COUNT;
|
||||
for (; i < size_unrolled; i += UNROLL_COUNT)
|
||||
{
|
||||
for (size_t j = 0; j < UNROLL_COUNT; ++j)
|
||||
{
|
||||
@ -676,7 +743,7 @@ public:
|
||||
|
||||
/// Process tails and add directly to the final destination.
|
||||
|
||||
for (; i < batch_size; ++i)
|
||||
for (; i < row_end; ++i)
|
||||
{
|
||||
size_t k = key[i];
|
||||
AggregateDataPtr & place = map[k];
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
#include <Disks/IDisk.h>
|
||||
#include <IO/ReadBufferFromFileBase.h>
|
||||
#include <IO/WriteBufferFromFileBase.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#include <Common/Exception.h>
|
||||
#include <Disks/IO/createReadBufferFromFileBase.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <base/logger_useful.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#include <Core/SettingsFields.h>
|
||||
#include <Parsers/ASTBackupQuery.h>
|
||||
#include <Parsers/ASTSetQuery.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -10,9 +12,10 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_PARSE_BACKUP_SETTINGS;
|
||||
extern const int WRONG_BACKUP_SETTINGS;
|
||||
}
|
||||
|
||||
/// List of backup settings except base_backup_name.
|
||||
/// List of backup settings except base_backup_name and cluster_host_ids.
|
||||
#define LIST_OF_BACKUP_SETTINGS(M) \
|
||||
M(String, compression_method) \
|
||||
M(Int64, compression_level) \
|
||||
@ -23,16 +26,13 @@ namespace ErrorCodes
|
||||
M(UInt64, replica_num) \
|
||||
M(Bool, allow_storing_multiple_replicas) \
|
||||
M(Bool, internal) \
|
||||
M(String, host_id) \
|
||||
M(String, coordination_zk_path)
|
||||
|
||||
|
||||
BackupSettings BackupSettings::fromBackupQuery(const ASTBackupQuery & query)
|
||||
{
|
||||
BackupSettings res;
|
||||
|
||||
if (query.base_backup_name)
|
||||
res.base_backup_info = BackupInfo::fromAST(*query.base_backup_name);
|
||||
|
||||
if (query.settings)
|
||||
{
|
||||
const auto & settings = query.settings->as<const ASTSetQuery &>().changes;
|
||||
@ -48,25 +48,149 @@ BackupSettings BackupSettings::fromBackupQuery(const ASTBackupQuery & query)
|
||||
}
|
||||
}
|
||||
|
||||
if (query.base_backup_name)
|
||||
res.base_backup_info = BackupInfo::fromAST(*query.base_backup_name);
|
||||
|
||||
if (query.cluster_host_ids)
|
||||
res.cluster_host_ids = Util::clusterHostIDsFromAST(*query.cluster_host_ids);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void BackupSettings::copySettingsToBackupQuery(ASTBackupQuery & query) const
|
||||
void BackupSettings::copySettingsToQuery(ASTBackupQuery & query) const
|
||||
{
|
||||
query.base_backup_name = base_backup_info ? base_backup_info->toAST() : nullptr;
|
||||
|
||||
auto query_settings = std::make_shared<ASTSetQuery>();
|
||||
query_settings->is_standalone = false;
|
||||
|
||||
static const BackupSettings default_settings;
|
||||
bool all_settings_are_default = true;
|
||||
|
||||
#define SET_SETTINGS_IN_BACKUP_QUERY_HELPER(TYPE, NAME) \
|
||||
if ((NAME) != default_settings.NAME) \
|
||||
query_settings->changes.emplace_back(#NAME, static_cast<Field>(SettingField##TYPE{NAME}));
|
||||
{ \
|
||||
query_settings->changes.emplace_back(#NAME, static_cast<Field>(SettingField##TYPE{NAME})); \
|
||||
all_settings_are_default = false; \
|
||||
}
|
||||
|
||||
LIST_OF_BACKUP_SETTINGS(SET_SETTINGS_IN_BACKUP_QUERY_HELPER)
|
||||
|
||||
if (all_settings_are_default)
|
||||
query_settings = nullptr;
|
||||
|
||||
query.settings = query_settings;
|
||||
|
||||
query.base_backup_name = base_backup_info ? base_backup_info->toAST() : nullptr;
|
||||
query.cluster_host_ids = !cluster_host_ids.empty() ? Util::clusterHostIDsToAST(cluster_host_ids) : nullptr;
|
||||
}
|
||||
|
||||
std::vector<Strings> BackupSettings::Util::clusterHostIDsFromAST(const IAST & ast)
|
||||
{
|
||||
std::vector<Strings> res;
|
||||
|
||||
const auto * array_of_shards = typeid_cast<const ASTFunction *>(&ast);
|
||||
if (!array_of_shards || (array_of_shards->name != "array"))
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS,
|
||||
"Setting cluster_host_ids has wrong format, must be array of arrays of string literals");
|
||||
|
||||
if (array_of_shards->arguments)
|
||||
{
|
||||
const ASTs shards = array_of_shards->arguments->children;
|
||||
res.resize(shards.size());
|
||||
|
||||
for (size_t i = 0; i != shards.size(); ++i)
|
||||
{
|
||||
const auto * array_of_replicas = typeid_cast<const ASTLiteral *>(shards[i].get());
|
||||
if (!array_of_replicas || (array_of_replicas->value.getType() != Field::Types::Array))
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS,
|
||||
"Setting cluster_host_ids has wrong format, must be array of arrays of string literals");
|
||||
const auto & replicas = array_of_replicas->value.get<const Array &>();
|
||||
res[i].resize(replicas.size());
|
||||
for (size_t j = 0; j != replicas.size(); ++j)
|
||||
{
|
||||
const auto & replica = replicas[j];
|
||||
if (replica.getType() != Field::Types::String)
|
||||
throw Exception(
|
||||
ErrorCodes::CANNOT_PARSE_BACKUP_SETTINGS,
|
||||
"Setting cluster_host_ids has wrong format, must be array of arrays of string literals");
|
||||
res[i][j] = replica.get<const String &>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ASTPtr BackupSettings::Util::clusterHostIDsToAST(const std::vector<Strings> & cluster_host_ids)
|
||||
{
|
||||
if (cluster_host_ids.empty())
|
||||
return nullptr;
|
||||
|
||||
auto res = std::make_shared<ASTFunction>();
|
||||
res->name = "array";
|
||||
auto res_replicas = std::make_shared<ASTExpressionList>();
|
||||
res->arguments = res_replicas;
|
||||
res->children.push_back(res_replicas);
|
||||
res_replicas->children.resize(cluster_host_ids.size());
|
||||
|
||||
for (size_t i = 0; i != cluster_host_ids.size(); ++i)
|
||||
{
|
||||
const auto & shard = cluster_host_ids[i];
|
||||
|
||||
Array res_shard;
|
||||
res_shard.resize(shard.size());
|
||||
for (size_t j = 0; j != shard.size(); ++j)
|
||||
res_shard[j] = Field{shard[j]};
|
||||
|
||||
res_replicas->children[i] = std::make_shared<ASTLiteral>(Field{std::move(res_shard)});
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> BackupSettings::Util::findShardNumAndReplicaNum(const std::vector<Strings> & cluster_host_ids, const String & host_id)
|
||||
{
|
||||
for (size_t i = 0; i != cluster_host_ids.size(); ++i)
|
||||
{
|
||||
for (size_t j = 0; j != cluster_host_ids[i].size(); ++j)
|
||||
if (cluster_host_ids[i][j] == host_id)
|
||||
return {i + 1, j + 1};
|
||||
}
|
||||
throw Exception(ErrorCodes::WRONG_BACKUP_SETTINGS, "Cannot determine shard number or replica number, the current host {} is not found in the cluster's hosts", host_id);
|
||||
}
|
||||
|
||||
Strings BackupSettings::Util::filterHostIDs(const std::vector<Strings> & cluster_host_ids, size_t only_shard_num, size_t only_replica_num)
|
||||
{
|
||||
Strings collected_host_ids;
|
||||
|
||||
auto collect_replicas = [&](size_t shard_index)
|
||||
{
|
||||
const auto & shard = cluster_host_ids[shard_index - 1];
|
||||
if (only_replica_num)
|
||||
{
|
||||
if (only_replica_num <= shard.size())
|
||||
collected_host_ids.push_back(shard[only_replica_num - 1]);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t replica_index = 1; replica_index <= shard.size(); ++replica_index)
|
||||
collected_host_ids.push_back(shard[replica_index - 1]);
|
||||
}
|
||||
};
|
||||
|
||||
if (only_shard_num)
|
||||
{
|
||||
if (only_shard_num <= cluster_host_ids.size())
|
||||
collect_replicas(only_shard_num);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t shard_index = 1; shard_index <= cluster_host_ids.size(); ++shard_index)
|
||||
collect_replicas(shard_index);
|
||||
}
|
||||
|
||||
return collected_host_ids;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -43,12 +43,28 @@ struct BackupSettings
|
||||
/// Whether this backup is a part of a distributed backup created by BACKUP ON CLUSTER.
|
||||
bool internal = false;
|
||||
|
||||
/// Internal, should not be specified by user.
|
||||
/// The current host's ID in the format 'escaped_host_name:port'.
|
||||
String host_id;
|
||||
|
||||
/// Internal, should not be specified by user.
|
||||
/// Cluster's hosts' IDs in the format 'escaped_host_name:port' for all shards and replicas in a cluster specified in BACKUP ON CLUSTER.
|
||||
std::vector<Strings> cluster_host_ids;
|
||||
|
||||
/// Internal, should not be specified by user.
|
||||
/// Path in Zookeeper used to coordinate a distributed backup created by BACKUP ON CLUSTER.
|
||||
String coordination_zk_path;
|
||||
|
||||
static BackupSettings fromBackupQuery(const ASTBackupQuery & query);
|
||||
void copySettingsToBackupQuery(ASTBackupQuery & query) const;
|
||||
void copySettingsToQuery(ASTBackupQuery & query) const;
|
||||
|
||||
struct Util
|
||||
{
|
||||
static std::vector<Strings> clusterHostIDsFromAST(const IAST & ast);
|
||||
static ASTPtr clusterHostIDsToAST(const std::vector<Strings> & cluster_host_ids);
|
||||
static std::pair<size_t, size_t> findShardNumAndReplicaNum(const std::vector<Strings> & cluster_host_ids, const String & host_id);
|
||||
static Strings filterHostIDs(const std::vector<Strings> & cluster_host_ids, size_t only_shard_num, size_t only_replica_num);
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -15,8 +15,6 @@ std::string_view toString(BackupStatus backup_status)
|
||||
{
|
||||
switch (backup_status)
|
||||
{
|
||||
case BackupStatus::PREPARING:
|
||||
return "PREPARING";
|
||||
case BackupStatus::MAKING_BACKUP:
|
||||
return "MAKING_BACKUP";
|
||||
case BackupStatus::BACKUP_COMPLETE:
|
||||
|
||||
@ -9,7 +9,6 @@ namespace DB
|
||||
enum class BackupStatus
|
||||
{
|
||||
/// Statuses of making backups
|
||||
PREPARING,
|
||||
MAKING_BACKUP,
|
||||
BACKUP_COMPLETE,
|
||||
FAILED_TO_BACKUP,
|
||||
|
||||
@ -88,7 +88,6 @@ namespace
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
using Kind = ASTBackupQuery::Kind;
|
||||
using Element = ASTBackupQuery::Element;
|
||||
using Elements = ASTBackupQuery::Elements;
|
||||
@ -107,8 +106,8 @@ namespace
|
||||
/// Prepares internal structures for making backup entries.
|
||||
void prepare(const ASTBackupQuery::Elements & elements)
|
||||
{
|
||||
String current_database = context->getCurrentDatabase();
|
||||
renaming_settings.setFromBackupQuery(elements, current_database);
|
||||
calculateShardNumAndReplicaNumInBackup();
|
||||
renaming_settings.setFromBackupQuery(elements);
|
||||
|
||||
for (const auto & element : elements)
|
||||
{
|
||||
@ -116,11 +115,7 @@ namespace
|
||||
{
|
||||
case ElementType::TABLE:
|
||||
{
|
||||
const String & table_name = element.name.second;
|
||||
String database_name = element.name.first;
|
||||
if (database_name.empty())
|
||||
database_name = current_database;
|
||||
prepareToBackupTable(DatabaseAndTableName{database_name, table_name}, element.partitions);
|
||||
prepareToBackupTable(element.name, element.partitions);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -155,7 +150,7 @@ namespace
|
||||
auto data_backup = info.storage->backupData(context, info.partitions);
|
||||
if (!data_backup.empty())
|
||||
{
|
||||
String data_path = PathsInBackup::getDataPath(*info.create_query, backup_settings.shard_num, backup_settings.replica_num);
|
||||
String data_path = PathsInBackup::getDataPath(*info.create_query, shard_num_in_backup, replica_num_in_backup);
|
||||
for (auto & [path_in_backup, backup_entry] : data_backup)
|
||||
res.emplace_back(data_path + path_in_backup, std::move(backup_entry));
|
||||
}
|
||||
@ -170,6 +165,19 @@ namespace
|
||||
}
|
||||
|
||||
private:
|
||||
void calculateShardNumAndReplicaNumInBackup()
|
||||
{
|
||||
size_t shard_num = 0;
|
||||
size_t replica_num = 0;
|
||||
if (!backup_settings.host_id.empty())
|
||||
{
|
||||
std::tie(shard_num, replica_num)
|
||||
= BackupSettings::Util::findShardNumAndReplicaNum(backup_settings.cluster_host_ids, backup_settings.host_id);
|
||||
}
|
||||
shard_num_in_backup = shard_num;
|
||||
replica_num_in_backup = replica_num;
|
||||
}
|
||||
|
||||
/// Prepares to backup a single table and probably its database's definition.
|
||||
void prepareToBackupTable(const DatabaseAndTableName & table_name_, const ASTs & partitions_)
|
||||
{
|
||||
@ -286,7 +294,7 @@ namespace
|
||||
std::pair<String, BackupEntryPtr> makeBackupEntryForMetadata(const IAST & create_query) const
|
||||
{
|
||||
auto metadata_entry = std::make_unique<BackupEntryFromMemory>(serializeAST(create_query));
|
||||
String metadata_path = PathsInBackup::getMetadataPath(create_query, backup_settings.shard_num, backup_settings.replica_num);
|
||||
String metadata_path = PathsInBackup::getMetadataPath(create_query, shard_num_in_backup, replica_num_in_backup);
|
||||
return {metadata_path, std::move(metadata_entry)};
|
||||
}
|
||||
|
||||
@ -307,6 +315,8 @@ namespace
|
||||
|
||||
ContextPtr context;
|
||||
BackupSettings backup_settings;
|
||||
size_t shard_num_in_backup = 0;
|
||||
size_t replica_num_in_backup = 0;
|
||||
DDLRenamingSettings renaming_settings;
|
||||
std::unordered_map<String /* db_name_in_backup */, CreateDatabaseInfo> databases;
|
||||
std::map<DatabaseAndTableName /* table_name_in_backup */, CreateTableInfo> tables;
|
||||
@ -322,16 +332,15 @@ BackupEntries makeBackupEntries(const ContextPtr & context, const Elements & ele
|
||||
}
|
||||
|
||||
|
||||
void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, size_t num_threads)
|
||||
void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, ThreadPool & thread_pool)
|
||||
{
|
||||
if (!num_threads || !backup->supportsWritingInMultipleThreads())
|
||||
num_threads = 1;
|
||||
std::vector<ThreadFromGlobalPool> threads;
|
||||
size_t num_active_threads = 0;
|
||||
size_t num_active_jobs = 0;
|
||||
std::mutex mutex;
|
||||
std::condition_variable cond;
|
||||
std::condition_variable event;
|
||||
std::exception_ptr exception;
|
||||
|
||||
bool always_single_threaded = !backup->supportsWritingInMultipleThreads();
|
||||
|
||||
for (auto & name_and_entry : backup_entries)
|
||||
{
|
||||
auto & name = name_and_entry.first;
|
||||
@ -341,14 +350,23 @@ void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries
|
||||
std::unique_lock lock{mutex};
|
||||
if (exception)
|
||||
break;
|
||||
cond.wait(lock, [&] { return num_active_threads < num_threads; });
|
||||
if (exception)
|
||||
break;
|
||||
++num_active_threads;
|
||||
++num_active_jobs;
|
||||
}
|
||||
|
||||
threads.emplace_back([backup, &name, &entry, &mutex, &cond, &num_active_threads, &exception]()
|
||||
auto job = [&]()
|
||||
{
|
||||
SCOPE_EXIT({
|
||||
std::lock_guard lock{mutex};
|
||||
if (!--num_active_jobs)
|
||||
event.notify_all();
|
||||
});
|
||||
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
if (exception)
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
backup->writeFile(name, std::move(entry));
|
||||
@ -359,17 +377,16 @@ void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries
|
||||
if (!exception)
|
||||
exception = std::current_exception();
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
std::lock_guard lock{mutex};
|
||||
--num_active_threads;
|
||||
cond.notify_all();
|
||||
}
|
||||
});
|
||||
if (always_single_threaded || !thread_pool.trySchedule(job))
|
||||
job();
|
||||
}
|
||||
|
||||
for (auto & thread : threads)
|
||||
thread.join();
|
||||
{
|
||||
std::unique_lock lock{mutex};
|
||||
event.wait(lock, [&] { return !num_active_jobs; });
|
||||
}
|
||||
|
||||
backup_entries.clear();
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user