Merge branch 'master' of github.com:ClickHouse/ClickHouse into refactor-schema-inference

This commit is contained in:
avogar 2022-12-09 14:45:10 +00:00
commit cd4fa00d2c
274 changed files with 3107 additions and 1665 deletions

View File

@ -16,6 +16,7 @@ Checks: '*,
-android-*,
-bugprone-assignment-in-if-condition,
-bugprone-branch-clone,
-bugprone-easily-swappable-parameters,
-bugprone-exception-escape,
@ -23,7 +24,6 @@ Checks: '*,
-bugprone-narrowing-conversions,
-bugprone-not-null-terminated-result,
-bugprone-unchecked-optional-access,
-bugprone-assignment-in-if-condition,
-cert-dcl16-c,
-cert-err58-cpp,
@ -34,7 +34,6 @@ Checks: '*,
-clang-analyzer-optin.performance.Padding,
-clang-analyzer-optin.portability.UnixAPI,
-clang-analyzer-security.insecureAPI.bzero,
-clang-analyzer-security.insecureAPI.strcpy,
@ -103,12 +102,13 @@ Checks: '*,
-openmp-*,
-misc-const-correctness,
-misc-no-recursion,
-misc-non-private-member-variables-in-classes,
-misc-const-correctness,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-macro-to-enum,
-modernize-pass-by-value,
-modernize-return-braced-init-list,
-modernize-use-auto,
@ -117,7 +117,6 @@ Checks: '*,
-modernize-use-nodiscard,
-modernize-use-override,
-modernize-use-trailing-return-type,
-modernize-macro-to-enum,
-performance-inefficient-string-concatenation,
-performance-no-int-to-ptr,
@ -135,17 +134,35 @@ Checks: '*,
-readability-magic-numbers,
-readability-named-parameter,
-readability-redundant-declaration,
-readability-simplify-boolean-expr,
-readability-static-accessed-through-instance,
-readability-suspicious-call-argument,
-readability-uppercase-literal-suffix,
-readability-use-anyofallof,
-readability-simplify-boolean-expr,
-zirkon-*,
-misc-*, # temporarily disabled due to being too slow
# also disable checks in other categories which are aliases of checks in misc-*:
# https://releases.llvm.org/15.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html
-cert-dcl54-cpp, # alias of misc-new-delete-overloads
-hicpp-new-delete-operators, # alias of misc-new-delete-overloads
-cert-fio38-c, # alias of misc-non-copyable-objects
-cert-dcl03-c, # alias of misc-static-assert
-hicpp-static-assert, # alias of misc-static-assert
-cert-err09-cpp, # alias of misc-throw-by-value-catch-by-reference
-cert-err61-cpp, # alias of misc-throw-by-value-catch-by-reference
-cppcoreguidelines-c-copy-assignment-signature, # alias of misc-unconventional-assign-operator
-cppcoreguidelines-non-private-member-variables-in-classes, # alias of misc-non-private-member-variables-in-classes
'
WarningsAsErrors: '*'
# TODO: use dictionary syntax for CheckOptions when minimum clang-tidy level rose to 15
# some-check.SomeOption: 'some value'
# instead of
# - key: some-check.SomeOption
# value: 'some value'
CheckOptions:
- key: readability-identifier-naming.ClassCase
value: CamelCase

View File

@ -61,6 +61,7 @@ jobs:
committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
commit-message: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
branch: auto/${{ env.GITHUB_TAG }}
assignees: ${{ github.event.sender.login }} # assign the PR to the tag pusher
delete-branch: true
title: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
labels: do not test

3
.gitmodules vendored
View File

@ -287,3 +287,6 @@
[submodule "contrib/xxHash"]
path = contrib/xxHash
url = https://github.com/Cyan4973/xxHash.git
[submodule "contrib/google-benchmark"]
path = contrib/google-benchmark
url = https://github.com/google/benchmark.git

View File

@ -111,6 +111,7 @@ if (ENABLE_FUZZING)
set (ENABLE_JEMALLOC 0)
set (ENABLE_CHECK_HEAVY_BUILDS 1)
set (GLIBC_COMPATIBILITY OFF)
set (ENABLE_BENCHMARKS 0)
# For codegen_select_fuzzer
set (ENABLE_PROTOBUF 1)
@ -168,6 +169,7 @@ endif ()
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF)
if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND USE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL)
# Only for Linux, x86_64 or aarch64.

View File

@ -16,8 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
## Upcoming events
* [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
* [**ClickHosue Meetup at the RELEX Solutions office in Stockholm**](https://www.meetup.com/clickhouse-stockholm-user-group/events/289492084/) - Dec 1 - Formulate by RELEX is a Swedish promotion planning and analytics company. They will share why they chose ClickHouse for their real time analytics and forecasting solution. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) - Dec 5 - Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management.
* [**ClickHouse Meetup at the Rokt offices in Manhattan**](https://www.meetup.com/clickhouse-new-york-user-group/events/289403909/) - Dec 6 - We are very excited to be holding our next in-person ClickHouse meetup at the Rokt offices in Manhattan. Featuring talks from Bloomberg, Disney Streaming, Prequel, Rokt, and ClickHouse
* [**v22.12 Release Webinar**](https://clickhouse.com/company/events/v22-12-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
* [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - Jan 16 - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion!
* **ClickHouse Meetup in Seattle* - Keep an eye on this space as we will be announcing a January meetup in Seattle soon!

View File

@ -17,6 +17,7 @@
#include <filesystem>
#include <fmt/format.h>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/classification.hpp> /// is_any_of
namespace
@ -38,7 +39,7 @@ std::string getEditor()
return editor;
}
std::string getFuzzyFinder()
std::pair<std::string, FuzzyFinderType> getFuzzyFinder()
{
const char * env_path = std::getenv("PATH"); // NOLINT(concurrency-mt-unsafe)
@ -52,14 +53,20 @@ std::string getFuzzyFinder()
std::filesystem::path path(path_str);
std::filesystem::path sk_bin_path = path / "sk";
if (!access(sk_bin_path.c_str(), X_OK))
return sk_bin_path;
return {sk_bin_path, FUZZY_FINDER_SKIM};
std::filesystem::path fzf_bin_path = path / "fzf";
if (!access(fzf_bin_path.c_str(), X_OK))
return fzf_bin_path;
return {fzf_bin_path, FUZZY_FINDER_FZF};
}
return {};
return {"", FUZZY_FINDER_NONE};
}
String escapeShellArgument(std::string arg)
{
boost::replace_all(arg, "'", "'\\''");
return fmt::format("'{}'", arg);
}
/// See comments in ShellCommand::executeImpl()
@ -305,11 +312,12 @@ ReplxxLineReader::ReplxxLineReader(
replxx::Replxx::highlighter_callback_t highlighter_)
: LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_))
, editor(getEditor())
, fuzzy_finder(getFuzzyFinder())
{
using namespace std::placeholders;
using Replxx = replxx::Replxx;
std::tie(fuzzy_finder, fuzzy_finder_type) = getFuzzyFinder();
if (!history_file_path.empty())
{
history_file_fd = open(history_file_path.c_str(), O_RDWR);
@ -415,11 +423,12 @@ ReplxxLineReader::ReplxxLineReader(
rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action);
/// interactive search in history (requires fzf/sk)
if (!fuzzy_finder.empty())
if (fuzzy_finder_type != FUZZY_FINDER_NONE)
{
auto interactive_history_search = [this](char32_t code)
{
openInteractiveHistorySearch();
rx.invoke(Replxx::ACTION::CLEAR_SELF, code);
return rx.invoke(Replxx::ACTION::REPAINT, code);
};
rx.bind_key(Replxx::KEY::control('R'), interactive_history_search);
@ -515,9 +524,22 @@ void ReplxxLineReader::openInteractiveHistorySearch()
///
/// And also note, that fzf and skim is 95% compatible (at least option
/// that is used here)
std::string fuzzy_finder_command = fmt::format(
"{} --read0 --tac --no-sort --tiebreak=index --bind=ctrl-r:toggle-sort --height=30% < {} > {}",
fuzzy_finder, history_file.getPath(), output_file.getPath());
std::string fuzzy_finder_command = fmt::format("{} --read0 --height=30%", fuzzy_finder);
switch (fuzzy_finder_type)
{
case FUZZY_FINDER_SKIM:
fuzzy_finder_command += " --tac --tiebreak=-score";
break;
case FUZZY_FINDER_FZF:
fuzzy_finder_command += " --tac --tiebreak=index";
break;
case FUZZY_FINDER_NONE:
/// assertion for !fuzzy_finder.empty() is enough
break;
}
fuzzy_finder_command += fmt::format(" < {} > {}",
escapeShellArgument(history_file.getPath()),
escapeShellArgument(output_file.getPath()));
char * const argv[] = {sh, sh_c, fuzzy_finder_command.data(), nullptr};
try

View File

@ -4,6 +4,14 @@
#include <replxx.hxx>
enum FuzzyFinderType
{
FUZZY_FINDER_NONE,
/// Use https://github.com/junegunn/fzf
FUZZY_FINDER_FZF,
/// Use https://github.com/lotabout/skim
FUZZY_FINDER_SKIM,
};
class ReplxxLineReader : public LineReader
{
@ -38,4 +46,5 @@ private:
std::string editor;
std::string fuzzy_finder;
FuzzyFinderType fuzzy_finder_type = FUZZY_FINDER_NONE;
};

View File

@ -187,8 +187,20 @@ struct integer<Bits, Signed>::_impl
static_assert(Bits % base_bits == 0);
/// Simple iteration in both directions
static constexpr unsigned little(unsigned idx) { return idx; }
static constexpr unsigned big(unsigned idx) { return item_count - 1 - idx; }
static constexpr unsigned little(unsigned idx)
{
if constexpr (std::endian::native == std::endian::little)
return idx;
else
return item_count - 1 - idx;
}
static constexpr unsigned big(unsigned idx)
{
if constexpr (std::endian::native == std::endian::little)
return item_count - 1 - idx;
else
return idx;
}
static constexpr unsigned any(unsigned idx) { return idx; }
template <class T>
@ -240,20 +252,20 @@ struct integer<Bits, Signed>::_impl
{
static_assert(sizeof(Integral) <= sizeof(base_type));
self.items[0] = _impl::to_Integral(rhs);
self.items[little(0)] = _impl::to_Integral(rhs);
if constexpr (std::is_signed_v<Integral>)
{
if (rhs < 0)
{
for (size_t i = 1; i < item_count; ++i)
self.items[i] = -1;
for (unsigned i = 1; i < item_count; ++i)
self.items[little(i)] = -1;
return;
}
}
for (size_t i = 1; i < item_count; ++i)
self.items[i] = 0;
for (unsigned i = 1; i < item_count; ++i)
self.items[little(i)] = 0;
}
template <typename TupleLike, size_t i = 0>
@ -348,7 +360,7 @@ struct integer<Bits, Signed>::_impl
constexpr const unsigned to_copy = min_bits / base_bits;
for (unsigned i = 0; i < to_copy; ++i)
self.items[i] = rhs.items[i];
self.items[little(i)] = rhs.items[little(i)];
if constexpr (Bits > Bits2)
{
@ -357,13 +369,13 @@ struct integer<Bits, Signed>::_impl
if (rhs < 0)
{
for (unsigned i = to_copy; i < item_count; ++i)
self.items[i] = -1;
self.items[little(i)] = -1;
return;
}
}
for (unsigned i = to_copy; i < item_count; ++i)
self.items[i] = 0;
self.items[little(i)] = 0;
}
}
@ -454,7 +466,7 @@ private:
{
if constexpr (sizeof(T) <= sizeof(base_type))
{
if (0 == idx)
if (little(0) == idx)
return static_cast<base_type>(x);
}
else if (idx * sizeof(base_type) < sizeof(T))
@ -475,7 +487,7 @@ private:
for (unsigned i = 0; i < op_items; ++i)
{
base_type rhs_item = get_item(rhs, i);
base_type rhs_item = get_item(rhs, little(i));
base_type & res_item = res.items[little(i)];
underflows[i] = res_item < rhs_item;
@ -508,7 +520,7 @@ private:
for (unsigned i = 0; i < op_items; ++i)
{
base_type rhs_item = get_item(rhs, i);
base_type rhs_item = get_item(rhs, little(i));
base_type & res_item = res.items[little(i)];
res_item += rhs_item;
@ -580,12 +592,12 @@ private:
else if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 a = (CompilerUInt128(lhs.items[little(1)]) << 64) + lhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(rhs.items[little(1)]) << 64) + rhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 c = a * b;
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
res.items[little(0)] = c;
res.items[little(1)] = c >> 64;
return res;
}
else
@ -597,7 +609,7 @@ private:
#endif
for (unsigned i = 0; i < item_count; ++i)
{
base_type rhs_item = get_item(rhs, i);
base_type rhs_item = get_item(rhs, little(i));
unsigned pos = i * base_bits;
while (rhs_item)
@ -792,7 +804,7 @@ public:
integer<Bits, Signed> res;
for (unsigned i = 0; i < item_count; ++i)
res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, i);
res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, little(i));
return res;
}
else
@ -810,7 +822,7 @@ public:
integer<Bits, Signed> res;
for (unsigned i = 0; i < item_count; ++i)
res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, i);
res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, little(i));
return res;
}
else
@ -845,17 +857,17 @@ public:
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 a = (CompilerUInt128(numerator.items[little(1)]) << 64) + numerator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(denominator.items[little(1)]) << 64) + denominator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 c = a / b; // NOLINT
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
res.items[little(0)] = c;
res.items[little(1)] = c >> 64;
CompilerUInt128 remainder = a - b * c;
numerator.items[0] = remainder;
numerator.items[1] = remainder >> 64;
numerator.items[little(0)] = remainder;
numerator.items[little(1)] = remainder >> 64;
return res;
}
@ -1039,15 +1051,15 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
else
{
auto it = il.begin();
for (size_t i = 0; i < _impl::item_count; ++i)
for (unsigned i = 0; i < _impl::item_count; ++i)
{
if (it < il.end())
{
items[i] = *it;
items[_impl::little(i)] = *it;
++it;
}
else
items[i] = 0;
items[_impl::little(i)] = 0;
}
}
}
@ -1208,7 +1220,7 @@ constexpr integer<Bits, Signed>::operator T() const noexcept
UnsignedT res{};
for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
res += UnsignedT(items[_impl::little(i)]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
return res;
}

View File

@ -5,21 +5,21 @@ if (ENABLE_CLANG_TIDY)
find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache")
if (CLANG_TIDY_CACHE_PATH)
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy")
# Why do we use ';' here?
# It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY
# The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax.
set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper")
else ()
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12")
find_program (CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy")
endif ()
if (CLANG_TIDY_PATH)
message (STATUS
"Using clang-tidy: ${CLANG_TIDY_PATH}.
The checks will be run during build process.
See the .clang-tidy file at the root directory to configure the checks.")
The checks will be run during the build process.
See the .clang-tidy file in the root directory to configure the checks.")
set (USE_CLANG_TIDY ON)

View File

@ -21,12 +21,12 @@ set (APPLE_CLANG_MINIMUM_VERSION 12.0.0)
set (GCC_MINIMUM_VERSION 11)
if (COMPILER_GCC)
message (FATAL_ERROR "Compilation with GCC is unsupported. Please use Clang instead.")
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION})
message (FATAL_ERROR "Compilation with GCC version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${GCC_MINIMUM_VERSION}.")
endif ()
message (WARNING "Compilation with GCC is unsupported. Please use Clang instead.")
elseif (COMPILER_CLANG)
if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
# (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it.
@ -83,7 +83,7 @@ if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME)
if (NOT LINKER_NAME)
if (GOLD_PATH)
message (WARNING "Linking with gold is not recommended. Please use lld.")
message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.")
if (COMPILER_GCC)
set (LINKER_NAME "gold")
else ()

View File

@ -171,6 +171,8 @@ add_contrib (annoy-cmake annoy)
add_contrib (xxHash-cmake xxHash)
add_contrib (google-benchmark-cmake google-benchmark)
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

1
contrib/google-benchmark vendored Submodule

@ -0,0 +1 @@
Subproject commit 2257fa4d6afb8e5a2ccd510a70f38fe7fcdf1edf

View File

@ -0,0 +1,34 @@
set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-benchmark/src")
set (SRCS
"${SRC_DIR}/benchmark.cc"
"${SRC_DIR}/benchmark_api_internal.cc"
"${SRC_DIR}/benchmark_name.cc"
"${SRC_DIR}/benchmark_register.cc"
"${SRC_DIR}/benchmark_runner.cc"
"${SRC_DIR}/check.cc"
"${SRC_DIR}/colorprint.cc"
"${SRC_DIR}/commandlineflags.cc"
"${SRC_DIR}/complexity.cc"
"${SRC_DIR}/console_reporter.cc"
"${SRC_DIR}/counter.cc"
"${SRC_DIR}/csv_reporter.cc"
"${SRC_DIR}/json_reporter.cc"
"${SRC_DIR}/perf_counters.cc"
"${SRC_DIR}/reporter.cc"
"${SRC_DIR}/sleep.cc"
"${SRC_DIR}/statistics.cc"
"${SRC_DIR}/string_util.cc"
"${SRC_DIR}/sysinfo.cc"
"${SRC_DIR}/timers.cc")
add_library(google_benchmark "${SRCS}")
target_include_directories(google_benchmark SYSTEM PUBLIC "${SRC_DIR}/../include")
add_library(google_benchmark_main "${SRC_DIR}/benchmark_main.cc")
target_link_libraries(google_benchmark_main PUBLIC google_benchmark)
add_library(google_benchmark_all INTERFACE)
target_link_libraries(google_benchmark_all INTERFACE google_benchmark google_benchmark_main)
add_library(ch_contrib::gbenchmark_all ALIAS google_benchmark_all)

2
contrib/qpl vendored

@ -1 +1 @@
Subproject commit cdc8442f7a5e7a6ff6eea39c69665e0c5034d85d
Subproject commit becb7a1b15bdb4845ec3721a550707ffa51d029d

View File

@ -15,7 +15,7 @@ set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")
set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl")
set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake")
set (EFFICIENT_WAIT ON)
set (EFFICIENT_WAIT OFF)
set (BLOCK_ON_FAULT ON)
set (LOG_HW_INIT OFF)
set (SANITIZE_MEMORY OFF)
@ -42,7 +42,7 @@ include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake")
include(CheckLanguage)
check_language(ASM_NASM)
if(NOT CMAKE_ASM_NASM_COMPILER)
message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!")
message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!")
endif()
# [SUBDIR]isal
@ -110,18 +110,18 @@ target_compile_options(isal PRIVATE
"$<$<CONFIG:Debug>:>"
"$<$<CONFIG:Release>:>")
target_compile_options(isal_asm PUBLIC "-I${QPL_SRC_DIR}/isal/include/"
PUBLIC "-I${QPL_SRC_DIR}/isal/igzip/"
PUBLIC "-I${QPL_SRC_DIR}/isal/crc/"
PUBLIC "-DQPL_LIB")
target_compile_options(isal_asm PRIVATE "-I${QPL_SRC_DIR}/isal/include/"
PRIVATE "-I${QPL_SRC_DIR}/isal/igzip/"
PRIVATE "-I${QPL_SRC_DIR}/isal/crc/"
PRIVATE "-DQPL_LIB")
# AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available".
# AS_FEATURE_LEVEL=5 means "Check SIMD capabilities of the target system at runtime and use up to AVX2 if available".
# HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system.
if (ENABLE_AVX512)
target_compile_options(isal_asm PUBLIC "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10")
target_compile_options(isal_asm PRIVATE "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10")
else()
target_compile_options(isal_asm PUBLIC "-DAS_FEATURE_LEVEL=5")
target_compile_options(isal_asm PRIVATE "-DAS_FEATURE_LEVEL=5")
endif()
# Here must remove "-fno-sanitize=undefined" from COMPILE_OPTIONS.
@ -315,7 +315,13 @@ target_compile_definitions(_qpl
PRIVATE -DQPL_BADARG_CHECK
PUBLIC -DENABLE_QPL_COMPRESSION)
find_library(LIBACCEL accel-config)
if(NOT LIBACCEL)
message(FATAL_ERROR "Please install QPL dependency library:libaccel-config from https://github.com/intel/idxd-config")
endif()
target_link_libraries(_qpl
PRIVATE ${LIBACCEL}
PRIVATE ${CMAKE_DL_LIBS})
add_library (ch_contrib::qpl ALIAS _qpl)

View File

@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="22.11.1.1360"
ARG VERSION="22.11.2.30"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.

View File

@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
ARG VERSION="22.11.1.1360"
ARG VERSION="22.11.2.30"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image

View File

@ -0,0 +1,23 @@
---
sidebar_position: 1
sidebar_label: 2022
---
# 2022 Changelog
### ClickHouse release v22.8.11.15-lts (65c9506d161) FIXME as compared to v22.8.10.29-lts (d568a57f7af)
#### Bug Fix
* Backported in [#43098](https://github.com/ClickHouse/ClickHouse/issues/43098): Updated normaliser to clone the alias ast. resolves [#42452](https://github.com/ClickHouse/ClickHouse/issues/42452) Implementation: * Updated QueryNormalizer to clone alias ast, when its replaced. Previously just assigning the same leads to exception in LogicalExpressinsOptimizer as it would be the same parent being inserted again. * This bug is not seen with new analyser (allow_experimental_analyzer), so no changes for it. I added a test for the same. [#42827](https://github.com/ClickHouse/ClickHouse/pull/42827) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
* Backported in [#43751](https://github.com/ClickHouse/ClickHouse/issues/43751): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)).
* Backported in [#43617](https://github.com/ClickHouse/ClickHouse/issues/43617): Fix sumMap() for Nullable(Decimal()). [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)).
* Backported in [#43886](https://github.com/ClickHouse/ClickHouse/issues/43886): Fixed `ALTER ... RESET SETTING` with `ON CLUSTER`. It could be applied to one replica only. Fixes [#43843](https://github.com/ClickHouse/ClickHouse/issues/43843). [#43848](https://github.com/ClickHouse/ClickHouse/pull/43848) ([Elena Torró](https://github.com/elenatorro)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Use only PRs to our repository in pr_info on push [#43895](https://github.com/ClickHouse/ClickHouse/pull/43895) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix tags workflow [#43942](https://github.com/ClickHouse/ClickHouse/pull/43942) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

View File

@ -9,7 +9,7 @@ slug: /en/install
You have two options for getting up and running with ClickHouse:
- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** the official ClickHouse as a service, - built by, maintained, and supported by the creators of ClickHouse
- **Self-managed ClickHouse:** ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture
- **[Self-managed ClickHouse](https://github.com/ClickHouse/ClickHouse):** ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture
## ClickHouse Cloud

View File

@ -1202,6 +1202,7 @@ SELECT * FROM json_each_row_nested
- [input_format_import_nested_json](../operations/settings/settings.md#input_format_import_nested_json) - map nested JSON data to nested tables (it works for JSONEachRow format). Default value - `false`.
- [input_format_json_read_bools_as_numbers](../operations/settings/settings.md#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`.
- [input_format_json_read_numbers_as_strings](../operations/settings/settings.md#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`.
- [input_format_json_read_objects_as_strings](../operations/settings/settings.md#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`.
- [output_format_json_quote_64bit_integers](../operations/settings/settings.md#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`.
- [output_format_json_quote_64bit_floats](../operations/settings/settings.md#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`.
- [output_format_json_quote_denormals](../operations/settings/settings.md#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`.

View File

@ -244,7 +244,7 @@ The username and password can be indicated in one of three ways:
$ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @-
```
1. In the user and password URL parameters. Example:
2. In the user and password URL parameters (*We do not recommend using this method as the parameter might be logged by web proxy and cached in the browser*). Example:
<!-- -->
@ -252,7 +252,7 @@ $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @-
$ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @-
```
1. Using X-ClickHouse-User and X-ClickHouse-Key headers. Example:
3. Using X-ClickHouse-User and X-ClickHouse-Key headers. Example:
<!-- -->

View File

@ -91,4 +91,21 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n
**Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until theyre overridden explicitly for these users.
## Constraints on Merge Tree Settings
It is possible to set constraints for [merge tree settings](merge-tree-settings.md). There constraints are applied when table with merge tree engine is created or its storage settings are altered. Name of merge tree setting must be prepended by `merge_tree_` prefix when referenced in `<constraint>` section.
**Example:** Forbid to create new tables with explicitly specified `storage_policy`
``` xml
<profiles>
<default>
<constraints>
<merge_tree_storage_policy>
<const/>
</merge_tree_storage_policy>
</constraints>
</default>
</profiles>
```
[Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) <!--hide-->

View File

@ -3760,6 +3760,29 @@ Allow parsing numbers as strings in JSON input formats.
Disabled by default.
### input_format_json_read_objects_as_strings {#input_format_json_read_objects_as_strings}
Allow parsing JSON objects as strings in JSON input formats.
Example:
```sql
SET input_format_json_read_objects_as_strings = 1;
CREATE TABLE test (id UInt64, obj String, date Date) ENGINE=Memory();
INSERT INTO test FORMAT JSONEachRow {"id" : 1, "obj" : {"a" : 1, "b" : "Hello"}, "date" : "2020-01-01"};
SELECT * FROM test;
```
Result:
```
┌─id─┬─obj──────────────────────┬───────date─┐
│ 1 │ {"a" : 1, "b" : "Hello"} │ 2020-01-01 │
└────┴──────────────────────────┴────────────┘
```
Disabled by default.
### input_format_json_validate_types_from_metadata {#input_format_json_validate_types_from_metadata}
For JSON/JSONCompact/JSONColumnsWithMetadata input formats, if this setting is set to 1,

View File

@ -1,10 +1,82 @@
---
slug: /en/operations/update
sidebar_title: Self-managed Upgrade
title: Self-managed Upgrade
---
# Update
## ClickHouse upgrade overview
## Self-managed ClickHouse Upgrade
This document contains:
- general guidelines
- a recommended plan
- specifics for upgrading the binaries on your systems
## General guidelines
These notes should help you with planning, and to understand why we make the recommendations that we do later in the document.
### Upgrade ClickHouse server separately from ClickHouse Keeper or ZooKeeper
Unless there is a security fix needed for ClickHouse Keeper or Apache ZooKeeper it is not necessary to upgrade Keeper when you upgrade ClickHouse server. Keeper stability is required during the upgrade process, so complete the ClickHouse server upgrades before considering an upgrade of Keeper.
### Minor version upgrades should be adopted often
It is highly recommended to always upgrade to the newest minor version as soon as it is released. Minor releases do not have breaking changes but do have important bug fixes (and may have security fixes).
### Test experimental features on a separate ClickHouse server running the target version
The compatibility of experimental features can be broken at any moment in any way. If you are using experimental features, then check the changelogs and consider setting up a separate ClickHouse server with the target version installed and test your use of the experimental features there.
### Downgrades
If you upgrade and then realize that the new version is not compatible with some feature that you depend on you may be able to downgrade to a recent (less than one year old) version if you have not started to use any of the new features. Once the new features are used the downgrade will not work.
### Multiple ClickHouse server versions in a cluster
We make an effort to maintain a one-year compatibility window (which includes 2 LTS versions). This means that any two versions should be able to work together in a cluster if the difference between them is less than one year (or if there are less than two LTS versions between them). However, it is recommended to upgrade all members of a cluster to the same version as quickly as possible, as some minor issues are possible (like slowdown of distributed queries, retriable errors in some background operations in ReplicatedMergeTree, etc).
We never recommend running different versions in the same cluster when the release dates are more than one year. While we do not expect that you will have data loss, the cluster may become unusable. The issues that you should expect if you have more than one year difference in versions include:
- the cluster may not work
- some (or even all) queries may fail with arbitrary errors
- arbitrary errors/warnings may appear in the logs
- it may be impossible to downgrade
### Incremental upgrades
If the difference between the current version and the target version is more than one year, then it is recommended to either:
- Upgrade with downtime (stop all servers, upgrade all servers, run all servers).
- Or to upgrade through an intermediate version (a version less than one year more recent than the current version).
## Recommended plan
These are the recommended steps for a zero-downtime ClickHouse upgrade:
1. Make sure that your configuration changes are not in the default `/etc/clickhouse-server/config.xml` file and that they are instead in `/etc/clickhouse-server/config.d/`, as `/etc/clickhouse-server/config.xml` could be overwritten during an upgrade.
2. Read through the [changelogs](/docs/en/whats-new/changelog/index.md) for breaking changes (going back from the target release to the release you are currently on).
3. Make any updates identified in the breaking changes that can be made before upgrading, and a list of the changes that will need to be made after the upgrade.
4. Identify one or more replicas for each shard to keep up while the rest of the replicas for each shard are upgraded.
5. On the replicas that will be upgraded, one at a time:
- shutdown ClickHouse server
- upgrade the server to the target version
- bring ClickHouse server up
- wait for the Keeper messages to indicate that the system is stable
- continue to the next replica
6. Check for errors in the Keeper log and the ClickHouse log
7. Upgrade the replicas identified in step 4 to the new version
8. Refer to the list of changes made in steps 1 through 3 and make the changes that need to be made after the upgrade.
:::note
This error message is expected when there are multiple versions of ClickHouse running in a replicated environment. You will stop seeing these when all replicas are upgraded to the same version.
```
MergeFromLogEntryTask: Code: 40. DB::Exception: Checksums of parts don't match:
hash of uncompressed files doesn't match. (CHECKSUM_DOESNT_MATCH) Data after merge is not
byte-identical to data on another replicas.
```
:::
## ClickHouse server binary upgrade process
If ClickHouse was installed from `deb` packages, execute the following commands on the server:

View File

@ -1,6 +1,7 @@
---
slug: /en/sql-reference/aggregate-functions/reference/exponentialmovingaverage
sidebar_position: 108
sidebar_title: exponentialMovingAverage
---
## exponentialMovingAverage

View File

@ -7,7 +7,7 @@ sidebar_label: UPDATE
# ALTER TABLE … UPDATE Statements
``` sql
ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] WHERE filter_expr
ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr
```
Manipulates data matching the specified filtering expression. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations).

View File

@ -10,7 +10,7 @@ Creates [settings profiles](../../../operations/access-rights.md#settings-profil
Syntax:
``` sql
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
```

View File

@ -11,7 +11,7 @@ sidebar_label: "Профиль настроек"
Синтаксис:
``` sql
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1]
CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...]
```

View File

@ -111,6 +111,8 @@ EOF
tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR"
fi
sha512sum "$TARBALL" > "$TARBALL".sha512
rm -r "$PKG_PATH"
}

View File

@ -58,22 +58,52 @@ namespace ErrorCodes
class Benchmark : public Poco::Util::Application
{
public:
Benchmark(unsigned concurrency_, double delay_,
Strings && hosts_, Ports && ports_, bool round_robin_,
bool cumulative_, bool secure_, const String & default_database_,
const String & user_, const String & password_, const String & quota_key_, const String & stage,
bool randomize_, size_t max_iterations_, double max_time_,
const String & json_path_, size_t confidence_,
const String & query_id_, const String & query_to_execute_, bool continue_on_errors_,
bool reconnect_, bool display_client_side_time_, bool print_stacktrace_, const Settings & settings_)
Benchmark(unsigned concurrency_,
double delay_,
Strings && hosts_,
Ports && ports_,
bool round_robin_,
bool cumulative_,
bool secure_,
const String & default_database_,
const String & user_,
const String & password_,
const String & quota_key_,
const String & stage,
bool randomize_,
size_t max_iterations_,
double max_time_,
const String & json_path_,
size_t confidence_,
const String & query_id_,
const String & query_to_execute_,
size_t max_consecutive_errors_,
bool continue_on_errors_,
bool reconnect_,
bool display_client_side_time_,
bool print_stacktrace_,
const Settings & settings_)
:
round_robin(round_robin_), concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_),
cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_),
json_path(json_path_), confidence(confidence_), query_id(query_id_),
query_to_execute(query_to_execute_), continue_on_errors(continue_on_errors_), reconnect(reconnect_),
round_robin(round_robin_),
concurrency(concurrency_),
delay(delay_),
queue(concurrency),
randomize(randomize_),
cumulative(cumulative_),
max_iterations(max_iterations_),
max_time(max_time_),
json_path(json_path_),
confidence(confidence_),
query_id(query_id_),
query_to_execute(query_to_execute_),
continue_on_errors(continue_on_errors_),
max_consecutive_errors(max_consecutive_errors_),
reconnect(reconnect_),
display_client_side_time(display_client_side_time_),
print_stacktrace(print_stacktrace_), settings(settings_),
shared_context(Context::createShared()), global_context(Context::createGlobal(shared_context.get())),
print_stacktrace(print_stacktrace_),
settings(settings_),
shared_context(Context::createShared()),
global_context(Context::createGlobal(shared_context.get())),
pool(concurrency)
{
const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable;
@ -166,6 +196,7 @@ private:
String query_id;
String query_to_execute;
bool continue_on_errors;
size_t max_consecutive_errors;
bool reconnect;
bool display_client_side_time;
bool print_stacktrace;
@ -174,6 +205,8 @@ private:
ContextMutablePtr global_context;
QueryProcessingStage::Enum query_processing_stage;
std::atomic<size_t> consecutive_errors{0};
/// Don't execute new queries after timelimit or SIGINT or exception
std::atomic<bool> shutdown{false};
@ -393,13 +426,14 @@ private:
try
{
execute(connection_entries, query, connection_index);
consecutive_errors = 0;
}
catch (...)
{
std::lock_guard lock(mutex);
std::cerr << "An error occurred while processing the query " << "'" << query << "'"
<< ": " << getCurrentExceptionMessage(false) << std::endl;
if (!continue_on_errors)
if (!(continue_on_errors || max_consecutive_errors > ++consecutive_errors))
{
shutdown = true;
throw;
@ -648,6 +682,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
("stacktrace", "print stack traces of exceptions")
("confidence", value<size_t>()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)")
("query_id", value<std::string>()->default_value(""), "")
("max-consecutive-errors", value<size_t>()->default_value(0), "set number of allowed consecutive errors")
("continue_on_errors", "continue testing even if a query fails")
("reconnect", "establish new connection for every query")
("client-side-time", "display the time including network communication instead of server-side time; note that for server versions before 22.8 we always display client-side time")
@ -702,6 +737,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
options["confidence"].as<size_t>(),
options["query_id"].as<std::string>(),
options["query"].as<std::string>(),
options["max-consecutive-errors"].as<size_t>(),
options.count("continue_on_errors"),
options.count("reconnect"),
options.count("client-side-time"),

View File

@ -348,17 +348,9 @@ void Client::connect()
}
catch (const Exception & e)
{
/// It is typical when users install ClickHouse, type some password and instantly forget it.
/// This problem can't be fixed with reconnection so it is not attempted
if ((connection_parameters.user.empty() || connection_parameters.user == "default")
&& e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED)
if (e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED)
{
std::cerr << std::endl
<< "If you have installed ClickHouse and forgot password you can reset it in the configuration file." << std::endl
<< "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml" << std::endl
<< "and deleting this file will reset the password." << std::endl
<< "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl
<< std::endl;
/// This problem can't be fixed with reconnection so it is not attempted
throw;
}
else

View File

@ -1,4 +1,15 @@
set (CLICKHOUSE_DISKS_SOURCES DisksApp.cpp ICommand.cpp)
set (CLICKHOUSE_DISKS_SOURCES
DisksApp.cpp
ICommand.cpp
CommandCopy.cpp
CommandLink.cpp
CommandList.cpp
CommandListDisks.cpp
CommandMkDir.cpp
CommandMove.cpp
CommandRead.cpp
CommandRemove.cpp
CommandWrite.cpp)
set (CLICKHOUSE_DISKS_LINK
PRIVATE

View File

@ -1,7 +1,6 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
#include <Common/TerminalSize.h>
namespace DB
{
@ -11,7 +10,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandCopy : public ICommand
class CommandCopy final : public ICommand
{
public:
CommandCopy()
@ -51,16 +50,16 @@ public:
String disk_name_from = config.getString("diskFrom", config.getString("disk", "default"));
String disk_name_to = config.getString("diskTo", config.getString("disk", "default"));
String path_from = command_arguments[0];
String path_to = command_arguments[1];
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk_from = global_context->getDisk(disk_name_from);
DiskPtr disk_to = global_context->getDisk(disk_name_to);
String full_path_from = fullPathWithValidate(disk_from, path_from);
String full_path_to = fullPathWithValidate(disk_to, path_to);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);
disk_from->copy(full_path_from, disk_to, full_path_to);
disk_from->copy(relative_path_from, disk_to, relative_path_to);
}
};
}

View File

@ -1,5 +1,3 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
@ -11,7 +9,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandLink : public ICommand
class CommandLink final : public ICommand
{
public:
CommandLink()
@ -40,15 +38,15 @@ public:
String disk_name = config.getString("disk", "default");
String path_from = command_arguments[0];
String path_to = command_arguments[1];
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path_from = fullPathWithValidate(disk, path_from);
String full_path_to = fullPathWithValidate(disk, path_to);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);
disk->createHardLink(full_path_from, full_path_to);
disk->createHardLink(relative_path_from, relative_path_to);
}
};
}

View File

@ -1,7 +1,6 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
#include <Common/TerminalSize.h>
namespace DB
{
@ -11,7 +10,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandList : public ICommand
class CommandList final : public ICommand
{
public:
CommandList()
@ -46,43 +45,47 @@ public:
String disk_name = config.getString("disk", "default");
String path = command_arguments[0];
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path = fullPathWithValidate(disk, path);
String relative_path = validatePathAndGetAsRelative(path);
bool recursive = config.getBool("recursive", false);
if (recursive)
listRecursive(disk, full_path);
listRecursive(disk, relative_path);
else
list(disk, full_path);
list(disk, relative_path);
}
private:
static void list(const DiskPtr & disk, const std::string & full_path)
static void list(const DiskPtr & disk, const std::string & relative_path)
{
std::vector<String> file_names;
disk->listFiles(full_path, file_names);
disk->listFiles(relative_path, file_names);
for (const auto & file_name : file_names)
std::cout << file_name << '\n';
}
static void listRecursive(const DiskPtr & disk, const std::string & full_path)
static void listRecursive(const DiskPtr & disk, const std::string & relative_path)
{
std::vector<String> file_names;
disk->listFiles(full_path, file_names);
disk->listFiles(relative_path, file_names);
std::cout << full_path << ":\n";
for (const auto & file_name : file_names)
std::cout << file_name << '\n';
std::cout << "\n";
std::cout << relative_path << ":\n";
if (!file_names.empty())
{
for (const auto & file_name : file_names)
std::cout << file_name << '\n';
std::cout << "\n";
}
for (const auto & file_name : file_names)
{
auto path = full_path + "/" + file_name;
auto path = relative_path + "/" + file_name;
if (disk->isDirectory(path))
listRecursive(disk, path);
}

View File

@ -1,5 +1,3 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
@ -11,7 +9,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandListDisks : public ICommand
class CommandListDisks final : public ICommand
{
public:
CommandListDisks()

View File

@ -1,7 +1,7 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
#include <Common/TerminalSize.h>
namespace DB
{
@ -11,7 +11,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandMkDir : public ICommand
class CommandMkDir final : public ICommand
{
public:
CommandMkDir()
@ -46,17 +46,17 @@ public:
String disk_name = config.getString("disk", "default");
String path = command_arguments[0];
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path = fullPathWithValidate(disk, path);
String relative_path = validatePathAndGetAsRelative(path);
bool recursive = config.getBool("recursive", false);
if (recursive)
disk->createDirectories(full_path);
disk->createDirectories(relative_path);
else
disk->createDirectory(full_path);
disk->createDirectory(relative_path);
}
};
}

View File

@ -1,5 +1,3 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
@ -11,7 +9,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandMove : public ICommand
class CommandMove final : public ICommand
{
public:
CommandMove()
@ -39,18 +37,18 @@ public:
String disk_name = config.getString("disk", "default");
String path_from = command_arguments[0];
String path_to = command_arguments[1];
const String & path_from = command_arguments[0];
const String & path_to = command_arguments[1];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path_from = fullPathWithValidate(disk, path_from);
String full_path_to = fullPathWithValidate(disk, path_to);
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);
if (disk->isFile(full_path_from))
disk->moveFile(full_path_from, full_path_to);
if (disk->isFile(relative_path_from))
disk->moveFile(relative_path_from, relative_path_to);
else
disk->moveDirectory(full_path_from, full_path_to);
disk->moveDirectory(relative_path_from, relative_path_to);
}
};
}

View File

@ -1,7 +1,9 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/copyData.h>
#include <Common/TerminalSize.h>
namespace DB
{
@ -11,7 +13,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandRead : public ICommand
class CommandRead final : public ICommand
{
public:
CommandRead()
@ -46,27 +48,25 @@ public:
String disk_name = config.getString("disk", "default");
String path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path = fullPathWithValidate(disk, path);
String relative_path = validatePathAndGetAsRelative(command_arguments[0]);
String path_output = config.getString("output", "");
if (!path_output.empty())
{
String full_path_output = fullPathWithValidate(disk, path_output);
String relative_path_output = validatePathAndGetAsRelative(path_output);
auto in = disk->readFile(full_path);
auto out = disk->writeFile(full_path_output);
auto in = disk->readFile(relative_path);
auto out = disk->writeFile(relative_path_output);
copyData(*in, *out);
out->finalize();
return;
}
else
{
auto in = disk->readFile(full_path);
auto in = disk->readFile(relative_path);
std::unique_ptr<WriteBufferFromFileBase> out = std::make_unique<WriteBufferFromFileDescriptor>(STDOUT_FILENO);
copyData(*in, *out);
}

View File

@ -1,5 +1,3 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
@ -11,7 +9,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandRemove : public ICommand
class CommandRemove final : public ICommand
{
public:
CommandRemove()
@ -39,13 +37,13 @@ public:
String disk_name = config.getString("disk", "default");
String path = command_arguments[0];
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path = fullPathWithValidate(disk, path);
String relative_path = validatePathAndGetAsRelative(path);
disk->removeRecursive(full_path);
disk->removeRecursive(relative_path);
}
};
}

View File

@ -1,8 +1,11 @@
#pragma once
#include "ICommand.h"
#include <Interpreters/Context.h>
#include <Common/TerminalSize.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/copyData.h>
namespace DB
{
@ -11,7 +14,7 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
class CommandWrite : public ICommand
class CommandWrite final : public ICommand
{
public:
CommandWrite()
@ -46,11 +49,11 @@ public:
String disk_name = config.getString("disk", "default");
String path = command_arguments[0];
const String & path = command_arguments[0];
DiskPtr disk = global_context->getDisk(disk_name);
String full_path = fullPathWithValidate(disk, path);
String relative_path = validatePathAndGetAsRelative(path);
String path_input = config.getString("input", "");
std::unique_ptr<ReadBufferFromFileBase> in;
@ -60,11 +63,11 @@ public:
}
else
{
String full_path_input = fullPathWithValidate(disk, path_input);
in = disk->readFile(full_path_input);
String relative_path_input = validatePathAndGetAsRelative(path_input);
in = disk->readFile(relative_path_input);
}
auto out = disk->writeFile(full_path);
auto out = disk->writeFile(relative_path);
copyData(*in, *out);
out->finalize();
}

View File

@ -1,11 +1,12 @@
#include "DisksApp.h"
#include "ICommand.h"
#include <Disks/registerDisks.h>
#include <base/argsToConfig.h>
#include <Common/TerminalSize.h>
#include <Formats/registerFormats.h>
namespace DB
{

View File

@ -1,28 +1,22 @@
#pragma once
#include "CommandCopy.cpp"
#include "CommandLink.cpp"
#include "CommandList.cpp"
#include "CommandListDisks.cpp"
#include "CommandMkDir.cpp"
#include "CommandMove.cpp"
#include "CommandRead.cpp"
#include "CommandRemove.cpp"
#include "CommandWrite.cpp"
#include <Loggers/Loggers.h>
#include <Common/ProgressIndication.h>
#include <Common/StatusFile.h>
#include <Common/InterruptListener.h>
#include <Core/Settings.h>
#include <Interpreters/Context.h>
#include <Poco/Util/Application.h>
#include <boost/program_options.hpp>
namespace DB
{
class ICommand;
using CommandPtr = std::unique_ptr<ICommand>;
namespace po = boost::program_options;
using ProgramOptionsDescription = boost::program_options::options_description;
using CommandLineOptions = boost::program_options::variables_map;
class DisksApp : public Poco::Util::Application, public Loggers
{
public:

View File

@ -30,19 +30,21 @@ void ICommand::addOptions(ProgramOptionsDescription & options_description)
options_description.add(*command_option_description);
}
String ICommand::fullPathWithValidate(const DiskPtr & disk, const String & path)
String ICommand::validatePathAndGetAsRelative(const String & path)
{
if (fs::path(path).lexically_normal().string() != path)
/// If path contain non-normalized symbols like . we will normalized them. If the resulting normalized path
/// still contain '..' it can be dangerous, disallow such paths. Also since clickhouse-disks
/// is not an interactive program (don't track you current path) it's OK to disallow .. paths.
String lexically_normal_path = fs::path(path).lexically_normal();
if (lexically_normal_path.find("..") != std::string::npos)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Path {} is not normalized", path);
String disk_path = fs::canonical(fs::path(disk->getPath())) / "";
String full_path = (fs::absolute(disk_path) / path).lexically_normal();
/// If path is absolute we should keep it as relative inside disk, so disk will look like
/// an ordinary filesystem with root.
if (fs::path(lexically_normal_path).is_absolute())
return lexically_normal_path.substr(1);
if (!full_path.starts_with(disk_path))
throw DB::Exception(
DB::ErrorCodes::BAD_ARGUMENTS, "Path {} must be inside disk path {}", full_path, disk_path);
return path;
return lexically_normal_path;
}
}

View File

@ -2,16 +2,10 @@
#include <Disks/IDisk.h>
#include <Poco/Util/Application.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/copyData.h>
#include <boost/program_options.hpp>
#include <Common/TerminalSize.h>
#include <Common/Config/ConfigProcessor.h>
#include <Poco/Util/Application.h>
#include <memory>
@ -43,7 +37,7 @@ public:
protected:
void printHelpMessage() const;
static String fullPathWithValidate(const DiskPtr & disk, const String & path);
static String validatePathAndGetAsRelative(const String & path);
public:
String command_name;
@ -55,14 +49,16 @@ protected:
po::positional_options_description positional_options_description;
};
using CommandPtr = std::unique_ptr<ICommand>;
}
std::unique_ptr <DB::ICommand> makeCommandCopy();
std::unique_ptr <DB::ICommand> makeCommandLink();
std::unique_ptr <DB::ICommand> makeCommandList();
std::unique_ptr <DB::ICommand> makeCommandListDisks();
std::unique_ptr <DB::ICommand> makeCommandMove();
std::unique_ptr <DB::ICommand> makeCommandRead();
std::unique_ptr <DB::ICommand> makeCommandRemove();
std::unique_ptr <DB::ICommand> makeCommandWrite();
std::unique_ptr <DB::ICommand> makeCommandMkDir();
DB::CommandPtr makeCommandCopy();
DB::CommandPtr makeCommandLink();
DB::CommandPtr makeCommandList();
DB::CommandPtr makeCommandListDisks();
DB::CommandPtr makeCommandMove();
DB::CommandPtr makeCommandRead();
DB::CommandPtr makeCommandRemove();
DB::CommandPtr makeCommandWrite();
DB::CommandPtr makeCommandMkDir();

View File

@ -16,11 +16,13 @@
#include <Access/ExternalAuthenticators.h>
#include <Access/AccessChangesNotifier.h>
#include <Access/AccessBackup.h>
#include <Access/resolveSetting.h>
#include <Backups/BackupEntriesCollector.h>
#include <Backups/RestorerFromBackup.h>
#include <Core/Settings.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <base/defines.h>
#include <base/find_symbols.h>
#include <IO/Operators.h>
#include <Poco/AccessExpireCache.h>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/split.hpp>
@ -38,7 +40,6 @@ namespace ErrorCodes
extern const int AUTHENTICATION_FAILED;
}
namespace
{
void checkForUsersNotInMainConfig(
@ -103,7 +104,7 @@ public:
bool isSettingNameAllowed(std::string_view setting_name) const
{
if (Settings::hasBuiltin(setting_name))
if (settingIsBuiltin(setting_name))
return true;
std::lock_guard lock{mutex};
@ -454,9 +455,21 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne
{
tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed");
WriteBufferFromOwnString message;
message << credentials.getUserName() << ": Authentication failed: password is incorrect, or there is no user with such name.";
/// Better exception message for usability.
/// It is typical when users install ClickHouse, type some password and instantly forget it.
if (credentials.getUserName().empty() || credentials.getUserName() == "default")
message << "\n\n"
<< "If you have installed ClickHouse and forgot password you can reset it in the configuration file.\n"
<< "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml\n"
<< "and deleting this file will reset the password.\n"
<< "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed.\n\n";
/// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
/// only the log will show the exact reason.
throw Exception(credentials.getUserName() + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED);
throw Exception(message.str(), ErrorCodes::AUTHENTICATION_FAILED);
}
}

View File

@ -174,7 +174,6 @@ private:
void initialize();
void setUser(const UserPtr & user_) const;
void setRolesInfo(const std::shared_ptr<const EnabledRolesInfo> & roles_info_) const;
void setSettingsAndConstraints() const;
void calculateAccessRights() const;
template <bool throw_if_denied, bool grant_option>

View File

@ -1,13 +1,15 @@
#include <string_view>
#include <Access/SettingsConstraints.h>
#include <Access/resolveSetting.h>
#include <Access/AccessControl.h>
#include <Core/Settings.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Common/FieldVisitorToString.h>
#include <Common/FieldVisitorsAccurateComparison.h>
#include <IO/WriteHelpers.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <boost/range/algorithm_ext/erase.hpp>
namespace DB
{
namespace ErrorCodes
@ -18,7 +20,6 @@ namespace ErrorCodes
extern const int UNKNOWN_SETTING;
}
SettingsConstraints::SettingsConstraints(const AccessControl & access_control_) : access_control(&access_control_)
{
}
@ -35,19 +36,28 @@ void SettingsConstraints::clear()
constraints.clear();
}
void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability)
void SettingsConstraints::set(const String & full_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability)
{
auto & constraint = constraints[setting_name];
auto & constraint = constraints[full_name];
if (!min_value.isNull())
constraint.min_value = Settings::castValueUtil(setting_name, min_value);
constraint.min_value = settingCastValueUtil(full_name, min_value);
if (!max_value.isNull())
constraint.max_value = Settings::castValueUtil(setting_name, max_value);
constraint.max_value = settingCastValueUtil(full_name, max_value);
constraint.writability = writability;
}
void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const
void SettingsConstraints::get(const Settings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const
{
auto checker = getChecker(current_settings, setting_name);
// NOTE: for `Settings` short name is equal to full name
auto checker = getChecker(current_settings, short_name);
min_value = checker.constraint.min_value;
max_value = checker.constraint.max_value;
writability = checker.constraint.writability;
}
void SettingsConstraints::get(const MergeTreeSettings &, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const
{
auto checker = getMergeTreeChecker(short_name);
min_value = checker.constraint.min_value;
max_value = checker.constraint.max_value;
writability = checker.constraint.writability;
@ -97,6 +107,17 @@ void SettingsConstraints::check(const Settings & current_settings, SettingsChang
});
}
void SettingsConstraints::check(const MergeTreeSettings & current_settings, const SettingChange & change) const
{
checkImpl(current_settings, const_cast<SettingChange &>(change), THROW_ON_VIOLATION);
}
void SettingsConstraints::check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const
{
for (const auto & change : changes)
check(current_settings, change);
}
void SettingsConstraints::clamp(const Settings & current_settings, SettingsChanges & changes) const
{
boost::range::remove_erase_if(
@ -107,6 +128,36 @@ void SettingsConstraints::clamp(const Settings & current_settings, SettingsChang
});
}
template <class T>
bool getNewValueToCheck(const T & current_settings, SettingChange & change, Field & new_value, bool throw_on_failure)
{
Field current_value;
bool has_current_value = current_settings.tryGet(change.name, current_value);
/// Setting isn't checked if value has not changed.
if (has_current_value && change.value == current_value)
return false;
if (throw_on_failure)
new_value = T::castValueUtil(change.name, change.value);
else
{
try
{
new_value = T::castValueUtil(change.name, change.value);
}
catch (...)
{
return false;
}
}
/// Setting isn't checked if value has not changed.
if (has_current_value && new_value == current_value)
return false;
return true;
}
bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const
{
@ -115,26 +166,6 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
if (setting_name == "profile")
return true;
bool cannot_cast;
auto cast_value = [&](const Field & x) -> Field
{
cannot_cast = false;
if (reaction == THROW_ON_VIOLATION)
return Settings::castValueUtil(setting_name, x);
else
{
try
{
return Settings::castValueUtil(setting_name, x);
}
catch (...)
{
cannot_cast = true;
return {};
}
}
};
if (reaction == THROW_ON_VIOLATION)
{
try
@ -156,27 +187,21 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh
else if (!access_control->isSettingNameAllowed(setting_name))
return false;
Field current_value, new_value;
if (current_settings.tryGet(setting_name, current_value))
{
/// Setting isn't checked if value has not changed.
if (change.value == current_value)
return false;
new_value = cast_value(change.value);
if ((new_value == current_value) || cannot_cast)
return false;
}
else
{
new_value = cast_value(change.value);
if (cannot_cast)
return false;
}
Field new_value;
if (!getNewValueToCheck(current_settings, change, new_value, reaction == THROW_ON_VIOLATION))
return false;
return getChecker(current_settings, setting_name).check(change, new_value, reaction);
}
bool SettingsConstraints::checkImpl(const MergeTreeSettings & current_settings, SettingChange & change, ReactionOnViolation reaction) const
{
Field new_value;
if (!getNewValueToCheck(current_settings, change, new_value, reaction == THROW_ON_VIOLATION))
return false;
return getMergeTreeChecker(change.name).check(change, new_value, reaction);
}
bool SettingsConstraints::Checker::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const
{
const String & setting_name = change.name;
@ -185,16 +210,13 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n
{
if (reaction == THROW_ON_VIOLATION)
return applyVisitor(FieldVisitorAccurateLess{}, left, right);
else
try
{
try
{
return applyVisitor(FieldVisitorAccurateLess{}, left, right);
}
catch (...)
{
return true;
}
return applyVisitor(FieldVisitorAccurateLess{}, left, right);
}
catch (...)
{
return true;
}
};
@ -280,6 +302,14 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu
return Checker(it->second);
}
SettingsConstraints::Checker SettingsConstraints::getMergeTreeChecker(std::string_view short_name) const
{
auto it = constraints.find(settingFullName<MergeTreeSettings>(short_name));
if (it == constraints.end())
return Checker(); // Allowed
return Checker(it->second);
}
bool SettingsConstraints::Constraint::operator==(const Constraint & other) const
{
return writability == other.writability && min_value == other.min_value && max_value == other.max_value;

View File

@ -12,6 +12,7 @@ namespace Poco::Util
namespace DB
{
struct Settings;
struct MergeTreeSettings;
struct SettingChange;
class SettingsChanges;
class AccessControl;
@ -65,8 +66,9 @@ public:
void clear();
bool empty() const { return constraints.empty(); }
void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability);
void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const;
void set(const String & full_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability);
void get(const Settings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const;
void get(const MergeTreeSettings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const;
void merge(const SettingsConstraints & other);
@ -75,6 +77,10 @@ public:
void check(const Settings & current_settings, const SettingsChanges & changes) const;
void check(const Settings & current_settings, SettingsChanges & changes) const;
/// Checks whether `change` violates these constraints and throws an exception if so. (setting short name is expected inside `changes`)
void check(const MergeTreeSettings & current_settings, const SettingChange & change) const;
void check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const;
/// Checks whether `change` violates these and clamps the `change` if so.
void clamp(const Settings & current_settings, SettingsChanges & changes) const;
@ -137,8 +143,10 @@ private:
};
bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
bool checkImpl(const MergeTreeSettings & current_settings, SettingChange & change, ReactionOnViolation reaction) const;
Checker getChecker(const Settings & current_settings, std::string_view setting_name) const;
Checker getMergeTreeChecker(std::string_view short_name) const;
// Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)`
using Constraints = std::unordered_map<String, Constraint, StringHash, std::equal_to<>>;

View File

@ -4,6 +4,7 @@
#include <Access/User.h>
#include <Access/SettingsProfile.h>
#include <Access/AccessControl.h>
#include <Access/resolveSetting.h>
#include <Access/AccessChangesNotifier.h>
#include <Dictionaries/IDictionary.h>
#include <Common/Config/ConfigReloader.h>
@ -451,9 +452,9 @@ namespace
for (const String & constraint_type : constraint_types)
{
if (constraint_type == "min")
profile_element.min_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type));
profile_element.min_value = settingStringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type));
else if (constraint_type == "max")
profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type));
profile_element.max_value = settingStringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type));
else if (constraint_type == "readonly" || constraint_type == "const")
{
writability_count++;
@ -517,7 +518,7 @@ namespace
SettingsProfileElement profile_element;
profile_element.setting_name = setting_name;
profile_element.value = Settings::stringToValueUtil(setting_name, config.getString(profile_config + "." + key));
profile_element.value = settingStringToValueUtil(setting_name, config.getString(profile_config + "." + key));
profile->elements.emplace_back(std::move(profile_element));
}

View File

@ -0,0 +1,90 @@
#pragma once
#include <Core/Settings.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
//
// Settings from different classes (Settings, MergeTreeSettings) can coexist in the same "namespace".
// This is, for example, required to define settings constraints inside user profiles.
// `resolveSetting(full_name)` is used to resolve setting name and choose which class is to be used.
// Templated lambda syntax should be used:
//
// return resolveSetting(name, [] <typename T> (std::string_view name, SettingsType<T>)
// {
// return T::castValueUtil(name, value); // T will be deduced into `Settings`, `MergeTreeSettings`, ...
// });
//
namespace DB
{
constexpr std::string_view MERGE_TREE_SETTINGS_PREFIX = "merge_tree_";
template <typename T> struct SettingsType {};
// Resolve setting name and call function `f` back with short name and class
template <typename F>
auto resolveSetting(std::string_view full_name, F && f)
{
if (full_name.starts_with(MERGE_TREE_SETTINGS_PREFIX))
{
std::string_view short_name = static_cast<std::string_view>(full_name).substr(MERGE_TREE_SETTINGS_PREFIX.size());
if (MergeTreeSettings::hasBuiltin(short_name)) // Check is required because `Settings` also contain names starting with 'merge_tree_' prefix
return f(short_name, SettingsType<MergeTreeSettings>());
}
// NOTE: other setting name resolution rules are to be added here
// If no rule works - use global namespace
return f(full_name, SettingsType<Settings>());
}
inline Field settingCastValueUtil(std::string_view full_name, const Field & value)
{
return resolveSetting(full_name, [&] <typename T> (std::string_view short_name, SettingsType<T>)
{
return T::castValueUtil(short_name, value);
});
}
inline String settingValueToStringUtil(std::string_view full_name, const Field & value)
{
return resolveSetting(full_name, [&] <typename T> (std::string_view short_name, SettingsType<T>)
{
return T::valueToStringUtil(short_name, value);
});
}
inline Field settingStringToValueUtil(std::string_view full_name, const String & str)
{
return resolveSetting(full_name, [&] <typename T> (std::string_view short_name, SettingsType<T>)
{
return T::stringToValueUtil(short_name, str);
});
}
inline bool settingIsBuiltin(std::string_view full_name)
{
return resolveSetting(full_name, [&] <typename T> (std::string_view short_name, SettingsType<T>)
{
return T::hasBuiltin(short_name);
});
}
template <typename T>
inline String settingFullName(std::string_view short_name);
template <>
inline String settingFullName<Settings>(std::string_view short_name)
{
return String(short_name);
}
template <>
inline String settingFullName<MergeTreeSettings>(std::string_view short_name)
{
String full_name(MERGE_TREE_SETTINGS_PREFIX);
full_name += short_name; // Just because you cannot concatenate `std::string_view` and `std::string` using operator+ in C++20 yet
return full_name;
}
}

View File

@ -477,7 +477,7 @@ public:
final_flags = std::make_unique<UInt8[]>(row_end);
final_flags_ptr = final_flags.get();
bool included_elements = 0;
size_t included_elements = 0;
const auto & flags = assert_cast<const ColumnUInt8 &>(*columns[if_argument_pos]).getData();
for (size_t i = row_begin; i < row_end; i++)
{

View File

@ -3,6 +3,8 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <Interpreters/Context.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/FunctionNode.h>
@ -56,7 +58,7 @@ public:
auto & count_distinct_argument_column_typed = count_distinct_argument_column->as<ColumnNode &>();
/// Build subquery SELECT count_distinct_argument_column FROM table_expression GROUP BY count_distinct_argument_column
auto subquery = std::make_shared<QueryNode>();
auto subquery = std::make_shared<QueryNode>(Context::createCopy(query_node->getContext()));
subquery->getJoinTree() = query_node->getJoinTree();
subquery->getProjection().getNodes().push_back(count_distinct_argument_column);
subquery->getGroupBy().getNodes().push_back(count_distinct_argument_column);

View File

@ -6,6 +6,7 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Interpreters/Context.h>
namespace DB
{
@ -16,7 +17,8 @@ namespace
class NormalizeCountVariantsVisitor : public InDepthQueryTreeVisitor<NormalizeCountVariantsVisitor>
{
public:
static void visitImpl(QueryTreeNodePtr & node)
explicit NormalizeCountVariantsVisitor(ContextPtr context_) : context(std::move(context_)) {}
void visitImpl(QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();
if (!function_node || !function_node->isAggregateFunction() || (function_node->getFunctionName() != "count" && function_node->getFunctionName() != "sum"))
@ -39,13 +41,16 @@ public:
}
else if (function_node->getFunctionName() == "sum" &&
first_argument_constant_literal.getType() == Field::Types::UInt64 &&
first_argument_constant_literal.get<UInt64>() == 1)
first_argument_constant_literal.get<UInt64>() == 1 &&
!context->getSettingsRef().aggregate_functions_null_for_empty)
{
resolveAsCountAggregateFunction(*function_node);
function_node->getArguments().getNodes().clear();
}
}
private:
ContextPtr context;
static inline void resolveAsCountAggregateFunction(FunctionNode & function_node)
{
auto function_result_type = function_node.getResultType();
@ -59,9 +64,9 @@ private:
}
void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr)
void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context)
{
NormalizeCountVariantsVisitor visitor;
NormalizeCountVariantsVisitor visitor(context);
visitor.visit(query_tree_node);
}

View File

@ -647,6 +647,11 @@ struct IdentifierResolveScope
subquery_depth = parent_scope->subquery_depth;
context = parent_scope->context;
}
if (auto * union_node = scope_node->as<UnionNode>())
context = union_node->getContext();
else if (auto * query_node = scope_node->as<QueryNode>())
context = query_node->getContext();
}
QueryTreeNodePtr scope_node;
@ -974,7 +979,9 @@ public:
void resolve(QueryTreeNodePtr node, const QueryTreeNodePtr & table_expression, ContextPtr context)
{
IdentifierResolveScope scope(node, nullptr /*parent_scope*/);
scope.context = context;
if (!scope.context)
scope.context = context;
auto node_type = node->getNodeType();
@ -4050,7 +4057,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
auto constant_data_type = std::make_shared<DataTypeUInt64>();
auto in_subquery = std::make_shared<QueryNode>();
auto in_subquery = std::make_shared<QueryNode>(Context::createCopy(scope.context));
in_subquery->getProjection().getNodes().push_back(std::make_shared<ConstantNode>(1UL, constant_data_type));
in_subquery->getJoinTree() = exists_subquery_argument;
in_subquery->getLimit() = std::make_shared<ConstantNode>(1UL, constant_data_type);
@ -4103,7 +4110,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
projection_columns.emplace_back(column.name, column.type);
}
auto in_second_argument_query_node = std::make_shared<QueryNode>();
auto in_second_argument_query_node = std::make_shared<QueryNode>(Context::createCopy(scope.context));
in_second_argument_query_node->setIsSubquery(true);
in_second_argument_query_node->getProjectionNode() = std::move(column_nodes_to_select);
in_second_argument_query_node->getJoinTree() = std::move(in_second_argument);
@ -5764,14 +5771,6 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
max_subquery_depth);
auto & query_node_typed = query_node->as<QueryNode &>();
if (query_node_typed.hasSettingsChanges())
{
auto updated_scope_context = Context::createCopy(scope.context);
updated_scope_context->applySettingsChanges(query_node_typed.getSettingsChanges());
scope.context = std::move(updated_scope_context);
}
const auto & settings = scope.context->getSettingsRef();
if (settings.group_by_use_nulls)

View File

@ -56,7 +56,7 @@ public:
if (!isInt64OrUInt64FieldType(constant_value_literal.getType()))
return;
if (constant_value_literal.get<UInt64>() != 1)
if (constant_value_literal.get<UInt64>() != 1 || context->getSettingsRef().aggregate_functions_null_for_empty)
return;
function_node_arguments_nodes[0] = std::move(function_node_arguments_nodes[1]);

View File

@ -21,8 +21,10 @@
namespace DB
{
QueryNode::QueryNode()
QueryNode::QueryNode(ContextMutablePtr context_, SettingsChanges settings_changes_)
: IQueryTreeNode(children_size)
, context(std::move(context_))
, settings_changes(std::move(settings_changes_))
{
children[with_child_index] = std::make_shared<ListNode>();
children[projection_child_index] = std::make_shared<ListNode>();
@ -32,6 +34,10 @@ QueryNode::QueryNode()
children[limit_by_child_index] = std::make_shared<ListNode>();
}
QueryNode::QueryNode(ContextMutablePtr context_)
: QueryNode(context_, {} /*settings_changes*/)
{}
void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const
{
buffer << std::string(indent, ' ') << "QUERY id: " << format_state.getNodeId(this);
@ -222,7 +228,7 @@ void QueryNode::updateTreeHashImpl(HashState & state) const
QueryTreeNodePtr QueryNode::cloneImpl() const
{
auto result_query_node = std::make_shared<QueryNode>();
auto result_query_node = std::make_shared<QueryNode>(context);
result_query_node->is_subquery = is_subquery;
result_query_node->is_cte = is_cte;

View File

@ -10,6 +10,8 @@
#include <Analyzer/ListNode.h>
#include <Analyzer/TableExpressionModifiers.h>
#include <Interpreters/Context_fwd.h>
namespace DB
{
@ -61,7 +63,41 @@ using QueryNodePtr = std::shared_ptr<QueryNode>;
class QueryNode final : public IQueryTreeNode
{
public:
explicit QueryNode();
/// Construct query node with context and changed settings
explicit QueryNode(ContextMutablePtr context_, SettingsChanges settings_changes_);
/// Construct query node with context
explicit QueryNode(ContextMutablePtr context_);
/// Get context
ContextPtr getContext() const
{
return context;
}
/// Get mutable context
const ContextMutablePtr & getMutableContext() const
{
return context;
}
/// Get mutable context
ContextMutablePtr & getMutableContext()
{
return context;
}
/// Returns true if query node has settings changes, false otherwise
bool hasSettingsChanges() const
{
return !settings_changes.empty();
}
/// Get query node settings changes
const SettingsChanges & getSettingsChanges() const
{
return settings_changes;
}
/// Returns true if query node is subquery, false otherwise
bool isSubquery() const
@ -513,24 +549,6 @@ public:
return children[offset_child_index];
}
/// Returns true if query node has settings changes specified, false otherwise
bool hasSettingsChanges() const
{
return !settings_changes.empty();
}
/// Get query node settings changes
const SettingsChanges & getSettingsChanges() const
{
return settings_changes;
}
/// Set query node settings changes value
void setSettingsChanges(SettingsChanges settings_changes_value)
{
settings_changes = std::move(settings_changes_value);
}
/// Get query node projection columns
const NamesAndTypes & getProjectionColumns() const
{
@ -572,6 +590,7 @@ private:
std::string cte_name;
NamesAndTypes projection_columns;
ContextMutablePtr context;
SettingsChanges settings_changes;
static constexpr size_t with_child_index = 0;

View File

@ -77,75 +77,90 @@ public:
}
private:
QueryTreeNodePtr buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, bool is_subquery, const std::string & cte_name) const;
QueryTreeNodePtr buildSelectOrUnionExpression(const ASTPtr & select_or_union_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const;
QueryTreeNodePtr buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, bool is_subquery, const std::string & cte_name) const;
QueryTreeNodePtr buildSelectWithUnionExpression(const ASTPtr & select_with_union_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const;
QueryTreeNodePtr buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, bool is_subquery, const std::string & cte_name) const;
QueryTreeNodePtr buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const;
QueryTreeNodePtr buildSelectExpression(const ASTPtr & select_query, bool is_subquery, const std::string & cte_name) const;
QueryTreeNodePtr buildSelectExpression(const ASTPtr & select_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const;
QueryTreeNodePtr buildSortList(const ASTPtr & order_by_expression_list) const;
QueryTreeNodePtr buildSortList(const ASTPtr & order_by_expression_list, const ContextPtr & context) const;
QueryTreeNodePtr buildInterpolateList(const ASTPtr & interpolate_expression_list) const;
QueryTreeNodePtr buildInterpolateList(const ASTPtr & interpolate_expression_list, const ContextPtr & context) const;
QueryTreeNodePtr buildWindowList(const ASTPtr & window_definition_list) const;
QueryTreeNodePtr buildWindowList(const ASTPtr & window_definition_list, const ContextPtr & context) const;
QueryTreeNodePtr buildExpressionList(const ASTPtr & expression_list) const;
QueryTreeNodePtr buildExpressionList(const ASTPtr & expression_list, const ContextPtr & context) const;
QueryTreeNodePtr buildExpression(const ASTPtr & expression) const;
QueryTreeNodePtr buildExpression(const ASTPtr & expression, const ContextPtr & context) const;
QueryTreeNodePtr buildWindow(const ASTPtr & window_definition) const;
QueryTreeNodePtr buildWindow(const ASTPtr & window_definition, const ContextPtr & context) const;
QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query) const;
QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const;
ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const;
ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const;
ASTPtr query;
ContextPtr context;
QueryTreeNodePtr query_tree_node;
};
QueryTreeBuilder::QueryTreeBuilder(ASTPtr query_, ContextPtr context_)
: query(query_->clone())
, context(std::move(context_))
{
if (query->as<ASTSelectWithUnionQuery>() ||
query->as<ASTSelectIntersectExceptQuery>() ||
query->as<ASTSelectQuery>())
query_tree_node = buildSelectOrUnionExpression(query, false /*is_subquery*/, {} /*cte_name*/);
query_tree_node = buildSelectOrUnionExpression(query, false /*is_subquery*/, {} /*cte_name*/, context_);
else if (query->as<ASTExpressionList>())
query_tree_node = buildExpressionList(query);
query_tree_node = buildExpressionList(query, context_);
else
query_tree_node = buildExpression(query);
query_tree_node = buildExpression(query, context_);
}
QueryTreeNodePtr QueryTreeBuilder::buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, bool is_subquery, const std::string & cte_name) const
QueryTreeNodePtr QueryTreeBuilder::buildSelectOrUnionExpression(const ASTPtr & select_or_union_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const
{
QueryTreeNodePtr query_node;
if (select_or_union_query->as<ASTSelectWithUnionQuery>())
query_node = buildSelectWithUnionExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/);
query_node = buildSelectWithUnionExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context);
else if (select_or_union_query->as<ASTSelectIntersectExceptQuery>())
query_node = buildSelectIntersectExceptQuery(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/);
query_node = buildSelectIntersectExceptQuery(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context);
else if (select_or_union_query->as<ASTSelectQuery>())
query_node = buildSelectExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/);
query_node = buildSelectExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context);
else
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "SELECT or UNION query {} is not supported", select_or_union_query->formatForErrorMessage());
return query_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, bool is_subquery, const std::string & cte_name) const
QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr & select_with_union_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const
{
auto & select_with_union_query_typed = select_with_union_query->as<ASTSelectWithUnionQuery &>();
auto & select_lists = select_with_union_query_typed.list_of_selects->as<ASTExpressionList &>();
if (select_lists.children.size() == 1)
return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name);
return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name, context);
auto union_node = std::make_shared<UnionNode>(select_with_union_query_typed.union_mode);
auto union_node = std::make_shared<UnionNode>(Context::createCopy(context), select_with_union_query_typed.union_mode);
union_node->setIsSubquery(is_subquery);
union_node->setIsCTE(!cte_name.empty());
union_node->setCTEName(cte_name);
@ -156,20 +171,23 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr &
for (size_t i = 0; i < select_lists_children_size; ++i)
{
auto & select_list_node = select_lists.children[i];
QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/);
QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/, context);
union_node->getQueries().getNodes().push_back(std::move(query_node));
}
return union_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, bool is_subquery, const std::string & cte_name) const
QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const
{
auto & select_intersect_except_query_typed = select_intersect_except_query->as<ASTSelectIntersectExceptQuery &>();
auto select_lists = select_intersect_except_query_typed.getListOfSelects();
if (select_lists.size() == 1)
return buildSelectExpression(select_lists[0], is_subquery, cte_name);
return buildSelectExpression(select_lists[0], is_subquery, cte_name, context);
SelectUnionMode union_mode;
if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_ALL)
@ -183,7 +201,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "UNION type is not initialized");
auto union_node = std::make_shared<UnionNode>(union_mode);
auto union_node = std::make_shared<UnionNode>(Context::createCopy(context), union_mode);
union_node->setIsSubquery(is_subquery);
union_node->setIsCTE(!cte_name.empty());
union_node->setCTEName(cte_name);
@ -194,17 +212,32 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr
for (size_t i = 0; i < select_lists_size; ++i)
{
auto & select_list_node = select_lists[i];
QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/);
QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/, context);
union_node->getQueries().getNodes().push_back(std::move(query_node));
}
return union_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_query, bool is_subquery, const std::string & cte_name) const
QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_query,
bool is_subquery,
const std::string & cte_name,
const ContextPtr & context) const
{
const auto & select_query_typed = select_query->as<ASTSelectQuery &>();
auto current_query_tree = std::make_shared<QueryNode>();
auto updated_context = Context::createCopy(context);
auto select_settings = select_query_typed.settings();
SettingsChanges settings_changes;
if (select_settings)
{
auto & set_query = select_settings->as<ASTSetQuery &>();
updated_context->applySettingsChanges(set_query.changes);
settings_changes = set_query.changes;
}
auto current_query_tree = std::make_shared<QueryNode>(std::move(updated_context), std::move(settings_changes));
current_query_tree->setIsSubquery(is_subquery);
current_query_tree->setIsCTE(!cte_name.empty());
@ -218,30 +251,25 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
current_query_tree->setIsGroupByAll(select_query_typed.group_by_all);
current_query_tree->setOriginalAST(select_query);
auto select_settings = select_query_typed.settings();
if (select_settings)
{
auto & set_query = select_settings->as<ASTSetQuery &>();
current_query_tree->setSettingsChanges(set_query.changes);
}
auto current_context = current_query_tree->getContext();
current_query_tree->getJoinTree() = buildJoinTree(select_query_typed.tables());
current_query_tree->getJoinTree() = buildJoinTree(select_query_typed.tables(), current_context);
auto select_with_list = select_query_typed.with();
if (select_with_list)
current_query_tree->getWithNode() = buildExpressionList(select_with_list);
current_query_tree->getWithNode() = buildExpressionList(select_with_list, current_context);
auto select_expression_list = select_query_typed.select();
if (select_expression_list)
current_query_tree->getProjectionNode() = buildExpressionList(select_expression_list);
current_query_tree->getProjectionNode() = buildExpressionList(select_expression_list, current_context);
auto prewhere_expression = select_query_typed.prewhere();
if (prewhere_expression)
current_query_tree->getPrewhere() = buildExpression(prewhere_expression);
current_query_tree->getPrewhere() = buildExpression(prewhere_expression, current_context);
auto where_expression = select_query_typed.where();
if (where_expression)
current_query_tree->getWhere() = buildExpression(where_expression);
current_query_tree->getWhere() = buildExpression(where_expression, current_context);
auto group_by_list = select_query_typed.groupBy();
if (group_by_list)
@ -254,56 +282,56 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
for (auto & grouping_sets_keys : group_by_children)
{
auto grouping_sets_keys_list_node = buildExpressionList(grouping_sets_keys);
auto grouping_sets_keys_list_node = buildExpressionList(grouping_sets_keys, current_context);
current_query_tree->getGroupBy().getNodes().emplace_back(std::move(grouping_sets_keys_list_node));
}
}
else
{
current_query_tree->getGroupByNode() = buildExpressionList(group_by_list);
current_query_tree->getGroupByNode() = buildExpressionList(group_by_list, current_context);
}
}
auto having_expression = select_query_typed.having();
if (having_expression)
current_query_tree->getHaving() = buildExpression(having_expression);
current_query_tree->getHaving() = buildExpression(having_expression, current_context);
auto window_list = select_query_typed.window();
if (window_list)
current_query_tree->getWindowNode() = buildWindowList(window_list);
current_query_tree->getWindowNode() = buildWindowList(window_list, current_context);
auto select_order_by_list = select_query_typed.orderBy();
if (select_order_by_list)
current_query_tree->getOrderByNode() = buildSortList(select_order_by_list);
current_query_tree->getOrderByNode() = buildSortList(select_order_by_list, current_context);
auto interpolate_list = select_query_typed.interpolate();
if (interpolate_list)
current_query_tree->getInterpolate() = buildInterpolateList(interpolate_list);
current_query_tree->getInterpolate() = buildInterpolateList(interpolate_list, current_context);
auto select_limit_by_limit = select_query_typed.limitByLength();
if (select_limit_by_limit)
current_query_tree->getLimitByLimit() = buildExpression(select_limit_by_limit);
current_query_tree->getLimitByLimit() = buildExpression(select_limit_by_limit, current_context);
auto select_limit_by_offset = select_query_typed.limitOffset();
if (select_limit_by_offset)
current_query_tree->getLimitByOffset() = buildExpression(select_limit_by_offset);
current_query_tree->getLimitByOffset() = buildExpression(select_limit_by_offset, current_context);
auto select_limit_by = select_query_typed.limitBy();
if (select_limit_by)
current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by);
current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by, current_context);
auto select_limit = select_query_typed.limitLength();
if (select_limit)
current_query_tree->getLimit() = buildExpression(select_limit);
current_query_tree->getLimit() = buildExpression(select_limit, current_context);
auto select_offset = select_query_typed.limitOffset();
if (select_offset)
current_query_tree->getOffset() = buildExpression(select_offset);
current_query_tree->getOffset() = buildExpression(select_offset, current_context);
return current_query_tree;
}
QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_expression_list) const
QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_expression_list, const ContextPtr & context) const
{
auto list_node = std::make_shared<ListNode>();
@ -324,7 +352,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express
collator = std::make_shared<Collator>(order_by_element.collation->as<ASTLiteral &>().value.get<String &>());
const auto & sort_expression_ast = order_by_element.children.at(0);
auto sort_expression = buildExpression(sort_expression_ast);
auto sort_expression = buildExpression(sort_expression_ast, context);
auto sort_node = std::make_shared<SortNode>(std::move(sort_expression),
sort_direction,
nulls_sort_direction,
@ -332,11 +360,11 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express
order_by_element.with_fill);
if (order_by_element.fill_from)
sort_node->getFillFrom() = buildExpression(order_by_element.fill_from);
sort_node->getFillFrom() = buildExpression(order_by_element.fill_from, context);
if (order_by_element.fill_to)
sort_node->getFillTo() = buildExpression(order_by_element.fill_to);
sort_node->getFillTo() = buildExpression(order_by_element.fill_to, context);
if (order_by_element.fill_step)
sort_node->getFillStep() = buildExpression(order_by_element.fill_step);
sort_node->getFillStep() = buildExpression(order_by_element.fill_step, context);
list_node->getNodes().push_back(std::move(sort_node));
}
@ -344,7 +372,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express
return list_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpolate_expression_list) const
QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpolate_expression_list, const ContextPtr & context) const
{
auto list_node = std::make_shared<ListNode>();
@ -355,7 +383,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpola
{
const auto & interpolate_element = expression->as<const ASTInterpolateElement &>();
auto expression_to_interpolate = std::make_shared<IdentifierNode>(Identifier(interpolate_element.column));
auto interpolate_expression = buildExpression(interpolate_element.expr);
auto interpolate_expression = buildExpression(interpolate_element.expr, context);
auto interpolate_node = std::make_shared<InterpolateNode>(std::move(expression_to_interpolate), std::move(interpolate_expression));
list_node->getNodes().push_back(std::move(interpolate_node));
@ -364,7 +392,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpola
return list_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definition_list) const
QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definition_list, const ContextPtr & context) const
{
auto list_node = std::make_shared<ListNode>();
@ -375,7 +403,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definit
{
const auto & window_list_element_typed = window_list_element->as<const ASTWindowListElement &>();
auto window_node = buildWindow(window_list_element_typed.definition);
auto window_node = buildWindow(window_list_element_typed.definition, context);
window_node->setAlias(window_list_element_typed.name);
list_node->getNodes().push_back(std::move(window_node));
@ -384,7 +412,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definit
return list_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression_list) const
QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression_list, const ContextPtr & context) const
{
auto list_node = std::make_shared<ListNode>();
@ -393,14 +421,14 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression
for (auto & expression : expression_list_typed.children)
{
auto expression_node = buildExpression(expression);
auto expression_node = buildExpression(expression, context);
list_node->getNodes().push_back(std::move(expression_node));
}
return list_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) const
QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, const ContextPtr & context) const
{
QueryTreeNodePtr result;
@ -411,13 +439,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
}
else if (const auto * asterisk = expression->as<ASTAsterisk>())
{
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(std::move(column_transformers));
}
else if (const auto * qualified_asterisk = expression->as<ASTQualifiedAsterisk>())
{
auto & qualified_identifier = qualified_asterisk->children.at(0)->as<ASTTableIdentifier &>();
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_transformers));
}
else if (const auto * ast_literal = expression->as<ASTLiteral>())
@ -466,7 +494,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
}
const auto & lambda_expression = lambda_arguments_and_expression.at(1);
auto lambda_expression_node = buildExpression(lambda_expression);
auto lambda_expression_node = buildExpression(lambda_expression, context);
result = std::make_shared<LambdaNode>(std::move(lambda_arguments), std::move(lambda_expression_node));
}
@ -478,20 +506,20 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
{
const auto & function_parameters_list = function->parameters->as<ASTExpressionList>()->children;
for (const auto & argument : function_parameters_list)
function_node->getParameters().getNodes().push_back(buildExpression(argument));
function_node->getParameters().getNodes().push_back(buildExpression(argument, context));
}
if (function->arguments)
{
const auto & function_arguments_list = function->arguments->as<ASTExpressionList>()->children;
for (const auto & argument : function_arguments_list)
function_node->getArguments().getNodes().push_back(buildExpression(argument));
function_node->getArguments().getNodes().push_back(buildExpression(argument, context));
}
if (function->is_window_function)
{
if (function->window_definition)
function_node->getWindowNode() = buildWindow(function->window_definition);
function_node->getWindowNode() = buildWindow(function->window_definition, context);
else
function_node->getWindowNode() = std::make_shared<IdentifierNode>(Identifier(function->window_name));
}
@ -502,20 +530,20 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
else if (const auto * subquery = expression->as<ASTSubquery>())
{
auto subquery_query = subquery->children[0];
auto query_node = buildSelectWithUnionExpression(subquery_query, true /*is_subquery*/, {} /*cte_name*/);
auto query_node = buildSelectWithUnionExpression(subquery_query, true /*is_subquery*/, {} /*cte_name*/, context);
result = std::move(query_node);
}
else if (const auto * with_element = expression->as<ASTWithElement>())
{
auto with_element_subquery = with_element->subquery->as<ASTSubquery &>().children.at(0);
auto query_node = buildSelectWithUnionExpression(with_element_subquery, true /*is_subquery*/, with_element->name /*cte_name*/);
auto query_node = buildSelectWithUnionExpression(with_element_subquery, true /*is_subquery*/, with_element->name /*cte_name*/, context);
result = std::move(query_node);
}
else if (const auto * columns_regexp_matcher = expression->as<ASTColumnsRegexpMatcher>())
{
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(columns_regexp_matcher->getMatcher(), std::move(column_transformers));
}
else if (const auto * columns_list_matcher = expression->as<ASTColumnsListMatcher>())
@ -529,13 +557,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
}
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(std::move(column_list_identifiers), std::move(column_transformers));
}
else if (const auto * qualified_columns_regexp_matcher = expression->as<ASTQualifiedColumnsRegexpMatcher>())
{
auto & qualified_identifier = qualified_columns_regexp_matcher->children.at(0)->as<ASTTableIdentifier &>();
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers));
}
else if (const auto * qualified_columns_list_matcher = expression->as<ASTQualifiedColumnsListMatcher>())
@ -551,7 +579,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts});
}
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/);
auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context);
result = std::make_shared<MatcherNode>(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers));
}
else
@ -567,7 +595,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co
return result;
}
QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition) const
QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition, const ContextPtr & context) const
{
const auto & window_definition_typed = window_definition->as<const ASTWindowDefinition &>();
WindowFrame window_frame;
@ -586,23 +614,23 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition)
window_node->setParentWindowName(window_definition_typed.parent_window_name);
if (window_definition_typed.partition_by)
window_node->getPartitionByNode() = buildExpressionList(window_definition_typed.partition_by);
window_node->getPartitionByNode() = buildExpressionList(window_definition_typed.partition_by, context);
if (window_definition_typed.order_by)
window_node->getOrderByNode() = buildSortList(window_definition_typed.order_by);
window_node->getOrderByNode() = buildSortList(window_definition_typed.order_by, context);
if (window_definition_typed.frame_begin_offset)
window_node->getFrameBeginOffsetNode() = buildExpression(window_definition_typed.frame_begin_offset);
window_node->getFrameBeginOffsetNode() = buildExpression(window_definition_typed.frame_begin_offset, context);
if (window_definition_typed.frame_end_offset)
window_node->getFrameEndOffsetNode() = buildExpression(window_definition_typed.frame_end_offset);
window_node->getFrameEndOffsetNode() = buildExpression(window_definition_typed.frame_end_offset, context);
window_node->setOriginalAST(window_definition);
return window_node;
}
QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select_query) const
QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const
{
if (!tables_in_select_query)
{
@ -668,7 +696,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
auto & subquery_expression = table_expression.subquery->as<ASTSubquery &>();
const auto & select_with_union_query = subquery_expression.children[0];
auto node = buildSelectWithUnionExpression(select_with_union_query, true /*is_subquery*/, {} /*cte_name*/);
auto node = buildSelectWithUnionExpression(select_with_union_query, true /*is_subquery*/, {} /*cte_name*/, context);
node->setAlias(subquery_expression.tryGetAlias());
node->setOriginalAST(select_with_union_query);
@ -694,9 +722,9 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
for (const auto & argument : function_arguments_list)
{
if (argument->as<ASTSelectQuery>() || argument->as<ASTSelectWithUnionQuery>() || argument->as<ASTSelectIntersectExceptQuery>())
node->getArguments().getNodes().push_back(buildSelectOrUnionExpression(argument, false /*is_subquery*/, {} /*cte_name*/));
node->getArguments().getNodes().push_back(buildSelectOrUnionExpression(argument, false /*is_subquery*/, {} /*cte_name*/, context));
else
node->getArguments().getNodes().push_back(buildExpression(argument));
node->getArguments().getNodes().push_back(buildExpression(argument, context));
}
}
@ -726,9 +754,9 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
QueryTreeNodePtr join_expression;
if (table_join.using_expression_list)
join_expression = buildExpressionList(table_join.using_expression_list);
join_expression = buildExpressionList(table_join.using_expression_list, context);
else if (table_join.on_expression)
join_expression = buildExpression(table_join.on_expression);
join_expression = buildExpression(table_join.on_expression, context);
const auto & settings = context->getSettingsRef();
auto join_default_strictness = settings.join_default_strictness;
@ -785,7 +813,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
auto last_table_expression = std::move(table_expressions.back());
table_expressions.pop_back();
auto array_join_expressions_list = buildExpressionList(array_join_expression.expression_list);
auto array_join_expressions_list = buildExpressionList(array_join_expression.expression_list, context);
auto array_join_node = std::make_shared<ArrayJoinNode>(std::move(last_table_expression), std::move(array_join_expressions_list), is_left_array_join);
/** Original AST is not set because it will contain only array join part and does
@ -805,7 +833,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
}
ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const
ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const
{
ColumnTransformersNodes column_transformers;
size_t children_size = matcher_expression->children.size();
@ -818,14 +846,14 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr &
{
if (apply_transformer->lambda)
{
auto lambda_query_tree_node = buildExpression(apply_transformer->lambda);
auto lambda_query_tree_node = buildExpression(apply_transformer->lambda, context);
column_transformers.emplace_back(std::make_shared<ApplyColumnTransformerNode>(std::move(lambda_query_tree_node)));
}
else
{
auto function_node = std::make_shared<FunctionNode>(apply_transformer->func_name);
if (apply_transformer->parameters)
function_node->getParametersNode() = buildExpressionList(apply_transformer->parameters);
function_node->getParametersNode() = buildExpressionList(apply_transformer->parameters, context);
column_transformers.emplace_back(std::make_shared<ApplyColumnTransformerNode>(std::move(function_node)));
}
@ -856,7 +884,7 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr &
for (const auto & replace_transformer_child : replace_transformer->children)
{
auto & replacement = replace_transformer_child->as<ASTColumnsReplaceTransformer::Replacement &>();
replacements.emplace_back(ReplaceColumnTransformerNode::Replacement{replacement.name, buildExpression(replacement.expr)});
replacements.emplace_back(ReplaceColumnTransformerNode::Replacement{replacement.name, buildExpression(replacement.expr, context)});
}
column_transformers.emplace_back(std::make_shared<ReplaceColumnTransformerNode>(replacements, replace_transformer->is_strict));

View File

@ -13,6 +13,8 @@ namespace DB
* AST that represent query ASTSelectWithUnionQuery, ASTSelectIntersectExceptQuery, ASTSelectQuery.
* AST that represent a list of expressions ASTExpressionList.
* AST that represent expression ASTIdentifier, ASTAsterisk, ASTLiteral, ASTFunction.
*
* For QUERY and UNION nodes contexts are created with respect to specified SETTINGS.
*/
QueryTreeNodePtr buildQueryTree(ASTPtr query, ContextPtr context);

View File

@ -3,8 +3,6 @@
#include <Common/SipHash.h>
#include <Common/FieldVisitorToString.h>
#include <Core/NamesAndTypes.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <IO/Operators.h>
@ -18,9 +16,12 @@
#include <Parsers/ASTFunction.h>
#include <Core/ColumnWithTypeAndName.h>
#include <Core/NamesAndTypes.h>
#include <DataTypes/getLeastSupertype.h>
#include <Interpreters/Context.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/Utils.h>
@ -33,8 +34,9 @@ namespace ErrorCodes
extern const int BAD_ARGUMENTS;
}
UnionNode::UnionNode(SelectUnionMode union_mode_)
UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_)
: IQueryTreeNode(children_size)
, context(std::move(context_))
, union_mode(union_mode_)
{
if (union_mode == SelectUnionMode::UNION_DEFAULT ||
@ -129,7 +131,7 @@ void UnionNode::updateTreeHashImpl(HashState & state) const
QueryTreeNodePtr UnionNode::cloneImpl() const
{
auto result_union_node = std::make_shared<UnionNode>(union_mode);
auto result_union_node = std::make_shared<UnionNode>(context, union_mode);
result_union_node->is_subquery = is_subquery;
result_union_node->is_cte = is_cte;

View File

@ -3,12 +3,14 @@
#include <Core/NamesAndTypes.h>
#include <Core/Field.h>
#include <Parsers/SelectUnionMode.h>
#include <Analyzer/Identifier.h>
#include <Analyzer/IQueryTreeNode.h>
#include <Analyzer/ListNode.h>
#include <Analyzer/TableExpressionModifiers.h>
#include <Parsers/SelectUnionMode.h>
#include <Interpreters/Context_fwd.h>
namespace DB
{
@ -37,8 +39,26 @@ using UnionNodePtr = std::shared_ptr<UnionNode>;
class UnionNode final : public IQueryTreeNode
{
public:
/// Construct union node with normalized union mode
explicit UnionNode(SelectUnionMode union_mode_);
/// Construct union node with context and normalized union mode
explicit UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_);
/// Get context
ContextPtr getContext() const
{
return context;
}
/// Get mutable context
const ContextMutablePtr & getMutableContext() const
{
return context;
}
/// Get mutable context
ContextMutablePtr & getMutableContext()
{
return context;
}
/// Returns true if union node is subquery, false otherwise
bool isSubquery() const
@ -129,6 +149,7 @@ private:
bool is_subquery = false;
bool is_cte = false;
std::string cte_name;
ContextMutablePtr context;
SelectUnionMode union_mode;
static constexpr size_t queries_child_index = 0;

View File

@ -1,5 +1,9 @@
add_subdirectory(StringUtils)
if (ENABLE_BENCHMARKS)
add_subdirectory(benchmarks)
endif()
if (ENABLE_EXAMPLES)
add_subdirectory(examples)
endif()

View File

@ -3,7 +3,6 @@
#include "Epoll.h"
#include <Common/Exception.h>
#include <unistd.h>
#include <Common/logger_useful.h>
namespace DB
{
@ -70,9 +69,6 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki
if (ready_size == -1 && errno != EINTR)
throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR);
if (errno == EINTR)
LOG_TEST(&Poco::Logger::get("Epoll"), "EINTR");
}
while (ready_size <= 0 && (ready_size != 0 || blocking));

View File

@ -531,6 +531,11 @@ void increment(Event event, Count amount)
DB::CurrentThread::getProfileEvents().increment(event, amount);
}
void incrementNoTrace(Event event, Count amount)
{
DB::CurrentThread::getProfileEvents().incrementNoTrace(event, amount);
}
void Counters::increment(Event event, Count amount)
{
Counters * current = this;
@ -547,6 +552,16 @@ void Counters::increment(Event event, Count amount)
DB::TraceSender::send(DB::TraceType::ProfileEvent, StackTrace(), {.event = event, .increment = amount});
}
void Counters::incrementNoTrace(Event event, Count amount)
{
Counters * current = this;
do
{
current->counters[event].fetch_add(amount, std::memory_order_relaxed);
current = current->parent;
} while (current != nullptr);
}
CountersIncrement::CountersIncrement(Counters::Snapshot const & snapshot)
{
init();

View File

@ -54,6 +54,7 @@ namespace ProfileEvents
}
void increment(Event event, Count amount = 1);
void incrementNoTrace(Event event, Count amount = 1);
struct Snapshot
{
@ -105,6 +106,10 @@ namespace ProfileEvents
/// Increment a counter for event. Thread-safe.
void increment(Event event, Count amount = 1);
/// The same as above but ignores value of setting 'trace_profile_events'
/// and never sends profile event to trace log.
void incrementNoTrace(Event event, Count amount = 1);
/// Get name of event by identifier. Returns statically allocated string.
const char * getName(Event event);

View File

@ -50,11 +50,11 @@ namespace
/// But pass with some frequency to avoid drop of all traces.
if (overrun_count > 0 && write_trace_iteration % (overrun_count + 1) == 0)
{
ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, overrun_count);
ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerSignalOverruns, overrun_count);
}
else
{
ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, std::max(0, overrun_count) + 1);
ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerSignalOverruns, std::max(0, overrun_count) + 1);
return;
}
}
@ -67,7 +67,7 @@ namespace
const StackTrace stack_trace(signal_context);
TraceSender::send(trace_type, stack_trace, {});
ProfileEvents::increment(ProfileEvents::QueryProfilerRuns);
ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerRuns);
errno = saved_errno;
}

View File

@ -164,8 +164,13 @@ public:
void get128(char * out)
{
finalize();
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
unalignedStore<UInt64>(out + 8, v0 ^ v1);
unalignedStore<UInt64>(out, v2 ^ v3);
#else
unalignedStore<UInt64>(out, v0 ^ v1);
unalignedStore<UInt64>(out + 8, v2 ^ v3);
#endif
}
template <typename T>

View File

@ -6,8 +6,6 @@
#include <fcntl.h>
#include <unistd.h>
#include <Common/logger_useful.h>
namespace DB
{
@ -72,8 +70,6 @@ void TimerDescriptor::drain() const
if (errno != EINTR)
throwFromErrno("Cannot drain timer_fd", ErrorCodes::CANNOT_READ_FROM_SOCKET);
else
LOG_TEST(&Poco::Logger::get("TimerDescriptor"), "EINTR");
}
}
}

View File

@ -0,0 +1,9 @@
clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
target_link_libraries (integer_hash_tables_and_hashes PRIVATE
ch_contrib::gbenchmark_all
dbms
ch_contrib::abseil_swiss_tables
ch_contrib::sparsehash
ch_contrib::wyhash
ch_contrib::farmhash
ch_contrib::xxHash)

View File

@ -1,5 +1,8 @@
#include <iostream>
#include <benchmark/benchmark.h>
#include <iomanip>
#include <iostream>
#include <random>
#include <vector>
#include <unordered_map>
@ -13,12 +16,23 @@
//#define DBMS_HASH_MAP_COUNT_COLLISIONS
//#define DBMS_HASH_MAP_DEBUG_RESIZES
#include <base/types.h>
#include <IO/ReadBufferFromFile.h>
#include <farmhash.h>
#include <wyhash.h>
#include <Compression/CompressedReadBuffer.h>
#include <IO/ReadBufferFromFile.h>
#include <base/types.h>
#include <Common/HashTable/HashMap.h>
#include <Common/SipHash.h>
#include <pcg-random/pcg_random.hpp>
#include <Common/randomSeed.h>
#ifdef __clang__
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wused-but-marked-unused"
#endif
#include <xxhash.h>
using Key = UInt64;
using Value = UInt64;
@ -282,98 +296,91 @@ namespace Hashes
return res;
}
};
struct FarmHash
{
size_t operator()(Key x) const { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash64(reinterpret_cast<const char *>(&x), sizeof(x)); }
};
struct WyHash
{
size_t operator()(Key x) const { return wyhash(reinterpret_cast<const char *>(&x), sizeof(x), 0, _wyp); }
};
struct XXH3Hash
{
size_t operator()(Key x) const { return XXH_INLINE_XXH3_64bits(reinterpret_cast<const char *>(&x), sizeof(x)); }
};
}
template <template <typename...> class Map, typename Hash>
void NO_INLINE test(const Key * data, size_t size, std::function<void(Map<Key, Value, Hash> &)> init = {})
{
Stopwatch watch;
Map<Key, Value, Hash> map;
if (init)
init(map);
for (const auto * end = data + size; data < end; ++data)
++map[*data];
watch.stop();
std::cerr << __PRETTY_FUNCTION__
<< ":\nElapsed: " << watch.elapsedSeconds()
<< " (" << size / watch.elapsedSeconds() << " elem/sec.)"
<< ", map size: " << map.size() << "\n";
}
template <template <typename...> class Map, typename Init>
void NO_INLINE testForEachHash(const Key * data, size_t size, Init && init)
template <template <typename...> typename Map, typename Hash>
struct TestRndInput : public benchmark::Fixture
{
test<Map, Hashes::IdentityHash>(data, size, init);
test<Map, Hashes::SimpleMultiplyHash>(data, size, init);
test<Map, Hashes::MultiplyAndMixHash>(data, size, init);
test<Map, Hashes::MixMultiplyMixHash>(data, size, init);
test<Map, Hashes::MurMurMixHash>(data, size, init);
test<Map, Hashes::MixAllBitsHash>(data, size, init);
test<Map, Hashes::IntHash32>(data, size, init);
test<Map, Hashes::ArcadiaNumericHash>(data, size, init);
test<Map, Hashes::MurMurButDifferentHash>(data, size, init);
test<Map, Hashes::TwoRoundsTwoVarsHash>(data, size, init);
test<Map, Hashes::TwoRoundsLessOpsHash>(data, size, init);
test<Map, Hashes::CRC32Hash>(data, size, init);
test<Map, Hashes::MulShiftHash>(data, size, init);
test<Map, Hashes::TabulationHash>(data, size, init);
test<Map, Hashes::CityHash>(data, size, init);
test<Map, Hashes::SipHash>(data, size, init);
}
static void NO_INLINE testForEachMapAndHash(const Key * data, size_t size)
{
auto nothing = [](auto &){};
testForEachHash<HashMap>(data, size, nothing);
testForEachHash<std::unordered_map>(data, size, nothing);
testForEachHash<::google::dense_hash_map>(data, size, [](auto & map){ map.set_empty_key(-1); });
testForEachHash<::google::sparse_hash_map>(data, size, nothing);
testForEachHash<::absl::flat_hash_map>(data, size, nothing);
}
int main(int argc, char ** argv)
{
if (argc < 2)
void SetUp(const ::benchmark::State & state) override
{
std::cerr << "Usage: program n\n";
return 1;
pcg64_fast rng(randomSeed());
std::normal_distribution<double> dist(0, 10);
const size_t elements = state.range(0);
data.resize(elements);
for (auto & elem : data)
elem = static_cast<Key>(dist(rng)) % elements;
}
size_t n = std::stol(argv[1]);
// size_t m = std::stol(argv[2]);
std::cerr << std::fixed << std::setprecision(3);
std::vector<Key> data(n);
std::cerr << "sizeof(Key) = " << sizeof(Key) << ", sizeof(Value) = " << sizeof(Value) << std::endl;
void test(benchmark::State & st)
{
Stopwatch watch;
DB::ReadBufferFromFileDescriptor in1(STDIN_FILENO);
DB::CompressedReadBuffer in2(in1);
in2.readStrict(reinterpret_cast<char*>(data.data()), sizeof(data[0]) * n);
watch.stop();
std::cerr
<< "Vector. Size: " << n
<< ", elapsed: " << watch.elapsedSeconds()
<< " (" << n / watch.elapsedSeconds() << " elem/sec.)"
<< std::endl;
for (auto _ : st)
::test<HashMap, Hash>(data.data(), data.size());
}
/** Actually we should not run multiple test within same invocation of binary,
* because order of test could alter test results (due to state of allocator and various minor reasons),
* but in this case it's Ok.
*/
std::vector<Key> data;
};
testForEachMapAndHash(data.data(), data.size());
return 0;
}
#define OK_GOOGLE(Fixture, Map, Hash, N) \
BENCHMARK_TEMPLATE_DEFINE_F(Fixture, Test##Map##Hash, Map, Hashes::Hash)(benchmark::State & st) \
{ \
test(st); \
} \
BENCHMARK_REGISTER_F(Fixture, Test##Map##Hash)->Arg(N);
constexpr size_t elements_to_insert = 10'000'000;
/// tldr: crc32 has almost the same speed as identity hash if the corresponding intrinsics are available
/// todo: extend benchmark with larger key sizes up to say 24 bytes
OK_GOOGLE(TestRndInput, HashMap, ArcadiaNumericHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, CRC32Hash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, CityHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, FarmHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, IdentityHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, IntHash32, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MixAllBitsHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MixMultiplyMixHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MulShiftHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MultiplyAndMixHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MurMurButDifferentHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, MurMurMixHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, SimpleMultiplyHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, SipHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, TabulationHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, TwoRoundsLessOpsHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, TwoRoundsTwoVarsHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, WyHash, elements_to_insert)
OK_GOOGLE(TestRndInput, HashMap, XXH3Hash, elements_to_insert)
#ifdef __clang__
# pragma clang diagnostic pop
#endif

View File

@ -40,9 +40,6 @@ target_link_libraries (array_cache PRIVATE clickhouse_common_io)
clickhouse_add_executable (space_saving space_saving.cpp)
target_link_libraries (space_saving PRIVATE clickhouse_common_io)
clickhouse_add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
target_link_libraries (integer_hash_tables_and_hashes PRIVATE dbms ch_contrib::abseil_swiss_tables ch_contrib::sparsehash)
clickhouse_add_executable (integer_hash_tables_benchmark integer_hash_tables_benchmark.cpp)
target_link_libraries (integer_hash_tables_benchmark PRIVATE dbms ch_contrib::abseil_swiss_tables ch_contrib::sparsehash)

View File

@ -58,9 +58,13 @@ inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table)
value = uint_;
/// Use little endian
for (size_t i = 0; i < sizeof(TUInt); ++i)
memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
{
if constexpr (std::endian::native == std::endian::little)
memcpy(out + i * 2, &table[static_cast<size_t>(uint8[sizeof(TUInt) - 1 - i]) * 2], 2);
else
memcpy(out + i * 2, &table[static_cast<size_t>(uint8[i]) * 2], 2);
}
}
template <typename TUInt>

View File

@ -62,7 +62,7 @@ GTEST_TEST(WideInteger, Conversions)
zero += minus_one;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&zero, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
ASSERT_EQ(0, memcmp(&zero, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE", sizeof(zero)));
#else
ASSERT_EQ(0, memcmp(&zero, "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
#endif
@ -160,7 +160,7 @@ GTEST_TEST(WideInteger, Arithmetic)
zero += minus_one;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&zero, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
ASSERT_EQ(0, memcmp(&zero, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFE", sizeof(zero)));
#else
ASSERT_EQ(0, memcmp(&zero, "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
#endif
@ -244,7 +244,7 @@ GTEST_TEST(WideInteger, Shift)
auto y = x << 64;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", sizeof(Int128)));
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00", sizeof(Int128)));
#else
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00", sizeof(Int128)));
#endif
@ -261,7 +261,7 @@ GTEST_TEST(WideInteger, Shift)
y = x << 16;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00", sizeof(Int128)));
#else
ASSERT_EQ(0, memcmp(&y, "\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
#endif
@ -269,18 +269,21 @@ GTEST_TEST(WideInteger, Shift)
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
y <<= 64;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00\x00\x00\x00\x00", sizeof(Int128)));
#else
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
#endif
y >>= 32;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00\x00\x00\x00", sizeof(Int128)));
#else
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
#endif
y <<= 64;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF\x00\x00\x00\x00", sizeof(Int128)));
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", sizeof(Int128)));
#else
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF", sizeof(Int128)));
#endif

View File

@ -44,10 +44,10 @@ DeflateQplJobHWPool::DeflateQplJobHWPool()
for (UInt32 index = 0; index < MAX_HW_JOB_NUMBER; ++index)
{
qpl_job * qpl_job_ptr = reinterpret_cast<qpl_job *>(hw_jobs_buffer.get() + index * job_size);
if (qpl_init_job(qpl_path_hardware, qpl_job_ptr) != QPL_STS_OK)
if (auto status = qpl_init_job(qpl_path_hardware, qpl_job_ptr); status != QPL_STS_OK)
{
job_pool_ready = false;
LOG_WARNING(log, "Initialization of hardware-assisted DeflateQpl codec failed, falling back to software DeflateQpl codec. Please check if Intel In-Memory Analytics Accelerator (IAA) is properly set up. QPL Version: {}.",qpl_version);
LOG_WARNING(log, "Initialization of hardware-assisted DeflateQpl codec failed: {} , falling back to software DeflateQpl codec. Please check if Intel In-Memory Analytics Accelerator (IAA) is properly set up. QPL Version: {}.", static_cast<UInt32>(status), qpl_version);
return;
}
hw_job_ptr_pool[index] = qpl_job_ptr;
@ -165,7 +165,7 @@ Int32 HardwareCodecDeflateQpl::doCompressData(const char * source, UInt32 source
}
else
{
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doCompressData->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doCompressData->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
DeflateQplJobHWPool::instance().releaseJob(job_id);
return RET_ERROR;
}
@ -193,7 +193,7 @@ Int32 HardwareCodecDeflateQpl::doDecompressDataSynchronous(const char * source,
if (auto status = qpl_submit_job(job_ptr); status != QPL_STS_OK)
{
DeflateQplJobHWPool::instance().releaseJob(job_id);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataSynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataSynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
return RET_ERROR;
}
/// Busy waiting till job complete.
@ -233,7 +233,7 @@ Int32 HardwareCodecDeflateQpl::doDecompressDataAsynchronous(const char * source,
else
{
DeflateQplJobHWPool::instance().releaseJob(job_id);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataAsynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
LOG_WARNING(log, "DeflateQpl HW codec failed, falling back to SW codec.(Details: doDecompressDataAsynchronous->qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
return RET_ERROR;
}
}
@ -289,7 +289,7 @@ qpl_job * SoftwareCodecDeflateQpl::getJobCodecPtr()
// Job initialization
if (auto status = qpl_init_job(qpl_path_software, sw_job); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_COMPRESS,
"Initialization of DeflateQpl software fallback codec failed. (Details: qpl_init_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
"Initialization of DeflateQpl software fallback codec failed. (Details: qpl_init_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
}
return sw_job;
}
@ -308,7 +308,7 @@ UInt32 SoftwareCodecDeflateQpl::doCompressData(const char * source, UInt32 sourc
if (auto status = qpl_execute_job(job_ptr); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_COMPRESS,
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
return job_ptr->total_out;
}
@ -327,7 +327,7 @@ void SoftwareCodecDeflateQpl::doDecompressData(const char * source, UInt32 sourc
if (auto status = qpl_execute_job(job_ptr); status != QPL_STS_OK)
throw Exception(ErrorCodes::CANNOT_DECOMPRESS,
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", status);
"Execution of DeflateQpl software fallback codec failed. (Details: qpl_execute_job with error code: {} - please refer to qpl_status in ./contrib/qpl/include/qpl/c_api/status.h)", static_cast<UInt32>(status));
}
CompressionCodecDeflateQpl::CompressionCodecDeflateQpl()

View File

@ -445,7 +445,7 @@ UInt8 getDataBytesSize(const IDataType * column_type)
if (max_size == 1 || max_size == 2 || max_size == 4 || max_size == 8)
return static_cast<UInt8>(max_size);
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Delta is only applicable for data types of size 1, 2, 4, 8 bytes. Given type {}",
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec DoubleDelta is only applicable for data types of size 1, 2, 4, 8 bytes. Given type {}",
column_type->getName());
}

View File

@ -344,7 +344,7 @@ UInt8 getDataBytesSize(const IDataType * column_type)
if (max_size == 1 || max_size == 2 || max_size == 4 || max_size == 8)
return static_cast<UInt8>(max_size);
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Delta is only applicable for data types of size 1, 2, 4, 8 bytes. Given type {}",
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla is only applicable for data types of size 1, 2, 4, 8 bytes. Given type {}",
column_type->getName());
}

View File

@ -546,7 +546,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
\
M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \
M(UInt64, max_entries_for_hash_table_stats, 10'000, "How many entries hash table statistics collected during aggregation is allowed to have", 0) \
M(UInt64, max_size_to_preallocate_for_aggregation, 10'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \
M(UInt64, max_size_to_preallocate_for_aggregation, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \
\
M(Bool, kafka_disable_num_consumers_limit, false, "Disable limit on kafka_num_consumers that depends on the number of available CPU cores", 0) \
M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \
@ -606,6 +606,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, wait_for_async_insert, true, "If true wait for processing of asynchronous insertion", 0) \
M(Seconds, wait_for_async_insert_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "Timeout for waiting for processing asynchronous insertion", 0) \
M(UInt64, async_insert_max_data_size, 1000000, "Maximum size in bytes of unparsed data collected per query before being inserted", 0) \
M(UInt64, async_insert_max_query_number, 450, "Maximum number of insert queries before being inserted", 0) \
M(Milliseconds, async_insert_busy_timeout_ms, 200, "Maximum time to wait before dumping collected data per query since the first data appeared", 0) \
\
M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \
@ -764,6 +765,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(Bool, input_format_json_try_infer_numbers_from_strings, true, "Try to infer numbers from string fields while schema inference", 0) \
M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \
M(Bool, input_format_json_read_numbers_as_strings, false, "Allow to parse numbers as strings in JSON input formats", 0) \
M(Bool, input_format_json_read_objects_as_strings, false, "Allow to parse JSON objects as strings in JSON input formats", 0) \
M(Bool, input_format_try_infer_integers, true, "Try to infer numbers from string fields while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \

View File

@ -311,13 +311,20 @@ void SerializationString::serializeTextJSON(const IColumn & column, size_t row_n
void SerializationString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.json.read_numbers_as_strings && !istr.eof() && *istr.position() != '"')
if (settings.json.read_objects_as_strings && !istr.eof() && *istr.position() == '{')
{
String field;
readJSONObjectPossiblyInvalid(field, istr);
ReadBufferFromString buf(field);
read(column, [&](ColumnString::Chars & data) { data.insert(field.begin(), field.end()); });
}
else if (settings.json.read_numbers_as_strings && !istr.eof() && *istr.position() != '"')
{
String field;
readJSONField(field, istr);
Float64 tmp;
ReadBufferFromString buf(field);
if (tryReadFloatText(tmp, buf))
if (tryReadFloatText(tmp, buf) && buf.eof())
read(column, [&](ColumnString::Chars & data) { data.insert(field.begin(), field.end()); });
else
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse JSON String value here: {}", field);

View File

@ -18,7 +18,6 @@
#include <sys/stat.h>
#include <Disks/DiskFactory.h>
#include <Disks/DiskMemory.h>
#include <Disks/DiskRestartProxy.h>
#include <Common/randomSeed.h>
#include <IO/ReadHelpers.h>

View File

@ -1,481 +0,0 @@
#include "DiskMemory.h"
#include "DiskFactory.h"
#include <IO/ReadBufferFromFileBase.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromFileBase.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
#include <Disks/ObjectStorages/LocalObjectStorage.h>
#include <Disks/ObjectStorages/FakeMetadataStorageFromDisk.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int FILE_DOESNT_EXIST;
extern const int FILE_ALREADY_EXISTS;
extern const int DIRECTORY_DOESNT_EXIST;
extern const int CANNOT_DELETE_DIRECTORY;
}
class DiskMemoryDirectoryIterator final : public IDirectoryIterator
{
public:
explicit DiskMemoryDirectoryIterator(std::vector<fs::path> && dir_file_paths_)
: dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin())
{
}
void next() override { ++iter; }
bool isValid() const override { return iter != dir_file_paths.end(); }
String path() const override { return iter->string(); }
String name() const override { return iter->filename(); }
private:
std::vector<fs::path> dir_file_paths;
std::vector<fs::path>::iterator iter;
};
/// Adapter with actual behaviour as ReadBufferFromString.
class ReadIndirectBuffer final : public ReadBufferFromFileBase
{
public:
ReadIndirectBuffer(String path_, const String & data_)
: impl(ReadBufferFromString(data_)), path(std::move(path_))
{
internal_buffer = impl.buffer();
working_buffer = internal_buffer;
pos = working_buffer.begin();
}
std::string getFileName() const override { return path; }
off_t seek(off_t off, int whence) override
{
impl.swap(*this);
off_t result = impl.seek(off, whence);
impl.swap(*this);
return result;
}
off_t getPosition() override { return pos - working_buffer.begin(); }
private:
ReadBufferFromString impl;
const String path;
};
/// This class is responsible to update files metadata after buffer is finalized.
class WriteIndirectBuffer final : public WriteBufferFromFileBase
{
public:
WriteIndirectBuffer(DiskMemory * disk_, String path_, WriteMode mode_, size_t buf_size)
: WriteBufferFromFileBase(buf_size, nullptr, 0), disk(disk_), path(std::move(path_)), mode(mode_)
{
}
~WriteIndirectBuffer() override
{
try
{
finalize();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
void finalizeImpl() override
{
if (impl.isFinished())
return;
next();
/// str() finalizes buffer.
String value = impl.str();
std::lock_guard lock(disk->mutex);
auto iter = disk->files.find(path);
if (iter == disk->files.end())
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
/// Resize to the actual number of bytes written to string.
value.resize(count());
if (mode == WriteMode::Rewrite)
disk->files.insert_or_assign(path, DiskMemory::FileData{iter->second.type, value});
else if (mode == WriteMode::Append)
disk->files.insert_or_assign(path, DiskMemory::FileData{iter->second.type, iter->second.data + value});
}
std::string getFileName() const override { return path; }
void sync() override {}
private:
void nextImpl() override
{
if (!offset())
return;
impl.write(working_buffer.begin(), offset());
}
WriteBufferFromOwnString impl;
DiskMemory * disk;
const String path;
const WriteMode mode;
};
DiskMemory::DiskMemory(const String & name_)
: IDisk(name_)
, disk_path("memory(" + name_ + ')')
{}
ReservationPtr DiskMemory::reserve(UInt64 /*bytes*/)
{
throw Exception("Method reserve is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
UInt64 DiskMemory::getTotalSpace() const
{
return 0;
}
UInt64 DiskMemory::getAvailableSpace() const
{
return 0;
}
UInt64 DiskMemory::getUnreservedSpace() const
{
return 0;
}
bool DiskMemory::exists(const String & path) const
{
std::lock_guard lock(mutex);
return files.find(path) != files.end();
}
bool DiskMemory::isFile(const String & path) const
{
std::lock_guard lock(mutex);
auto iter = files.find(path);
if (iter == files.end())
return false;
return iter->second.type == FileType::File;
}
bool DiskMemory::isDirectory(const String & path) const
{
std::lock_guard lock(mutex);
auto iter = files.find(path);
if (iter == files.end())
return false;
return iter->second.type == FileType::Directory;
}
size_t DiskMemory::getFileSize(const String & path) const
{
std::lock_guard lock(mutex);
auto iter = files.find(path);
if (iter == files.end())
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
return iter->second.data.length();
}
void DiskMemory::createDirectory(const String & path)
{
std::lock_guard lock(mutex);
if (files.find(path) != files.end())
return;
String parent_path = parentPath(path);
if (!parent_path.empty() && files.find(parent_path) == files.end())
throw Exception(
"Failed to create directory '" + path + "'. Parent directory " + parent_path + " does not exist",
ErrorCodes::DIRECTORY_DOESNT_EXIST);
files.emplace(path, FileData{FileType::Directory});
}
void DiskMemory::createDirectories(const String & path)
{
std::lock_guard lock(mutex);
createDirectoriesImpl(path);
}
void DiskMemory::createDirectoriesImpl(const String & path)
{
if (files.find(path) != files.end())
return;
String parent_path = parentPath(path);
if (!parent_path.empty())
createDirectoriesImpl(parent_path);
files.emplace(path, FileData{FileType::Directory});
}
void DiskMemory::clearDirectory(const String & path)
{
std::lock_guard lock(mutex);
if (files.find(path) == files.end())
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
for (auto iter = files.begin(); iter != files.end();)
{
if (parentPath(iter->first) != path)
{
++iter;
continue;
}
if (iter->second.type == FileType::Directory)
throw Exception(
"Failed to clear directory '" + path + "'. " + iter->first + " is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
iter = files.erase(iter);
}
}
void DiskMemory::moveDirectory(const String & /*from_path*/, const String & /*to_path*/)
{
throw Exception("Method moveDirectory is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
DirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path) const
{
std::lock_guard lock(mutex);
if (!path.empty() && files.find(path) == files.end())
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
std::vector<fs::path> dir_file_paths;
for (const auto & file : files)
if (parentPath(file.first) == path)
dir_file_paths.emplace_back(file.first);
return std::make_unique<DiskMemoryDirectoryIterator>(std::move(dir_file_paths));
}
void DiskMemory::moveFile(const String & from_path, const String & to_path)
{
std::lock_guard lock(mutex);
if (files.find(to_path) != files.end())
throw Exception(
"Failed to move file from " + from_path + " to " + to_path + ". File " + to_path + " already exist",
ErrorCodes::FILE_ALREADY_EXISTS);
replaceFileImpl(from_path, to_path);
}
void DiskMemory::replaceFile(const String & from_path, const String & to_path)
{
std::lock_guard lock(mutex);
replaceFileImpl(from_path, to_path);
}
void DiskMemory::replaceFileImpl(const String & from_path, const String & to_path)
{
String to_parent_path = parentPath(to_path);
if (!to_parent_path.empty() && files.find(to_parent_path) == files.end())
throw Exception(
"Failed to move file from " + from_path + " to " + to_path + ". Directory " + to_parent_path + " does not exist",
ErrorCodes::DIRECTORY_DOESNT_EXIST);
auto iter = files.find(from_path);
if (iter == files.end())
throw Exception(
"Failed to move file from " + from_path + " to " + to_path + ". File " + from_path + " does not exist",
ErrorCodes::FILE_DOESNT_EXIST);
auto node = files.extract(iter);
node.key() = to_path;
files.insert(std::move(node));
}
std::unique_ptr<ReadBufferFromFileBase> DiskMemory::readFile(const String & path, const ReadSettings &, std::optional<size_t>, std::optional<size_t>) const
{
std::lock_guard lock(mutex);
auto iter = files.find(path);
if (iter == files.end())
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
return std::make_unique<ReadIndirectBuffer>(path, iter->second.data);
}
std::unique_ptr<WriteBufferFromFileBase> DiskMemory::writeFile(const String & path, size_t buf_size, WriteMode mode, const WriteSettings &)
{
std::lock_guard lock(mutex);
auto iter = files.find(path);
if (iter == files.end())
{
String parent_path = parentPath(path);
if (!parent_path.empty() && files.find(parent_path) == files.end())
throw Exception(
"Failed to create file '" + path + "'. Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
files.emplace(path, FileData{FileType::File});
}
return std::make_unique<WriteIndirectBuffer>(this, path, mode, buf_size);
}
void DiskMemory::removeFile(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
if (file_it->second.type == FileType::Directory)
throw Exception("Path '" + path + "' is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
else
files.erase(file_it);
}
void DiskMemory::removeDirectory(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
if (file_it->second.type == FileType::Directory)
{
files.erase(file_it);
if (std::any_of(files.begin(), files.end(), [path](const auto & file) { return parentPath(file.first) == path; }))
throw Exception("Directory '" + path + "' is not empty", ErrorCodes::CANNOT_DELETE_DIRECTORY);
}
else
{
throw Exception("Path '" + path + "' is not a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
}
}
void DiskMemory::removeFileIfExists(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
return;
if (file_it->second.type == FileType::Directory)
throw Exception("Path '" + path + "' is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
else
files.erase(file_it);
}
void DiskMemory::removeRecursive(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
for (auto iter = files.begin(); iter != files.end();)
{
if (iter->first.size() >= path.size() && std::string_view(iter->first.data(), path.size()) == path)
iter = files.erase(iter);
else
++iter;
}
}
void DiskMemory::listFiles(const String & path, std::vector<String> & file_names) const
{
std::lock_guard lock(mutex);
for (auto it = iterateDirectory(path); it->isValid(); it->next())
file_names.push_back(it->name());
}
void DiskMemory::createHardLink(const String &, const String &)
{
throw Exception("Method createHardLink is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
void DiskMemory::createFile(const String &)
{
throw Exception("Method createFile is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
void DiskMemory::setReadOnly(const String &)
{
throw Exception("Method setReadOnly is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
void DiskMemory::truncateFile(const String & path, size_t size)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
file_it->second.data.resize(size);
}
MetadataStoragePtr DiskMemory::getMetadataStorage()
{
auto object_storage = std::make_shared<LocalObjectStorage>();
return std::make_shared<FakeMetadataStorageFromDisk>(
std::static_pointer_cast<IDisk>(shared_from_this()), object_storage, getPath());
}
using DiskMemoryPtr = std::shared_ptr<DiskMemory>;
void registerDiskMemory(DiskFactory & factory, bool global_skip_access_check)
{
auto creator = [global_skip_access_check](
const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context,
const DisksMap & /*map*/) -> DiskPtr
{
bool skip_access_check = global_skip_access_check || config.getBool(config_prefix + ".skip_access_check", false);
DiskPtr disk = std::make_shared<DiskMemory>(name);
disk->startup(context, skip_access_check);
return disk;
};
factory.registerDiskType("memory", creator);
}
}

View File

@ -1,127 +0,0 @@
#pragma once
#include <memory>
#include <mutex>
#include <unordered_map>
#include <utility>
#include <Disks/IDisk.h>
namespace DB
{
class ReadBufferFromFileBase;
class WriteBufferFromFileBase;
/** Implementation of Disk intended only for testing purposes.
* All filesystem objects are stored in memory and lost on server restart.
*
* NOTE Work in progress. Currently the interface is not viable enough to support MergeTree or even StripeLog tables.
* Please delete this interface if it will not be finished after 2020-06-18.
*/
class DiskMemory : public IDisk
{
public:
explicit DiskMemory(const String & name_);
const String & getPath() const override { return disk_path; }
ReservationPtr reserve(UInt64 bytes) override;
UInt64 getTotalSpace() const override;
UInt64 getAvailableSpace() const override;
UInt64 getUnreservedSpace() const override;
bool exists(const String & path) const override;
bool isFile(const String & path) const override;
bool isDirectory(const String & path) const override;
size_t getFileSize(const String & path) const override;
void createDirectory(const String & path) override;
void createDirectories(const String & path) override;
void clearDirectory(const String & path) override;
void moveDirectory(const String & from_path, const String & to_path) override;
DirectoryIteratorPtr iterateDirectory(const String & path) const override;
void createFile(const String & path) override;
void moveFile(const String & from_path, const String & to_path) override;
void replaceFile(const String & from_path, const String & to_path) override;
void listFiles(const String & path, std::vector<String> & file_names) const override;
std::unique_ptr<ReadBufferFromFileBase> readFile(
const String & path,
const ReadSettings & settings,
std::optional<size_t> read_hint,
std::optional<size_t> file_size) const override;
std::unique_ptr<WriteBufferFromFileBase> writeFile(
const String & path,
size_t buf_size,
WriteMode mode,
const WriteSettings & settings) override;
void removeFile(const String & path) override;
void removeFileIfExists(const String & path) override;
void removeDirectory(const String & path) override;
void removeRecursive(const String & path) override;
void setLastModified(const String &, const Poco::Timestamp &) override {}
Poco::Timestamp getLastModified(const String &) const override { return Poco::Timestamp(); }
time_t getLastChanged(const String &) const override { return {}; }
void setReadOnly(const String & path) override;
void createHardLink(const String & src_path, const String & dst_path) override;
void truncateFile(const String & path, size_t size) override;
DataSourceDescription getDataSourceDescription() const override { return DataSourceDescription{DataSourceType::RAM, "", false, false}; }
bool isRemote() const override { return false; }
bool supportZeroCopyReplication() const override { return false; }
MetadataStoragePtr getMetadataStorage() override;
private:
void createDirectoriesImpl(const String & path);
void replaceFileImpl(const String & from_path, const String & to_path);
friend class WriteIndirectBuffer;
enum class FileType
{
File,
Directory
};
struct FileData
{
FileType type;
String data;
FileData(FileType type_, String data_) : type(type_), data(std::move(data_)) {}
explicit FileData(FileType type_) : type(type_) {}
};
using Files = std::unordered_map<String, FileData>; /// file path -> file data
const String disk_path;
Files files;
mutable std::mutex mutex;
};
}

View File

@ -463,6 +463,8 @@ inline String fullPath(const DiskPtr & disk, const String & path)
/// Return parent path for the specified path.
inline String parentPath(const String & path)
{
if (path == "/")
return "/";
if (path.ends_with('/'))
return fs::path(path).parent_path().parent_path() / "";
return fs::path(path).parent_path() / "";

View File

@ -8,7 +8,6 @@ namespace DB
{
void registerDiskLocal(DiskFactory & factory, bool global_skip_access_check);
void registerDiskMemory(DiskFactory & factory, bool global_skip_access_check);
#if USE_AWS_S3
void registerDiskS3(DiskFactory & factory, bool global_skip_access_check);
@ -35,7 +34,6 @@ void registerDisks(bool global_skip_access_check)
auto & factory = DiskFactory::instance();
registerDiskLocal(factory, global_skip_access_check);
registerDiskMemory(factory, global_skip_access_check);
#if USE_AWS_S3
registerDiskS3(factory, global_skip_access_check);

View File

@ -10,11 +10,6 @@ namespace fs = std::filesystem;
template <typename T>
DB::DiskPtr createDisk();
template <>
DB::DiskPtr createDisk<DB::DiskMemory>()
{
return std::make_shared<DB::DiskMemory>("memory_disk");
}
template <>
DB::DiskPtr createDisk<DB::DiskLocal>()
@ -30,11 +25,6 @@ void destroyDisk(DB::DiskPtr & disk)
disk.reset();
}
template <>
void destroyDisk<DB::DiskMemory>(DB::DiskPtr & disk)
{
disk.reset();
}
template <>
void destroyDisk<DB::DiskLocal>(DB::DiskPtr & disk)
@ -55,7 +45,7 @@ public:
};
using DiskImplementations = testing::Types<DB::DiskMemory, DB::DiskLocal>;
using DiskImplementations = testing::Types<DB::DiskLocal>;
TYPED_TEST_SUITE(DiskTest, DiskImplementations);

View File

@ -1,15 +1,11 @@
#pragma once
#include <Disks/DiskLocal.h>
#include <Disks/DiskMemory.h>
#include <Disks/IDisk.h>
template <typename T>
DB::DiskPtr createDisk();
template <>
DB::DiskPtr createDisk<DB::DiskMemory>();
template <>
DB::DiskPtr createDisk<DB::DiskLocal>();
@ -18,6 +14,3 @@ void destroyDisk(DB::DiskPtr & disk);
template <>
void destroyDisk<DB::DiskLocal>(DB::DiskPtr & disk);
template <>
void destroyDisk<DB::DiskMemory>(DB::DiskPtr & disk);

View File

@ -98,6 +98,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
format_settings.json.quote_decimals = settings.output_format_json_quote_decimals;
format_settings.json.read_bools_as_numbers = settings.input_format_json_read_bools_as_numbers;
format_settings.json.read_numbers_as_strings = settings.input_format_json_read_numbers_as_strings;
format_settings.json.read_objects_as_strings = settings.input_format_json_read_objects_as_strings;
format_settings.json.try_infer_numbers_from_strings = settings.input_format_json_try_infer_numbers_from_strings;
format_settings.json.validate_types_from_metadata = settings.input_format_json_validate_types_from_metadata;
format_settings.json.validate_utf8 = settings.output_format_json_validate_utf8;

View File

@ -154,6 +154,7 @@ struct FormatSettings
bool serialize_as_strings = false;
bool read_bools_as_numbers = true;
bool read_numbers_as_strings = true;
bool read_objects_as_strings = true;
bool try_infer_numbers_from_strings = false;
bool validate_types_from_metadata = true;
bool validate_utf8 = false;

View File

@ -323,6 +323,31 @@ static void transformMapsAndObjectsToObjects(DataTypes & data_types)
}
}
static void transformMapsObjectsAndStringsToStrings(DataTypes & data_types)
{
bool have_maps = false;
bool have_objects = false;
bool have_strings = false;
for (const auto & type : data_types)
{
if (isMap(type))
have_maps = true;
else if (isObject(type))
have_objects = true;
else if (isString(type))
have_strings = true;
}
if (have_strings && (have_maps || have_objects))
{
for (auto & type : data_types)
{
if (isMap(type) || isObject(type))
type = std::make_shared<DataTypeString>();
}
}
}
template <bool is_json>
static void transformInferredTypesIfNeededImpl(DataTypes & types, const FormatSettings & settings, JSONInferenceInfo * json_info)
{
@ -374,6 +399,9 @@ static void transformInferredTypesIfNeededImpl(DataTypes & types, const FormatSe
/// Convert Maps to Objects if needed.
if (settings.json.try_infer_objects)
transformMapsAndObjectsToObjects(data_types);
if (settings.json.read_objects_as_strings)
transformMapsObjectsAndStringsToStrings(data_types);
};
transformTypesRecursively(types, transform_simple_types, transform_complex_types);
@ -772,6 +800,8 @@ static DataTypePtr tryInferMapOrObject(ReadBuffer & buf, const FormatSettings &
{
if (settings.json.try_infer_objects)
return std::make_shared<DataTypeObject>("json", true);
if (settings.json.read_objects_as_strings)
return makeNullable(std::make_shared<DataTypeString>());
return nullptr;
}

View File

@ -34,7 +34,6 @@ public:
}
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return true; }
/// Some functions may return different values on different shards/replicas, so it's not constant for distributed query
bool isSuitableForConstantFolding() const override { return !is_distributed; }

View File

@ -65,13 +65,27 @@ struct HexImpl
}
}
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out, bool reverse_order = false)
{
while (pos < end)
if (!reverse_order)
{
writeHexByteUppercase(*pos, out);
++pos;
out += word_size;
while (pos < end)
{
writeHexByteUppercase(*pos, out);
++pos;
out += word_size;
}
}
else
{
const auto * start_pos = pos;
pos = end - 1;
while (pos >= start_pos)
{
writeHexByteUppercase(*pos, out);
--pos;
out += word_size;
}
}
*out = '\0';
++out;
@ -95,7 +109,8 @@ struct HexImpl
for (size_t i = 0; i < size; ++i)
{
const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
executeOneString(in_pos, in_pos + type_size_in_bytes, out);
bool reverse_order = (std::endian::native == std::endian::big);
executeOneString(in_pos, in_pos + type_size_in_bytes, out, reverse_order);
pos += hex_length;
out_offsets[i] = pos;
@ -174,7 +189,9 @@ struct BinImpl
for (size_t i = 0; i < size; ++i)
{
const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
executeOneString(in_pos, in_pos + type_size_in_bytes, out);
bool reverse_order = (std::endian::native == std::endian::big);
executeOneString(in_pos, in_pos + type_size_in_bytes, out, reverse_order);
pos += hex_length;
out_offsets[i] = pos;
@ -182,13 +199,27 @@ struct BinImpl
col_res = std::move(col_str);
}
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out, bool reverse_order = false)
{
while (pos < end)
if (!reverse_order)
{
writeBinByte(*pos, out);
++pos;
out += word_size;
while (pos < end)
{
writeBinByte(*pos, out);
++pos;
out += word_size;
}
}
else
{
const auto * start_pos = pos;
pos = end - 1;
while (pos >= start_pos)
{
writeBinByte(*pos, out);
--pos;
out += word_size;
}
}
*out = '\0';
++out;

View File

@ -2670,8 +2670,6 @@ public:
String getName() const override { return cast_name; }
bool isDeterministic() const override { return true; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
bool hasInformationAboutMonotonicity() const override

View File

@ -79,8 +79,6 @@ public:
String getName() const override { return "FunctionExpression"; }
bool isDeterministic() const override { return true; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
const DataTypes & getArgumentTypes() const override { return argument_types; }
@ -176,8 +174,6 @@ public:
String getName() const override { return name; }
bool isDeterministic() const override { return true; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
const DataTypes & getArgumentTypes() const override { return capture->captured_types; }

View File

@ -59,7 +59,6 @@ public:
}
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
private:

View File

@ -38,7 +38,10 @@ public:
}
/** It could return many different values for single argument. */
bool isDeterministic() const override { return false; }
bool isDeterministic() const override
{
return false;
}
bool isDeterministicInScopeOfQuery() const override
{

View File

@ -26,7 +26,10 @@ public:
return name;
}
bool isDeterministic() const override { return false; }
bool isDeterministic() const override
{
return false;
}
bool isDeterministicInScopeOfQuery() const override
{

View File

@ -50,11 +50,6 @@ public:
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
bool isDeterministicInScopeOfQuery() const override
{
return true;
}
/// getMacro may return different values on different shards/replicas, so it's not constant for distributed query
bool isSuitableForConstantFolding() const override { return !is_distributed; }

View File

@ -105,11 +105,6 @@ public:
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override
{
return true;
}
bool isSuitableForConstantFolding() const override { return !is_distributed; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }

View File

@ -56,7 +56,6 @@ public:
}
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForConstantFolding() const override { return !is_distributed; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
@ -89,7 +88,6 @@ public:
size_t getNumberOfArguments() const override { return 1; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override
{

View File

@ -62,9 +62,15 @@ public:
return std::make_unique<ExecutableFunctionNow>(time_value);
}
bool isDeterministic() const override { return false; }
bool isDeterministicInScopeOfQuery() const override { return true; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
bool isDeterministic() const override
{
return false;
}
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override
{
return false;
}
private:
time_t time_value;

Some files were not shown because too many files have changed in this diff Show More