From 0760759dc6b1f14fbc3fe5b18ee9b9967bdc15a7 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Tue, 9 Mar 2021 16:44:56 +0800 Subject: [PATCH 001/290] Add DataTypeMap support LowCardinality and FixedString type --- .../DataTypeLowCardinalityHelpers.cpp | 17 +++++++++ src/Functions/array/arrayElement.cpp | 37 ++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index a68dc30d5c2..80d154b6234 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -1,11 +1,13 @@ #include #include #include +#include #include #include #include #include +#include #include @@ -39,6 +41,11 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type) return std::make_shared(elements); } + if (const auto * map_type = typeid_cast(type.get())) + { + return std::make_shared(recursiveRemoveLowCardinality(map_type->getKeyType()), recursiveRemoveLowCardinality(map_type->getValueType())); + } + if (const auto * low_cardinality_type = typeid_cast(type.get())) return low_cardinality_type->getDictionaryType(); @@ -78,6 +85,16 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) return ColumnTuple::create(columns); } + if (const auto * column_map = typeid_cast(column.get())) + { + auto nested = column_map->getNestedColumnPtr(); + auto nested_no_lc = recursiveRemoveLowCardinality(nested); + if (nested.get() == nested_no_lc.get()) + return column; + + return ColumnMap::create(nested_no_lc); + } + if (const auto * column_low_cardinality = typeid_cast(column.get())) return column_low_cardinality->convertToFullColumn(); diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 7d053988cae..cfe8e7839b1 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,9 @@ private: static bool matchKeyToIndexString(const IColumn & data, const Offsets & offsets, const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs); + static bool matchKeyToIndexFixedString(const IColumn & data, const Offsets & offsets, + const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs); + static bool matchKeyToIndexStringConst(const IColumn & data, const Offsets & offsets, const Field & index, PaddedPODArray & matched_idxs); @@ -767,6 +771,19 @@ struct MatcherString } }; +struct MatcherFixedString +{ + const ColumnFixedString & data; + const ColumnFixedString & index; + + bool match(size_t row_data, size_t row_index) const + { + auto data_ref = data.getDataAt(row_data); + auto index_ref = index.getDataAt(row_index); + return memequalSmallAllowOverflow15(index_ref.data, index_ref.size, data_ref.data, data_ref.size); + } +}; + struct MatcherStringConst { const ColumnString & data; @@ -863,6 +880,23 @@ bool FunctionArrayElement::matchKeyToIndexString( return true; } +bool FunctionArrayElement::matchKeyToIndexFixedString( + const IColumn & data, const Offsets & offsets, + const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs) +{ + const auto * index_string = checkAndGetColumn(arguments[1].column.get()); + if (!index_string) + return false; + + const auto * data_string = checkAndGetColumn(&data); + if (!data_string) + return false; + + MatcherFixedString matcher{*data_string, *index_string}; + executeMatchKeyToIndex(offsets, matched_idxs, matcher); + return true; +} + template bool FunctionArrayElement::matchKeyToIndexNumberConst( const IColumn & data, const Offsets & offsets, @@ -910,7 +944,8 @@ bool FunctionArrayElement::matchKeyToIndex( || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) - || matchKeyToIndexString(data, offsets, arguments, matched_idxs); + || matchKeyToIndexString(data, offsets, arguments, matched_idxs) + || matchKeyToIndexFixedString(data, offsets, arguments, matched_idxs); } bool FunctionArrayElement::matchKeyToIndexConst( From 37749eecde99021924b0eb5459462a58a77165a8 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Tue, 9 Mar 2021 16:50:13 +0800 Subject: [PATCH 002/290] Add test cases --- .../01763_support_map_lowcardinality_type.reference | 2 ++ .../01763_support_map_lowcardinality_type.sql | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference create mode 100644 tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql diff --git a/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference new file mode 100644 index 00000000000..8fdcdf3d8d5 --- /dev/null +++ b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference @@ -0,0 +1,2 @@ +b +{'1':1} 1 0 diff --git a/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql new file mode 100644 index 00000000000..ccade153ca1 --- /dev/null +++ b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS map_lc; +SET allow_experimental_map_type = 1; +CREATE TABLE map_lc +( + `kv` Map(LowCardinality(String), LowCardinality(String)) +) +ENGINE = Memory; + +INSERT INTO map_lc select map('a', 'b'); +SELECT kv['a'] FROM map_lc; +DROP TABLE map_lc; +SELECT map(toFixedString('1',1),1) AS m, m[toFixedString('1',1)],m[toFixedString('1',2)]; From 072a68f8ab9cd5c043cc147eef945791d686bf45 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Fri, 12 Mar 2021 16:08:16 +0800 Subject: [PATCH 003/290] Fix build error --- src/DataTypes/DataTypeLowCardinalityHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index 80d154b6234..41ba81814d0 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -87,7 +87,7 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) if (const auto * column_map = typeid_cast(column.get())) { - auto nested = column_map->getNestedColumnPtr(); + const auto & nested = column_map->getNestedColumnPtr(); auto nested_no_lc = recursiveRemoveLowCardinality(nested); if (nested.get() == nested_no_lc.get()) return column; From a681b2484b5695a09d4a77cbf541cd23b21742ad Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 12 Apr 2021 13:32:38 +0300 Subject: [PATCH 004/290] Update arrayElement.cpp --- src/Functions/array/arrayElement.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 87f91d5446c..9aa1e637b31 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -953,7 +953,7 @@ bool FunctionArrayElement::matchKeyToIndex( || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) - || matchKeyToIndexString(data, offsets, arguments, matched_idxs); + || matchKeyToIndexString(data, offsets, arguments, matched_idxs) || matchKeyToIndexFixedString(data, offsets, arguments, matched_idxs); } From 7e413675a26b959db831f75e5df765d493ca1435 Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 19 Apr 2021 11:18:46 +0800 Subject: [PATCH 005/290] Fix Map table create error --- src/DataTypes/DataTypeMap.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 1d580761362..4fa92247a6a 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -55,9 +56,19 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu void DataTypeMap::assertKeyType() const { - if (!key_type->isValueRepresentedByInteger() && !isStringOrFixedString(*key_type) && !WhichDataType(key_type).isNothing()) + bool type_error = false; + if (key_type->getTypeId() == TypeIndex::LowCardinality) + { + const auto & low_cardinality_data_type = assert_cast(*key_type); + if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType()))) + type_error = true; + } + else if (!key_type->isValueRepresentedByInteger() && !isStringOrFixedString(*key_type) && !WhichDataType(key_type).isNothing()) + type_error = true; + + if (type_error) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Type of Map key must be a type, that can be represented by integer or string," + "Type of Map key must be a type, that can be represented by integer or [LowCardinality]string," " but {} given", key_type->getName()); } From aa27b9b8b51d2178fcc00f34308aa69ee569ba34 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Wed, 5 May 2021 09:00:24 +0000 Subject: [PATCH 006/290] Add untested ProcessorStatisticsOS class --- src/Common/ProcessorStatisticsOS.cpp | 208 +++++++++++++++++++++++++++ src/Common/ProcessorStatisticsOS.h | 80 +++++++++++ 2 files changed, 288 insertions(+) create mode 100644 src/Common/ProcessorStatisticsOS.cpp create mode 100644 src/Common/ProcessorStatisticsOS.h diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp new file mode 100644 index 00000000000..e83cc0bdf6f --- /dev/null +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -0,0 +1,208 @@ +#if defined(OS_LINUX) + +#include +#include +#include +#include +#include +#include + +#include "ProcessorStatisticsOS.h" + + +#include + +#include + +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; + extern const int CANNOT_CLOSE_FILE; +} + +static constexpr auto loadavg_filename = "/proc/loadavg"; +static constexpr auto procst_filename = "/proc/stat"; +static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; + +ProcessorStatisticsOS::ProcessorStatisticsOS() + : loadavg_fd(openWithCheck(loadavg_filename, O_RDONLY | O_CLOEXEC)) + , procst_fd(openWithCheck(procst_filename, O_RDONLY | O_CLOEXEC)) + , cpuinfo_fd(openWithCheck(cpuinfo_filename, O_RDONLY | O_CLOEXEC)) +{} + +ProcessorStatisticsOS::~ProcessorStatisticsOS() +{ + closeFD(loadavg_fd, String(loadavg_filename)); + closeFD(procst_fd, String(procst_filename)); + closeFD(cpuinfo_fd, String(cpuinfo_filename)); +} + +int ProcessorStatisticsOS::openWithCheck(const String & filename, int flags) +{ + int fd = ::open(filename.c_str(), flags); + checkFDAfterOpen(fd, filename); + return fd; +} + +void ProcessorStatisticsOS::checkFDAfterOpen(int fd, const String & filename) +{ + if (-1 == fd) + throwFromErrno( + "Cannot open file" + String(filename), + errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); +} + +void ProcessorStatisticsOS::closeFD(int fd, const String & filename) +{ + if (0 != ::close(fd)) + { + try + { + throwFromErrno( + "File descriptor for \"" + filename + "\" could not be closed. " + "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE); + } catch(const ErrnoException&) + { + DB::tryLogCurrentException(__PRETTY_FUNCTION__); + } + } +} + +ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() const +{ + Data data; + readLoadavg(data); + readProcst(data); + readCpuinfo(data); + return data; +} + +void ProcessorStatisticsOS::readLoadavg(Data & data) const +{ + constexpr size_t buf_size = 1024; + char buf[buf_size]; + + ssize_t res = 0; + + do + { + res = ::pread(loadavg_fd, buf, buf_size, 0); + + if (-1 == res) + { + if (errno == EINTR) + continue; + + throwFromErrno("Cannot read from file " + String(loadavg_filename), + ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); + } + + assert(res >= 0); + break; + } while (true); + + ReadBufferFromMemory in(buf, res); + + readFloatAndSkipWhitespaceIfAny(data.avg1, in); + readFloatAndSkipWhitespaceIfAny(data.avg5, in); + readFloatAndSkipWhitespaceIfAny(data.avg15, in); +} + +void ProcessorStatisticsOS::readProcst(Data & data) const +{ + MMappedFileDescriptor mapped_procst(procst_fd, 0); + ReadBufferFromMemory in(mapped_procst.getData(), + mapped_procst.getLength()); + + String field_name, field_val; + uint64_t unused; + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + + readIntTextAndSkipWhitespaceIfAny(data.user_time, in); + readIntTextAndSkipWhitespaceIfAny(data.nice_time, in); + readIntTextAndSkipWhitespaceIfAny(data.system_time, in); + readIntTextAndSkipWhitespaceIfAny(data.idle_time, in); + readIntTextAndSkipWhitespaceIfAny(data.iowait_time, in); + + readIntTextAndSkipWhitespaceIfAny(unused, in); + readIntTextAndSkipWhitespaceIfAny(unused, in); + + readIntTextAndSkipWhitespaceIfAny(data.steal_time, in); + readIntTextAndSkipWhitespaceIfAny(data.guest_time, in); + readIntTextAndSkipWhitespaceIfAny(data.nice_time, in); + + do + { + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, in); + } while (field_name != String("processes")); + + data.processes = static_cast(std::stoul(field_val)); + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + readIntTextAndSkipWhitespaceIfAny(data.procs_running, in); + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + readIntTextAndSkipWhitespaceIfAny(data.procs_blocked, in); +} + +void ProcessorStatisticsOS::readCpuinfo(Data & data) const +{ + MMappedFileDescriptor mapped_cpuinfo(cpuinfo_fd, 0); + ReadBufferFromMemory in(mapped_cpuinfo.getData(), + mapped_cpuinfo.getLength()); + + String field_name, field_val; + char unused; + + do + { + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + readCharAndSkipWhitespaceIfAny(unused, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, in); + } while (field_name != String("cpu MHz")); + + data.freq = stof(field_val); +} + +template +void ProcessorStatisticsOS::readIntTextAndSkipWhitespaceIfAny(T& x, ReadBuffer& buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); +} + +void ProcessorStatisticsOS::readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); +} + +void ProcessorStatisticsOS::readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) +{ + readChar(c, buf); + skipWhitespaceIfAny(buf); +} + +void ProcessorStatisticsOS::readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) +{ + readFloatText(f, buf); + skipWhitespaceIfAny(buf); +} + +} + +#endif diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h new file mode 100644 index 00000000000..1ae9f6ba760 --- /dev/null +++ b/src/Common/ProcessorStatisticsOS.h @@ -0,0 +1,80 @@ +#pragma once +#if defined(OS_LINUX) + +#include +#include + +#include + +#include + +namespace DB +{ + +/** Opens files: /proc/loadav, /proc/stat, /proc/cpuinfo. Keeps it open and reads processor statistics. + * This is Linux specific. + * See: man procfs + */ + +class ProcessorStatisticsOS +{ +public: + struct Data + { + float avg1; + float avg5; + float avg15; + + /** The amount of time, measured in units of USER_HZ + * (1/100ths of a second on most architectures, use sysconf(_SC_CLK_TCK) to obtain the right value) + */ + uint64_t user_time; + uint64_t nice_time; + uint64_t system_time; + uint64_t idle_time; + uint64_t iowait_time; + uint64_t steal_time; + uint64_t guest_time; + uint64_t guest_nice_time; + + uint32_t processes; + uint32_t procs_running; + uint32_t procs_blocked; + + float freq; + }; + + ProcessorStatisticsOS(); + ~ProcessorStatisticsOS(); + + Data get() const; + +private: + static int openWithCheck(const String & filename, int flags); + + static void checkFDAfterOpen(int fd, const String & filename); + + static void closeFD(int fd, const String & filename); + + template + static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); + + static void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf); + + static void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf); + + static void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf); + + void readLoadavg(Data & data) const; + void readProcst(Data & data) const; + void readCpuinfo(Data & data) const; + +private: + int loadavg_fd; + int procst_fd; + int cpuinfo_fd; +}; + +} + +#endif From 6765858e96ca3c2a99ccf67a1375878acae0f2d1 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Thu, 6 May 2021 10:12:01 +0000 Subject: [PATCH 007/290] Update logic and functionality (untested) --- src/Common/ProcessorStatisticsOS.cpp | 212 +++++++++++++-------------- src/Common/ProcessorStatisticsOS.h | 76 ++++++---- 2 files changed, 152 insertions(+), 136 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index e83cc0bdf6f..252b6b776e9 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -1,11 +1,9 @@ #if defined(OS_LINUX) -#include -#include -#include #include #include #include +#include #include "ProcessorStatisticsOS.h" @@ -16,9 +14,8 @@ #include -#include +#include #include -#include namespace DB { @@ -35,147 +32,144 @@ static constexpr auto loadavg_filename = "/proc/loadavg"; static constexpr auto procst_filename = "/proc/stat"; static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; +static const long USER_HZ = sysconf(_SC_CLK_TCK); + ProcessorStatisticsOS::ProcessorStatisticsOS() - : loadavg_fd(openWithCheck(loadavg_filename, O_RDONLY | O_CLOEXEC)) - , procst_fd(openWithCheck(procst_filename, O_RDONLY | O_CLOEXEC)) - , cpuinfo_fd(openWithCheck(cpuinfo_filename, O_RDONLY | O_CLOEXEC)) -{} - -ProcessorStatisticsOS::~ProcessorStatisticsOS() + : loadavg_in(loadavg_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) + , procst_in(procst_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) + , cpuinfo_in(cpuinfo_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) { - closeFD(loadavg_fd, String(loadavg_filename)); - closeFD(procst_fd, String(procst_filename)); - closeFD(cpuinfo_fd, String(cpuinfo_filename)); + ProcStLoad unused; + calcStLoad(unused); } -int ProcessorStatisticsOS::openWithCheck(const String & filename, int flags) -{ - int fd = ::open(filename.c_str(), flags); - checkFDAfterOpen(fd, filename); - return fd; -} +ProcessorStatisticsOS::~ProcessorStatisticsOS() {} -void ProcessorStatisticsOS::checkFDAfterOpen(int fd, const String & filename) -{ - if (-1 == fd) - throwFromErrno( - "Cannot open file" + String(filename), - errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); -} - -void ProcessorStatisticsOS::closeFD(int fd, const String & filename) -{ - if (0 != ::close(fd)) - { - try - { - throwFromErrno( - "File descriptor for \"" + filename + "\" could not be closed. " - "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE); - } catch(const ErrnoException&) - { - DB::tryLogCurrentException(__PRETTY_FUNCTION__); - } - } -} - -ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() const +ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() { Data data; - readLoadavg(data); - readProcst(data); - readCpuinfo(data); + readLoadavg(data.loadavg); + calcStLoad(data.stload); + readFreq(data.freq); return data; } -void ProcessorStatisticsOS::readLoadavg(Data & data) const +void ProcessorStatisticsOS::readLoadavg(ProcLoadavg& loadavg) { - constexpr size_t buf_size = 1024; - char buf[buf_size]; - - ssize_t res = 0; - - do - { - res = ::pread(loadavg_fd, buf, buf_size, 0); - - if (-1 == res) - { - if (errno == EINTR) - continue; - - throwFromErrno("Cannot read from file " + String(loadavg_filename), - ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); - } - - assert(res >= 0); - break; - } while (true); - - ReadBufferFromMemory in(buf, res); + loadavg_in.seek(0, SEEK_SET); - readFloatAndSkipWhitespaceIfAny(data.avg1, in); - readFloatAndSkipWhitespaceIfAny(data.avg5, in); - readFloatAndSkipWhitespaceIfAny(data.avg15, in); + readFloatAndSkipWhitespaceIfAny(loadavg.avg1, loadavg_in); + readFloatAndSkipWhitespaceIfAny(loadavg.avg5, loadavg_in); + readFloatAndSkipWhitespaceIfAny(loadavg.avg15, loadavg_in); } -void ProcessorStatisticsOS::readProcst(Data & data) const +void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) { - MMappedFileDescriptor mapped_procst(procst_fd, 0); - ReadBufferFromMemory in(mapped_procst.getData(), - mapped_procst.getLength()); + ProcTime cur_proc_time; + readProcTimeAndProcesses(cur_proc_time, stload); + + std::time_t cur_time = std::time(nullptr); + float time_dif = static_cast(cur_time - last_stload_call_time); + + stload.user_time = + (cur_proc_time.user - last_proc_time.user) / time_dif; + stload.nice_time = + (cur_proc_time.nice - last_proc_time.nice) / time_dif; + stload.system_time = + (cur_proc_time.system - last_proc_time.system) / time_dif; + stload.idle_time = + (cur_proc_time.idle - last_proc_time.idle) / time_dif; + stload.iowait_time = + (cur_proc_time.iowait - last_proc_time.iowait) / time_dif; + stload.steal_time = + (cur_proc_time.steal - last_proc_time.steal) / time_dif; + stload.guest_time = + (cur_proc_time.guest - last_proc_time.guest) / time_dif; + stload.guest_nice_time = + (cur_proc_time.guest_nice - last_proc_time.guest_nice) / time_dif; + + last_stload_call_time = cur_time; + last_proc_time = cur_proc_time; +} + +void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload) +{ + procst_in.seek(0, SEEK_SET); String field_name, field_val; uint64_t unused; - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readIntTextAndSkipWhitespaceIfAny(data.user_time, in); - readIntTextAndSkipWhitespaceIfAny(data.nice_time, in); - readIntTextAndSkipWhitespaceIfAny(data.system_time, in); - readIntTextAndSkipWhitespaceIfAny(data.idle_time, in); - readIntTextAndSkipWhitespaceIfAny(data.iowait_time, in); + readIntTextAndSkipWhitespaceIfAny(proc_time.user, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.nice, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.system, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.idle, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.iowait, procst_in); + proc_time.user *= USER_HZ; + proc_time.nice *= USER_HZ; + proc_time.system *= USER_HZ; + proc_time.idle *= USER_HZ; + proc_time.iowait *= USER_HZ; - readIntTextAndSkipWhitespaceIfAny(unused, in); - readIntTextAndSkipWhitespaceIfAny(unused, in); + readIntTextAndSkipWhitespaceIfAny(unused, procst_in); + readIntTextAndSkipWhitespaceIfAny(unused, procst_in); - readIntTextAndSkipWhitespaceIfAny(data.steal_time, in); - readIntTextAndSkipWhitespaceIfAny(data.guest_time, in); - readIntTextAndSkipWhitespaceIfAny(data.nice_time, in); + readIntTextAndSkipWhitespaceIfAny(proc_time.steal, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.guest, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.guest_nice, procst_in); + proc_time.steal *= USER_HZ; + proc_time.guest *= USER_HZ; + proc_time.guest_nice *= USER_HZ; do { - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, procst_in); } while (field_name != String("processes")); - data.processes = static_cast(std::stoul(field_val)); + stload.processes = static_cast(std::stoul(field_val)); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); - readIntTextAndSkipWhitespaceIfAny(data.procs_running, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); + readIntTextAndSkipWhitespaceIfAny(stload.procs_running, procst_in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); - readIntTextAndSkipWhitespaceIfAny(data.procs_blocked, in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); + readIntTextAndSkipWhitespaceIfAny(stload.procs_blocked, procst_in); } -void ProcessorStatisticsOS::readCpuinfo(Data & data) const -{ - MMappedFileDescriptor mapped_cpuinfo(cpuinfo_fd, 0); - ReadBufferFromMemory in(mapped_cpuinfo.getData(), - mapped_cpuinfo.getLength()); +void ProcessorStatisticsOS::readFreq(ProcFreq & freq) +{ + cpuinfo_in.seek(0, SEEK_SET); String field_name, field_val; char unused; + int cpu_count = 0; - do + do { + do + { + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, cpuinfo_in); + } while (!cpuinfo_in.eof() && field_name != String("cpu MHz")); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, in); - readCharAndSkipWhitespaceIfAny(unused, in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, in); - } while (field_name != String("cpu MHz")); - - data.freq = stof(field_val); + if (cpuinfo_in.eof()) + break; + + readCharAndSkipWhitespaceIfAny(unused, cpuinfo_in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, cpuinfo_in); + + cpu_count++; + + float cur_cpu_freq = stof(field_val); + + freq.avg += cur_cpu_freq; + freq.max = (cpu_count == 1 ? cur_cpu_freq : + std::max(freq.max, cur_cpu_freq)); + freq.min = (cpu_count == 1 ? cur_cpu_freq : + std::min(freq.min, cur_cpu_freq)); + } while (true); + + freq.avg /= static_cast(cpu_count); } template diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 1ae9f6ba760..7bc77496f4a 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -6,7 +6,7 @@ #include -#include +#include namespace DB { @@ -19,42 +19,59 @@ namespace DB class ProcessorStatisticsOS { public: - struct Data - { + struct ProcLoadavg { float avg1; float avg5; float avg15; + }; - /** The amount of time, measured in units of USER_HZ - * (1/100ths of a second on most architectures, use sysconf(_SC_CLK_TCK) to obtain the right value) - */ - uint64_t user_time; - uint64_t nice_time; - uint64_t system_time; - uint64_t idle_time; - uint64_t iowait_time; - uint64_t steal_time; - uint64_t guest_time; - uint64_t guest_nice_time; + struct ProcStLoad { + float user_time; + float nice_time; + float system_time; + float idle_time; + float iowait_time; + float steal_time; + float guest_time; + float guest_nice_time; uint32_t processes; uint32_t procs_running; uint32_t procs_blocked; + }; - float freq; + struct ProcFreq { + float max; + float min; + float avg; + }; + + struct Data + { + ProcLoadavg loadavg; + ProcStLoad stload; + ProcFreq freq; }; ProcessorStatisticsOS(); ~ProcessorStatisticsOS(); - Data get() const; + Data get(); private: - static int openWithCheck(const String & filename, int flags); - - static void checkFDAfterOpen(int fd, const String & filename); - - static void closeFD(int fd, const String & filename); + struct ProcTime { + /** The amount of time, measured in units of USER_HZ + * (1/100ths of a second on most architectures, use sysconf(_SC_CLK_TCK) to obtain the right value) + */ + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; + uint64_t iowait; + uint64_t steal; + uint64_t guest; + uint64_t guest_nice; + }; template static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); @@ -65,14 +82,19 @@ private: static void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf); - void readLoadavg(Data & data) const; - void readProcst(Data & data) const; - void readCpuinfo(Data & data) const; + void readLoadavg(ProcLoadavg & loadavg); + void calcStLoad(ProcStLoad & stload); + void readFreq(ProcFreq & freq); + + void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload); private: - int loadavg_fd; - int procst_fd; - int cpuinfo_fd; + ReadBufferFromFile loadavg_in; + ReadBufferFromFile procst_in; + ReadBufferFromFile cpuinfo_in; + + std::time_t last_stload_call_time; + ProcTime last_proc_time; }; } From 69ccdb3aa97d038a3857ad0d9773f61f6b2b59a3 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Thu, 6 May 2021 11:14:51 +0000 Subject: [PATCH 008/290] Changed comment for ProcTime structure --- src/Common/ProcessorStatisticsOS.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 7bc77496f4a..51bb8c3a157 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -60,9 +60,7 @@ public: private: struct ProcTime { - /** The amount of time, measured in units of USER_HZ - * (1/100ths of a second on most architectures, use sysconf(_SC_CLK_TCK) to obtain the right value) - */ + // The amount of time, measured in seconds uint64_t user; uint64_t nice; uint64_t system; From e4f2f36c1dfd019c3eaf67720e1aaf6072fc6e15 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Thu, 6 May 2021 11:35:11 +0000 Subject: [PATCH 009/290] Changed "*= USER_HZ" to "/= USER_HZ" --- src/Common/ProcessorStatisticsOS.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 252b6b776e9..e05cb589f95 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -106,11 +106,11 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS readIntTextAndSkipWhitespaceIfAny(proc_time.system, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.idle, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.iowait, procst_in); - proc_time.user *= USER_HZ; - proc_time.nice *= USER_HZ; - proc_time.system *= USER_HZ; - proc_time.idle *= USER_HZ; - proc_time.iowait *= USER_HZ; + proc_time.user /= USER_HZ; + proc_time.nice /= USER_HZ; + proc_time.system /= USER_HZ; + proc_time.idle /= USER_HZ; + proc_time.iowait /= USER_HZ; readIntTextAndSkipWhitespaceIfAny(unused, procst_in); readIntTextAndSkipWhitespaceIfAny(unused, procst_in); @@ -118,9 +118,9 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS readIntTextAndSkipWhitespaceIfAny(proc_time.steal, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.guest, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.guest_nice, procst_in); - proc_time.steal *= USER_HZ; - proc_time.guest *= USER_HZ; - proc_time.guest_nice *= USER_HZ; + proc_time.steal /= USER_HZ; + proc_time.guest /= USER_HZ; + proc_time.guest_nice /= USER_HZ; do { From a163eeb12e7cd0690b1ca2cb2ae39138e4440cab Mon Sep 17 00:00:00 2001 From: elevankoff Date: Thu, 6 May 2021 11:55:14 +0000 Subject: [PATCH 010/290] Delete whitespace --- src/Common/ProcessorStatisticsOS.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index e05cb589f95..9046db431e7 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -7,7 +7,6 @@ #include "ProcessorStatisticsOS.h" - #include #include From 505b0516778cb62cc1ffec0c5bf1948932a43f2b Mon Sep 17 00:00:00 2001 From: elevankoff Date: Fri, 7 May 2021 10:16:32 +0000 Subject: [PATCH 011/290] Fixed some bugs --- src/Common/ProcessorStatisticsOS.cpp | 19 ++++++++++++++----- src/Common/ProcessorStatisticsOS.h | 2 ++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 9046db431e7..7b341cbea44 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -7,6 +7,8 @@ #include "ProcessorStatisticsOS.h" +#include "Poco/String.h" + #include #include @@ -31,7 +33,7 @@ static constexpr auto loadavg_filename = "/proc/loadavg"; static constexpr auto procst_filename = "/proc/stat"; static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; -static const long USER_HZ = sysconf(_SC_CLK_TCK); +static const uint64_t USER_HZ = static_cast(sysconf(_SC_CLK_TCK)); ProcessorStatisticsOS::ProcessorStatisticsOS() : loadavg_in(loadavg_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) @@ -97,7 +99,7 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS String field_name, field_val; uint64_t unused; - + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.user, procst_in); @@ -124,7 +126,8 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS do { readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, procst_in); + readString(field_val, procst_in); + skipWhitespaceIfAny(procst_in); } while (field_name != String("processes")); stload.processes = static_cast(std::stoul(field_val)); @@ -148,7 +151,7 @@ void ProcessorStatisticsOS::readFreq(ProcFreq & freq) { do { - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, cpuinfo_in); + readStringAndSkipWhitespaceIfAny(field_name, cpuinfo_in); } while (!cpuinfo_in.eof() && field_name != String("cpu MHz")); if (cpuinfo_in.eof()) @@ -172,12 +175,18 @@ void ProcessorStatisticsOS::readFreq(ProcFreq & freq) } template -void ProcessorStatisticsOS::readIntTextAndSkipWhitespaceIfAny(T& x, ReadBuffer& buf) +void ProcessorStatisticsOS::readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) { readIntText(x, buf); skipWhitespaceIfAny(buf); } +void ProcessorStatisticsOS::readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readString(s, buf); + skipWhitespaceIfAny(buf); +} + void ProcessorStatisticsOS::readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) { readStringUntilWhitespace(s, buf); diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 51bb8c3a157..cd0d15770ed 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -75,6 +75,8 @@ private: static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); static void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf); + + static void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer& buf); static void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf); From 44fb1ebc37c106f6abc8c9b8b47cd1073c2f2e16 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Fri, 7 May 2021 12:39:20 +0000 Subject: [PATCH 012/290] Small fix --- src/Common/ProcessorStatisticsOS.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 7b341cbea44..d3124ebddd3 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -126,8 +126,7 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS do { readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readString(field_val, procst_in); - skipWhitespaceIfAny(procst_in); + readStringAndSkipWhitespaceIfAny(field_val, procst_in); } while (field_name != String("processes")); stload.processes = static_cast(std::stoul(field_val)); From 69efc15f2ac8ab798cca3eeb8e77faae6eecc50a Mon Sep 17 00:00:00 2001 From: elevankoff Date: Fri, 7 May 2021 18:06:12 +0000 Subject: [PATCH 013/290] Add untested "MemoryInfo" class --- src/Common/MemoryInfoOS.cpp | 78 +++++++++++++++++++++++++++++++++++++ src/Common/MemoryInfoOS.h | 55 ++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/Common/MemoryInfoOS.cpp create mode 100644 src/Common/MemoryInfoOS.h diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp new file mode 100644 index 00000000000..02edccf579f --- /dev/null +++ b/src/Common/MemoryInfoOS.cpp @@ -0,0 +1,78 @@ +#if defined(OS_LINUX) + +#include +#include +#include + +#include "MemoryInfoOS.h" + +#include + +#include +#include + +namespace DB +{ + +static constexpr auto meminfo_filename = "/proc/meminfo"; + +MemoryInfoOS::MemoryInfoOS() + : meminfo_in(meminfo_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) +{} + +MemoryInfoOS::~MemoryInfoOS() {} + +MemoryInfoOS::Data MemoryInfoOS::get() +{ + meminfo_in.seek(0, SEEK_SET); + + MemoryInfoOS::Data data; + String field_name; + + assert(readField(data.total, String("MemTotal"))); + assert(readField(data.free, String("MemFree"))); + skipField(); + assert(readField(data.buffers, String("Buffers"))); + assert(readField(data.cached, String("Cached"))); + + data.free_and_cached = data.free + data.cached; + + assert(readField(data.swap_cached, String("SwapCached"))); + + while (!readField(data.swap_total, String("SwapTotal"))) {} + + assert(readField(data.swap_free, String("SwapFree"))); + + return data; +} + +bool MemoryInfoOS::readField(unsigned long & field_val, const String & field_name_target) +{ + String field_name; + + readStringAndSkipWhitespaceIfAny(field_name, meminfo_in); + readIntTextAndSkipWhitespaceIfAny(field_val, meminfo_in); + return (field_name == (field_name_target + String(":"))); +} + +void MemoryInfoOS::skipField() +{ + skipToNextLineOrEOF(meminfo_in); +} + +void MemoryInfoOS::readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readString(s, buf); + skipWhitespaceIfAny(buf); +} + +template +void MemoryInfoOS::readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); +} + +} + +#endif diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h new file mode 100644 index 00000000000..d6d07850ead --- /dev/null +++ b/src/Common/MemoryInfoOS.h @@ -0,0 +1,55 @@ +#pragma once +#if defined(OS_LINUX) + +#include +#include + +#include + +#include + +namespace DB +{ + +/** Opens file /proc/meminfo. Keeps it open and reads statistics about memory usage. + * This is Linux specific. + * See: man procfs + */ + +class MemoryInfoOS +{ +public: + // In kB + struct Data { + unsigned long total; + unsigned long free; + unsigned long buffers; + unsigned long cached; + unsigned long free_and_cached; + + unsigned long swap_total; + unsigned long swap_free; + unsigned long swap_cached; + }; + + MemoryInfoOS(); + ~MemoryInfoOS(); + + Data get(); + +private: + ReadBufferFromFile meminfo_in; + + bool readField(unsigned long & field_val, const String & field_name_target); + + void skipField(); + + static void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf); + + template + static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); +}; + +} + +#endif From 6ab7dd9f29f47eaaef944f77ed12839869243f17 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Fri, 7 May 2021 19:36:19 +0000 Subject: [PATCH 014/290] Change unsigned long -> uint64_t; delete private static functions from .h; another small fixes --- src/Common/MemoryInfoOS.cpp | 34 +++++++++++++++++----------------- src/Common/MemoryInfoOS.h | 23 +++++++++-------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index 02edccf579f..d2e3929b264 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -1,7 +1,5 @@ #if defined(OS_LINUX) -#include -#include #include #include "MemoryInfoOS.h" @@ -15,9 +13,24 @@ namespace DB { static constexpr auto meminfo_filename = "/proc/meminfo"; - + +static constexpr int READ_BUFFER_BUF_SIZE = (64 << 10); + +void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readString(s, buf); + skipWhitespaceIfAny(buf); +} + +template +void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); +} + MemoryInfoOS::MemoryInfoOS() - : meminfo_in(meminfo_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) + : meminfo_in(meminfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) {} MemoryInfoOS::~MemoryInfoOS() {} @@ -60,19 +73,6 @@ void MemoryInfoOS::skipField() skipToNextLineOrEOF(meminfo_in); } -void MemoryInfoOS::readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readString(s, buf); - skipWhitespaceIfAny(buf); -} - -template -void MemoryInfoOS::readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} - } #endif diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index d6d07850ead..8c98a11692d 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -21,15 +21,15 @@ class MemoryInfoOS public: // In kB struct Data { - unsigned long total; - unsigned long free; - unsigned long buffers; - unsigned long cached; - unsigned long free_and_cached; + uint64_t total; + uint64_t free; + uint64_t buffers; + uint64_t cached; + uint64_t free_and_cached; - unsigned long swap_total; - unsigned long swap_free; - unsigned long swap_cached; + uint64_t swap_total; + uint64_t swap_free; + uint64_t swap_cached; }; MemoryInfoOS(); @@ -40,14 +40,9 @@ public: private: ReadBufferFromFile meminfo_in; - bool readField(unsigned long & field_val, const String & field_name_target); + bool readField(uint64_t & field_val, const String & field_name_target); void skipField(); - - static void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf); - - template - static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); }; } From fd5827f735767bd7c05e979a76a6dfb34cd4c527 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Fri, 7 May 2021 19:38:22 +0000 Subject: [PATCH 015/290] Change data type of READ_BUFFER_BUF_SIZE --- src/Common/MemoryInfoOS.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index d2e3929b264..b2cfd1609e3 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -14,7 +14,7 @@ namespace DB static constexpr auto meminfo_filename = "/proc/meminfo"; -static constexpr int READ_BUFFER_BUF_SIZE = (64 << 10); +static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) { From c5c9e95744e712fbdaa452afee329bfe2da90f3c Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 8 May 2021 06:37:06 +0000 Subject: [PATCH 016/290] Change size of beffer for ReadBuffer; delete private static functions from class --- src/Common/ProcessorStatisticsOS.cpp | 70 ++++++++++++++-------------- src/Common/ProcessorStatisticsOS.h | 11 ----- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index d3124ebddd3..d7d308916b7 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -35,10 +35,43 @@ static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; static const uint64_t USER_HZ = static_cast(sysconf(_SC_CLK_TCK)); +static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); + +template +void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); +} + +void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readString(s, buf); + skipWhitespaceIfAny(buf); +} + +void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); +} + +void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) +{ + readChar(c, buf); + skipWhitespaceIfAny(buf); +} + +void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) +{ + readFloatText(f, buf); + skipWhitespaceIfAny(buf); +} + ProcessorStatisticsOS::ProcessorStatisticsOS() - : loadavg_in(loadavg_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) - , procst_in(procst_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) - , cpuinfo_in(cpuinfo_filename, DBMS_DEFAULT_BUFFER_SIZE, O_RDONLY | O_CLOEXEC) + : loadavg_in(loadavg_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) + , procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) + , cpuinfo_in(cpuinfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) { ProcStLoad unused; calcStLoad(unused); @@ -173,37 +206,6 @@ void ProcessorStatisticsOS::readFreq(ProcFreq & freq) freq.avg /= static_cast(cpu_count); } -template -void ProcessorStatisticsOS::readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} - -void ProcessorStatisticsOS::readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readString(s, buf); - skipWhitespaceIfAny(buf); -} - -void ProcessorStatisticsOS::readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} - -void ProcessorStatisticsOS::readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) -{ - readChar(c, buf); - skipWhitespaceIfAny(buf); -} - -void ProcessorStatisticsOS::readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) -{ - readFloatText(f, buf); - skipWhitespaceIfAny(buf); -} - } #endif diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index cd0d15770ed..123f9385113 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -71,17 +71,6 @@ private: uint64_t guest_nice; }; - template - static void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf); - - static void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf); - - static void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer& buf); - - static void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf); - - static void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf); - void readLoadavg(ProcLoadavg & loadavg); void calcStLoad(ProcStLoad & stload); void readFreq(ProcFreq & freq); From 6066d557a8819b0cf5a04dcaf62d4debaef81f79 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 8 May 2021 07:41:47 +0000 Subject: [PATCH 017/290] Made "get()" method order-independent and fixed bug of reading in the "readField" method --- src/Common/MemoryInfoOS.cpp | 47 ++++++++++++++++++++----------------- src/Common/MemoryInfoOS.h | 5 ++-- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index b2cfd1609e3..d8160561ee8 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -1,6 +1,8 @@ #if defined(OS_LINUX) #include +#include +#include #include "MemoryInfoOS.h" @@ -16,9 +18,9 @@ static constexpr auto meminfo_filename = "/proc/meminfo"; static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); -void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) { - readString(s, buf); + readStringUntilWhitespace(s, buf); skipWhitespaceIfAny(buf); } @@ -42,35 +44,36 @@ MemoryInfoOS::Data MemoryInfoOS::get() MemoryInfoOS::Data data; String field_name; - assert(readField(data.total, String("MemTotal"))); - assert(readField(data.free, String("MemFree"))); - skipField(); - assert(readField(data.buffers, String("Buffers"))); - assert(readField(data.cached, String("Cached"))); + std::unordered_map meminfo; + + while (!meminfo_in.eof()) + meminfo.insert(readField()); + + data.total = meminfo["MemTotal"]; + data.free = meminfo["MemFree"]; + data.buffers = meminfo["Buffers"]; + data.cached = meminfo["Cached"]; + data.swap_total = meminfo["SwapTotal"]; + data.swap_cached = meminfo["SwapCached"]; + data.swap_free = meminfo["SwapFree"]; data.free_and_cached = data.free + data.cached; - assert(readField(data.swap_cached, String("SwapCached"))); - - while (!readField(data.swap_total, String("SwapTotal"))) {} - - assert(readField(data.swap_free, String("SwapFree"))); - return data; } -bool MemoryInfoOS::readField(unsigned long & field_val, const String & field_name_target) +std::pair MemoryInfoOS::readField() { - String field_name; + String key; + uint64_t val; - readStringAndSkipWhitespaceIfAny(field_name, meminfo_in); - readIntTextAndSkipWhitespaceIfAny(field_val, meminfo_in); - return (field_name == (field_name_target + String(":"))); -} + readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); + readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); -void MemoryInfoOS::skipField() -{ - skipToNextLineOrEOF(meminfo_in); + // Delete the read ":" from the end + key.pop_back(); + + return std::make_pair(key, val); } } diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index 8c98a11692d..e1bf1dcfde4 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -40,9 +41,7 @@ public: private: ReadBufferFromFile meminfo_in; - bool readField(uint64_t & field_val, const String & field_name_target); - - void skipField(); + std::pair readField(); }; } From dc7b84a3cc55ad73c8a8c9e6a397bf2e556555b6 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 8 May 2021 13:04:08 +0000 Subject: [PATCH 018/290] Fix bug --- src/Common/MemoryInfoOS.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index d8160561ee8..5eb2501e322 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -69,6 +69,7 @@ std::pair MemoryInfoOS::readField() readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); + skipToNextLineOrEOF(meminfo_in); // Delete the read ":" from the end key.pop_back(); From 7bc0d846b7a6804e212c7485775a89679524f9bb Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 8 May 2021 20:38:10 +0000 Subject: [PATCH 019/290] Add "DiskStatisticsOS" class --- src/Common/DiskStatisticsOS.cpp | 76 +++++++++++++++++++++++++++++++++ src/Common/DiskStatisticsOS.h | 38 +++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/Common/DiskStatisticsOS.cpp create mode 100644 src/Common/DiskStatisticsOS.h diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp new file mode 100644 index 00000000000..40ba15ac6b8 --- /dev/null +++ b/src/Common/DiskStatisticsOS.cpp @@ -0,0 +1,76 @@ +#if defined(OS_LINUX) + +#include "DiskStatisticsOS.h" + +#include + +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_STATVFS; +} + +static constexpr auto mounts_filename = "/proc/mounts"; + +static constexpr std::size_t READ_BUFFER_BUF_SIZE = (64 << 10); + +void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); +} + +DiskStatisticsOS::DiskStatisticsOS() + : mounts_in(mounts_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) +{} + +DiskStatisticsOS::~DiskStatisticsOS() {} + +DiskStatisticsOS::Data DiskStatisticsOS::get() +{ + mounts_in.seek(0, SEEK_SET); + + DiskStatisticsOS::Data data = {0, 0}; + + while (!mounts_in.eof()) + { + String filesystem = readNextFilesystem(); + + struct statvfs stat; + + if (statvfs(filesystem.c_str(), &stat)) + throwFromErrno("Cannot statvfs", ErrorCodes::CANNOT_STATVFS); + + uint64_t total_blocks = static_cast(stat.f_blocks); + uint64_t free_blocks = static_cast(stat.f_bfree); + uint64_t used_blocks = total_blocks - free_blocks; + uint64_t block_size = static_cast(stat.f_bsize); + + data.total += total_blocks * block_size; + data.used += used_blocks * block_size; + } + + return data; +} + +String DiskStatisticsOS::readNextFilesystem() +{ + String filesystem, unused; + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(unused, mounts_in); + readStringUntilWhitespace(filesystem, mounts_in); + skipToNextLineOrEOF(mounts_in); + + return filesystem; +} + +} + +#endif diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h new file mode 100644 index 00000000000..a1c260f24c3 --- /dev/null +++ b/src/Common/DiskStatisticsOS.h @@ -0,0 +1,38 @@ +#if defined (OS_LINUX) + +#include + +#include + +#include + +namespace DB +{ + +/** Opens file /proc/mounts. Keeps it open, reads all mounted filesytems and + * calculates disk usage. + */ +class DiskStatisticsOS +{ +public: + // In bytes + struct Data { + uint64_t total; + uint64_t used; + }; + + DiskStatisticsOS(); + ~DiskStatisticsOS(); + + Data get(); + +private: + String readNextFilesystem(); + +private: + ReadBufferFromFile mounts_in; +}; + +} + +#endif From f570b1274e5ffb28e994ea90b4dfc218b55f209a Mon Sep 17 00:00:00 2001 From: elevankoff Date: Tue, 11 May 2021 11:53:25 +0000 Subject: [PATCH 020/290] Fix typo in the comment --- src/Common/ProcessorStatisticsOS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 123f9385113..20ba680b6dd 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -11,7 +11,7 @@ namespace DB { -/** Opens files: /proc/loadav, /proc/stat, /proc/cpuinfo. Keeps it open and reads processor statistics. +/** Opens files: /proc/loadavg, /proc/stat, /proc/cpuinfo. Keeps it open and reads processor statistics. * This is Linux specific. * See: man procfs */ From 2d6d05386a6d403f7ea720168252e53545d2b180 Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Wed, 24 Mar 2021 22:47:28 +0300 Subject: [PATCH 021/290] Just Works Just works (remastered) First steps First steps fixed First steps first fails Research first steps Tokenizer created Sprint to the moon Rename Rename 2.0 Rename 3.0 Work in progress Update Oops Oops x2 Try this Now surely works Maybe now? Now? Cmake first try Restore to previous Cmake second try Make this work Correct mistakes Third try cmake Exclude simd Better Try Add std::cerr More std::cerr More and more std::cerr Maybe fix? A B C D E F G H I J K L M N O P AA AB AC AD AE AF AAA AAB AAC AAD AAF AAE AAF AAG AAH AAI AAJ AAK AAAA AAAB AAAC AAAD AAAE AAAF AAAG AAAH AAAAA AAAAB First try v2 First try v2.1 First try v2.2 First try v2.3 First try v2.4 First try v2.5 First try v2.6 First try v2.7 First try v2.8 First try v2.9 First try v2.10 First try v2.11 First try v2.12 First try v2.13 First try v2.14 First try v2.15 First try v2.16 First try v2.16 First try v2.17 First try v2.18 First try v2.19 First try v2.20 First try v2.21 First try v2.22 First try v2.23 First try v2.24 First try v2.25 First try v2.26 First try v2.27 First try v2.28 First try v2.29 First try v2.30 First try v2.31 First try v2.32 First try v2.33 First try v2.34 First try v2.35 First try v2.36 First try v2.37 Second try v2.00 Second try v2.01 Second try v2.02 Second try v2.03 Second try v2.04 Second try v2.05 Second try v2.06 Second try v2.07 Second try v2.08 Second try v2.09 Second try v2.10 Second try v2.11 Second try v2.12 Second try v2.13 Second try v2.14 Second try v2.15 Second try v2.16 Second try v2.17 Cleanup Link SQLJSON only in simdjson build Fix? Fix?1.1 Fix Revert "Fix" This reverts commit 9df7aa977c880ec130062bceece7e215190b4837. Revert "Fix?1.1" This reverts commit 37429ecc9003fd73c106344186e39ff6603dde6c. Revert "Fix?" This reverts commit c1236fb8f4b5a799a5564aecf81136301f226e33. Revert "Link SQLJSON only in simdjson build" This reverts commit 8795cd8b143f3cfd312ddbf1b98e10d0d6fcaf51. Revert "Cleanup" This reverts commit e100dbc545f54421276be2e5d44f99f52fe1d87c. Third try v2.0 Third try v2.1 Third try v2.2 Third try v2.3 Third try v2.4 Third try v2.5 Third try v2.6 Third try v2.7 Third try v2.8 Third try v2.9 Third try v2.10 Third try v2.11 Third try v2.12 Third try v2.13 Third try v2.14 Third try v2.15 Pre-intermediate touches v1.0 Pre-intermediate touches v1.1 Pre-intermediate touches v1.2 Pre-intermediate touches v1.3 Last changes --- src/Functions/CMakeLists.txt | 2 + src/Functions/FunctionSQLJSON.cpp | 19 ++ src/Functions/FunctionSQLJSON.h | 277 ++++++++++++++++++ src/Functions/FunctionsJSON.h | 2 - src/Functions/JSONPath/ASTs/ASTJSONPath.h | 26 ++ .../JSONPath/ASTs/ASTJSONPathMemberAccess.h | 25 ++ .../JSONPath/ASTs/ASTJSONPathQuery.h | 23 ++ src/Functions/JSONPath/ASTs/CMakeLists.txt | 8 + src/Functions/JSONPath/CMakeLists.txt | 8 + .../JSONPath/Generators/CMakeLists.txt | 8 + .../JSONPath/Generators/GeneratorJSONPath.h | 98 +++++++ .../JSONPath/Generators/IGenerator.h | 30 ++ .../JSONPath/Generators/IGenerator_fwd.h | 16 + src/Functions/JSONPath/Generators/IVisitor.h | 40 +++ .../Generators/VisitorJSONPathMemberAccess.h | 39 +++ .../JSONPath/Generators/VisitorStatus.h | 11 + src/Functions/JSONPath/Parsers/CMakeLists.txt | 8 + .../JSONPath/Parsers/ParserJSONPath.cpp | 34 +++ .../JSONPath/Parsers/ParserJSONPath.h | 22 ++ .../Parsers/ParserJSONPathMemberAccess.cpp | 42 +++ .../Parsers/ParserJSONPathMemberAccess.h | 12 + .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 39 +++ .../JSONPath/Parsers/ParserJSONPathQuery.h | 17 ++ src/Functions/RapidJSONParser.h | 3 +- src/Functions/SimdJSONParser.h | 64 +++- src/Functions/registerFunctions.cpp | 2 + src/Parsers/Lexer.cpp | 3 + src/Parsers/Lexer.h | 1 + 28 files changed, 860 insertions(+), 19 deletions(-) create mode 100644 src/Functions/FunctionSQLJSON.cpp create mode 100644 src/Functions/FunctionSQLJSON.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPath.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h create mode 100644 src/Functions/JSONPath/ASTs/CMakeLists.txt create mode 100644 src/Functions/JSONPath/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Generators/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Generators/GeneratorJSONPath.h create mode 100644 src/Functions/JSONPath/Generators/IGenerator.h create mode 100644 src/Functions/JSONPath/Generators/IGenerator_fwd.h create mode 100644 src/Functions/JSONPath/Generators/IVisitor.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/Generators/VisitorStatus.h create mode 100644 src/Functions/JSONPath/Parsers/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPath.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPath.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 1c3beb2e47d..24add0f4f0a 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -114,6 +114,8 @@ target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_url) add_subdirectory(array) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) +add_subdirectory(JSONPath) + if (USE_STATS) target_link_libraries(clickhouse_functions PRIVATE stats) endif() diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp new file mode 100644 index 00000000000..ddcca12835f --- /dev/null +++ b/src/Functions/FunctionSQLJSON.cpp @@ -0,0 +1,19 @@ +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + +void registerFunctionsSQLJSON(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h new file mode 100644 index 00000000000..24749099e57 --- /dev/null +++ b/src/Functions/FunctionSQLJSON.h @@ -0,0 +1,277 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include +#include + +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; +} + +class FunctionSQLJSONHelpers +{ +public: + template typename Impl, class JSONParser> + class Executor + { + public: + static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) + { + MutableColumnPtr to{result_type->createColumn()}; + to->reserve(input_rows_count); + + if (arguments.size() < 2) + { + throw Exception{"JSONPath functions require at least 2 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; + } + + /// Check 1 argument: must be of type String (JSONPath) + const auto & first_column = arguments[0]; + if (!isString(first_column.type)) + { + throw Exception{ + "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + /// Check 2 argument: must be of type String (JSON) + const auto & second_column = arguments[1]; + if (!isString(second_column.type)) + { + throw Exception{ + "JSONPath functions require 2 argument to be JSON of string, illegal type: " + second_column.type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + /// If argument is successfully cast to (ColumnConst *) then it is quoted string + /// Example: + /// SomeFunction('some string argument') + /// + /// Otherwise it is a column + /// Example: + /// SomeFunction(database.table.column) + + /// Check 1 argument: must be const String (JSONPath) + const ColumnPtr & arg_jsonpath = first_column.column; + const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); + if (!arg_jsonpath_const) + { + throw Exception{"JSONPath argument must be of type const String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + /// Retrieve data from 1 argument + const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); + if (!arg_jsonpath_string) + { + throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; + } + + /// Check 2 argument: must be const or non-const String (JSON) + const ColumnPtr & arg_json = second_column.column; + const auto * col_json_const = typeid_cast(arg_json.get()); + const auto * col_json_string + = typeid_cast(col_json_const ? col_json_const->getDataColumnPtr().get() : arg_json.get()); + + /// Get data and offsets for 1 argument (JSONPath) + const ColumnString::Chars & chars_path = arg_jsonpath_string->getChars(); + const ColumnString::Offsets & offsets_path = arg_jsonpath_string->getOffsets(); + + /// Get data and offsets for 1 argument (JSON) + const char * query_begin = reinterpret_cast(&chars_path[0]); + const char * query_end = query_begin + offsets_path[0] - 1; + + /// Tokenize query + Tokens tokens(query_begin, query_end); + /// Max depth 0 indicates that depth is not limited + IParser::Pos token_iterator(tokens, 0); + + /// Parse query and create AST tree + Expected expected; + ASTPtr res; + ParserJSONPath parser; + const bool parse_res = parser.parse(token_iterator, res, expected); + if (!parse_res) + { + throw Exception{"Unable to parse JSONPath", ErrorCodes::BAD_ARGUMENTS}; + } + + /// Get data and offsets for 1 argument (JSON) + const ColumnString::Chars & chars_json = col_json_string->getChars(); + const ColumnString::Offsets & offsets_json = col_json_string->getOffsets(); + + JSONParser json_parser; + using Element = typename JSONParser::Element; + Element document; + bool document_ok = false; + + /// Parse JSON for every row + Impl impl; + for (const auto i : ext::range(0, input_rows_count)) + { + std::string_view json{ + reinterpret_cast(&chars_json[offsets_json[i - 1]]), offsets_json[i] - offsets_json[i - 1] - 1}; + document_ok = json_parser.parse(json, document); + + bool added_to_column = false; + if (document_ok) + { + added_to_column = impl.insertResultToColumn(*to, document, res); + } + if (!added_to_column) + { + to->insertDefault(); + } + } + return to; + } + }; + +private: +}; + +template typename Impl> +class FunctionSQLJSON : public IFunction +{ +public: + static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } + FunctionSQLJSON(const Context & context_) : context(context_) { } + + static constexpr auto name = Name::name; + String getName() const override { return Name::name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + return Impl::getReturnType(Name::name, arguments); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + /// Choose JSONParser. + /// 1. Lexer(path) -> Tokens + /// 2. Create ASTPtr + /// 3. Parser(Tokens, ASTPtr) -> complete AST + /// 4. Execute functions, call interpreter for each json (in function) +#if USE_SIMDJSON + if (context.getSettingsRef().allow_simdjson) + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); +#endif + +#if USE_RAPIDJSON + throw Exception{"RapidJSON is not supported :(", ErrorCodes::BAD_ARGUMENTS}; +#else + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); +#endif + } + +private: + const Context & context; +}; + +struct NameSQLJSONTest +{ + static constexpr auto name{"SQLJSONTest"}; +}; + +struct NameSQLJSONMemberAccess +{ + static constexpr auto name{"SQLJSONMemberAccess"}; +}; + +/** + * Function to test logic before function calling, will be removed in final PR + * @tparam JSONParser parser + */ +template +class SQLJSONTestImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element &, ASTPtr &) + { + String str = "I am working:-)"; + ColumnString & col_str = assert_cast(dest); + col_str.insertData(str.data(), str.size()); + return true; + } +}; + +/** + * Function to test jsonpath member access, will be removed in final PR + * @tparam JSONParser parser + */ +template +class SQLJSONMemberAccessImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) + { + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + while ((status = generator_json_path.getNextItem(current_element)) == VisitorStatus::Ok) + { + /// No-op + } + if (status == VisitorStatus::Error) + { + return false; + } + ColumnString & col_str = assert_cast(dest); + std::stringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr << current_element.getElement(); + auto output_str = ostr.str(); + col_str.insertData(output_str.data(), output_str.size()); + return true; + } +}; + +} diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index f066bb1029a..581cc2015aa 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -80,8 +80,6 @@ public: const ColumnString::Chars & chars = col_json_string->getChars(); const ColumnString::Offsets & offsets = col_json_string->getOffsets(); - size_t num_index_arguments = Impl::getNumberOfIndexArguments(arguments); - std::vector moves = prepareMoves(Name::name, arguments, 1, num_index_arguments); /// Preallocate memory in parser if necessary. JSONParser parser; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h new file mode 100644 index 00000000000..cd73cd14257 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace DB +{ +class ASTJSONPath : public IAST +{ +public: + String getID(char) const override + { + std::cerr << "in ASTJSONPath: getID\n"; + return "ASTJSONPath"; + } + + ASTPtr clone() const override + { + std::cerr << "in " << "ASTJSONPath" << ": clone\n"; + return std::make_shared(*this); + } + + ASTJSONPathQuery * jsonpath_query; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h new file mode 100644 index 00000000000..663859f566f --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathMemberAccess : public IAST +{ +public: + String getID(char) const override + { + return "ASTJSONPathMemberAccess"; + } + + ASTPtr clone() const override + { + return std::make_shared(*this); + } + +public: + /// Member name to lookup in json document (in path: $.some_key.another_key. ...) + String member_name; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h new file mode 100644 index 00000000000..6b952c2519d --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathQuery : public IAST +{ +public: + String getID(char) const override + { + std::cerr << "in ASTJSONPathQuery: getID\n"; + return "ASTJSONPathQuery"; + } + + ASTPtr clone() const override + { + std::cerr << "in " << getID(' ') << ": clone\n"; + return std::make_shared(*this); + } +}; + +} diff --git a/src/Functions/JSONPath/ASTs/CMakeLists.txt b/src/Functions/JSONPath/ASTs/CMakeLists.txt new file mode 100644 index 00000000000..c671dbbc001 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_asts .) +add_library(clickhouse_functions_jsonpath_asts ${clickhouse_functions_jsonpath_asts_sources} ${clickhouse_functions_jsonpath_asts_headers}) +target_link_libraries(clickhouse_functions_jsonpath_asts PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_asts PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt new file mode 100644 index 00000000000..8a46909f555 --- /dev/null +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -0,0 +1,8 @@ +add_subdirectory(ASTs) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_asts) + +add_subdirectory(Generators) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) + +add_subdirectory(Parsers) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) \ No newline at end of file diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt new file mode 100644 index 00000000000..0d1a289e8b4 --- /dev/null +++ b/src/Functions/JSONPath/Generators/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_generators .) +add_library(clickhouse_functions_jsonpath_generators ${clickhouse_functions_jsonpath_generators_sources} ${clickhouse_functions_jsonpath_generators_headers}) +target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h new file mode 100644 index 00000000000..dd4354a4613 --- /dev/null +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -0,0 +1,98 @@ +#include +#include +#include + +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +template +class GeneratorJSONPath : public IGenerator +{ +public: + GeneratorJSONPath(ASTPtr query_ptr_) + { + query_ptr = query_ptr_; + const auto * path = query_ptr->as(); + if (!path) { + throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); + } + const auto * query = path->jsonpath_query; + if (!path || !query) + { + throw Exception("Something went terribly wrong", ErrorCodes::LOGICAL_ERROR); + } + + for (auto child_ast : query->children) + { + if (child_ast->getID() == "ASTJSONPathMemberAccess") + { + auto member_access_generator = std::make_shared>(child_ast); + if (member_access_generator) { + visitors.push_back(member_access_generator); + } else { + throw Exception("member_access_generator could not be nullptr", ErrorCodes::LOGICAL_ERROR); + } + } + } + } + + const char * getName() const override { return "GeneratorJSONPath"; } + + /** + * The only generator which is called from JSONPath functions. + * @param element root of JSON document + * @return is the generator exhausted + */ + VisitorStatus getNextItem(typename JSONParser::Element & element) override + { + if (visitors[current_visitor]->isExhausted()) { + if (!backtrace()) { + return VisitorStatus::Exhausted; + } + } + + /// Apply all non-exhausted visitors + for (int i = 0; i < current_visitor; ++i) { + VisitorStatus status = visitors[i]->apply(element); + /// on fail return immediately + if (status == VisitorStatus::Error) { + return status; + } + } + + /// Visit newly initialized (for the first time or through reinitialize) visitors + for (size_t i = current_visitor; i < visitors.size(); ++i) { + VisitorStatus status = visitors[i]->visit(element); + current_visitor = i; + /// on fail return immediately + if (status == VisitorStatus::Error) { + return status; + } + } + return VisitorStatus::Ok; + } + +private: + bool backtrace() { + while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { + visitors[current_visitor]->reinitialize(); + current_visitor--; + } + return current_visitor >= 0; + } + + int current_visitor = 0; + ASTPtr query_ptr; + VisitorList visitors; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h new file mode 100644 index 00000000000..31d9e167f24 --- /dev/null +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +template +class IGenerator +{ +public: + IGenerator() = default; + + virtual const char * getName() const = 0; + + /** + * Used to yield next element in JSONPath query. Does so by recursively calling getNextItem + * on its children Generators one by one. + * + * @param element to be extracted into + * @return true if generator is not exhausted + */ + virtual VisitorStatus getNextItem(typename JSONParser::Element & element) = 0; + + virtual ~IGenerator() = default; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generators/IGenerator_fwd.h new file mode 100644 index 00000000000..27c3976b95b --- /dev/null +++ b/src/Functions/JSONPath/Generators/IGenerator_fwd.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace DB { + +template +class IGenerator; + +template +using IVisitorPtr = std::shared_ptr>; + +template +using VisitorList = std::vector>; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h new file mode 100644 index 00000000000..fdd254478a5 --- /dev/null +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +namespace DB { + +template +class IVisitor { +public: + /** + * Applies this visitor to document and mutates its state + * @param element simdjson element + */ + virtual VisitorStatus visit(typename JSONParser::Element & element) = 0; + + /** + * Applies this visitor to document, but does not mutate state + * @param element simdjson element + */ + virtual VisitorStatus apply(typename JSONParser::Element & element) const = 0; + + /** + * Restores visitor's initial state for later use + */ + virtual void reinitialize() = 0; + + bool isExhausted() { + return is_exhausted; + } + + void setExhausted(bool exhausted) { + is_exhausted = exhausted; + } + + virtual ~IVisitor() = default; +private: + bool is_exhausted = false; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h new file mode 100644 index 00000000000..50b814eeaeb --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -0,0 +1,39 @@ +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathMemberAccess : public IVisitor +{ +public: + VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + + VisitorStatus apply(typename JSONParser::Element & element) const override { + const auto * member_access = member_access_ptr->as(); + typename JSONParser::Element result; + bool result_ok = element.getObject().find(std::string_view(member_access->member_name), result); + if (result_ok) + { + element = result; + return VisitorStatus::Ok; + } + return VisitorStatus::Error; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + this->setExhausted(true); + return apply(element); + } + + void reinitialize() override { + this->setExhausted(false); + } + +private: + ASTPtr member_access_ptr; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h new file mode 100644 index 00000000000..51d795efbf7 --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -0,0 +1,11 @@ +#pragma once + +namespace DB { + +enum VisitorStatus { + Ok, + Exhausted, + Error +}; + +} diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt new file mode 100644 index 00000000000..f2f94298576 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_parsers .) +add_library(clickhouse_functions_jsonpath_parsers ${clickhouse_functions_jsonpath_parsers_sources} ${clickhouse_functions_jsonpath_parsers_headers}) +target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp new file mode 100644 index 00000000000..bf62f44fade --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -0,0 +1,34 @@ +#include + +#include + +#include + +#include + +namespace DB +{ + +/** + * Entry parser for JSONPath + */ +bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto ast_jsonpath = std::make_shared(); + ParserJSONPathQuery parser_jsonpath_query; + + /// Push back dot AST and brackets AST to query->children + ASTPtr query; + + bool res = parser_jsonpath_query.parse(pos, query, expected); + + if (res) { + /// Set ASTJSONPathQuery of ASTJSONPath + ast_jsonpath->set(ast_jsonpath->jsonpath_query, query); + } + + node = ast_jsonpath; + return res; +} + +} // namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.h b/src/Functions/JSONPath/Parsers/ParserJSONPath.h new file mode 100644 index 00000000000..5defc76b515 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.h @@ -0,0 +1,22 @@ +#pragma once + +#include + + +namespace DB +{ + +/** + * Entry parser for JSONPath + */ +class ParserJSONPath : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPath"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPath() = default; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp new file mode 100644 index 00000000000..10ae128616b --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -0,0 +1,42 @@ +#include +#include + +#include +#include +#include + +namespace DB +{ + +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathMemberAccess + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto member_access = std::make_shared(); + node = member_access; + if (pos->type != TokenType::Dot) { + return false; + } + ++pos; + + if (pos->type != TokenType::BareWord) { + return false; + } + ParserIdentifier name_p; + ASTPtr member_name; + if (!name_p.parse(pos, member_name, expected)) { + return false; + } + + if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { + return false; + } + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h new file mode 100644 index 00000000000..49fda6f1ac8 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -0,0 +1,12 @@ +#include + +namespace DB +{ +class ParserJSONPathMemberAccess : public IParserBase +{ + const char * getName() const override {return "ParserJSONPathMemberAccess";} + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp new file mode 100644 index 00000000000..c0831780fc4 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -0,0 +1,39 @@ +#include + +#include + +#include + +namespace DB + +{ +/** + * + * @param pos token iterator + * @param query node of ASTJSONPathQuery + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expected) +{ + query = std::make_shared(); + ParserJSONPathMemberAccess parser_jsonpath_member_access; + + if (pos->type != TokenType::DollarSign) { + return false; + } + ++pos; + + bool res = false; + ASTPtr member_access; + while (parser_jsonpath_member_access.parse(pos, member_access, expected)) + { + query->children.push_back(member_access); + member_access = nullptr; + res = true; + } + /// true in case of at least one success + return res; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h new file mode 100644 index 00000000000..cffec125c70 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathQuery : public IParserBase +{ +protected: + const char * getName() const override { return "ParserJSONPathQuery"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + /// backlog: strict or lax mode +}; +} diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index 992480d64f7..5604a8c9fe0 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -12,7 +12,6 @@ namespace DB { - /// This class can be used as an argument for the template class FunctionJSON. /// It provides ability to parse JSONs using rapidjson library. struct RapidJSONParser @@ -45,6 +44,8 @@ struct RapidJSONParser Array getArray() const; Object getObject() const; + ALWAYS_INLINE rapidjson::Value * getDom() const { return nullptr; } + private: const rapidjson::Value * ptr = nullptr; }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a9adfa27e2c..2dd952d920f 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -5,10 +5,10 @@ #endif #if USE_SIMDJSON -# include +# include # include # include -# include +# include namespace DB @@ -30,8 +30,8 @@ struct SimdJSONParser class Element { public: - ALWAYS_INLINE Element() {} - ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} + ALWAYS_INLINE Element() { } + ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) { } ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; } ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; } @@ -50,6 +50,8 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; + ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } + private: simdjson::dom::element element; }; @@ -61,21 +63,35 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) {} + ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) { } ALWAYS_INLINE Element operator*() const { return *it; } - ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } - ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } + ALWAYS_INLINE Iterator & operator++() + { + ++it; + return *this; + } + ALWAYS_INLINE Iterator operator++(int) + { + auto res = *this; + ++it; + return res; + } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } + private: simdjson::dom::array::iterator it; }; - ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) {} + ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) { } ALWAYS_INLINE Iterator begin() const { return array.begin(); } ALWAYS_INLINE Iterator end() const { return array.end(); } ALWAYS_INLINE size_t size() const { return array.size(); } - ALWAYS_INLINE Element operator[](size_t index) const { assert(index < size()); return array.at(index).first; } + ALWAYS_INLINE Element operator[](size_t index) const + { + assert(index < size()); + return array.at(index).first; + } private: simdjson::dom::array array; @@ -90,17 +106,31 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) {} - ALWAYS_INLINE KeyValuePair operator*() const { const auto & res = *it; return {res.key, res.value}; } - ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } - ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } + ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) { } + ALWAYS_INLINE KeyValuePair operator*() const + { + const auto & res = *it; + return {res.key, res.value}; + } + ALWAYS_INLINE Iterator & operator++() + { + ++it; + return *this; + } + ALWAYS_INLINE Iterator operator++(int) + { + auto res = *this; + ++it; + return res; + } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } + private: simdjson::dom::object::iterator it; }; - ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) {} + ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) { } ALWAYS_INLINE Iterator begin() const { return object.begin(); } ALWAYS_INLINE Iterator end() const { return object.end(); } ALWAYS_INLINE size_t size() const { return object.size(); } @@ -126,6 +156,8 @@ struct SimdJSONParser return {res.key, res.value}; } + ALWAYS_INLINE simdjson::dom::object getDom() const { return object; } + private: simdjson::dom::object object; }; @@ -145,8 +177,8 @@ struct SimdJSONParser void reserve(size_t max_size) { if (parser.allocate(max_size) != simdjson::error_code::SUCCESS) - throw Exception{"Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", - ErrorCodes::CANNOT_ALLOCATE_MEMORY}; + throw Exception{ + "Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", ErrorCodes::CANNOT_ALLOCATE_MEMORY}; } private: diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index d827cc40a86..116a2ae0324 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -40,6 +40,7 @@ void registerFunctionsGeo(FunctionFactory &); void registerFunctionsIntrospection(FunctionFactory &); void registerFunctionsNull(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); +void registerFunctionsSQLJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); void registerFunctionsUnixTimestamp64(FunctionFactory & factory); void registerFunctionBitHammingDistance(FunctionFactory & factory); @@ -97,6 +98,7 @@ void registerFunctions() registerFunctionsGeo(factory); registerFunctionsNull(factory); registerFunctionsJSON(factory); + registerFunctionsSQLJSON(factory); registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); registerFunctionsUnixTimestamp64(factory); diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index ffa8250a3f3..33cecab208f 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -240,6 +240,9 @@ Token Lexer::nextTokenImpl() case '*': ++pos; return Token(TokenType::Asterisk, token_begin, pos); + case '$': + ++pos; + return Token(TokenType::DollarSign, token_begin, pos); case '/': /// division (/) or start of comment (//, /*) { ++pos; diff --git a/src/Parsers/Lexer.h b/src/Parsers/Lexer.h index dc1c9824b6b..e9885d8df83 100644 --- a/src/Parsers/Lexer.h +++ b/src/Parsers/Lexer.h @@ -33,6 +33,7 @@ namespace DB \ M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \ \ + M(DollarSign) \ M(Plus) \ M(Minus) \ M(Slash) \ From d8f8fb2ae573da122d1bf54ba5024eac25a5d01f Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 22 May 2021 09:54:03 +0000 Subject: [PATCH 022/290] Some decorative changes --- src/Common/DiskStatisticsOS.cpp | 25 +++++----- src/Common/DiskStatisticsOS.h | 7 +-- src/Common/MemoryInfoOS.cpp | 39 ++++++++------- src/Common/MemoryInfoOS.h | 7 +-- src/Common/ProcessorStatisticsOS.cpp | 74 ++++++++++++++-------------- src/Common/ProcessorStatisticsOS.h | 7 +-- 6 files changed, 75 insertions(+), 84 deletions(-) diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp index 40ba15ac6b8..3654f843c3a 100644 --- a/src/Common/DiskStatisticsOS.cpp +++ b/src/Common/DiskStatisticsOS.cpp @@ -17,31 +17,32 @@ namespace ErrorCodes extern const int CANNOT_STATVFS; } +namespace +{ + void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); + } +} + static constexpr auto mounts_filename = "/proc/mounts"; static constexpr std::size_t READ_BUFFER_BUF_SIZE = (64 << 10); -void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} - -DiskStatisticsOS::DiskStatisticsOS() - : mounts_in(mounts_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) -{} +DiskStatisticsOS::DiskStatisticsOS() {} DiskStatisticsOS::~DiskStatisticsOS() {} DiskStatisticsOS::Data DiskStatisticsOS::get() { - mounts_in.seek(0, SEEK_SET); + ReadBufferFromFile mounts_in(mounts_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); DiskStatisticsOS::Data data = {0, 0}; while (!mounts_in.eof()) { - String filesystem = readNextFilesystem(); + String filesystem = readNextFilesystem(mounts_in); struct statvfs stat; @@ -60,7 +61,7 @@ DiskStatisticsOS::Data DiskStatisticsOS::get() return data; } -String DiskStatisticsOS::readNextFilesystem() +String DiskStatisticsOS::readNextFilesystem(ReadBuffer& mounts_in) { String filesystem, unused; diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h index a1c260f24c3..d14cf273ccd 100644 --- a/src/Common/DiskStatisticsOS.h +++ b/src/Common/DiskStatisticsOS.h @@ -9,7 +9,7 @@ namespace DB { -/** Opens file /proc/mounts. Keeps it open, reads all mounted filesytems and +/** Opens file /proc/mounts, reads all mounted filesytems and * calculates disk usage. */ class DiskStatisticsOS @@ -27,10 +27,7 @@ public: Data get(); private: - String readNextFilesystem(); - -private: - ReadBufferFromFile mounts_in; + String readNextFilesystem(ReadBuffer& mounts_in); }; } diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index 5eb2501e322..b8641809ae9 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -14,32 +14,33 @@ namespace DB { +namespace +{ + template + void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) + { + readIntText(x, buf); + skipWhitespaceIfAny(buf); + } + + void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); + } +} + static constexpr auto meminfo_filename = "/proc/meminfo"; static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); -void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} - -template -void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} - -MemoryInfoOS::MemoryInfoOS() - : meminfo_in(meminfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) -{} +MemoryInfoOS::MemoryInfoOS() {} MemoryInfoOS::~MemoryInfoOS() {} MemoryInfoOS::Data MemoryInfoOS::get() { - meminfo_in.seek(0, SEEK_SET); + ReadBufferFromFile meminfo_in(meminfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); MemoryInfoOS::Data data; String field_name; @@ -47,7 +48,7 @@ MemoryInfoOS::Data MemoryInfoOS::get() std::unordered_map meminfo; while (!meminfo_in.eof()) - meminfo.insert(readField()); + meminfo.insert(readField(meminfo_in)); data.total = meminfo["MemTotal"]; data.free = meminfo["MemFree"]; @@ -62,7 +63,7 @@ MemoryInfoOS::Data MemoryInfoOS::get() return data; } -std::pair MemoryInfoOS::readField() +std::pair MemoryInfoOS::readField(ReadBuffer& meminfo_in) { String key; uint64_t val; diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index e1bf1dcfde4..a868d4bc23d 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -12,11 +12,10 @@ namespace DB { -/** Opens file /proc/meminfo. Keeps it open and reads statistics about memory usage. +/** Opens file /proc/meminfo and reads statistics about memory usage. * This is Linux specific. * See: man procfs */ - class MemoryInfoOS { public: @@ -39,9 +38,7 @@ public: Data get(); private: - ReadBufferFromFile meminfo_in; - - std::pair readField(); + std::pair readField(ReadBuffer& meminfo_in); }; } diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index d7d308916b7..78353cfeeab 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -29,6 +29,40 @@ namespace ErrorCodes extern const int CANNOT_CLOSE_FILE; } +namespace +{ + template + void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) + { + readIntText(x, buf); + skipWhitespaceIfAny(buf); + } + + void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readString(s, buf); + skipWhitespaceIfAny(buf); + } + + void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); + } + + void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) + { + readChar(c, buf); + skipWhitespaceIfAny(buf); + } + + void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) + { + readFloatText(f, buf); + skipWhitespaceIfAny(buf); + } +} + static constexpr auto loadavg_filename = "/proc/loadavg"; static constexpr auto procst_filename = "/proc/stat"; static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; @@ -37,41 +71,7 @@ static const uint64_t USER_HZ = static_cast(sysconf(_SC_CLK_TCK)); static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); -template -void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} - -void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readString(s, buf); - skipWhitespaceIfAny(buf); -} - -void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} - -void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) -{ - readChar(c, buf); - skipWhitespaceIfAny(buf); -} - -void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) -{ - readFloatText(f, buf); - skipWhitespaceIfAny(buf); -} - ProcessorStatisticsOS::ProcessorStatisticsOS() - : loadavg_in(loadavg_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) - , procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) - , cpuinfo_in(cpuinfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC) { ProcStLoad unused; calcStLoad(unused); @@ -90,7 +90,7 @@ ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() void ProcessorStatisticsOS::readLoadavg(ProcLoadavg& loadavg) { - loadavg_in.seek(0, SEEK_SET); + ReadBufferFromFile loadavg_in(loadavg_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); readFloatAndSkipWhitespaceIfAny(loadavg.avg1, loadavg_in); readFloatAndSkipWhitespaceIfAny(loadavg.avg5, loadavg_in); @@ -128,7 +128,7 @@ void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload) { - procst_in.seek(0, SEEK_SET); + ReadBufferFromFile procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); String field_name, field_val; uint64_t unused; @@ -173,7 +173,7 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS void ProcessorStatisticsOS::readFreq(ProcFreq & freq) { - cpuinfo_in.seek(0, SEEK_SET); + ReadBufferFromFile cpuinfo_in(cpuinfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); String field_name, field_val; char unused; diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 20ba680b6dd..ba95b006e9a 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -11,11 +11,10 @@ namespace DB { -/** Opens files: /proc/loadavg, /proc/stat, /proc/cpuinfo. Keeps it open and reads processor statistics. +/** Opens files: /proc/loadavg, /proc/stat, /proc/cpuinfo and reads processor statistics in get() method. * This is Linux specific. * See: man procfs */ - class ProcessorStatisticsOS { public: @@ -78,10 +77,6 @@ private: void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload); private: - ReadBufferFromFile loadavg_in; - ReadBufferFromFile procst_in; - ReadBufferFromFile cpuinfo_in; - std::time_t last_stload_call_time; ProcTime last_proc_time; }; From ee3223b9440f9d4ed9c4feed029ccda6e19721cd Mon Sep 17 00:00:00 2001 From: elevankoff Date: Sat, 22 May 2021 09:57:51 +0000 Subject: [PATCH 023/290] Add ProcessorStatisticsOS, MemoryInfoOS and DiskStatisticsOS --- src/Interpreters/AsynchronousMetrics.cpp | 54 ++++++++++++++++++++++++ src/Interpreters/AsynchronousMetrics.h | 7 ++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 6eb143d17df..28c7be9ea2a 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -236,6 +236,60 @@ void AsynchronousMetrics::update() } #endif + /// Process memory information according to OS +#if defined(OS_LINUX) + { + MemoryInfoOS::Data data = memory_info.get(); + + new_values["MemoryTotal"] = data.total; + new_values["MemoryFree"] = data.free; + new_values["MemoryBuffers"] = data.buffers; + new_values["MemoryCached"] = data.cached; + new_values["MemoryFreeAndCached"] = data.free_and_cached; + new_values["MemorySwapTotal"] = data.swap_total; + new_values["MemorySwapFree"] = data.swap_free; + new_values["MemorySwapCached"] = data.swap_cached; + } +#endif + + /// Process processor usage according to OS +#if defined(OS_LINUX) + { + ProcessorStatisticsOS::Data data = proc_stat.get(); + + new_values["LoadAvg1"] = data.loadavg.avg1; + new_values["LoadAvg5"] = data.loadavg.avg5; + new_values["LoadAvg15"] = data.loadavg.avg15; + + new_values["FreqMin"] = data.freq.min; + new_values["FreqMax"] = data.freq.max; + new_values["FreqAvg"] = data.freq.avg; + + new_values["TimeLoadUser"] = data.stload.user_time; + new_values["TimeLoadNice"] = data.stload.nice_time; + new_values["TimeLoadSystem"] = data.stload.system_time; + new_values["TimeLoadIDLE"] = data.stload.idle_time; + new_values["TimeLoadIowait"] = data.stload.iowait_time; + new_values["TimeLoadSteal"] = data.stload.steal_time; + new_values["TimeLoadGuest"] = data.stload.guest_time; + new_values["TimeLoadGuestNice"] = data.stload.guest_nice_time; + + new_values["Processess"] = data.stload.processes; + new_values["ProcessesRunning"] = data.stload.procs_running; + new_values["ProcessesBlocked"] = data.stload.procs_blocked; + } +#endif + + /// Process disk usage according to OS +#if defined(OS_LINUX) + { + DiskStatisticsOS::Data data = disk_stat.get(); + + new_values["DiskTotal"] = data.total; + new_values["DiskUsed"] = data.used; + } +#endif + { auto databases = DatabaseCatalog::instance().getDatabases(); diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 38875c21edd..36e0fabd8a9 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include #include @@ -10,7 +13,6 @@ #include #include - namespace DB { @@ -80,6 +82,9 @@ private: #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; + MemoryInfoOS memory_info; + ProcessorStatisticsOS proc_stat; + DiskStatisticsOS disk_stat; #endif std::unique_ptr thread; From 4c3882f2f9a4374648638880b7ea94795eab1c02 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Fri, 21 May 2021 15:51:24 +0100 Subject: [PATCH 024/290] Config in-place include --- src/Common/Config/ConfigProcessor.cpp | 57 ++++++++++++------- .../configs/config_zk.xml | 3 + .../test_config_substitutions/test.py | 7 +++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index fa9e9b72087..851d8d9d250 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -322,33 +322,48 @@ void ConfigProcessor::doIncludesRecursive( } else { - Element & element = dynamic_cast(*node); - - for (const auto & attr_name : SUBSTITUTION_ATTRS) - element.removeAttribute(attr_name); - - if (replace) + /// Replace the whole node not just contents. + if (node->nodeName() == "include") { - while (Node * child = node->firstChild()) - node->removeChild(child); + const NodeListPtr children = node_to_include->childNodes(); + for (size_t i = 0, size = children->length(); i < size; ++i) + { + NodePtr new_node = config->importNode(children->item(i), true); + node->parentNode()->insertBefore(new_node, node); + } - element.removeAttribute("replace"); + node->parentNode()->removeChild(node); } - - const NodeListPtr children = node_to_include->childNodes(); - for (size_t i = 0, size = children->length(); i < size; ++i) + else { - NodePtr new_node = config->importNode(children->item(i), true); - node->appendChild(new_node); - } + Element & element = dynamic_cast(*node); - const NamedNodeMapPtr from_attrs = node_to_include->attributes(); - for (size_t i = 0, size = from_attrs->length(); i < size; ++i) - { - element.setAttributeNode(dynamic_cast(config->importNode(from_attrs->item(i), true))); - } + for (const auto & attr_name : SUBSTITUTION_ATTRS) + element.removeAttribute(attr_name); - included_something = true; + if (replace) + { + while (Node * child = node->firstChild()) + node->removeChild(child); + + element.removeAttribute("replace"); + } + + const NodeListPtr children = node_to_include->childNodes(); + for (size_t i = 0, size = children->length(); i < size; ++i) + { + NodePtr new_node = config->importNode(children->item(i), true); + node->appendChild(new_node); + } + + const NamedNodeMapPtr from_attrs = node_to_include->attributes(); + for (size_t i = 0, size = from_attrs->length(); i < size; ++i) + { + element.setAttributeNode(dynamic_cast(config->importNode(from_attrs->item(i), true))); + } + + included_something = true; + } } }; diff --git a/tests/integration/test_config_substitutions/configs/config_zk.xml b/tests/integration/test_config_substitutions/configs/config_zk.xml index aa589e9f9d3..9fad5658445 100644 --- a/tests/integration/test_config_substitutions/configs/config_zk.xml +++ b/tests/integration/test_config_substitutions/configs/config_zk.xml @@ -10,5 +10,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 565cd1c0e97..47154efec36 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -20,6 +20,8 @@ def start_cluster(): try: def create_zk_roots(zk): zk.create(path="/setting/max_query_size", value=b"77777", makepath=True) + zk.create(path="/users_from_zk_1", value=b"default", makepath=True) + zk.create(path="/users_from_zk_2", value=b"default", makepath=True) cluster.add_zookeeper_startup_command(create_zk_roots) @@ -37,6 +39,11 @@ def test_config(start_cluster): assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" +def test_include_config(start_cluster): + assert node3.query("select 1", user="user_1") + assert node3.query("select 1", user="user_2") + + def test_allow_databases(start_cluster): node5.query("CREATE DATABASE db1") node5.query( From 700850a9705d814e3799d87d7ce6978b169eb0c9 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 24 May 2021 13:34:39 +0100 Subject: [PATCH 025/290] Add more tests and handle correctly missing include --- src/Common/Config/ConfigProcessor.cpp | 5 +++++ .../configs/config_env.xml | 3 +++ .../configs/config_incl.xml | 5 ++++- .../configs/config_include_from_env.xml | 2 ++ .../configs/include_from_source.xml | 17 +++++++++++++++++ .../configs/max_query_size.xml | 3 --- .../test_config_substitutions/test.py | 13 ++++++++++--- 7 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_config_substitutions/configs/include_from_source.xml delete mode 100644 tests/integration/test_config_substitutions/configs/max_query_size.xml diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 851d8d9d250..5af13d8a9df 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -318,7 +318,12 @@ void ConfigProcessor::doIncludesRecursive( else if (throw_on_bad_incl) throw Poco::Exception(error_msg + name); else + { + if (node->nodeName() == "include") + node->parentNode()->removeChild(node); + LOG_WARNING(log, "{}{}", error_msg, name); + } } else { diff --git a/tests/integration/test_config_substitutions/configs/config_env.xml b/tests/integration/test_config_substitutions/configs/config_env.xml index 712855c47c0..2d63b9c688d 100644 --- a/tests/integration/test_config_substitutions/configs/config_env.xml +++ b/tests/integration/test_config_substitutions/configs/config_env.xml @@ -10,5 +10,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/configs/config_incl.xml b/tests/integration/test_config_substitutions/configs/config_incl.xml index 383a23af1ff..43ec78ff8ef 100644 --- a/tests/integration/test_config_substitutions/configs/config_incl.xml +++ b/tests/integration/test_config_substitutions/configs/config_incl.xml @@ -1,5 +1,5 @@ - /etc/clickhouse-server/config.d/max_query_size.xml + /etc/clickhouse-server/config.d/include_from_source.xml @@ -11,5 +11,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/configs/config_include_from_env.xml b/tests/integration/test_config_substitutions/configs/config_include_from_env.xml index 71e11235749..79b650f3d9e 100644 --- a/tests/integration/test_config_substitutions/configs/config_include_from_env.xml +++ b/tests/integration/test_config_substitutions/configs/config_include_from_env.xml @@ -11,5 +11,7 @@ default default + + diff --git a/tests/integration/test_config_substitutions/configs/include_from_source.xml b/tests/integration/test_config_substitutions/configs/include_from_source.xml new file mode 100644 index 00000000000..6095180bb59 --- /dev/null +++ b/tests/integration/test_config_substitutions/configs/include_from_source.xml @@ -0,0 +1,17 @@ + + 99999 + + + + + default + + + + + + + default + + + diff --git a/tests/integration/test_config_substitutions/configs/max_query_size.xml b/tests/integration/test_config_substitutions/configs/max_query_size.xml deleted file mode 100644 index 9ec61368be9..00000000000 --- a/tests/integration/test_config_substitutions/configs/max_query_size.xml +++ /dev/null @@ -1,3 +0,0 @@ - - 99999 - diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 47154efec36..aec3f1d3635 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -8,11 +8,11 @@ node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], env_variables={"MAX_QUERY_SIZE": "55555"}) node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], - main_configs=['configs/max_query_size.xml']) # include value 77777 + main_configs=['configs/include_from_source.xml']) # include value 77777 node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], - env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, - main_configs=['configs/max_query_size.xml']) + env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/include_from_source.xml"}, + main_configs=['configs/include_from_source.xml']) @pytest.fixture(scope="module") @@ -40,6 +40,13 @@ def test_config(start_cluster): def test_include_config(start_cluster): + # + assert node4.query("select 1") + assert node4.query("select 1", user="user_1") + assert node4.query("select 1", user="user_2") + + # Date: Mon, 24 May 2021 13:53:02 +0100 Subject: [PATCH 026/290] Document new include element --- docs/en/operations/configuration-files.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md index a52d82f21d0..c5114ae19bd 100644 --- a/docs/en/operations/configuration-files.md +++ b/docs/en/operations/configuration-files.md @@ -22,6 +22,23 @@ Some settings specified in the main configuration file can be overridden in othe The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)). +If you want to replace an entire element with a substitution use `include` as element name. + +XML substitution example: + +```xml + + + + + + + + + + +``` + Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element. ## User Settings {#user-settings} @@ -32,6 +49,8 @@ Users configuration can be splitted into separate files similar to `config.xml` Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`. Directory `users.d` is used by default, as `users_config` defaults to `users.xml`. +Note that configuration files are first merged taking into account [Override](#override) settings and includes are processed after that. + ## XML example {#example} For example, you can have separate config file for each user like this: From 399e998ecb535ace36c253a31f9e3240235fdb9b Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 24 May 2021 14:07:21 +0100 Subject: [PATCH 027/290] Count non-empty substitutions, easier to interpret --- src/Common/Config/ConfigProcessor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 5af13d8a9df..44f5ffc5536 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -296,10 +296,10 @@ void ConfigProcessor::doIncludesRecursive( { const auto * subst = attributes->getNamedItem(attr_name); attr_nodes[attr_name] = subst; - substs_count += static_cast(subst == nullptr); + substs_count += static_cast(subst != nullptr); } - if (substs_count < SUBSTITUTION_ATTRS.size() - 1) /// only one substitution is allowed + if (substs_count > 1) /// only one substitution is allowed throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">"); /// Replace the original contents, not add to it. From 027fc70acef44a41ca9ec565f499246f04d3dbe5 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 24 May 2021 14:07:38 +0100 Subject: [PATCH 028/290] Validate that include element is used properly --- src/Common/Config/ConfigProcessor.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 44f5ffc5536..2b6e3287461 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -302,6 +302,14 @@ void ConfigProcessor::doIncludesRecursive( if (substs_count > 1) /// only one substitution is allowed throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">"); + if (node->nodeName() == "include") + { + if (node->hasChildNodes()) + throw Poco::Exception(" element must have no children"); + if (substs_count == 0) + throw Poco::Exception("no substitution attributes set for element , must have one"); + } + /// Replace the original contents, not add to it. bool replace = attributes->getNamedItem("replace"); From 067ec0855cfeb16392e5bef121328abb5b6e7957 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Mon, 24 May 2021 17:16:15 +0000 Subject: [PATCH 029/290] Decorative fixes --- src/Common/DiskStatisticsOS.cpp | 28 ++--- src/Common/DiskStatisticsOS.h | 12 ++- src/Common/MemoryInfoOS.cpp | 46 ++++---- src/Common/MemoryInfoOS.h | 7 +- src/Common/ProcessorStatisticsOS.cpp | 152 ++++++++++++--------------- src/Common/ProcessorStatisticsOS.h | 21 ++-- 6 files changed, 127 insertions(+), 139 deletions(-) diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp index 3654f843c3a..0485d129ecc 100644 --- a/src/Common/DiskStatisticsOS.cpp +++ b/src/Common/DiskStatisticsOS.cpp @@ -9,15 +9,15 @@ #include #include -namespace DB +namespace DB { -namespace ErrorCodes +namespace ErrorCodes { extern const int CANNOT_STATVFS; } -namespace +namespace { void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) { @@ -34,35 +34,35 @@ DiskStatisticsOS::DiskStatisticsOS() {} DiskStatisticsOS::~DiskStatisticsOS() {} -DiskStatisticsOS::Data DiskStatisticsOS::get() +DiskStatisticsOS::Data DiskStatisticsOS::get() { ReadBufferFromFile mounts_in(mounts_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); DiskStatisticsOS::Data data = {0, 0}; - while (!mounts_in.eof()) + while (!mounts_in.eof()) { String filesystem = readNextFilesystem(mounts_in); - + struct statvfs stat; - if (statvfs(filesystem.c_str(), &stat)) + if (statvfs(filesystem.c_str(), &stat)) throwFromErrno("Cannot statvfs", ErrorCodes::CANNOT_STATVFS); - + uint64_t total_blocks = static_cast(stat.f_blocks); - uint64_t free_blocks = static_cast(stat.f_bfree); - uint64_t used_blocks = total_blocks - free_blocks; - uint64_t block_size = static_cast(stat.f_bsize); + uint64_t free_blocks = static_cast(stat.f_bfree); + uint64_t used_blocks = total_blocks - free_blocks; + uint64_t block_size = static_cast(stat.f_bsize); data.total += total_blocks * block_size; - data.used += used_blocks * block_size; + data.used += used_blocks * block_size; } return data; } -String DiskStatisticsOS::readNextFilesystem(ReadBuffer& mounts_in) -{ +String DiskStatisticsOS::readNextFilesystem(ReadBuffer& mounts_in) +{ String filesystem, unused; readStringUntilWhitespaceAndSkipWhitespaceIfAny(unused, mounts_in); diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h index d14cf273ccd..05f53a421d2 100644 --- a/src/Common/DiskStatisticsOS.h +++ b/src/Common/DiskStatisticsOS.h @@ -1,3 +1,4 @@ +#pragma once #if defined (OS_LINUX) #include @@ -6,17 +7,18 @@ #include -namespace DB +namespace DB { -/** Opens file /proc/mounts, reads all mounted filesytems and +/** Opens file /proc/mounts, reads all mounted filesystems and * calculates disk usage. - */ -class DiskStatisticsOS + */ +class DiskStatisticsOS { public: // In bytes - struct Data { + struct Data + { uint64_t total; uint64_t used; }; diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index b8641809ae9..17036d115e8 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -11,23 +11,23 @@ #include #include -namespace DB +namespace DB { -namespace +namespace { - template - void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) - { - readIntText(x, buf); - skipWhitespaceIfAny(buf); - } +template +void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); +} - void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); - } +void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); +} } static constexpr auto meminfo_filename = "/proc/meminfo"; @@ -38,10 +38,10 @@ MemoryInfoOS::MemoryInfoOS() {} MemoryInfoOS::~MemoryInfoOS() {} -MemoryInfoOS::Data MemoryInfoOS::get() +MemoryInfoOS::Data MemoryInfoOS::get() { ReadBufferFromFile meminfo_in(meminfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - + MemoryInfoOS::Data data; String field_name; @@ -49,14 +49,14 @@ MemoryInfoOS::Data MemoryInfoOS::get() while (!meminfo_in.eof()) meminfo.insert(readField(meminfo_in)); - - data.total = meminfo["MemTotal"]; - data.free = meminfo["MemFree"]; - data.buffers = meminfo["Buffers"]; - data.cached = meminfo["Cached"]; - data.swap_total = meminfo["SwapTotal"]; + + data.total = meminfo["MemTotal"]; + data.free = meminfo["MemFree"]; + data.buffers = meminfo["Buffers"]; + data.cached = meminfo["Cached"]; + data.swap_total = meminfo["SwapTotal"]; data.swap_cached = meminfo["SwapCached"]; - data.swap_free = meminfo["SwapFree"]; + data.swap_free = meminfo["SwapFree"]; data.free_and_cached = data.free + data.cached; @@ -67,7 +67,7 @@ std::pair MemoryInfoOS::readField(ReadBuffer& meminfo_in) { String key; uint64_t val; - + readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); skipToNextLineOrEOF(meminfo_in); diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index a868d4bc23d..ae630e4ee70 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -9,18 +9,19 @@ #include -namespace DB +namespace DB { /** Opens file /proc/meminfo and reads statistics about memory usage. * This is Linux specific. * See: man procfs */ -class MemoryInfoOS +class MemoryInfoOS { public: // In kB - struct Data { + struct Data + { uint64_t total; uint64_t free; uint64_t buffers; diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 78353cfeeab..0deea56e7fc 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -21,46 +21,38 @@ namespace DB { -namespace ErrorCodes +namespace { - extern const int FILE_DOESNT_EXIST; - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; - extern const int CANNOT_CLOSE_FILE; +template +void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + skipWhitespaceIfAny(buf); } -namespace +void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) { - template - void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) - { - readIntText(x, buf); - skipWhitespaceIfAny(buf); - } + readString(s, buf); + skipWhitespaceIfAny(buf); +} - void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readString(s, buf); - skipWhitespaceIfAny(buf); - } +void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) +{ + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); +} - void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); - } +void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) +{ + readChar(c, buf); + skipWhitespaceIfAny(buf); +} - void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) - { - readChar(c, buf); - skipWhitespaceIfAny(buf); - } - - void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) - { - readFloatText(f, buf); - skipWhitespaceIfAny(buf); - } +void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) +{ + readFloatText(f, buf); + skipWhitespaceIfAny(buf); +} } static constexpr auto loadavg_filename = "/proc/loadavg"; @@ -84,20 +76,20 @@ ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() Data data; readLoadavg(data.loadavg); calcStLoad(data.stload); - readFreq(data.freq); + readFreq(data.freq); return data; } void ProcessorStatisticsOS::readLoadavg(ProcLoadavg& loadavg) { ReadBufferFromFile loadavg_in(loadavg_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - + readFloatAndSkipWhitespaceIfAny(loadavg.avg1, loadavg_in); readFloatAndSkipWhitespaceIfAny(loadavg.avg5, loadavg_in); readFloatAndSkipWhitespaceIfAny(loadavg.avg15, loadavg_in); } -void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) +void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) { ProcTime cur_proc_time; readProcTimeAndProcesses(cur_proc_time, stload); @@ -105,23 +97,15 @@ void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) std::time_t cur_time = std::time(nullptr); float time_dif = static_cast(cur_time - last_stload_call_time); - stload.user_time = - (cur_proc_time.user - last_proc_time.user) / time_dif; - stload.nice_time = - (cur_proc_time.nice - last_proc_time.nice) / time_dif; - stload.system_time = - (cur_proc_time.system - last_proc_time.system) / time_dif; - stload.idle_time = - (cur_proc_time.idle - last_proc_time.idle) / time_dif; - stload.iowait_time = - (cur_proc_time.iowait - last_proc_time.iowait) / time_dif; - stload.steal_time = - (cur_proc_time.steal - last_proc_time.steal) / time_dif; - stload.guest_time = - (cur_proc_time.guest - last_proc_time.guest) / time_dif; - stload.guest_nice_time = - (cur_proc_time.guest_nice - last_proc_time.guest_nice) / time_dif; - + stload.user_time = (cur_proc_time.user - last_proc_time.user) / time_dif; + stload.nice_time = (cur_proc_time.nice - last_proc_time.nice) / time_dif; + stload.system_time = (cur_proc_time.system - last_proc_time.system) / time_dif; + stload.idle_time = (cur_proc_time.idle - last_proc_time.idle) / time_dif; + stload.iowait_time = (cur_proc_time.iowait - last_proc_time.iowait) / time_dif; + stload.steal_time = (cur_proc_time.steal - last_proc_time.steal) / time_dif; + stload.guest_time = (cur_proc_time.guest - last_proc_time.guest) / time_dif; + stload.guest_nice_time = (cur_proc_time.guest_nice - last_proc_time.guest_nice) / time_dif; + last_stload_call_time = cur_time; last_proc_time = cur_proc_time; } @@ -131,76 +115,72 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS ReadBufferFromFile procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); String field_name, field_val; - uint64_t unused; - + uint64_t unused; + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.user, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.nice, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.user, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.nice, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.system, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.idle, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.idle, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.iowait, procst_in); - proc_time.user /= USER_HZ; - proc_time.nice /= USER_HZ; + proc_time.user /= USER_HZ; + proc_time.nice /= USER_HZ; proc_time.system /= USER_HZ; - proc_time.idle /= USER_HZ; + proc_time.idle /= USER_HZ; proc_time.iowait /= USER_HZ; - + readIntTextAndSkipWhitespaceIfAny(unused, procst_in); readIntTextAndSkipWhitespaceIfAny(unused, procst_in); - - readIntTextAndSkipWhitespaceIfAny(proc_time.steal, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.guest, procst_in); + + readIntTextAndSkipWhitespaceIfAny(proc_time.steal, procst_in); + readIntTextAndSkipWhitespaceIfAny(proc_time.guest, procst_in); readIntTextAndSkipWhitespaceIfAny(proc_time.guest_nice, procst_in); - proc_time.steal /= USER_HZ; - proc_time.guest /= USER_HZ; + proc_time.steal /= USER_HZ; + proc_time.guest /= USER_HZ; proc_time.guest_nice /= USER_HZ; - do - { + do { readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); readStringAndSkipWhitespaceIfAny(field_val, procst_in); } while (field_name != String("processes")); - + stload.processes = static_cast(std::stoul(field_val)); - + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); readIntTextAndSkipWhitespaceIfAny(stload.procs_running, procst_in); - + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); readIntTextAndSkipWhitespaceIfAny(stload.procs_blocked, procst_in); } void ProcessorStatisticsOS::readFreq(ProcFreq & freq) -{ +{ ReadBufferFromFile cpuinfo_in(cpuinfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - + String field_name, field_val; char unused; int cpu_count = 0; + freq.max = freq.min = freq.avg = 0; - do - { - do - { + do { + do { readStringAndSkipWhitespaceIfAny(field_name, cpuinfo_in); } while (!cpuinfo_in.eof() && field_name != String("cpu MHz")); - - if (cpuinfo_in.eof()) + + if (cpuinfo_in.eof()) break; readCharAndSkipWhitespaceIfAny(unused, cpuinfo_in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, cpuinfo_in); + readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, cpuinfo_in); cpu_count++; - + float cur_cpu_freq = stof(field_val); freq.avg += cur_cpu_freq; - freq.max = (cpu_count == 1 ? cur_cpu_freq : - std::max(freq.max, cur_cpu_freq)); - freq.min = (cpu_count == 1 ? cur_cpu_freq : - std::min(freq.min, cur_cpu_freq)); + freq.max = (cpu_count == 1 ? cur_cpu_freq : std::max(freq.max, cur_cpu_freq)); + freq.min = (cpu_count == 1 ? cur_cpu_freq : std::min(freq.min, cur_cpu_freq)); } while (true); freq.avg /= static_cast(cpu_count); diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index ba95b006e9a..f29e5156bfe 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -8,23 +8,26 @@ #include -namespace DB +namespace DB { /** Opens files: /proc/loadavg, /proc/stat, /proc/cpuinfo and reads processor statistics in get() method. * This is Linux specific. * See: man procfs */ -class ProcessorStatisticsOS +class ProcessorStatisticsOS { public: - struct ProcLoadavg { + + struct ProcLoadavg + { float avg1; float avg5; float avg15; }; - struct ProcStLoad { + struct ProcStLoad + { float user_time; float nice_time; float system_time; @@ -39,7 +42,8 @@ public: uint32_t procs_blocked; }; - struct ProcFreq { + struct ProcFreq + { float max; float min; float avg; @@ -54,11 +58,12 @@ public: ProcessorStatisticsOS(); ~ProcessorStatisticsOS(); - + Data get(); private: - struct ProcTime { + struct ProcTime + { // The amount of time, measured in seconds uint64_t user; uint64_t nice; @@ -73,7 +78,7 @@ private: void readLoadavg(ProcLoadavg & loadavg); void calcStLoad(ProcStLoad & stload); void readFreq(ProcFreq & freq); - + void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload); private: From 6bf0840562dd140d01699c0f22120876bf3a132a Mon Sep 17 00:00:00 2001 From: elevankoff Date: Mon, 24 May 2021 17:24:29 +0000 Subject: [PATCH 030/290] More decorative fixes --- src/Common/MemoryInfoOS.cpp | 22 +++++----- src/Common/ProcessorStatisticsOS.cpp | 61 +++++++++++++++------------- src/Common/ProcessorStatisticsOS.h | 1 - 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index 17036d115e8..8cf2a0b44f4 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -16,18 +16,18 @@ namespace DB namespace { -template -void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} + template + void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) + { + readIntText(x, buf); + skipWhitespaceIfAny(buf); + } -void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} + void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); + } } static constexpr auto meminfo_filename = "/proc/meminfo"; diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 0deea56e7fc..69bce5f5b51 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -23,36 +23,36 @@ namespace DB namespace { -template -void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) -{ - readIntText(x, buf); - skipWhitespaceIfAny(buf); -} + template + void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) + { + readIntText(x, buf); + skipWhitespaceIfAny(buf); + } -void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readString(s, buf); - skipWhitespaceIfAny(buf); -} + void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readString(s, buf); + skipWhitespaceIfAny(buf); + } -void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) -{ - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); -} + void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) + { + readStringUntilWhitespace(s, buf); + skipWhitespaceIfAny(buf); + } -void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) -{ - readChar(c, buf); - skipWhitespaceIfAny(buf); -} + void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) + { + readChar(c, buf); + skipWhitespaceIfAny(buf); + } -void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) -{ - readFloatText(f, buf); - skipWhitespaceIfAny(buf); -} + void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) + { + readFloatText(f, buf); + skipWhitespaceIfAny(buf); + } } static constexpr auto loadavg_filename = "/proc/loadavg"; @@ -140,7 +140,8 @@ void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcS proc_time.guest /= USER_HZ; proc_time.guest_nice /= USER_HZ; - do { + do + { readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); readStringAndSkipWhitespaceIfAny(field_val, procst_in); } while (field_name != String("processes")); @@ -163,8 +164,10 @@ void ProcessorStatisticsOS::readFreq(ProcFreq & freq) int cpu_count = 0; freq.max = freq.min = freq.avg = 0; - do { - do { + do + { + do + { readStringAndSkipWhitespaceIfAny(field_name, cpuinfo_in); } while (!cpuinfo_in.eof() && field_name != String("cpu MHz")); diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index f29e5156bfe..70edfceb2ca 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -18,7 +18,6 @@ namespace DB class ProcessorStatisticsOS { public: - struct ProcLoadavg { float avg1; From 7d1524561e8e588c63b38db02012a92a4c667a67 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Mon, 24 May 2021 17:35:38 +0000 Subject: [PATCH 031/290] Delete extra whitespaces --- src/Interpreters/AsynchronousMetrics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 28c7be9ea2a..92ff4931481 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -236,7 +236,7 @@ void AsynchronousMetrics::update() } #endif - /// Process memory information according to OS + /// Process memory information according to OS #if defined(OS_LINUX) { MemoryInfoOS::Data data = memory_info.get(); @@ -252,7 +252,7 @@ void AsynchronousMetrics::update() } #endif - /// Process processor usage according to OS + /// Process processor usage according to OS #if defined(OS_LINUX) { ProcessorStatisticsOS::Data data = proc_stat.get(); @@ -280,7 +280,7 @@ void AsynchronousMetrics::update() } #endif - /// Process disk usage according to OS + /// Process disk usage according to OS #if defined(OS_LINUX) { DiskStatisticsOS::Data data = disk_stat.get(); From 9d3c24c9c0413faaf0148ed94c68e40cacd2a4a0 Mon Sep 17 00:00:00 2001 From: elevankoff Date: Mon, 24 May 2021 19:48:29 +0000 Subject: [PATCH 032/290] Fix typo --- src/Interpreters/AsynchronousMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 92ff4931481..2b6d552b179 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -274,7 +274,7 @@ void AsynchronousMetrics::update() new_values["TimeLoadGuest"] = data.stload.guest_time; new_values["TimeLoadGuestNice"] = data.stload.guest_nice_time; - new_values["Processess"] = data.stload.processes; + new_values["Processes"] = data.stload.processes; new_values["ProcessesRunning"] = data.stload.procs_running; new_values["ProcessesBlocked"] = data.stload.procs_blocked; } From 8199b45e95b169e86adea0ab806b464c120c941f Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 15 May 2021 13:10:19 +0300 Subject: [PATCH 033/290] Fix style Fifth try v2.0 Fifth try v2.1 Fifth try v2.2 Fifth try v2.3 Fifth try v2.4 Fifth try v2.5 Fifth try v2.6 Fifth try v2.7 Fifth try v2.8 Fifth try v2.9 Fifth try v2.10 Fifth try v2.11 Fifth try v2.12 Fifth try v2.13 Fifth try v2.14 Fifth try v2.15 Fifth try v2.16 Fifth try v2.17 Fifth try v2.18 Fifth try v2.19 Fifth try v2.20 Fifth try v2.21 Fifth try v2.22 Fifth try v2.23 Fifth try v2.24 Fifth try v2.25 Fifth try v2.26 Fifth try v2.27 Fifth try v2.28 Add ranges Add ranges try v1.1 Add ranges try v1.2 Add ranges try v1.3 Add ranges try v1.4 Add ranges try v1.5 Add ranges try v1.6 Add ranges try v1.7 Add ranges try v1.8 Add ranges try v1.9 Add ranges try v1.10 Add ranges try v1.11 Add ranges try v1.12 Add ranges try v1.13 Add ranges try v1.14 Add ranges try v1.15 Add ranges try v1.16 Add ranges try v1.17 Add ranges try v1.18 Add ranges try v1.19 Add ranges try v1.20 Add ranges try v1.21 Add ranges try v1.22 Add ranges try v1.23 Add ranges try v1.24 Add ranges try v1.25 Add ranges try v1.26 Add ranges try v1.27 Add ranges try v1.28 Add ranges try v1.29 Add ranges try v1.30 Add ranges try v1.31 Add ranges try v1.32 Add ranges try v1.33 Add ranges try v1.34 Add ranges try v1.35 Add ranges try v1.36 Add ranges try v1.37 Add ranges try v1.38 Add ranges try v1.39 Add ranges try v1.40 Add ranges try v1.41 Add ranges try v1.42 Add ranges try v1.43 Add ranges try v1.44 Add ranges try v1.45 Add ranges try v1.46 Add ranges try v1.47 Leftover comment Try wildcard Try wildcard v1.1 Try wildcard v1.2 Try wildcard v1.3 New functions New functions 1.1 New functions 1.2 New functions 1.3 New functions 1.4 New functions 1.5 New functions 1.6 New functions 1.7 New functions 1.8 New functions 1.9 New functions 1.10 New functions 1.11 New functions 1.12 New functions 1.13 New functions 1.14 New functions 1.15 New functions 1.16 Final steps Final steps v1.1 Final steps v1.2 --- src/Functions/DummyJSONParser.h | 2 + src/Functions/FunctionSQLJSON.cpp | 6 +- src/Functions/FunctionSQLJSON.h | 145 +++++++++++++----- src/Functions/FunctionsJSON.h | 2 + src/Functions/JSONPath/ASTs/ASTJSONPath.h | 2 - .../JSONPath/ASTs/ASTJSONPathQuery.h | 2 - .../JSONPath/ASTs/ASTJSONPathRange.h | 30 ++++ src/Functions/JSONPath/ASTs/CMakeLists.txt | 2 +- src/Functions/JSONPath/CMakeLists.txt | 2 +- .../JSONPath/Generators/CMakeLists.txt | 2 +- .../JSONPath/Generators/GeneratorJSONPath.h | 65 ++++---- .../JSONPath/Generators/IGenerator.h | 3 +- src/Functions/JSONPath/Generators/IVisitor.h | 5 + .../Generators/VisitorJSONPathMemberAccess.h | 27 +++- .../Generators/VisitorJSONPathRange.h | 93 +++++++++++ .../JSONPath/Generators/VisitorStatus.h | 3 +- src/Functions/JSONPath/Parsers/CMakeLists.txt | 2 +- .../JSONPath/Parsers/ParserJSONPath.cpp | 3 - .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 20 ++- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 98 ++++++++++++ .../JSONPath/Parsers/ParserJSONPathRange.h | 19 +++ src/Functions/RapidJSONParser.h | 3 +- src/Functions/SimdJSONParser.h | 62 ++------ 23 files changed, 445 insertions(+), 153 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathRange.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathRange.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRange.h diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index a71c90e4a19..3010347e4c4 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -39,6 +39,8 @@ struct DummyJSONParser std::string_view getString() const { return {}; } Array getArray() const { return {}; } Object getObject() const { return {}; } + + Element getElement() {return {}; } }; /// References an array in a JSON document. diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp index ddcca12835f..7d558dd0950 100644 --- a/src/Functions/FunctionSQLJSON.cpp +++ b/src/Functions/FunctionSQLJSON.cpp @@ -9,11 +9,11 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } - void registerFunctionsSQLJSON(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 24749099e57..1fc6986256d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,16 +1,12 @@ #pragma once #include -#include -#include -#include -#include #include -#include -#include -#include +#include +#include #include #include +#include #include #include #include @@ -21,12 +17,7 @@ #include #include #include -#include -#include -#include -#include #include -//#include #include #if !defined(ARCADIA_BUILD) @@ -37,11 +28,11 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; +extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -141,6 +132,7 @@ public: /// Parse JSON for every row Impl impl; + for (const auto i : ext::range(0, input_rows_count)) { std::string_view json{ @@ -205,45 +197,58 @@ private: const Context & context; }; -struct NameSQLJSONTest +struct NameJSONExists { - static constexpr auto name{"SQLJSONTest"}; + static constexpr auto name{"JSON_EXISTS"}; }; -struct NameSQLJSONMemberAccess +struct NameJSONValue { - static constexpr auto name{"SQLJSONMemberAccess"}; + static constexpr auto name{"JSON_VALUE"}; +}; + +struct NameJSONQuery +{ + static constexpr auto name{"JSON_QUERY"}; }; -/** - * Function to test logic before function calling, will be removed in final PR - * @tparam JSONParser parser - */ template -class SQLJSONTestImpl +class JSONExistsImpl { public: using Element = typename JSONParser::Element; - static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element &, ASTPtr &) + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) { - String str = "I am working:-)"; - ColumnString & col_str = assert_cast(dest); - col_str.insertData(str.data(), str.size()); + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) + { + if (status == VisitorStatus::Ok) { + break; + } + current_element = root; + } + + /// insert result, status can be either Ok (if we found the item) + /// or Exhausted (if we never found the item) + ColumnUInt8 & col_bool = assert_cast(dest); + if (status == VisitorStatus::Ok) { + col_bool.insert(0); + } else { + col_bool.insert(1); + } return true; } }; -/** - * Function to test jsonpath member access, will be removed in final PR - * @tparam JSONParser parser - */ template -class SQLJSONMemberAccessImpl +class JSONValueImpl { public: using Element = typename JSONParser::Element; @@ -257,18 +262,74 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - while ((status = generator_json_path.getNextItem(current_element)) == VisitorStatus::Ok) + Element res; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - /// No-op + if (status == VisitorStatus::Ok) { + if (!(current_element.isArray() || current_element.isObject())) { + break; + } + } else if (status == VisitorStatus::Error) { + /// ON ERROR + } + current_element = root; } - if (status == VisitorStatus::Error) + + if (status == VisitorStatus::Exhausted) { + return false; + } + + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out << current_element.getElement(); + auto output_str = out.str(); + ColumnString & col_str = assert_cast(dest); + col_str.insertData(output_str.data(), output_str.size()); + return true; + } +}; + +/** + * Function to test jsonpath member access, will be removed in final PR + * @tparam JSONParser parser + */ +template +class JSONQueryImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) + { + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + /// Create json array of results: [res1, res2, ...] + out << "["; + bool success = false; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { + if (status == VisitorStatus::Ok) { + if (success) { + out << ", "; + } + success = true; + out << current_element.getElement(); + } else if (status == VisitorStatus::Error) { + /// ON ERROR + } + current_element = root; + } + out << "]"; + if (!success) { return false; } ColumnString & col_str = assert_cast(dest); - std::stringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - ostr << current_element.getElement(); - auto output_str = ostr.str(); + auto output_str = out.str(); col_str.insertData(output_str.data(), output_str.size()); return true; } diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 581cc2015aa..f066bb1029a 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -80,6 +80,8 @@ public: const ColumnString::Chars & chars = col_json_string->getChars(); const ColumnString::Offsets & offsets = col_json_string->getOffsets(); + size_t num_index_arguments = Impl::getNumberOfIndexArguments(arguments); + std::vector moves = prepareMoves(Name::name, arguments, 1, num_index_arguments); /// Preallocate memory in parser if necessary. JSONParser parser; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h index cd73cd14257..092fe16bd9e 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPath.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -10,13 +10,11 @@ class ASTJSONPath : public IAST public: String getID(char) const override { - std::cerr << "in ASTJSONPath: getID\n"; return "ASTJSONPath"; } ASTPtr clone() const override { - std::cerr << "in " << "ASTJSONPath" << ": clone\n"; return std::make_shared(*this); } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h index 6b952c2519d..8da8a7baafb 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -9,13 +9,11 @@ class ASTJSONPathQuery : public IAST public: String getID(char) const override { - std::cerr << "in ASTJSONPathQuery: getID\n"; return "ASTJSONPathQuery"; } ASTPtr clone() const override { - std::cerr << "in " << getID(' ') << ": clone\n"; return std::make_shared(*this); } }; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h new file mode 100644 index 00000000000..21af3cff363 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ASTJSONPathRange : public IAST +{ +public: + String getID(char) const override + { + return "ASTJSONPathRange"; + } + + ASTPtr clone() const override + { + return std::make_shared(*this); + } + +public: + /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) + /// Range is represented as + /// Single index is represented as + std::vector> ranges; + bool is_star = false; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/CMakeLists.txt b/src/Functions/JSONPath/ASTs/CMakeLists.txt index c671dbbc001..ef56e3b0072 100644 --- a/src/Functions/JSONPath/ASTs/CMakeLists.txt +++ b/src/Functions/JSONPath/ASTs/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_asts PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_asts PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index 8a46909f555..8e65f7c8c6d 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -5,4 +5,4 @@ add_subdirectory(Generators) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) add_subdirectory(Parsers) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) \ No newline at end of file +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt index 0d1a289e8b4..76a116132fd 100644 --- a/src/Functions/JSONPath/Generators/CMakeLists.txt +++ b/src/Functions/JSONPath/Generators/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index dd4354a4613..68ea5a2a3c5 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -1,11 +1,10 @@ #include #include +#include #include #include -#include - namespace DB { @@ -26,20 +25,23 @@ public: throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); } const auto * query = path->jsonpath_query; - if (!path || !query) - { - throw Exception("Something went terribly wrong", ErrorCodes::LOGICAL_ERROR); - } for (auto child_ast : query->children) { if (child_ast->getID() == "ASTJSONPathMemberAccess") { - auto member_access_generator = std::make_shared>(child_ast); - if (member_access_generator) { - visitors.push_back(member_access_generator); + auto member_access_visitor = std::make_shared>(child_ast); + if (member_access_visitor) { + visitors.push_back(member_access_visitor); } else { - throw Exception("member_access_generator could not be nullptr", ErrorCodes::LOGICAL_ERROR); + throw Exception("member_access_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); + } + } else if (child_ast->getID() == "ASTJSONPathRange") { + auto range_visitor = std::make_shared>(child_ast); + if (range_visitor) { + visitors.push_back(range_visitor); + } else { + throw Exception("range_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); } } } @@ -54,39 +56,42 @@ public: */ VisitorStatus getNextItem(typename JSONParser::Element & element) override { - if (visitors[current_visitor]->isExhausted()) { - if (!backtrace()) { + while (true) { + auto root = element; + if (current_visitor < 0) { return VisitorStatus::Exhausted; } - } - /// Apply all non-exhausted visitors - for (int i = 0; i < current_visitor; ++i) { - VisitorStatus status = visitors[i]->apply(element); - /// on fail return immediately - if (status == VisitorStatus::Error) { + for (int i = 0; i < current_visitor; ++i) { + visitors[i]->apply(root); + } + + VisitorStatus status = VisitorStatus::Error; + for (size_t i = current_visitor; i < visitors.size(); ++i) { + status = visitors[i]->visit(root); + current_visitor = i; + if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { + break; + } + } + updateVisitorsForNextRun(); + + if (status != VisitorStatus::Ignore) { + element = root; return status; } } - - /// Visit newly initialized (for the first time or through reinitialize) visitors - for (size_t i = current_visitor; i < visitors.size(); ++i) { - VisitorStatus status = visitors[i]->visit(element); - current_visitor = i; - /// on fail return immediately - if (status == VisitorStatus::Error) { - return status; - } - } - return VisitorStatus::Ok; } private: - bool backtrace() { + bool updateVisitorsForNextRun() { while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { visitors[current_visitor]->reinitialize(); current_visitor--; } + if (current_visitor >= 0) { + visitors[current_visitor]->updateState(); + } return current_visitor >= 0; } diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h index 31d9e167f24..18c0ac7da67 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -16,8 +16,7 @@ public: virtual const char * getName() const = 0; /** - * Used to yield next element in JSONPath query. Does so by recursively calling getNextItem - * on its children Generators one by one. + * Used to yield next non-ignored element describes by JSONPath query. * * @param element to be extracted into * @return true if generator is not exhausted diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index fdd254478a5..78c1efe64fc 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -7,6 +7,9 @@ namespace DB { template class IVisitor { public: + + virtual const char * getName() const = 0; + /** * Applies this visitor to document and mutates its state * @param element simdjson element @@ -24,6 +27,8 @@ public: */ virtual void reinitialize() = 0; + virtual void updateState() = 0; + bool isExhausted() { return is_exhausted; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index 50b814eeaeb..cad36e40e4d 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -10,28 +10,39 @@ class VisitorJSONPathMemberAccess : public IVisitor public: VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + const char * getName() const override { return "VisitorJSONPathMemberAccess"; } + VisitorStatus apply(typename JSONParser::Element & element) const override { const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - bool result_ok = element.getObject().find(std::string_view(member_access->member_name), result); - if (result_ok) - { - element = result; - return VisitorStatus::Ok; - } - return VisitorStatus::Error; + element.getObject().find(std::string_view(member_access->member_name), result); + element = result; + return VisitorStatus::Ok; } VisitorStatus visit(typename JSONParser::Element & element) override { + if (!element.isObject()) { + this->setExhausted(true); + return VisitorStatus::Error; + } + const auto * member_access = member_access_ptr->as(); + typename JSONParser::Element result; + if (!element.getObject().find(std::string_view(member_access->member_name), result)) { + this->setExhausted(true); + return VisitorStatus::Error; + } + apply(element); this->setExhausted(true); - return apply(element); + return VisitorStatus::Ok; } void reinitialize() override { this->setExhausted(false); } + void updateState() override {} + private: ASTPtr member_access_ptr; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h new file mode 100644 index 00000000000..0858d9b70da --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -0,0 +1,93 @@ +#include +#include +#include + +namespace DB { + +template +class VisitorJSONPathRange : public IVisitor +{ +public: + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) { + const auto * range = range_ptr->as(); + current_range = 0; + if (range->is_star) { + current_index = 0; + } else { + current_index = range->ranges[current_range].first; + } + } + + const char * getName() const override { return "VisitorJSONPathRange"; } + + VisitorStatus apply(typename JSONParser::Element & element) const override { + typename JSONParser::Element result; + typename JSONParser::Array array = element.getArray(); + if (current_index >= array.size()) { + return VisitorStatus::Error; + } + result = array[current_index]; + element = result; + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + if (!element.isArray()) { + this->setExhausted(true); + return VisitorStatus::Error; + } + + const auto * range = range_ptr->as(); + VisitorStatus status; + if (current_index < element.getArray().size()) { + apply(element); + status = VisitorStatus::Ok; + } else if (!range->is_star) { + status = VisitorStatus::Ignore; + } else { + status = VisitorStatus::Ignore; + this->setExhausted(true); + } + + if (!range->is_star) { + if (current_index + 1 == range->ranges[current_range].second) { + if (current_range + 1 == range->ranges.size()) { + this->setExhausted(true); + } + } + } + + return status; + } + + void reinitialize() override { + const auto * range = range_ptr->as(); + current_range = 0; + if (range->is_star) { + current_index = 0; + } else { + current_index = range->ranges[current_range].first; + } + this->setExhausted(false); + } + + void updateState() override { + const auto * range = range_ptr->as(); + current_index++; + if (range->is_star) { + return; + } + if (current_index == range->ranges[current_range].second) { + current_range++; + current_index = range->ranges[current_range].first; + } + } + +private: + ASTPtr range_ptr; + size_t current_range; + UInt32 current_index; +}; + +} // namespace diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h index 51d795efbf7..17b424a3bf6 100644 --- a/src/Functions/JSONPath/Generators/VisitorStatus.h +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -5,7 +5,8 @@ namespace DB { enum VisitorStatus { Ok, Exhausted, - Error + Error, + Ignore }; } diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt index f2f94298576..ecabe5cc13b 100644 --- a/src/Functions/JSONPath/Parsers/CMakeLists.txt +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp index bf62f44fade..b65de621f9a 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -1,9 +1,6 @@ #include - #include - #include - #include namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index c0831780fc4..10cf6f2915c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,7 +1,6 @@ #include - #include - +#include #include namespace DB @@ -18,6 +17,7 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect { query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; + ParserJSONPathRange parser_jsonpath_range; if (pos->type != TokenType::DollarSign) { return false; @@ -25,15 +25,19 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect ++pos; bool res = false; - ASTPtr member_access; - while (parser_jsonpath_member_access.parse(pos, member_access, expected)) + ASTPtr subquery; + while (parser_jsonpath_member_access.parse(pos, subquery, expected) || + parser_jsonpath_range.parse(pos, subquery, expected)) { - query->children.push_back(member_access); - member_access = nullptr; + if (subquery) + { + query->children.push_back(subquery); + subquery = nullptr; + } res = true; } - /// true in case of at least one success - return res; + /// if we had at least one success and no fails + return res && pos->type == TokenType::EndOfStream; } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp new file mode 100644 index 00000000000..4f4a87f15ce --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -0,0 +1,98 @@ +#include +#include +#include + +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; +} +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathQuery + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto range = std::make_shared(); + node = range; + + if (pos->type != TokenType::OpeningSquareBracket) { + return false; + } + ++pos; + + while (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) + { + return false; + } + if (pos->type == TokenType::Asterisk) { + if (range->is_star) { + throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; + } + range->is_star = true; + ++pos; + continue; + } + + std::pair range_indices; + ParserNumber number_p; + ASTPtr number_ptr; + if (!number_p.parse(pos, number_ptr, expected)) + { + return false; + } + range_indices.first = number_ptr->as()->value.get(); + + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) { + /// Single index case + range_indices.second = range_indices.first + 1; + } else if (pos->type == TokenType::BareWord) { + /// Range case + ParserIdentifier name_p; + ASTPtr word; + if (!name_p.parse(pos, word, expected)) { + return false; + } + String to_identifier; + if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") { + return false; + } + if (!number_p.parse(pos, number_ptr, expected)) + { + return false; + } + range_indices.second = number_ptr->as()->value.get(); + } else { + return false; + } + + if (range_indices.first >= range_indices.second) { + throw Exception{ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", + range_indices.first, range_indices.second}; + } + + range->ranges.push_back(std::move(range_indices)); + if (pos->type != TokenType::ClosingSquareBracket) { + ++pos; + } + } + ++pos; + + /// We cant have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true + return !range->ranges.empty() != range->is_star; +} + +} // namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h new file mode 100644 index 00000000000..95708e5e7b8 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h @@ -0,0 +1,19 @@ +#pragma once + +#include + + +namespace DB +{ + +class ParserJSONPathRange : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathRange"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathRange() = default; +}; + +} diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index 5604a8c9fe0..992480d64f7 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -12,6 +12,7 @@ namespace DB { + /// This class can be used as an argument for the template class FunctionJSON. /// It provides ability to parse JSONs using rapidjson library. struct RapidJSONParser @@ -44,8 +45,6 @@ struct RapidJSONParser Array getArray() const; Object getObject() const; - ALWAYS_INLINE rapidjson::Value * getDom() const { return nullptr; } - private: const rapidjson::Value * ptr = nullptr; }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index 2dd952d920f..b9df8b142e3 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -5,10 +5,10 @@ #endif #if USE_SIMDJSON -# include +# include # include # include -# include +# include namespace DB @@ -30,8 +30,8 @@ struct SimdJSONParser class Element { public: - ALWAYS_INLINE Element() { } - ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) { } + ALWAYS_INLINE Element() {} + ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; } ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; } @@ -63,35 +63,21 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) { } + ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) {} ALWAYS_INLINE Element operator*() const { return *it; } - ALWAYS_INLINE Iterator & operator++() - { - ++it; - return *this; - } - ALWAYS_INLINE Iterator operator++(int) - { - auto res = *this; - ++it; - return res; - } + ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } + ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } - private: simdjson::dom::array::iterator it; }; - ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) { } + ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) {} ALWAYS_INLINE Iterator begin() const { return array.begin(); } ALWAYS_INLINE Iterator end() const { return array.end(); } ALWAYS_INLINE size_t size() const { return array.size(); } - ALWAYS_INLINE Element operator[](size_t index) const - { - assert(index < size()); - return array.at(index).first; - } + ALWAYS_INLINE Element operator[](size_t index) const { assert(index < size()); return array.at(index).first; } private: simdjson::dom::array array; @@ -106,31 +92,17 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) { } - ALWAYS_INLINE KeyValuePair operator*() const - { - const auto & res = *it; - return {res.key, res.value}; - } - ALWAYS_INLINE Iterator & operator++() - { - ++it; - return *this; - } - ALWAYS_INLINE Iterator operator++(int) - { - auto res = *this; - ++it; - return res; - } + ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) {} + ALWAYS_INLINE KeyValuePair operator*() const { const auto & res = *it; return {res.key, res.value}; } + ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } + ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } - private: simdjson::dom::object::iterator it; }; - ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) { } + ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) {} ALWAYS_INLINE Iterator begin() const { return object.begin(); } ALWAYS_INLINE Iterator end() const { return object.end(); } ALWAYS_INLINE size_t size() const { return object.size(); } @@ -156,8 +128,6 @@ struct SimdJSONParser return {res.key, res.value}; } - ALWAYS_INLINE simdjson::dom::object getDom() const { return object; } - private: simdjson::dom::object object; }; @@ -177,8 +147,8 @@ struct SimdJSONParser void reserve(size_t max_size) { if (parser.allocate(max_size) != simdjson::error_code::SUCCESS) - throw Exception{ - "Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", ErrorCodes::CANNOT_ALLOCATE_MEMORY}; + throw Exception{"Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", + ErrorCodes::CANNOT_ALLOCATE_MEMORY}; } private: From b9b28c3b7f2d8f0a4f01d8125c8405a9508ff324 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 15:34:39 +0300 Subject: [PATCH 034/290] Fix include --- src/Functions/FunctionSQLJSON.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 1fc6986256d..b3f9db87ac9 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include From 2035c0cd6aa0338ecabb81fa8d5c3c5d260a62ea Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 16:15:58 +0300 Subject: [PATCH 035/290] Change to IFunction --- src/Functions/FunctionsJSON.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index f066bb1029a..1032ab15ff9 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include From 819ad748a4016b35ad682fae09599f1e6f8282b1 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 16:53:55 +0300 Subject: [PATCH 036/290] Fix IFunction --- src/Functions/FunctionSQLJSON.h | 2 +- src/Functions/FunctionsJSON.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b3f9db87ac9..cee7390d67d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 1032ab15ff9..f066bb1029a 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include From 4b85c8e31f1be21f9b21e89c13ec0138d6ed6aab Mon Sep 17 00:00:00 2001 From: elevankoff Date: Wed, 2 Jun 2021 08:00:10 +0000 Subject: [PATCH 037/290] Small style changes --- src/Common/DiskStatisticsOS.cpp | 2 +- src/Common/DiskStatisticsOS.h | 2 +- src/Common/MemoryInfoOS.cpp | 2 +- src/Common/MemoryInfoOS.h | 2 +- src/Common/ProcessorStatisticsOS.cpp | 2 +- src/Common/ProcessorStatisticsOS.h | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp index 0485d129ecc..69f15b30a9e 100644 --- a/src/Common/DiskStatisticsOS.cpp +++ b/src/Common/DiskStatisticsOS.cpp @@ -61,7 +61,7 @@ DiskStatisticsOS::Data DiskStatisticsOS::get() return data; } -String DiskStatisticsOS::readNextFilesystem(ReadBuffer& mounts_in) +String DiskStatisticsOS::readNextFilesystem(ReadBuffer & mounts_in) { String filesystem, unused; diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h index 05f53a421d2..d4ec2417924 100644 --- a/src/Common/DiskStatisticsOS.h +++ b/src/Common/DiskStatisticsOS.h @@ -29,7 +29,7 @@ public: Data get(); private: - String readNextFilesystem(ReadBuffer& mounts_in); + String readNextFilesystem(ReadBuffer & mounts_in); }; } diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index 8cf2a0b44f4..301fcb6ad15 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -63,7 +63,7 @@ MemoryInfoOS::Data MemoryInfoOS::get() return data; } -std::pair MemoryInfoOS::readField(ReadBuffer& meminfo_in) +std::pair MemoryInfoOS::readField(ReadBuffer & meminfo_in) { String key; uint64_t val; diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index ae630e4ee70..63cda5b5c37 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -39,7 +39,7 @@ public: Data get(); private: - std::pair readField(ReadBuffer& meminfo_in); + std::pair readField(ReadBuffer & meminfo_in); }; } diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp index 69bce5f5b51..9b43fa428a9 100644 --- a/src/Common/ProcessorStatisticsOS.cpp +++ b/src/Common/ProcessorStatisticsOS.cpp @@ -110,7 +110,7 @@ void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) last_proc_time = cur_proc_time; } -void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload) +void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad & stload) { ReadBufferFromFile procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h index 70edfceb2ca..10b6d050b8c 100644 --- a/src/Common/ProcessorStatisticsOS.h +++ b/src/Common/ProcessorStatisticsOS.h @@ -75,10 +75,10 @@ private: }; void readLoadavg(ProcLoadavg & loadavg); - void calcStLoad(ProcStLoad & stload); - void readFreq(ProcFreq & freq); + void calcStLoad(ProcStLoad & stload); + void readFreq(ProcFreq & freq); - void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad& stload); + void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad & stload); private: std::time_t last_stload_call_time; From b00c3d8f5a911bd6ef3af9a1af50ae0794041734 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Thu, 3 Jun 2021 20:47:53 +0300 Subject: [PATCH 038/290] Fix build (probably) --- src/Functions/DummyJSONParser.h | 3 ++- src/Functions/FunctionSQLJSON.h | 15 ++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 3010347e4c4..21b4ebef150 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -40,7 +40,8 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - Element getElement() {return {}; } + Element getElement() { return {}; } + std::ostream & operator<<(std::ostream & os) { return os; } }; /// References an array in a JSON document. diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index cee7390d67d..cf33cf804ce 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -157,11 +157,11 @@ private: }; template typename Impl> -class FunctionSQLJSON : public IFunction +class FunctionSQLJSON : public IFunction, WithConstContext { public: - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - FunctionSQLJSON(const Context & context_) : context(context_) { } + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } + FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) {} static constexpr auto name = Name::name; String getName() const override { return Name::name; } @@ -182,7 +182,7 @@ public: /// 3. Parser(Tokens, ASTPtr) -> complete AST /// 4. Execute functions, call interpreter for each json (in function) #if USE_SIMDJSON - if (context.getSettingsRef().allow_simdjson) + if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif @@ -192,9 +192,6 @@ public: return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif } - -private: - const Context & context; }; struct NameJSONExists @@ -239,9 +236,9 @@ public: /// or Exhausted (if we never found the item) ColumnUInt8 & col_bool = assert_cast(dest); if (status == VisitorStatus::Ok) { - col_bool.insert(0); - } else { col_bool.insert(1); + } else { + col_bool.insert(0); } return true; } From 04daa9b5ed79c358dee587afe009bf8229fe79ef Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Fri, 4 Jun 2021 00:17:51 +0300 Subject: [PATCH 039/290] Second try to fix build --- src/Functions/DummyJSONParser.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 21b4ebef150..e74ab57ded1 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -41,7 +41,7 @@ struct DummyJSONParser Object getObject() const { return {}; } Element getElement() { return {}; } - std::ostream & operator<<(std::ostream & os) { return os; } + std::ostream & operator<<(std::ostream & os) const { return os; } }; /// References an array in a JSON document. @@ -100,4 +100,9 @@ struct DummyJSONParser #endif }; +ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +{ + return out; +} + } From bef8b4c2c6b205d639b69571e039fe82f5e9b713 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Fri, 4 Jun 2021 00:56:06 +0300 Subject: [PATCH 040/290] Add 'inline' to DummyParser's << operator; don't throw exceptioin on USE_RAPIDJSON --- src/Functions/DummyJSONParser.h | 3 +-- src/Functions/FunctionSQLJSON.h | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index e74ab57ded1..01fdab1abb6 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -41,7 +41,6 @@ struct DummyJSONParser Object getObject() const { return {}; } Element getElement() { return {}; } - std::ostream & operator<<(std::ostream & os) const { return os; } }; /// References an array in a JSON document. @@ -100,7 +99,7 @@ struct DummyJSONParser #endif }; -ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index cf33cf804ce..2d4f51e63c4 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -152,8 +152,6 @@ public: return to; } }; - -private: }; template typename Impl> @@ -184,10 +182,6 @@ public: #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); -#endif - -#if USE_RAPIDJSON - throw Exception{"RapidJSON is not supported :(", ErrorCodes::BAD_ARGUMENTS}; #else return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif From 14687eccf7fb29efd8cf92b1354f6bca58c075c6 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 00:04:53 +0300 Subject: [PATCH 041/290] Fix dollar token for lexer --- src/Functions/ya.make | 5 +++++ src/Parsers/Lexer.cpp | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 2a541369ff4..76fa893791e 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -42,6 +42,7 @@ SRCS( FunctionFile.cpp FunctionHelpers.cpp FunctionJoinGet.cpp + FunctionSQLJSON.cpp FunctionsAES.cpp FunctionsCoding.cpp FunctionsConversion.cpp @@ -74,6 +75,10 @@ SRCS( GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp GeoHash.cpp IFunction.cpp + JSONPath/Parsers/ParserJSONPath.cpp + JSONPath/Parsers/ParserJSONPathMemberAccess.cpp + JSONPath/Parsers/ParserJSONPathQuery.cpp + JSONPath/Parsers/ParserJSONPathRange.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index c3b3cf98a2e..3a6e7a26700 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -240,9 +240,6 @@ Token Lexer::nextTokenImpl() case '*': ++pos; return Token(TokenType::Asterisk, token_begin, pos); - case '$': - ++pos; - return Token(TokenType::DollarSign, token_begin, pos); case '/': /// division (/) or start of comment (//, /*) { ++pos; @@ -341,6 +338,9 @@ Token Lexer::nextTokenImpl() } default: + if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) { + return Token(TokenType::DollarSign, token_begin, ++pos); + } if (isWordCharASCII(*pos) || *pos == '$') { ++pos; From cdd13b5ab42a53e65fea32235943749f73a5ef8d Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 01:31:55 +0300 Subject: [PATCH 042/290] Style --- src/Functions/FunctionSQLJSON.h | 54 ++++++++++------ src/Functions/JSONPath/ASTs/ASTJSONPath.h | 12 +--- .../JSONPath/ASTs/ASTJSONPathMemberAccess.h | 10 +-- .../JSONPath/ASTs/ASTJSONPathQuery.h | 10 +-- .../JSONPath/ASTs/ASTJSONPathRange.h | 13 +--- .../JSONPath/Generators/GeneratorJSONPath.h | 2 +- .../JSONPath/Generators/IGenerator.h | 2 +- .../JSONPath/Generators/IGenerator_fwd.h | 6 +- src/Functions/JSONPath/Generators/IVisitor.h | 19 +++--- .../Generators/VisitorJSONPathMemberAccess.h | 17 ++--- .../Generators/VisitorJSONPathRange.h | 62 +++++++++++++------ .../JSONPath/Generators/VisitorStatus.h | 7 ++- .../JSONPath/Parsers/ParserJSONPath.cpp | 10 +-- .../JSONPath/Parsers/ParserJSONPath.h | 1 - .../Parsers/ParserJSONPathMemberAccess.h | 2 +- .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 13 ++-- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 44 ++++++++----- .../JSONPath/Parsers/ParserJSONPathRange.h | 1 - 18 files changed, 155 insertions(+), 130 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 2d4f51e63c4..5f478cebad8 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,14 +1,15 @@ #pragma once -#include +#include #include -#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -18,7 +19,6 @@ #include #include #include -#include #if !defined(ARCADIA_BUILD) # include "config_functions.h" @@ -28,11 +28,11 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_COLUMN; -extern const int ILLEGAL_TYPE_OF_ARGUMENT; -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; -extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -159,7 +159,7 @@ class FunctionSQLJSON : public IFunction, WithConstContext { public: static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } - FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) {} + FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) { } static constexpr auto name = Name::name; String getName() const override { return Name::name; } @@ -220,7 +220,8 @@ public: VisitorStatus status; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { + if (status == VisitorStatus::Ok) + { break; } current_element = root; @@ -229,9 +230,12 @@ public: /// insert result, status can be either Ok (if we found the item) /// or Exhausted (if we never found the item) ColumnUInt8 & col_bool = assert_cast(dest); - if (status == VisitorStatus::Ok) { + if (status == VisitorStatus::Ok) + { col_bool.insert(1); - } else { + } + else + { col_bool.insert(0); } return true; @@ -256,17 +260,22 @@ public: Element res; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { - if (!(current_element.isArray() || current_element.isObject())) { + if (status == VisitorStatus::Ok) + { + if (!(current_element.isArray() || current_element.isObject())) + { break; } - } else if (status == VisitorStatus::Error) { + } + else if (status == VisitorStatus::Error) + { /// ON ERROR } current_element = root; } - if (status == VisitorStatus::Exhausted) { + if (status == VisitorStatus::Exhausted) + { return false; } @@ -304,19 +313,24 @@ public: bool success = false; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { - if (success) { + if (status == VisitorStatus::Ok) + { + if (success) + { out << ", "; } success = true; out << current_element.getElement(); - } else if (status == VisitorStatus::Error) { + } + else if (status == VisitorStatus::Error) + { /// ON ERROR } current_element = root; } out << "]"; - if (!success) { + if (!success) + { return false; } ColumnString & col_str = assert_cast(dest); diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h index 092fe16bd9e..dfc117db846 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPath.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -1,22 +1,16 @@ #pragma once -#include #include +#include namespace DB { class ASTJSONPath : public IAST { public: - String getID(char) const override - { - return "ASTJSONPath"; - } + String getID(char) const override { return "ASTJSONPath"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } ASTJSONPathQuery * jsonpath_query; }; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h index 663859f566f..2c9482b665e 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h @@ -7,15 +7,9 @@ namespace DB class ASTJSONPathMemberAccess : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathMemberAccess"; - } + String getID(char) const override { return "ASTJSONPathMemberAccess"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } public: /// Member name to lookup in json document (in path: $.some_key.another_key. ...) diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h index 8da8a7baafb..ed2992777b2 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -7,15 +7,9 @@ namespace DB class ASTJSONPathQuery : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathQuery"; - } + String getID(char) const override { return "ASTJSONPathQuery"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } }; } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h index 21af3cff363..8a963d7fc6b 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -1,23 +1,16 @@ #pragma once -#include #include +#include namespace DB { - class ASTJSONPathRange : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathRange"; - } + String getID(char) const override { return "ASTJSONPathRange"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } public: /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 68ea5a2a3c5..6eea19cb516 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -100,4 +100,4 @@ private: VisitorList visitors; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h index 18c0ac7da67..d2cef9fe27b 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -26,4 +26,4 @@ public: virtual ~IGenerator() = default; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generators/IGenerator_fwd.h index 27c3976b95b..57ed04d0f6f 100644 --- a/src/Functions/JSONPath/Generators/IGenerator_fwd.h +++ b/src/Functions/JSONPath/Generators/IGenerator_fwd.h @@ -2,8 +2,8 @@ #include -namespace DB { - +namespace DB +{ template class IGenerator; @@ -13,4 +13,4 @@ using IVisitorPtr = std::shared_ptr>; template using VisitorList = std::vector>; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index 78c1efe64fc..d9917087cb0 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -2,12 +2,12 @@ #include -namespace DB { - +namespace DB +{ template -class IVisitor { +class IVisitor +{ public: - virtual const char * getName() const = 0; /** @@ -29,17 +29,14 @@ public: virtual void updateState() = 0; - bool isExhausted() { - return is_exhausted; - } + bool isExhausted() { return is_exhausted; } - void setExhausted(bool exhausted) { - is_exhausted = exhausted; - } + void setExhausted(bool exhausted) { is_exhausted = exhausted; } virtual ~IVisitor() = default; + private: bool is_exhausted = false; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index cad36e40e4d..10ee2a0c5d6 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -12,7 +12,8 @@ public: const char * getName() const override { return "VisitorJSONPathMemberAccess"; } - VisitorStatus apply(typename JSONParser::Element & element) const override { + VisitorStatus apply(typename JSONParser::Element & element) const override + { const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; element.getObject().find(std::string_view(member_access->member_name), result); @@ -22,13 +23,15 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { - if (!element.isObject()) { + if (!element.isObject()) + { this->setExhausted(true); return VisitorStatus::Error; } const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - if (!element.getObject().find(std::string_view(member_access->member_name), result)) { + if (!element.getObject().find(std::string_view(member_access->member_name), result)) + { this->setExhausted(true); return VisitorStatus::Error; } @@ -37,14 +40,12 @@ public: return VisitorStatus::Ok; } - void reinitialize() override { - this->setExhausted(false); - } + void reinitialize() override { this->setExhausted(false); } - void updateState() override {} + void updateState() override { } private: ASTPtr member_access_ptr; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index 0858d9b70da..d3313331593 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -2,28 +2,34 @@ #include #include -namespace DB { - +namespace DB +{ template class VisitorJSONPathRange : public IVisitor { public: - VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) { + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) + { const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) { + if (range->is_star) + { current_index = 0; - } else { + } + else + { current_index = range->ranges[current_range].first; } } const char * getName() const override { return "VisitorJSONPathRange"; } - VisitorStatus apply(typename JSONParser::Element & element) const override { + VisitorStatus apply(typename JSONParser::Element & element) const override + { typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); - if (current_index >= array.size()) { + if (current_index >= array.size()) + { return VisitorStatus::Error; } result = array[current_index]; @@ -33,26 +39,35 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { - if (!element.isArray()) { + if (!element.isArray()) + { this->setExhausted(true); return VisitorStatus::Error; } const auto * range = range_ptr->as(); VisitorStatus status; - if (current_index < element.getArray().size()) { + if (current_index < element.getArray().size()) + { apply(element); status = VisitorStatus::Ok; - } else if (!range->is_star) { + } + else if (!range->is_star) + { status = VisitorStatus::Ignore; - } else { + } + else + { status = VisitorStatus::Ignore; this->setExhausted(true); } - if (!range->is_star) { - if (current_index + 1 == range->ranges[current_range].second) { - if (current_range + 1 == range->ranges.size()) { + if (!range->is_star) + { + if (current_index + 1 == range->ranges[current_range].second) + { + if (current_range + 1 == range->ranges.size()) + { this->setExhausted(true); } } @@ -61,24 +76,31 @@ public: return status; } - void reinitialize() override { + void reinitialize() override + { const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) { + if (range->is_star) + { current_index = 0; - } else { + } + else + { current_index = range->ranges[current_range].first; } this->setExhausted(false); } - void updateState() override { + void updateState() override + { const auto * range = range_ptr->as(); current_index++; - if (range->is_star) { + if (range->is_star) + { return; } - if (current_index == range->ranges[current_range].second) { + if (current_index == range->ranges[current_range].second) + { current_range++; current_index = range->ranges[current_range].first; } diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h index 17b424a3bf6..96b2ea72f18 100644 --- a/src/Functions/JSONPath/Generators/VisitorStatus.h +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -1,8 +1,9 @@ #pragma once -namespace DB { - -enum VisitorStatus { +namespace DB +{ +enum VisitorStatus +{ Ok, Exhausted, Error, diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp index b65de621f9a..003e97af38b 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -1,11 +1,10 @@ +#include +#include #include #include -#include -#include namespace DB { - /** * Entry parser for JSONPath */ @@ -19,7 +18,8 @@ bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool res = parser_jsonpath_query.parse(pos, query, expected); - if (res) { + if (res) + { /// Set ASTJSONPathQuery of ASTJSONPath ast_jsonpath->set(ast_jsonpath->jsonpath_query, query); } @@ -28,4 +28,4 @@ bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return res; } -} // namespace DB +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.h b/src/Functions/JSONPath/Parsers/ParserJSONPath.h index 5defc76b515..7d2c2ad642c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.h @@ -5,7 +5,6 @@ namespace DB { - /** * Entry parser for JSONPath */ diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h index 49fda6f1ac8..000f20c8551 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -4,7 +4,7 @@ namespace DB { class ParserJSONPathMemberAccess : public IParserBase { - const char * getName() const override {return "ParserJSONPathMemberAccess";} + const char * getName() const override { return "ParserJSONPathMemberAccess"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 10cf6f2915c..0f171c0a82c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,7 +1,7 @@ -#include -#include -#include #include +#include +#include +#include namespace DB @@ -19,15 +19,16 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; - if (pos->type != TokenType::DollarSign) { + if (pos->type != TokenType::DollarSign) + { return false; } ++pos; bool res = false; ASTPtr subquery; - while (parser_jsonpath_member_access.parse(pos, subquery, expected) || - parser_jsonpath_range.parse(pos, subquery, expected)) + while (parser_jsonpath_member_access.parse(pos, subquery, expected) + || parser_jsonpath_range.parse(pos, subquery, expected)) { if (subquery) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 4f4a87f15ce..4027ec67ecb 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -28,18 +28,22 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte auto range = std::make_shared(); node = range; - if (pos->type != TokenType::OpeningSquareBracket) { + if (pos->type != TokenType::OpeningSquareBracket) + { return false; } ++pos; - while (pos->type != TokenType::ClosingSquareBracket) { + while (pos->type != TokenType::ClosingSquareBracket) + { if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) { return false; } - if (pos->type == TokenType::Asterisk) { - if (range->is_star) { + if (pos->type == TokenType::Asterisk) + { + if (range->is_star) + { throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; } range->is_star = true; @@ -56,18 +60,23 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } range_indices.first = number_ptr->as()->value.get(); - if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) { + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) + { /// Single index case range_indices.second = range_indices.first + 1; - } else if (pos->type == TokenType::BareWord) { + } + else if (pos->type == TokenType::BareWord) + { /// Range case ParserIdentifier name_p; ASTPtr word; - if (!name_p.parse(pos, word, expected)) { + if (!name_p.parse(pos, word, expected)) + { return false; } String to_identifier; - if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") { + if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") + { return false; } if (!number_p.parse(pos, number_ptr, expected)) @@ -75,17 +84,24 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return false; } range_indices.second = number_ptr->as()->value.get(); - } else { + } + else + { return false; } - if (range_indices.first >= range_indices.second) { - throw Exception{ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", - range_indices.first, range_indices.second}; + if (range_indices.first >= range_indices.second) + { + throw Exception{ + ErrorCodes::BAD_ARGUMENTS, + "Start of range must be greater than end of range, however {} >= {}", + range_indices.first, + range_indices.second}; } range->ranges.push_back(std::move(range_indices)); - if (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::ClosingSquareBracket) + { ++pos; } } @@ -95,4 +111,4 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return !range->ranges.empty() != range->is_star; } -} // namespace DB +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h index 95708e5e7b8..94db29577ab 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h @@ -5,7 +5,6 @@ namespace DB { - class ParserJSONPathRange : public IParserBase { private: From 3f469fe0cde1f843286e7aaaa1db62da29098a9c Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 15:13:46 +0300 Subject: [PATCH 043/290] Style --- src/Functions/FunctionSQLJSON.h | 8 +-- .../JSONPath/Generators/GeneratorJSONPath.h | 51 ++++++++++--------- .../Parsers/ParserJSONPathMemberAccess.cpp | 5 +- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 9 ++-- src/Parsers/Lexer.cpp | 3 +- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 5f478cebad8..67380a11235 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -56,18 +56,18 @@ public: const auto & first_column = arguments[0]; if (!isString(first_column.type)) { - throw Exception{ + throw Exception( "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } /// Check 2 argument: must be of type String (JSON) const auto & second_column = arguments[1]; if (!isString(second_column.type)) { - throw Exception{ + throw Exception( "JSONPath functions require 2 argument to be JSON of string, illegal type: " + second_column.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } /// If argument is successfully cast to (ColumnConst *) then it is quoted string diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 6eea19cb516..450a4f31dae 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -21,28 +21,22 @@ public: { query_ptr = query_ptr_; const auto * path = query_ptr->as(); - if (!path) { + if (!path) + { throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); } const auto * query = path->jsonpath_query; for (auto child_ast : query->children) { - if (child_ast->getID() == "ASTJSONPathMemberAccess") + if (typeid_cast(child_ast.get())) { - auto member_access_visitor = std::make_shared>(child_ast); - if (member_access_visitor) { - visitors.push_back(member_access_visitor); - } else { - throw Exception("member_access_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); - } - } else if (child_ast->getID() == "ASTJSONPathRange") { - auto range_visitor = std::make_shared>(child_ast); - if (range_visitor) { - visitors.push_back(range_visitor); - } else { - throw Exception("range_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); - } + visitors.push_back(std::make_shared>(child_ast)); + } + else if (child_ast->getID() == "ASTJSONPathRange") + { + + visitors.push_back(std::make_shared>(child_ast)); } } } @@ -56,27 +50,33 @@ public: */ VisitorStatus getNextItem(typename JSONParser::Element & element) override { - while (true) { + while (true) + { auto root = element; - if (current_visitor < 0) { + if (current_visitor < 0) + { return VisitorStatus::Exhausted; } - for (int i = 0; i < current_visitor; ++i) { + for (int i = 0; i < current_visitor; ++i) + { visitors[i]->apply(root); } VisitorStatus status = VisitorStatus::Error; - for (size_t i = current_visitor; i < visitors.size(); ++i) { + for (size_t i = current_visitor; i < visitors.size(); ++i) + { status = visitors[i]->visit(root); current_visitor = i; - if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { + if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) + { break; } } updateVisitorsForNextRun(); - if (status != VisitorStatus::Ignore) { + if (status != VisitorStatus::Ignore) + { element = root; return status; } @@ -84,12 +84,15 @@ public: } private: - bool updateVisitorsForNextRun() { - while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { + bool updateVisitorsForNextRun() + { + while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) + { visitors[current_visitor]->reinitialize(); current_visitor--; } - if (current_visitor >= 0) { + if (current_visitor >= 0) + { visitors[current_visitor]->updateState(); } return current_visitor >= 0; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 10ae128616b..0acb800dc1b 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -17,8 +17,6 @@ namespace DB */ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - auto member_access = std::make_shared(); - node = member_access; if (pos->type != TokenType::Dot) { return false; } @@ -27,12 +25,15 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & if (pos->type != TokenType::BareWord) { return false; } + ParserIdentifier name_p; ASTPtr member_name; if (!name_p.parse(pos, member_name, expected)) { return false; } + auto member_access = std::make_shared(); + node = member_access; if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { return false; } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 4027ec67ecb..25c5c76dd40 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -25,8 +25,6 @@ namespace ErrorCodes */ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - auto range = std::make_shared(); - node = range; if (pos->type != TokenType::OpeningSquareBracket) { @@ -34,6 +32,9 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } ++pos; + auto range = std::make_shared(); + node = range; + while (pos->type != TokenType::ClosingSquareBracket) { if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) @@ -107,8 +108,8 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } ++pos; - /// We cant have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true - return !range->ranges.empty() != range->is_star; + /// We can't have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true + return !range->ranges.empty() ^ range->is_star; } } diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 3a6e7a26700..4de72ebc2fd 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -338,7 +338,8 @@ Token Lexer::nextTokenImpl() } default: - if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) { + if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) + { return Token(TokenType::DollarSign, token_begin, ++pos); } if (isWordCharASCII(*pos) || *pos == '$') From 582cc3daa91ecccf8c2337ac5e1c306897c839ea Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 15:27:37 +0300 Subject: [PATCH 044/290] Make 1st arg always be const, fix style --- src/Functions/FunctionSQLJSON.cpp | 4 ---- src/Functions/FunctionSQLJSON.h | 24 +++++++++---------- .../JSONPath/Generators/GeneratorJSONPath.h | 2 ++ .../Generators/VisitorJSONPathRange.h | 4 +++- .../Parsers/ParserJSONPathMemberAccess.cpp | 23 +++++++++++------- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 6 +---- 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp index 7d558dd0950..a316d9de7ab 100644 --- a/src/Functions/FunctionSQLJSON.cpp +++ b/src/Functions/FunctionSQLJSON.cpp @@ -4,10 +4,6 @@ namespace DB { -namespace ErrorCodes -{ -extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} void registerFunctionsSQLJSON(FunctionFactory & factory) { diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 67380a11235..b13b03cb089 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -30,7 +30,6 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; } @@ -52,17 +51,24 @@ public: throw Exception{"JSONPath functions require at least 2 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; } - /// Check 1 argument: must be of type String (JSONPath) const auto & first_column = arguments[0]; + + /// Check 1 argument: must be of type String (JSONPath) if (!isString(first_column.type)) { throw Exception( "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + /// Check 1 argument: must be const (JSONPath) + if (!isColumnConst(*first_column.column)) + { + throw Exception("1 argument (JSONPath) must be const", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + const auto & second_column = arguments[1]; /// Check 2 argument: must be of type String (JSON) - const auto & second_column = arguments[1]; if (!isString(second_column.type)) { throw Exception( @@ -78,21 +84,15 @@ public: /// Example: /// SomeFunction(database.table.column) - /// Check 1 argument: must be const String (JSONPath) const ColumnPtr & arg_jsonpath = first_column.column; const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); - if (!arg_jsonpath_const) - { - throw Exception{"JSONPath argument must be of type const String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - /// Retrieve data from 1 argument const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); + if (!arg_jsonpath_string) { throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; } - /// Check 2 argument: must be const or non-const String (JSON) const ColumnPtr & arg_json = second_column.column; const auto * col_json_const = typeid_cast(arg_json.get()); const auto * col_json_string @@ -166,6 +166,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -182,9 +183,8 @@ public: #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); -#else - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); } }; diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 450a4f31dae..39f385fafc2 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index d3313331593..601c5ea80b9 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -112,4 +114,4 @@ private: UInt32 current_index; }; -} // namespace +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 0acb800dc1b..841e3fc4adc 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -1,13 +1,14 @@ -#include -#include +#pragma once + +#include +#include -#include -#include #include +#include +#include namespace DB { - /** * * @param pos token iterator @@ -17,24 +18,28 @@ namespace DB */ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (pos->type != TokenType::Dot) { + if (pos->type != TokenType::Dot) + { return false; } ++pos; - if (pos->type != TokenType::BareWord) { + if (pos->type != TokenType::BareWord) + { return false; } ParserIdentifier name_p; ASTPtr member_name; - if (!name_p.parse(pos, member_name, expected)) { + if (!name_p.parse(pos, member_name, expected)) + { return false; } auto member_access = std::make_shared(); node = member_access; - if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { + if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) + { return false; } return true; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 25c5c76dd40..f0d1af47b15 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -10,10 +10,6 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; } /** @@ -45,7 +41,7 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte { if (range->is_star) { - throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; + throw Exception("Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS); } range->is_star = true; ++pos; From dfba5a479b5eaa1e0f972be7977b463ee1347549 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sun, 6 Jun 2021 14:50:10 +0300 Subject: [PATCH 045/290] Remove pragma once --- src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 841e3fc4adc..85b43217867 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -1,5 +1,3 @@ -#pragma once - #include #include From bad7d56aaa69feb74982159b343483e233a61629 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sun, 6 Jun 2021 16:00:46 +0300 Subject: [PATCH 046/290] =?UTF-8?q?Style=20again=20=F0=9F=98=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JSONPath/Generators/VisitorJSONPathMemberAccess.h | 2 ++ src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h | 2 ++ src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index 10ee2a0c5d6..fd83c478227 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h index 000f20c8551..b28bf37d5ef 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -1,3 +1,5 @@ +#pragma once + #include namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index f0d1af47b15..3da0d508b27 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -89,11 +89,11 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte if (range_indices.first >= range_indices.second) { - throw Exception{ + throw Exception( ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", range_indices.first, - range_indices.second}; + range_indices.second); } range->ranges.push_back(std::move(range_indices)); From ba37ebb1335a36bd5b708870de9c02e10e29e500 Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Mon, 7 Jun 2021 23:54:44 +0300 Subject: [PATCH 047/290] Added functional tests --- .../01889_sql_json_functions.reference | 33 +++++++++++++++++ .../0_stateless/01889_sql_json_functions.sql | 35 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/queries/0_stateless/01889_sql_json_functions.reference create mode 100644 tests/queries/0_stateless/01889_sql_json_functions.sql diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference new file mode 100644 index 00000000000..b8dc5d8e416 --- /dev/null +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -0,0 +1,33 @@ +--JSON_VALUE-- +1 +1.2 +true +"world" +null + + + + +--JSON_QUERY-- +[1] +[1.2] +[true] +["world"] +[null] +[["world","world2"]] +[{"world":"!"}] + + +--JSON_EXISTS-- +1 +1 +0 +1 +0 +0 +1 +1 +0 +1 +0 +1 \ No newline at end of file diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql new file mode 100644 index 00000000000..6d67b73f305 --- /dev/null +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -0,0 +1,35 @@ +SELECT '--JSON_VALUE--'; +SELECT JSON_VALUE('$.hello', '{"hello":1}'); +SELECT JSON_VALUE('$.hello', '{"hello":1.2}'); +SELECT JSON_VALUE('$.hello', '{"hello":true}'); +SELECT JSON_VALUE('$.hello', '{"hello":"world"}'); +SELECT JSON_VALUE('$.hello', '{"hello":null}'); +SELECT JSON_VALUE('$.hello', '{"hello":["world","world2"]}'); +SELECT JSON_VALUE('$.hello', '{"hello":{"world":"!"}}'); +SELECT JSON_VALUE('$.hello', '{hello:world}'); -- invalid json => default value (empty string) +SELECT JSON_VALUE('$.hello', ''); + +SELECT '--JSON_QUERY--'; +SELECT JSON_QUERY('$.hello', '{"hello":1}'); +SELECT JSON_QUERY('$.hello', '{"hello":1.2}'); +SELECT JSON_QUERY('$.hello', '{"hello":true}'); +SELECT JSON_QUERY('$.hello', '{"hello":"world"}'); +SELECT JSON_QUERY('$.hello', '{"hello":null}'); +SELECT JSON_QUERY('$.hello', '{"hello":["world","world2"]}'); +SELECT JSON_QUERY('$.hello', '{"hello":{"world":"!"}}'); +SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => default value (empty string) +SELECT JSON_QUERY('$.hello', ''); + +SELECT '--JSON_EXISTS--'; +SELECT JSON_EXISTS('$.hello', '{"hello":1}'); +SELECT JSON_EXISTS('$.world', '{"hello":1,"world":2}'); +SELECT JSON_EXISTS('$.world', '{"hello":{"world":1}}'); +SELECT JSON_EXISTS('$.hello.world', '{"hello":{"world":1}}'); +SELECT JSON_EXISTS('$.hello', '{hello:world}'); -- invalid json => default value (zero integer) +SELECT JSON_EXISTS('$.hello', ''); +SELECT JSON_EXISTS('$.hello[*]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.hello[0]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.hello[1]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); +SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); +SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); \ No newline at end of file From 00ce6f69dfc431442e106da62aae7c581bed324d Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Wed, 9 Jun 2021 12:46:17 +0300 Subject: [PATCH 048/290] Added newline --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index b8dc5d8e416..7457aca18ed 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -30,4 +30,4 @@ null 0 1 0 -1 \ No newline at end of file +1 From ff3857fbe73d354bf446d85466b127462212e5b3 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Thu, 17 Jun 2021 12:24:10 +0300 Subject: [PATCH 049/290] Add root parsing --- src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h | 15 ++++++++ .../JSONPath/Generators/GeneratorJSONPath.h | 21 ++++++++--- src/Functions/JSONPath/Generators/IVisitor.h | 4 +++ .../JSONPath/Generators/VisitorJSONPathRoot.h | 35 +++++++++++++++++++ .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 28 +++++++-------- .../JSONPath/Parsers/ParserJSONPathRoot.cpp | 28 +++++++++++++++ .../JSONPath/Parsers/ParserJSONPathRoot.h | 18 ++++++++++ src/Parsers/Lexer.cpp | 3 +- .../01889_sql_json_functions.reference | 5 +++ .../0_stateless/01889_sql_json_functions.sql | 5 +++ 10 files changed, 143 insertions(+), 19 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h new file mode 100644 index 00000000000..1c6469c5b75 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathRoot : public IAST +{ +public: + String getID(char) const override { return "ASTJSONPathRoot"; } + + ASTPtr clone() const override { return std::make_shared(*this); } +}; + +} diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 39f385fafc2..2583ef8c921 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,10 @@ template class GeneratorJSONPath : public IGenerator { public: + /** + * Traverses children ASTs of ASTJSONPathQuery and creates a vector of corresponding visitors + * @param query_ptr_ pointer to ASTJSONPathQuery + */ GeneratorJSONPath(ASTPtr query_ptr_) { query_ptr = query_ptr_; @@ -31,13 +36,15 @@ public: for (auto child_ast : query->children) { - if (typeid_cast(child_ast.get())) + if (typeid_cast(child_ast.get())) { + visitors.push_back(std::make_shared>(child_ast)); + } + else if (typeid_cast(child_ast.get())) { visitors.push_back(std::make_shared>(child_ast)); } - else if (child_ast->getID() == "ASTJSONPathRange") + else if (typeid_cast(child_ast.get())) { - visitors.push_back(std::make_shared>(child_ast)); } } @@ -46,7 +53,13 @@ public: const char * getName() const override { return "GeneratorJSONPath"; } /** - * The only generator which is called from JSONPath functions. + * This method exposes API of traversing all paths, described by JSONPath, + * to SQLJSON Functions. + * Expected usage is to iteratively call this method from inside the function + * and to execute custom logic with received element or handle an error. + * On each such call getNextItem will yield next item into element argument + * and modify its internal state to prepare for next call. + * * @param element root of JSON document * @return is the generator exhausted */ diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index d9917087cb0..1461b842829 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -36,6 +36,10 @@ public: virtual ~IVisitor() = default; private: + /** + * This variable is for detecting whether a visitor's next visit will be able + * to yield a new item. + */ bool is_exhausted = false; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h new file mode 100644 index 00000000000..d8b88ce0255 --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathRoot : public IVisitor +{ +public: + VisitorJSONPathRoot(ASTPtr) { } + + const char * getName() const override { return "VisitorJSONPathRoot"; } + + VisitorStatus apply(typename JSONParser::Element & /*element*/) const override + { + /// No-op on document, since we are already passed document's root + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + apply(element); + this->setExhausted(true); + return VisitorStatus::Ok; + } + + void reinitialize() override { this->setExhausted(false); } + + void updateState() override { } +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 0f171c0a82c..0ab09733890 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include #include namespace DB @@ -18,27 +19,26 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; + ParserJSONPathRoot parser_jsonpath_root; - if (pos->type != TokenType::DollarSign) - { + ASTPtr path_root; + if (!parser_jsonpath_root.parse(pos, path_root, expected)) { return false; } - ++pos; + query->children.push_back(path_root); - bool res = false; - ASTPtr subquery; - while (parser_jsonpath_member_access.parse(pos, subquery, expected) - || parser_jsonpath_range.parse(pos, subquery, expected)) + ASTPtr accessor; + while (parser_jsonpath_member_access.parse(pos, accessor, expected) + || parser_jsonpath_range.parse(pos, accessor, expected)) { - if (subquery) + if (accessor) { - query->children.push_back(subquery); - subquery = nullptr; + query->children.push_back(accessor); + accessor = nullptr; } - res = true; } - /// if we had at least one success and no fails - return res && pos->type == TokenType::EndOfStream; + /// parsing was successful if we reached the end of query by this point + return pos->type == TokenType::EndOfStream; } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp new file mode 100644 index 00000000000..a67d284e40c --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp @@ -0,0 +1,28 @@ +#include +#include + +#include + +namespace DB +{ +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathRoot + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathRoot::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (pos->type != TokenType::DollarSign) + { + expected.add(pos, "dollar sign (start of jsonpath)"); + return false; + } + auto path_root = std::make_shared(); + node = path_root; + ++pos; + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h new file mode 100644 index 00000000000..59fed28d63e --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathRoot : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathRoot"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathRoot() = default; +}; + +} diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 4de72ebc2fd..be956ee705a 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -338,8 +338,9 @@ Token Lexer::nextTokenImpl() } default: - if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) + if (*pos == '$' && ((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end)) { + /// Capture standalone dollar sign return Token(TokenType::DollarSign, token_begin, ++pos); } if (isWordCharASCII(*pos) || *pos == '$') diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 7457aca18ed..e38058ffc50 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -1,4 +1,5 @@ --JSON_VALUE-- + 1 1.2 true @@ -9,6 +10,7 @@ null --JSON_QUERY-- +[{"hello":1}] [1] [1.2] [true] @@ -20,6 +22,9 @@ null --JSON_EXISTS-- 1 +0 +1 +1 1 0 1 diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 6d67b73f305..a1749b3be24 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -1,4 +1,5 @@ SELECT '--JSON_VALUE--'; +SELECT JSON_VALUE('$', '{"hello":1}'); -- root is a complex object => default value (empty string) SELECT JSON_VALUE('$.hello', '{"hello":1}'); SELECT JSON_VALUE('$.hello', '{"hello":1.2}'); SELECT JSON_VALUE('$.hello', '{"hello":true}'); @@ -10,6 +11,7 @@ SELECT JSON_VALUE('$.hello', '{hello:world}'); -- invalid json => default value SELECT JSON_VALUE('$.hello', ''); SELECT '--JSON_QUERY--'; +SELECT JSON_QUERY('$', '{"hello":1}'); SELECT JSON_QUERY('$.hello', '{"hello":1}'); SELECT JSON_QUERY('$.hello', '{"hello":1.2}'); SELECT JSON_QUERY('$.hello', '{"hello":true}'); @@ -21,6 +23,9 @@ SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => defau SELECT JSON_QUERY('$.hello', ''); SELECT '--JSON_EXISTS--'; +SELECT JSON_EXISTS('$', '{"hello":1}'); +SELECT JSON_EXISTS('$', ''); +SELECT JSON_EXISTS('$', '{}'); SELECT JSON_EXISTS('$.hello', '{"hello":1}'); SELECT JSON_EXISTS('$.world', '{"hello":1,"world":2}'); SELECT JSON_EXISTS('$.world', '{"hello":{"world":1}}'); From f44d6792b07733c39980f4b0ff633610ccd4e3fa Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Thu, 17 Jun 2021 14:48:05 +0300 Subject: [PATCH 050/290] Fix ext::range --- src/Functions/FunctionSQLJSON.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b13b03cb089..b605645499e 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include #if !defined(ARCADIA_BUILD) # include "config_functions.h" @@ -133,7 +133,7 @@ public: /// Parse JSON for every row Impl impl; - for (const auto i : ext::range(0, input_rows_count)) + for (const auto i : collections::range(0, input_rows_count)) { std::string_view json{ reinterpret_cast(&chars_json[offsets_json[i - 1]]), offsets_json[i] - offsets_json[i - 1] - 1}; From e2765991b04d3ef312c9e0e3370d1b23ad49cf52 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 01:37:59 +0300 Subject: [PATCH 051/290] Separate star from ranges for better code --- src/Functions/JSONPath/ASTs/ASTJSONPathStar.h | 15 +++++ .../JSONPath/Generators/GeneratorJSONPath.h | 8 ++- .../Generators/VisitorJSONPathMemberAccess.h | 11 ++-- .../Generators/VisitorJSONPathRange.h | 57 +++------------- .../JSONPath/Generators/VisitorJSONPathStar.h | 66 +++++++++++++++++++ .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 8 ++- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 28 ++------ .../JSONPath/Parsers/ParserJSONPathStar.cpp | 31 +++++++++ .../JSONPath/Parsers/ParserJSONPathStar.h | 18 +++++ 9 files changed, 163 insertions(+), 79 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathStar.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathStar.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathStar.h diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h b/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h new file mode 100644 index 00000000000..2aada47c459 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathStar : public IAST +{ +public: + String getID(char) const override { return "ASTJSONPathStar"; } + + ASTPtr clone() const override { return std::make_shared(*this); } +}; + +} diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 2583ef8c921..071a7ac3089 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,8 @@ public: for (auto child_ast : query->children) { - if (typeid_cast(child_ast.get())) { + if (typeid_cast(child_ast.get())) + { visitors.push_back(std::make_shared>(child_ast)); } else if (typeid_cast(child_ast.get())) @@ -47,6 +49,10 @@ public: { visitors.push_back(std::make_shared>(child_ast)); } + else if (typeid_cast(child_ast.get())) + { + visitors.push_back(std::make_shared>(child_ast)); + } } } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index fd83c478227..b0c601458b6 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -10,15 +10,15 @@ template class VisitorJSONPathMemberAccess : public IVisitor { public: - VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) + : member_access_ptr(member_access_ptr_->as()) { } const char * getName() const override { return "VisitorJSONPathMemberAccess"; } VisitorStatus apply(typename JSONParser::Element & element) const override { - const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - element.getObject().find(std::string_view(member_access->member_name), result); + element.getObject().find(std::string_view(member_access_ptr->member_name), result); element = result; return VisitorStatus::Ok; } @@ -30,9 +30,8 @@ public: this->setExhausted(true); return VisitorStatus::Error; } - const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - if (!element.getObject().find(std::string_view(member_access->member_name), result)) + if (!element.getObject().find(std::string_view(member_access_ptr->member_name), result)) { this->setExhausted(true); return VisitorStatus::Error; @@ -47,7 +46,7 @@ public: void updateState() override { } private: - ASTPtr member_access_ptr; + ASTJSONPathMemberAccess * member_access_ptr; }; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index 601c5ea80b9..57e208271d0 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -10,18 +10,10 @@ template class VisitorJSONPathRange : public IVisitor { public: - VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_->as()) { - const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) - { - current_index = 0; - } - else - { - current_index = range->ranges[current_range].first; - } + current_index = range_ptr->ranges[current_range].first; } const char * getName() const override { return "VisitorJSONPathRange"; } @@ -30,12 +22,7 @@ public: { typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); - if (current_index >= array.size()) - { - return VisitorStatus::Error; - } - result = array[current_index]; - element = result; + element = array[current_index]; return VisitorStatus::Ok; } @@ -47,32 +34,21 @@ public: return VisitorStatus::Error; } - const auto * range = range_ptr->as(); VisitorStatus status; if (current_index < element.getArray().size()) { apply(element); status = VisitorStatus::Ok; } - else if (!range->is_star) - { - status = VisitorStatus::Ignore; - } else { status = VisitorStatus::Ignore; - this->setExhausted(true); } - if (!range->is_star) + if (current_index + 1 == range_ptr->ranges[current_range].second + && current_range + 1 == range_ptr->ranges.size()) { - if (current_index + 1 == range->ranges[current_range].second) - { - if (current_range + 1 == range->ranges.size()) - { - this->setExhausted(true); - } - } + this->setExhausted(true); } return status; @@ -80,36 +56,23 @@ public: void reinitialize() override { - const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) - { - current_index = 0; - } - else - { - current_index = range->ranges[current_range].first; - } + current_index = range_ptr->ranges[current_range].first; this->setExhausted(false); } void updateState() override { - const auto * range = range_ptr->as(); current_index++; - if (range->is_star) - { - return; - } - if (current_index == range->ranges[current_range].second) + if (current_index == range_ptr->ranges[current_range].second) { current_range++; - current_index = range->ranges[current_range].first; + current_index = range_ptr->ranges[current_range].first; } } private: - ASTPtr range_ptr; + ASTJSONPathRange * range_ptr; size_t current_range; UInt32 current_index; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h b/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h new file mode 100644 index 00000000000..bc840597f2a --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathStar : public IVisitor +{ +public: + VisitorJSONPathStar(ASTPtr) + { + current_index = 0; + } + + const char * getName() const override { return "VisitorJSONPathStar"; } + + VisitorStatus apply(typename JSONParser::Element & element) const override + { + typename JSONParser::Element result; + typename JSONParser::Array array = element.getArray(); + element = array[current_index]; + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + if (!element.isArray()) + { + this->setExhausted(true); + return VisitorStatus::Error; + } + + VisitorStatus status; + if (current_index < element.getArray().size()) + { + apply(element); + status = VisitorStatus::Ok; + } + else + { + status = VisitorStatus::Ignore; + this->setExhausted(true); + } + + return status; + } + + void reinitialize() override + { + current_index = 0; + this->setExhausted(false); + } + + void updateState() override + { + current_index++; + } + +private: + UInt32 current_index; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 0ab09733890..c18b2ad9b31 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -19,17 +20,20 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; + ParserJSONPathStar parser_jsonpath_star; ParserJSONPathRoot parser_jsonpath_root; ASTPtr path_root; - if (!parser_jsonpath_root.parse(pos, path_root, expected)) { + if (!parser_jsonpath_root.parse(pos, path_root, expected)) + { return false; } query->children.push_back(path_root); ASTPtr accessor; while (parser_jsonpath_member_access.parse(pos, accessor, expected) - || parser_jsonpath_range.parse(pos, accessor, expected)) + || parser_jsonpath_range.parse(pos, accessor, expected) + || parser_jsonpath_star.parse(pos, accessor, expected)) { if (accessor) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 3da0d508b27..f8496cd67d0 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -31,26 +32,16 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte auto range = std::make_shared(); node = range; + ParserNumber number_p; + ASTPtr number_ptr; while (pos->type != TokenType::ClosingSquareBracket) { - if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) + if (pos->type != TokenType::Number) { return false; } - if (pos->type == TokenType::Asterisk) - { - if (range->is_star) - { - throw Exception("Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS); - } - range->is_star = true; - ++pos; - continue; - } std::pair range_indices; - ParserNumber number_p; - ASTPtr number_ptr; if (!number_p.parse(pos, number_ptr, expected)) { return false; @@ -64,16 +55,7 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } else if (pos->type == TokenType::BareWord) { - /// Range case - ParserIdentifier name_p; - ASTPtr word; - if (!name_p.parse(pos, word, expected)) - { - return false; - } - String to_identifier; - if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") - { + if (!ParserKeyword("TO").ignore(pos, expected)) { return false; } if (!number_p.parse(pos, number_ptr, expected)) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp new file mode 100644 index 00000000000..c0d2b376794 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -0,0 +1,31 @@ +#include + +#include + +namespace DB +{ +bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + + if (pos->type != TokenType::OpeningSquareBracket) + { + return false; + } + ++pos; + if (pos->type != TokenType::Asterisk) { + return false; + } + ++pos; + if (pos->type != TokenType::ClosingSquareBracket) { + expected.add(pos, "Closing square bracket"); + return false; + } + ++pos; + + auto star = std::make_shared(); + node = star; + + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h new file mode 100644 index 00000000000..543823357de --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathStar : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathStar"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathStar() = default; +}; + +} From 1863a9beb044104e7dd7bab0f97508f46a41a3bb Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 02:15:15 +0300 Subject: [PATCH 052/290] Change stringstream to WriteBuffer --- src/Functions/FunctionSQLJSON.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b605645499e..9bfb4291ba8 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -279,11 +281,11 @@ public: return false; } - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + String result; + WriteBufferFromString out(result); out << current_element.getElement(); - auto output_str = out.str(); ColumnString & col_str = assert_cast(dest); - col_str.insertData(output_str.data(), output_str.size()); + col_str.insertData(result.data(), result.size()); return true; } }; @@ -307,7 +309,9 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + String result; + WriteBufferFromString out(result); + /// Create json array of results: [res1, res2, ...] out << "["; bool success = false; @@ -334,8 +338,7 @@ public: return false; } ColumnString & col_str = assert_cast(dest); - auto output_str = out.str(); - col_str.insertData(output_str.data(), output_str.size()); + col_str.insertData(reinterpret_cast(result.data()), result.size()); return true; } }; From a1e3a3f2c11c6d389eb28ac3a73ddc33eb640b57 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:21:53 +0300 Subject: [PATCH 053/290] update h3 --- contrib/h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/h3 b/contrib/h3 index e209086ae1b..9cb6ff75836 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit e209086ae1b5477307f545a0f6111780edc59940 +Subproject commit 9cb6ff758365b9cf4cb5d669b664d2d448a14373 From 115edd3e42afb2aba40154036a288bacde99d6d7 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 21 Jun 2021 12:21:26 +0100 Subject: [PATCH 054/290] Fix hang and incorrect exit code returned from clickhouse-test Variables aren't shared when using multiprocessing, use shared memory instead https://docs.python.org/3/library/multiprocessing.html#shared-ctypes-objects. There appears to be a deadlock when multiple threads try to send sigterm signal at the same time. Avoid it by making sure sigterm is sent only once for the process group. --- tests/clickhouse-test | 47 ++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e508abab70c..dc8c5dbd2f6 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -48,15 +48,23 @@ MAX_RETRIES = 5 class Terminated(KeyboardInterrupt): pass + def signal_handler(sig, frame): raise Terminated(f'Terminated with {sig} signal') def stop_tests(): - # send signal to all processes in group to avoid hung check triggering - # (to avoid terminating clickhouse-test itself, the signal should be ignored) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - os.killpg(os.getpgid(os.getpid()), signal.SIGTERM) - signal.signal(signal.SIGTERM, signal.SIG_DFL) + global stop_tests_triggered_lock + global stop_tests_triggered + + with stop_tests_triggered_lock: + if not stop_tests_triggered.is_set(): + stop_tests_triggered.set() + + # send signal to all processes in group to avoid hung check triggering + # (to avoid terminating clickhouse-test itself, the signal should be ignored) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + os.killpg(os.getpgid(os.getpid()), signal.SIGTERM) + signal.signal(signal.SIGTERM, signal.SIG_DFL) def json_minify(string): """ @@ -326,18 +334,20 @@ def colored(text, args, color=None, on_color=None, attrs=None): return text -SERVER_DIED = False -exit_code = 0 stop_time = None +exit_code = multiprocessing.Value("i", 0) +server_died = multiprocessing.Event() +stop_tests_triggered_lock = multiprocessing.Lock() +stop_tests_triggered = multiprocessing.Event() queue = multiprocessing.Queue(maxsize=1) restarted_tests = [] # (test, stderr) # def run_tests_array(all_tests, suite, suite_dir, suite_tmp_dir, run_total): def run_tests_array(all_tests_with_params): all_tests, num_tests, suite, suite_dir, suite_tmp_dir = all_tests_with_params - global exit_code - global SERVER_DIED global stop_time + global exit_code + global server_died OP_SQUARE_BRACKET = colored("[", args, attrs=['bold']) CL_SQUARE_BRACKET = colored("]", args, attrs=['bold']) @@ -379,7 +389,7 @@ def run_tests_array(all_tests_with_params): else: break - if SERVER_DIED: + if server_died.is_set(): stop_tests() break @@ -441,7 +451,7 @@ def run_tests_array(all_tests_with_params): if failed_to_check or clickhouse_proc.returncode != 0: failures += 1 print("Server does not respond to health check") - SERVER_DIED = True + server_died.set() stop_tests() break @@ -494,10 +504,10 @@ def run_tests_array(all_tests_with_params): # Stop on fatal errors like segmentation fault. They are sent to client via logs. if ' ' in stderr: - SERVER_DIED = True + server_died.set() if testcase_args.stop and ('Connection refused' in stderr or 'Attempt to read after eof' in stderr) and not 'Received exception from server' in stderr: - SERVER_DIED = True + server_died.set() if os.path.isfile(stdout_file): status += ", result:\n\n" @@ -583,7 +593,7 @@ def run_tests_array(all_tests_with_params): f" {skipped_total} tests skipped. {(datetime.now() - start_time).total_seconds():.2f} s elapsed" f' ({multiprocessing.current_process().name}).', args, "red", attrs=["bold"])) - exit_code = 1 + exit_code.value = 1 else: print(colored(f"\n{passed_total} tests passed. {skipped_total} tests skipped." f" {(datetime.now() - start_time).total_seconds():.2f} s elapsed" @@ -750,7 +760,7 @@ def do_run_tests(jobs, suite, suite_dir, suite_tmp_dir, all_tests, parallel_test def main(args): - global SERVER_DIED + global server_died global stop_time global exit_code global server_logs_level @@ -853,7 +863,7 @@ def main(args): total_tests_run = 0 for suite in sorted(os.listdir(base_dir), key=sute_key_func): - if SERVER_DIED: + if server_died.is_set(): break suite_dir = os.path.join(base_dir, suite) @@ -953,8 +963,7 @@ def main(args): else: print(bt) - - exit_code = 1 + exit_code.value = 1 else: print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) @@ -971,7 +980,7 @@ def main(args): else: print("All tests have finished.") - sys.exit(exit_code) + sys.exit(exit_code.value) def find_binary(name): From 0adad2425a98c5dba656cb90bebb175d230f4c16 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 18 Jun 2021 15:09:04 +0800 Subject: [PATCH 055/290] json extract string or raw --- src/Functions/FunctionsJSON.h | 7 ++++++- tests/queries/0_stateless/00918_json_functions.reference | 4 ++-- .../0_stateless/01915_json_extract_raw_string.reference | 1 + .../queries/0_stateless/01915_json_extract_raw_string.sql | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01915_json_extract_raw_string.reference create mode 100644 tests/queries/0_stateless/01915_json_extract_raw_string.sql diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index b6bdf1be013..eec0a15c7a2 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -600,6 +600,8 @@ public: } }; +template +class JSONExtractRawImpl; /// Nodes of the extract tree. We need the extract tree to extract from JSON complex values containing array, tuples or nullables. template @@ -630,7 +632,10 @@ struct JSONExtractTree public: bool insertResultToColumn(IColumn & dest, const Element & element) override { - return JSONExtractStringImpl::insertResultToColumn(dest, element, {}); + if (element.isString()) + return JSONExtractStringImpl::insertResultToColumn(dest, element, {}); + else + return JSONExtractRawImpl::insertResultToColumn(dest, element, {}); } }; diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index a3beb2967d4..4a971bbad42 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -58,7 +58,7 @@ Friday (3,5) (3,0) --JSONExtractKeysAndValues-- -[('a','hello')] +[('a','hello'),('b','[-100,200,300]')] [('b',[-100,200,300])] [('a','hello'),('b','world')] [('a',5),('b',7),('c',11)] @@ -160,7 +160,7 @@ Friday (3,5) (3,0) --JSONExtractKeysAndValues-- -[('a','hello')] +[('a','hello'),('b','[-100,200,300]')] [('b',[-100,200,300])] [('a','hello'),('b','world')] [('a',5),('b',7),('c',11)] diff --git a/tests/queries/0_stateless/01915_json_extract_raw_string.reference b/tests/queries/0_stateless/01915_json_extract_raw_string.reference new file mode 100644 index 00000000000..839cb33f5f2 --- /dev/null +++ b/tests/queries/0_stateless/01915_json_extract_raw_string.reference @@ -0,0 +1 @@ +('123','456','[7,8,9]') diff --git a/tests/queries/0_stateless/01915_json_extract_raw_string.sql b/tests/queries/0_stateless/01915_json_extract_raw_string.sql new file mode 100644 index 00000000000..6ba94ac6dfd --- /dev/null +++ b/tests/queries/0_stateless/01915_json_extract_raw_string.sql @@ -0,0 +1 @@ +select JSONExtract('{"a": "123", "b": 456, "c": [7, 8, 9]}', 'Tuple(a String, b String, c String)'); From 4688f9e038cb0cff49ed2842c82582be68391480 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 13:50:09 +0000 Subject: [PATCH 056/290] hdfs truncate table --- src/Storages/HDFS/StorageHDFS.cpp | 27 ++++++++++++++------- src/Storages/HDFS/StorageHDFS.h | 2 ++ tests/integration/test_storage_hdfs/test.py | 16 ++++++++++-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index e3fd287bad8..9de4ca4650f 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace fs = std::filesystem; @@ -280,15 +281,7 @@ Pipe StorageHDFS::read( size_t max_block_size, unsigned num_streams) { - size_t begin_of_path; - /// This uri is checked for correctness in constructor of StorageHDFS and never modified afterwards - auto two_slash = uri.find("//"); - - if (two_slash == std::string::npos) - begin_of_path = uri.find('/'); - else - begin_of_path = uri.find('/', two_slash + 2); - + const size_t begin_of_path = uri.find('/', uri.find("//") + 2); const String path_from_uri = uri.substr(begin_of_path); const String uri_without_path = uri.substr(0, begin_of_path); @@ -330,6 +323,22 @@ BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const StorageM chooseCompressionMethod(uri, compression_method)); } +void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr context_, TableExclusiveLockHolder &) +{ + const size_t begin_of_path = uri.find('/', uri.find("//") + 2); + const String path = uri.substr(begin_of_path); + const String url = uri.substr(0, begin_of_path); + + HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", context_->getGlobalContext()->getConfigRef()); + HDFSFSPtr fs = createHDFSFS(builder.get()); + + int wait; + int ret = hdfsTruncate(fs.get(), path.data(), 0, &wait); + if (ret) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); +} + + void registerStorageHDFS(StorageFactory & factory) { factory.registerStorage("HDFS", [](const StorageFactory::Arguments & args) diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 397e147e7cd..da77b397adf 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -34,6 +34,8 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; + void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, TableExclusiveLockHolder &) override; + NamesAndTypesList getVirtuals() const override; protected: diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 34ced652a01..2dac7bc19d4 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -15,7 +15,6 @@ def started_cluster(): finally: cluster.shutdown() - def test_read_write_storage(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -235,7 +234,7 @@ def test_virtual_columns(started_cluster): expected = "1\tfile1\thdfs://hdfs1:9000//file1\n2\tfile2\thdfs://hdfs1:9000//file2\n3\tfile3\thdfs://hdfs1:9000//file3\n" assert node1.query("select id, _file as file_name, _path as file_path from virtual_cols order by id") == expected - + def test_read_files_with_spaces(started_cluster): hdfs_api = started_cluster.hdfs_api @@ -246,6 +245,19 @@ def test_read_files_with_spaces(started_cluster): assert node1.query("select * from test order by id") == "1\n2\n3\n" +def test_truncate_table(started_cluster): + hdfs_api = started_cluster.hdfs_api + node1.query( + "create table test_truncate (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/tr', 'TSV')") + node1.query("insert into test_truncate values (1, 'Mark', 72.53)") + assert hdfs_api.read_data("/tr") == "1\tMark\t72.53\n" + assert node1.query("select * from test_truncate") == "1\tMark\t72.53\n" + node1.query("truncate table test_truncate") + assert hdfs_api.read_data("/tr") == "" + assert node1.query("select * from test_truncate") == "" + node1.query("drop table test_truncate") + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 660e824851aafd38d29416c910ea02702a32eac4 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Mon, 21 Jun 2021 15:13:23 +0100 Subject: [PATCH 057/290] Missed one server_died.set() --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index dc8c5dbd2f6..c3ca1ec5953 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -489,7 +489,7 @@ def run_tests_array(all_tests_with_params): if MAX_RETRIES < counter: if args.replicated_database: if DISTRIBUTED_DDL_TIMEOUT_MSG in stderr: - SERVER_DIED = True + server_died.set() break if proc.returncode != 0: From ac0f86cdbf308bdad3e5d76b9fdf38cb9dc597b8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 15:44:36 +0000 Subject: [PATCH 058/290] Truncate for s3 --- src/Storages/HDFS/StorageHDFS.cpp | 3 +- src/Storages/HDFS/StorageHDFS.h | 2 +- src/Storages/StorageS3.cpp | 26 +++++++++++ src/Storages/StorageS3.h | 2 + tests/integration/test_storage_hdfs/test.py | 1 - tests/integration/test_storage_s3/test.py | 49 ++++++++++++++++----- 6 files changed, 67 insertions(+), 16 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 9de4ca4650f..c878fd4e1f8 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -332,8 +332,7 @@ void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", context_->getGlobalContext()->getConfigRef()); HDFSFSPtr fs = createHDFSFS(builder.get()); - int wait; - int ret = hdfsTruncate(fs.get(), path.data(), 0, &wait); + int ret = hdfsDelete(fs.get(), path.data(), 0); if (ret) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); } diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index da77b397adf..4a6614be2e0 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -34,7 +34,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; - void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, TableExclusiveLockHolder &) override; + void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context_, TableExclusiveLockHolder &) override; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 290a585128e..12ec405771e 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include @@ -434,6 +436,30 @@ BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMet max_single_part_upload_size); } + +void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) +{ + updateClientAndAuthSettings(local_context, client_auth); + + Aws::S3::Model::ObjectIdentifier obj; + obj.SetKey(client_auth.uri.key); + + Aws::S3::Model::Delete delkeys; + delkeys.AddObjects(std::move(obj)); + + Aws::S3::Model::DeleteObjectsRequest request; + request.SetBucket(client_auth.uri.bucket); + request.SetDelete(delkeys); + + auto response = client_auth.client->DeleteObjects(request); + if (!response.IsSuccess()) + { + const auto & err = response.GetError(); + throw Exception(std::to_string(static_cast(err.GetErrorType())) + ": " + err.GetMessage(), ErrorCodes::S3_ERROR); + } +} + + void StorageS3::updateClientAndAuthSettings(ContextPtr ctx, StorageS3::ClientAuthentificaiton & upd) { auto settings = ctx->getStorageS3Settings().getSettings(upd.uri.uri.toString()); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 73becc2aa57..240327fba6f 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -130,6 +130,8 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; + void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, TableExclusiveLockHolder &) override; + NamesAndTypesList getVirtuals() const override; private: diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 2dac7bc19d4..f60dc836608 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -253,7 +253,6 @@ def test_truncate_table(started_cluster): assert hdfs_api.read_data("/tr") == "1\tMark\t72.53\n" assert node1.query("select * from test_truncate") == "1\tMark\t72.53\n" node1.query("truncate table test_truncate") - assert hdfs_api.read_data("/tr") == "" assert node1.query("select * from test_truncate") == "" node1.query("drop table test_truncate") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 545ca4256f3..3f5254af49a 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -276,28 +276,28 @@ def test_put_get_with_redirect(started_cluster): # Test put with restricted S3 server redirect. def test_put_with_zero_redirect(started_cluster): - # type: (ClickHouseCluster) -> None + # type: (clickhousecluster) -> none bucket = started_cluster.minio_bucket - instance = started_cluster.instances["s3_max_redirects"] # type: ClickHouseInstance - table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + instance = started_cluster.instances["s3_max_redirects"] # type: clickhouseinstance + table_format = "column1 uint32, column2 uint32, column3 uint32" values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" filename = "test.csv" - # Should work without redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( - started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, table_format, values) + # should work without redirect + query = "insert into table function s3('http://{}:{}/{}/{}', 'csv', '{}') values {}".format( + started_cluster.minio_ip, minio_internal_port, bucket, filename, table_format, values) run_query(instance, query) - # Should not work with redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + # should not work with redirect + query = "insert into table function s3('http://{}:{}/{}/{}', 'csv', '{}') values {}".format( started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) - exception_raised = False + exception_raised = false try: run_query(instance, query) - except Exception as e: - assert str(e).find("Too many redirects while trying to access") != -1 - exception_raised = True + except exception as e: + assert str(e).find("too many redirects while trying to access") != -1 + exception_raised = true finally: assert exception_raised @@ -645,3 +645,28 @@ def test_storage_s3_put_gzip(started_cluster, extension, method): f = gzip.GzipFile(fileobj=buf, mode="rb") uncompressed_content = f.read().decode() assert sum([ int(i.split(',')[1]) for i in uncompressed_content.splitlines() ]) == 708 + + +def test_truncate_table(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] # type: ClickHouseInstance + name = "truncate" + + instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format( + name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, name)) + + instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name)) + result = instance.query("SELECT * FROM {}".format(name)) + assert result == instance.query("SELECT number FROM numbers(10)") + instance.query("TRUNCATE TABLE {}".format(name)) + + minio = started_cluster.minio_client + timeout = 30 + while timeout > 0: + if len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0: + return + timeout -= 1 + time.sleep(1) + assert(len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0) + assert instance.query("SELECT * FROM {}".format(name)) == "" + From f47dd116c4471f3e6bbf6bcee69136353d1483e5 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 21 Jun 2021 19:07:17 +0300 Subject: [PATCH 059/290] Update test.py --- tests/integration/test_storage_s3/test.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3f5254af49a..8a5708f5e8e 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -276,28 +276,28 @@ def test_put_get_with_redirect(started_cluster): # Test put with restricted S3 server redirect. def test_put_with_zero_redirect(started_cluster): - # type: (clickhousecluster) -> none + # type: (ClickHouseCluster) -> None bucket = started_cluster.minio_bucket - instance = started_cluster.instances["s3_max_redirects"] # type: clickhouseinstance - table_format = "column1 uint32, column2 uint32, column3 uint32" + instance = started_cluster.instances["s3_max_redirects"] # type: ClickHouseInstance + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" filename = "test.csv" - # should work without redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'csv', '{}') values {}".format( - started_cluster.minio_ip, minio_internal_port, bucket, filename, table_format, values) + # Should work without redirect + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, table_format, values) run_query(instance, query) - # should not work with redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'csv', '{}') values {}".format( + # Should not work with redirect + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) - exception_raised = false + exception_raised = False try: run_query(instance, query) - except exception as e: - assert str(e).find("too many redirects while trying to access") != -1 - exception_raised = true + except Exception as e: + assert str(e).find("Too many redirects while trying to access") != -1 + exception_raised = True finally: assert exception_raised From f3b0f11b59acacb8f5ac8304dc590c5000729662 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 21 Jun 2021 23:04:58 +0300 Subject: [PATCH 060/290] Update StorageHDFS.cpp --- src/Storages/HDFS/StorageHDFS.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index c878fd4e1f8..578da239c20 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -26,7 +26,6 @@ #include #include #include -#include namespace fs = std::filesystem; @@ -35,6 +34,7 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ACCESS_DENIED; } StorageHDFS::StorageHDFS( @@ -334,7 +334,7 @@ void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr int ret = hdfsDelete(fs.get(), path.data(), 0); if (ret) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); + throw Exception(ErrorCodes::ACCESS_DENIED, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); } From 8a70a9ba7c218315f8585f27bf9029561a75d7f6 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 22 Jun 2021 10:39:27 +0300 Subject: [PATCH 061/290] update cmake --- contrib/h3-cmake/CMakeLists.txt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/contrib/h3-cmake/CMakeLists.txt b/contrib/h3-cmake/CMakeLists.txt index 6b184a175b0..f4c70dc476f 100644 --- a/contrib/h3-cmake/CMakeLists.txt +++ b/contrib/h3-cmake/CMakeLists.txt @@ -3,21 +3,22 @@ set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib") set(SRCS "${H3_SOURCE_DIR}/lib/algos.c" -"${H3_SOURCE_DIR}/lib/baseCells.c" -"${H3_SOURCE_DIR}/lib/bbox.c" "${H3_SOURCE_DIR}/lib/coordijk.c" -"${H3_SOURCE_DIR}/lib/faceijk.c" -"${H3_SOURCE_DIR}/lib/geoCoord.c" -"${H3_SOURCE_DIR}/lib/h3Index.c" -"${H3_SOURCE_DIR}/lib/h3UniEdge.c" -"${H3_SOURCE_DIR}/lib/linkedGeo.c" -"${H3_SOURCE_DIR}/lib/localij.c" -"${H3_SOURCE_DIR}/lib/mathExtensions.c" +"${H3_SOURCE_DIR}/lib/bbox.c" "${H3_SOURCE_DIR}/lib/polygon.c" +"${H3_SOURCE_DIR}/lib/h3Index.c" "${H3_SOURCE_DIR}/lib/vec2d.c" "${H3_SOURCE_DIR}/lib/vec3d.c" "${H3_SOURCE_DIR}/lib/vertex.c" +"${H3_SOURCE_DIR}/lib/linkedGeo.c" +"${H3_SOURCE_DIR}/lib/localij.c" +"${H3_SOURCE_DIR}/lib/latLng.c" +"${H3_SOURCE_DIR}/lib/directedEdge.c" +"${H3_SOURCE_DIR}/lib/mathExtensions.c" +"${H3_SOURCE_DIR}/lib/iterators.c" "${H3_SOURCE_DIR}/lib/vertexGraph.c" +"${H3_SOURCE_DIR}/lib/faceijk.c" +"${H3_SOURCE_DIR}/lib/baseCells.c" ) configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h") From f2486ac8e93978d0b8efa86bacdd6e457c67bc64 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 22 Jun 2021 12:23:27 +0300 Subject: [PATCH 062/290] changes --- src/Functions/geoToH3.cpp | 15 +++++++++++---- src/Functions/h3EdgeAngle.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3GetBaseCell.cpp | 2 +- src/Functions/h3GetResolution.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- src/Functions/h3IndexesAreNeighbors.cpp | 2 +- src/Functions/h3IsValid.cpp | 2 +- src/Functions/h3ToChildren.cpp | 4 ++-- src/Functions/h3ToParent.cpp | 2 +- src/Functions/h3ToString.cpp | 2 +- src/Functions/h3kRing.cpp | 6 +++--- 12 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 7edb3faf62d..4fa20d0ad62 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -21,6 +21,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; } namespace @@ -79,11 +80,17 @@ public: const double lat = col_lat->getFloat64(row); const UInt8 res = col_res->getUInt(row); - GeoCoord coord; - coord.lon = degsToRads(lon); + LatLng coord; + coord.lng = degsToRads(lon); coord.lat = degsToRads(lat); - - H3Index hindex = geoToH3(&coord, res); + + H3Index hindex; + H3Error err = latLngToCell(&coord, res, &hindex); + if (err) { + throw Exception( + "Incorrect coorinates lat:" + std::to_string(coord.lat) + " lng:" + std::to_string(coord.lng) + " err:" + std::to_string(err), + ErrorCodes::INCORRECT_DATA); + } dst_data[row] = hindex; } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index 0fdafff9eed..071581a7c60 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -66,7 +66,7 @@ public: + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); // Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources - Float64 res = 8.99320592271288084e-6 * edgeLengthM(resolution); + Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 5ec57510e54..56374e10077 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -70,7 +70,7 @@ public: throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - Float64 res = edgeLengthM(resolution); + Float64 res = getHexagonEdgeLengthAvgM(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index 7f3843ed792..b73245f751b 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -59,7 +59,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 res = h3GetBaseCell(hindex); + UInt8 res = getBaseCellNumber(hindex); dst_data[row] = res; } diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 074e07e4277..49ade509934 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -59,7 +59,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 res = h3GetResolution(hindex); + UInt8 res = getResolution(hindex); dst_data[row] = res; } diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index e630fb7bd70..7f41348a14b 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -65,7 +65,7 @@ public: throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - Float64 res = hexAreaM2(resolution); + Float64 res = getHexagonAreaAvgM2(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 3c03d3d1adb..6507998e24c 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -67,7 +67,7 @@ public: const UInt64 hindex_origin = col_hindex_origin->getUInt(row); const UInt64 hindex_dest = col_hindex_dest->getUInt(row); - UInt8 res = h3IndexesAreNeighbors(hindex_origin, hindex_dest); + UInt8 res = areNeighborCells(hindex_origin, hindex_dest); dst_data[row] = res; } diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index d7f5a2c0771..bc140450b71 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -59,7 +59,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 is_valid = h3IsValid(hindex) == 0 ? 0 : 1; + UInt8 is_valid = isValidCell(hindex) == 0 ? 0 : 1; dst_data[row] = is_valid; } diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index d472c298432..88ac3056e72 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -84,14 +84,14 @@ public: throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName() + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution); + const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution); if (vec_size > MAX_ARRAY_SIZE) throw Exception("The result of function" + getName() + " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = " + toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE); hindex_vec.resize(vec_size); - h3ToChildren(parent_hindex, child_resolution, hindex_vec.data()); + cellToChildren(parent_hindex, child_resolution, hindex_vec.data()); dst_data.reserve(dst_data.size() + vec_size); for (auto hindex : hindex_vec) diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 6719d9f3456..9755184d63c 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -74,7 +74,7 @@ public: throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - UInt64 res = h3ToParent(hindex, resolution); + UInt64 res = cellToParent(hindex, resolution); dst_data[row] = res; } diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index dcd0951f67f..8ac97db0621 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -66,7 +66,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(i); - if (!h3IsValid(hindex)) + if (!isValidCell(hindex)) { throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index b54ed48ef3f..8b91f2fa1c7 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -77,7 +77,7 @@ public: const H3Index origin_hindex = col_hindex->getUInt(row); const int k = col_k->getInt(row); - /// Overflow is possible. The function maxKringSize does not check for overflow. + /// Overflow is possible. The function maxGridDiskSize does not check for overflow. /// The calculation is similar to square of k but several times more. /// Let's use huge underestimation as the safe bound. We should not allow to generate too large arrays nevertheless. constexpr auto max_k = 10000; @@ -86,9 +86,9 @@ public: if (k < 0) throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Argument 'k' for {} function must be non negative", getName()); - const auto vec_size = maxKringSize(k); + const auto vec_size = maxGridDiskSize(k); hindex_vec.resize(vec_size); - kRing(origin_hindex, k, hindex_vec.data()); + gridDisk(origin_hindex, k, hindex_vec.data()); dst_data.reserve(dst_data.size() + vec_size); for (auto hindex : hindex_vec) From 6a79fef8bd4d2a8b252ee02f72792b8826acaeb5 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 22 Jun 2021 12:35:56 +0300 Subject: [PATCH 063/290] submodule --- contrib/h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/h3 b/contrib/h3 index 9cb6ff75836..c7f46cfd71f 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit 9cb6ff758365b9cf4cb5d669b664d2d448a14373 +Subproject commit c7f46cfd71fb60e2fefc90e28abe81657deff735 From 63db58710d8ddfac43bbe20b41e4909471b0ec79 Mon Sep 17 00:00:00 2001 From: Zijie Lu Date: Tue, 22 Jun 2021 19:25:14 +0800 Subject: [PATCH 064/290] Support for DISTINCT ON (columns) Signed-off-by: Zijie Lu --- src/Common/ErrorCodes.cpp | 1 + src/Parsers/ParserSelectQuery.cpp | 15 +++++++++++++++ .../0_stateless/01917_distinct_on.reference | 3 +++ tests/queries/0_stateless/01917_distinct_on.sql | 9 +++++++++ 4 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/01917_distinct_on.reference create mode 100644 tests/queries/0_stateless/01917_distinct_on.sql diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d840830bf28..5afba23657d 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -554,6 +554,7 @@ M(584, PROJECTION_NOT_USED) \ M(585, CANNOT_PARSE_YAML) \ M(586, CANNOT_CREATE_FILE) \ + M(587, DISTINCT_ON_AND_LIMIT_BY_TOGETHER) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 548ec8879bd..12e83486af8 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -21,6 +22,7 @@ namespace ErrorCodes extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED; extern const int ROW_AND_ROWS_TOGETHER; extern const int FIRST_AND_NEXT_TOGETHER; + extern const int DISTINCT_ON_AND_LIMIT_BY_TOGETHER; } @@ -32,6 +34,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_select("SELECT"); ParserKeyword s_all("ALL"); ParserKeyword s_distinct("DISTINCT"); + ParserKeyword s_distinct_on("DISTINCT ON"); ParserKeyword s_from("FROM"); ParserKeyword s_prewhere("PREWHERE"); ParserKeyword s_where("WHERE"); @@ -94,6 +97,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } + bool has_distinct_on = false; + /// SELECT [ALL/DISTINCT] [TOP N [WITH TIES]] expr list { bool has_all = false; @@ -103,6 +108,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_all.ignore(pos, expected)) has_all = true; + if (s_distinct_on.ignore(pos, expected)) { + has_distinct_on = true; + if (!exp_list.parse(pos, limit_by_expression_list, expected)) + return false; + limit_by_length = std::make_shared(Field{UInt8(1)}); + } + if (s_distinct.ignore(pos, expected)) select_query->distinct = true; @@ -264,6 +276,9 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (limit_with_ties_occured) throw Exception("Can not use WITH TIES alongside LIMIT BY", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); + if (has_distinct_on) + throw Exception("Can not use distinct on alongside LIMIT BY", ErrorCodes::DISTINCT_ON_AND_LIMIT_BY_TOGETHER); + limit_by_length = limit_length; limit_by_offset = limit_offset; limit_length = nullptr; diff --git a/tests/queries/0_stateless/01917_distinct_on.reference b/tests/queries/0_stateless/01917_distinct_on.reference new file mode 100644 index 00000000000..09e5879c7f6 --- /dev/null +++ b/tests/queries/0_stateless/01917_distinct_on.reference @@ -0,0 +1,3 @@ +1 1 1 +2 2 2 +1 2 2 diff --git a/tests/queries/0_stateless/01917_distinct_on.sql b/tests/queries/0_stateless/01917_distinct_on.sql new file mode 100644 index 00000000000..0940d8566bd --- /dev/null +++ b/tests/queries/0_stateless/01917_distinct_on.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS t1; + +CREATE TABLE t1 (`a` UInt32, `b` UInt32, `c` UInt32 ) ENGINE = Memory; +INSERT INTO t1 VALUES (1, 1, 1), (1, 1, 2), (2, 2, 2), (1, 2, 2); + +SELECT DISTINCT ON (a, b) a, b, c FROM t1; + +DROP TABLE IF EXISTS t1; + From 3578a79e08f627254446b53bebf65b889303390f Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 22 Jun 2021 17:41:20 +0300 Subject: [PATCH 065/290] fix style --- src/Functions/geoToH3.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 4fa20d0ad62..90e29248d32 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -83,12 +83,12 @@ public: LatLng coord; coord.lng = degsToRads(lon); coord.lat = degsToRads(lat); - + H3Index hindex; H3Error err = latLngToCell(&coord, res, &hindex); if (err) { throw Exception( - "Incorrect coorinates lat:" + std::to_string(coord.lat) + " lng:" + std::to_string(coord.lng) + " err:" + std::to_string(err), + "Incorrect coordinates lat:" + std::to_string(coord.lat) + " lng:" + std::to_string(coord.lng) + " err:" + std::to_string(err), ErrorCodes::INCORRECT_DATA); } From a5d3600f202d70aaf62231fbf231325270c7f881 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 10:07:53 +0300 Subject: [PATCH 066/290] Fix compile errors with WriteBuffer --- src/Functions/DummyJSONParser.h | 6 ++++-- src/Functions/FunctionSQLJSON.h | 4 ++-- src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp | 3 ++- src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp | 3 +-- src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp | 4 +--- src/Functions/SimdJSONParser.h | 6 +++++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 01fdab1abb6..128ee88e0ca 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -2,6 +2,8 @@ #include #include +#include +#include namespace DB { @@ -40,7 +42,7 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - Element getElement() { return {}; } + ALWAYS_INLINE Element getUnderlyingElement() const { return {}; } }; /// References an array in a JSON document. @@ -99,7 +101,7 @@ struct DummyJSONParser #endif }; -inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +inline ALWAYS_INLINE WriteBufferFromString& operator<<(WriteBufferFromString& out, const DB::DummyJSONParser::Element &) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 9bfb4291ba8..3ff1b575bfc 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -283,7 +283,7 @@ public: String result; WriteBufferFromString out(result); - out << current_element.getElement(); + out << current_element; ColumnString & col_str = assert_cast(dest); col_str.insertData(result.data(), result.size()); return true; @@ -324,7 +324,7 @@ public: out << ", "; } success = true; - out << current_element.getElement(); + out << current_element; } else if (status == VisitorStatus::Error) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index f8496cd67d0..bc153b9d747 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -55,7 +55,8 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } else if (pos->type == TokenType::BareWord) { - if (!ParserKeyword("TO").ignore(pos, expected)) { + if (!ParserKeyword("TO").ignore(pos, expected)) + { return false; } if (!number_p.parse(pos, number_ptr, expected)) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp index a67d284e40c..86cf793fb52 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp @@ -19,8 +19,7 @@ bool ParserJSONPathRoot::parseImpl(Pos & pos, ASTPtr & node, Expected & expected expected.add(pos, "dollar sign (start of jsonpath)"); return false; } - auto path_root = std::make_shared(); - node = path_root; + node = std::make_shared(); ++pos; return true; } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp index c0d2b376794..97ab9ffec36 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -6,7 +6,6 @@ namespace DB { bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (pos->type != TokenType::OpeningSquareBracket) { return false; @@ -22,8 +21,7 @@ bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } ++pos; - auto star = std::make_shared(); - node = star; + node = std::make_shared(); return true; } diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index c5793088baf..a176f2c5961 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -50,7 +50,7 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; - ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } + ALWAYS_INLINE simdjson::dom::element getUnderlyingElement() const { return element; } private: simdjson::dom::element element; @@ -165,6 +165,10 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } +inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) { + return out << element.getUnderlyingElement(); +} + } #endif From ec7ec63a40605b7ba4afdd1ffb436d352b07a0d8 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Wed, 23 Jun 2021 13:22:38 +0300 Subject: [PATCH 067/290] Fix style --- src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp | 6 ++++-- src/Functions/SimdJSONParser.h | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp index 97ab9ffec36..1338a2064f1 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -11,11 +11,13 @@ bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected return false; } ++pos; - if (pos->type != TokenType::Asterisk) { + if (pos->type != TokenType::Asterisk) + { return false; } ++pos; - if (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::ClosingSquareBracket) + { expected.add(pos, "Closing square bracket"); return false; } diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a176f2c5961..c11fca3272c 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -165,7 +165,8 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } -inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) { +inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) +{ return out << element.getUnderlyingElement(); } From abe7e4195ebce6a6a54d18f3e67d3b5712c2e602 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Wed, 23 Jun 2021 15:02:47 +0300 Subject: [PATCH 068/290] . --- src/Functions/FunctionSQLJSON.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 3ff1b575bfc..1e9b25ee508 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -283,7 +283,7 @@ public: String result; WriteBufferFromString out(result); - out << current_element; + out << current_element.getUnderlyingElement(); ColumnString & col_str = assert_cast(dest); col_str.insertData(result.data(), result.size()); return true; @@ -324,7 +324,7 @@ public: out << ", "; } success = true; - out << current_element; + out << current_element.getUnderlyingElement(); } else if (status == VisitorStatus::Error) { From b00efaf3d1b0d9e2341e8b50f523573926a1b614 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 23 Jun 2021 17:03:39 +0300 Subject: [PATCH 069/290] Add materialized columns to joined columns --- src/Interpreters/JoinedTables.cpp | 2 +- src/Interpreters/TreeRewriter.cpp | 9 ++- src/Interpreters/getTableExpressions.cpp | 66 ++++++++----------- src/Interpreters/getTableExpressions.h | 12 +++- .../01925_join_materialized_columns.reference | 7 ++ .../01925_join_materialized_columns.sql | 14 ++++ 6 files changed, 67 insertions(+), 43 deletions(-) create mode 100644 tests/queries/0_stateless/01925_join_materialized_columns.reference create mode 100644 tests/queries/0_stateless/01925_join_materialized_columns.sql diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 45466ae5ca1..5e53074d24f 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -187,7 +187,7 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { - tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); + tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, true); if (tables_with_columns.size() != table_expressions.size()) throw Exception("Unexpected tables count", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 76093a14d45..1f94cda6b0f 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -896,9 +896,14 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (tables_with_columns.size() > 1) { - result.analyzed_join->columns_from_joined_table = tables_with_columns[1].columns; + const auto & right_table = tables_with_columns[1]; + auto & cols_from_joined = result.analyzed_join->columns_from_joined_table; + cols_from_joined = right_table.columns; + cols_from_joined.insert( + cols_from_joined.end(), right_table.materialized_columns.begin(), right_table.materialized_columns.end()); + result.analyzed_join->deduplicateAndQualifyColumnNames( - source_columns_set, tables_with_columns[1].table.getQualifiedNamePrefix()); + source_columns_set, right_table.table.getQualifiedNamePrefix()); } translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 22eb307071c..f7d82a8f599 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -113,50 +113,42 @@ static NamesAndTypesList getColumnsFromTableExpression( return names_and_type_list; } -NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression & table_expression, ContextPtr context) -{ - NamesAndTypesList materialized; - NamesAndTypesList aliases; - NamesAndTypesList virtuals; - return getColumnsFromTableExpression(table_expression, context, materialized, aliases, virtuals); -} - -TablesWithColumns getDatabaseAndTablesWithColumns(const std::vector & table_expressions, ContextPtr context) +TablesWithColumns getDatabaseAndTablesWithColumns( + const ASTTableExprConstPtrs & table_expressions, + ContextPtr context, + bool add_materialized) { TablesWithColumns tables_with_columns; - if (!table_expressions.empty()) + String current_database = context->getCurrentDatabase(); + bool include_alias_cols = context->getSettingsRef().asterisk_include_alias_columns; + bool include_materialized_cols = add_materialized || context->getSettingsRef().asterisk_include_materialized_columns; + + for (const ASTTableExpression * table_expression : table_expressions) { - String current_database = context->getCurrentDatabase(); - bool include_alias_cols = context->getSettingsRef().asterisk_include_alias_columns; - bool include_materialized_cols = context->getSettingsRef().asterisk_include_materialized_columns; + NamesAndTypesList materialized; + NamesAndTypesList aliases; + NamesAndTypesList virtuals; + NamesAndTypesList names_and_types = getColumnsFromTableExpression(*table_expression, context, materialized, aliases, virtuals); - for (const ASTTableExpression * table_expression : table_expressions) + removeDuplicateColumns(names_and_types); + + tables_with_columns.emplace_back( + DatabaseAndTableWithAlias(*table_expression, current_database), names_and_types); + + auto & table = tables_with_columns.back(); + table.addHiddenColumns(materialized); + table.addHiddenColumns(aliases); + table.addHiddenColumns(virtuals); + + if (include_alias_cols) { - NamesAndTypesList materialized; - NamesAndTypesList aliases; - NamesAndTypesList virtuals; - NamesAndTypesList names_and_types = getColumnsFromTableExpression(*table_expression, context, materialized, aliases, virtuals); + table.addAliasColumns(aliases); + } - removeDuplicateColumns(names_and_types); - - tables_with_columns.emplace_back( - DatabaseAndTableWithAlias(*table_expression, current_database), names_and_types); - - auto & table = tables_with_columns.back(); - table.addHiddenColumns(materialized); - table.addHiddenColumns(aliases); - table.addHiddenColumns(virtuals); - - if (include_alias_cols) - { - table.addAliasColumns(aliases); - } - - if (include_materialized_cols) - { - table.addMaterializedColumns(materialized); - } + if (include_materialized_cols) + { + table.addMaterializedColumns(materialized); } } diff --git a/src/Interpreters/getTableExpressions.h b/src/Interpreters/getTableExpressions.h index 961176437b5..19c27057c2f 100644 --- a/src/Interpreters/getTableExpressions.h +++ b/src/Interpreters/getTableExpressions.h @@ -10,13 +10,19 @@ namespace DB struct ASTTableExpression; class ASTSelectQuery; +using ASTTableExprConstPtrs = std::vector; + NameSet removeDuplicateColumns(NamesAndTypesList & columns); -std::vector getTableExpressions(const ASTSelectQuery & select_query); +ASTTableExprConstPtrs getTableExpressions(const ASTSelectQuery & select_query); + const ASTTableExpression * getTableExpression(const ASTSelectQuery & select, size_t table_number); + ASTPtr extractTableExpression(const ASTSelectQuery & select, size_t table_number); -NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression & table_expression, ContextPtr context); -TablesWithColumns getDatabaseAndTablesWithColumns(const std::vector & table_expressions, ContextPtr context); +TablesWithColumns getDatabaseAndTablesWithColumns( + const ASTTableExprConstPtrs & table_expressions, + ContextPtr context, + bool add_materialized = false); } diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference new file mode 100644 index 00000000000..90f754f6e7c --- /dev/null +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -0,0 +1,7 @@ +2020-02-02 13:00:00 fact2 t1_val2 2020-02-02 2020-02-05 13:00:00 fact2 t1_val2 2020-02-05 +- +2020-01-01 2020-01-01 +2020-02-02 2020-02-05 +- +2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 +2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql new file mode 100644 index 00000000000..9c4596f9915 --- /dev/null +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t1 (time DateTime, foo String, dimension_1 String, dt Date MATERIALIZED toDate(time)) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, foo); +CREATE TABLE t2 (time DateTime, bar String, dimension_2 String, dt Date MATERIALIZED toDate(time)) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, bar); + +INSERT INTO t1 VALUES ('2020-01-01 12:00:00', 'fact1', 't1_val1'), ('2020-02-02 13:00:00', 'fact2', 't1_val2'), ('2020-01-01 13:00:00', 'fact3', 't1_val3'); +INSERT INTO t2 VALUES ('2020-01-01 12:00:00', 'fact1', 't2_val2'), ('2020-02-05 13:00:00', 'fact2', 't1_val2'), ('2019-01-01 12:00:00', 'fact4', 't2_val2'); + +SELECT * FROM t1 JOIN t2 ON t1.foo = t2.bar WHERE t2.dt >= '2020-02-01'; +SELECT '-'; +SELECT t1.dt, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.dt; +SELECT '-'; +SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time; From e530a86d0f530aba4c5008a27df1119e37e289d6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 23 Jun 2021 17:08:54 +0300 Subject: [PATCH 070/290] Add query with USING to 01925_join_materialized_columns --- .../0_stateless/01925_join_materialized_columns.reference | 3 +++ tests/queries/0_stateless/01925_join_materialized_columns.sql | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference index 90f754f6e7c..e00de5f458d 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.reference +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -5,3 +5,6 @@ - 2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 +- +2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql index 9c4596f9915..7d5acc2cd25 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.sql +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -12,3 +12,5 @@ SELECT '-'; SELECT t1.dt, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.dt; SELECT '-'; SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time; +SELECT '-'; +SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time; From 84e02911cf4edcef68cac462ee8d74165ee77277 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 24 Jun 2021 17:14:36 +0300 Subject: [PATCH 071/290] Materialized columns for joined table, don't add to asterisk select --- src/Interpreters/DatabaseAndTableWithAlias.h | 2 -- src/Interpreters/ExpressionAnalyzer.cpp | 5 +++-- src/Interpreters/InterpreterSelectQuery.cpp | 5 ++--- src/Interpreters/JoinedTables.cpp | 4 ++-- src/Interpreters/JoinedTables.h | 3 +-- src/Interpreters/SelectQueryOptions.h | 15 ++++++++++++--- src/Interpreters/TreeRewriter.cpp | 5 ++--- src/Interpreters/getTableExpressions.cpp | 3 ++- .../01925_join_materialized_columns.reference | 10 ++++++---- .../01925_join_materialized_columns.sql | 2 ++ 10 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index b53cadce460..e60674d93c6 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -86,8 +86,6 @@ private: names.insert(col.name); } - -private: NameSet names; }; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index fe52b30da7b..2216f1b5818 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include @@ -900,8 +899,10 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( * in the subquery_for_set object this subquery is exposed as source and the temporary table _data1 as the `table`. * - this function shows the expression JOIN _data1. */ - auto interpreter = interpretSubquery(join_element.table_expression, getContext(), original_right_columns, query_options); + + auto interpreter = interpretSubquery( + join_element.table_expression, getContext(), original_right_columns, query_options.copy().setWithMaterialized()); { joined_plan = std::make_unique(); interpreter->buildQueryPlan(*joined_plan); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7cca527cbc1..173d363796e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -68,7 +68,6 @@ #include #include #include -#include #include #include @@ -330,7 +329,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( metadata_snapshot = storage->getInMemoryMetadataPtr(); } - if (has_input || !joined_tables.resolveTables()) + if (has_input || !joined_tables.resolveTables(options.with_materialized)) joined_tables.makeFakeTable(storage, metadata_snapshot, source_header); /// Rewrite JOINs @@ -339,7 +338,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), context->getSettingsRef()); joined_tables.reset(getSelectQuery()); - joined_tables.resolveTables(); + joined_tables.resolveTables(options.with_materialized); if (storage && joined_tables.isLeftTableSubquery()) { diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 5e53074d24f..c0c726b1e9b 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -185,9 +185,9 @@ StoragePtr JoinedTables::getLeftTableStorage() return DatabaseCatalog::instance().getTable(table_id, context); } -bool JoinedTables::resolveTables() +bool JoinedTables::resolveTables(bool with_materialized) { - tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, true); + tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, with_materialized); if (tables_with_columns.size() != table_expressions.size()) throw Exception("Unexpected tables count", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 52eb71e419d..6cbbb7c1400 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -30,14 +30,13 @@ public: } StoragePtr getLeftTableStorage(); - bool resolveTables(); + bool resolveTables(bool with_materialized); /// Make fake tables_with_columns[0] in case we have predefined input in InterpreterSelectQuery void makeFakeTable(StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const Block & source_header); std::shared_ptr makeTableJoin(const ASTSelectQuery & select_query); const TablesWithColumns & tablesWithColumns() const { return tables_with_columns; } - TablesWithColumns moveTablesWithColumns() { return std::move(tables_with_columns); } bool isLeftTableSubquery() const; bool isLeftTableFunction() const; diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index d723dbf4ff6..8050e184852 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -42,11 +42,14 @@ struct SelectQueryOptions bool ignore_alias = false; bool is_internal = false; bool is_subquery = false; // non-subquery can also have subquery_depth > 0, e.g. insert select + bool with_materialized = false; /// asterisk include materialized columns - SelectQueryOptions(QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, size_t depth = 0, bool is_subquery_ = false) + SelectQueryOptions( + QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, + size_t depth = 0, + bool is_subquery_ = false) : to_stage(stage), subquery_depth(depth), is_subquery(is_subquery_) - { - } + {} SelectQueryOptions copy() const { return *this; } @@ -114,6 +117,12 @@ struct SelectQueryOptions is_internal = value; return *this; } + + SelectQueryOptions & setWithMaterialized(bool value = true) + { + with_materialized = value; + return *this; + } }; } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 1f94cda6b0f..e2e7b68e757 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -32,7 +31,6 @@ #include #include -#include #include #include @@ -899,8 +897,9 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( const auto & right_table = tables_with_columns[1]; auto & cols_from_joined = result.analyzed_join->columns_from_joined_table; cols_from_joined = right_table.columns; + /// query can use materialized columns from right joined table, add it to columns_from_joined_table cols_from_joined.insert( - cols_from_joined.end(), right_table.materialized_columns.begin(), right_table.materialized_columns.end()); + cols_from_joined.end(), right_table.hidden_columns.begin(), right_table.hidden_columns.end()); result.analyzed_join->deduplicateAndQualifyColumnNames( source_columns_set, right_table.table.getQualifiedNamePrefix()); diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index f7d82a8f599..43f7030d06e 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -129,7 +129,8 @@ TablesWithColumns getDatabaseAndTablesWithColumns( NamesAndTypesList materialized; NamesAndTypesList aliases; NamesAndTypesList virtuals; - NamesAndTypesList names_and_types = getColumnsFromTableExpression(*table_expression, context, materialized, aliases, virtuals); + NamesAndTypesList names_and_types = getColumnsFromTableExpression( + *table_expression, context, materialized, aliases, virtuals); removeDuplicateColumns(names_and_types); diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference index e00de5f458d..8d93af00109 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.reference +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -1,10 +1,12 @@ +2020-02-02 13:00:00 fact2 t1_val2 2020-02-05 13:00:00 fact2 t1_val2 +- 2020-02-02 13:00:00 fact2 t1_val2 2020-02-02 2020-02-05 13:00:00 fact2 t1_val2 2020-02-05 - 2020-01-01 2020-01-01 2020-02-02 2020-02-05 - -2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 -2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 +2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 - -2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 -2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql index 7d5acc2cd25..91106a25436 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.sql +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -9,6 +9,8 @@ INSERT INTO t2 VALUES ('2020-01-01 12:00:00', 'fact1', 't2_val2'), ('2020-02-05 SELECT * FROM t1 JOIN t2 ON t1.foo = t2.bar WHERE t2.dt >= '2020-02-01'; SELECT '-'; +SELECT t1.*, t1.dt, t2.*, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar WHERE t2.dt >= '2020-02-01'; +SELECT '-'; SELECT t1.dt, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.dt; SELECT '-'; SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time; From af7776554b03e0299c8268397939234312085a66 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 24 Jun 2021 17:16:57 +0300 Subject: [PATCH 072/290] Fix space in ExpressionAnalyzer.cpp --- src/Interpreters/ExpressionAnalyzer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 2216f1b5818..a393440b1ae 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -899,8 +899,6 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( * in the subquery_for_set object this subquery is exposed as source and the temporary table _data1 as the `table`. * - this function shows the expression JOIN _data1. */ - - auto interpreter = interpretSubquery( join_element.table_expression, getContext(), original_right_columns, query_options.copy().setWithMaterialized()); { From 241b64d02ce7f2227e7d6b21e422b9a2c69e3394 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 24 Jun 2021 17:57:21 +0300 Subject: [PATCH 073/290] Support ALIASed columns for right joined table --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 4 ++-- src/Interpreters/JoinedTables.cpp | 4 ++-- src/Interpreters/JoinedTables.h | 2 +- src/Interpreters/SelectQueryOptions.h | 6 ++--- src/Interpreters/TreeRewriter.cpp | 6 ++--- src/Interpreters/getTableExpressions.cpp | 6 ++--- src/Interpreters/getTableExpressions.h | 2 +- .../01925_join_materialized_columns.reference | 10 ++++++++ .../01925_join_materialized_columns.sql | 24 +++++++++++++++++-- 10 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index a393440b1ae..00ffd540da0 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -900,7 +900,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( * - this function shows the expression JOIN _data1. */ auto interpreter = interpretSubquery( - join_element.table_expression, getContext(), original_right_columns, query_options.copy().setWithMaterialized()); + join_element.table_expression, getContext(), original_right_columns, query_options.copy().setWithAllColumns()); { joined_plan = std::make_unique(); interpreter->buildQueryPlan(*joined_plan); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 173d363796e..71181a84e1a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -329,7 +329,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( metadata_snapshot = storage->getInMemoryMetadataPtr(); } - if (has_input || !joined_tables.resolveTables(options.with_materialized)) + if (has_input || !joined_tables.resolveTables(options.with_all_cols)) joined_tables.makeFakeTable(storage, metadata_snapshot, source_header); /// Rewrite JOINs @@ -338,7 +338,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), context->getSettingsRef()); joined_tables.reset(getSelectQuery()); - joined_tables.resolveTables(options.with_materialized); + joined_tables.resolveTables(options.with_all_cols); if (storage && joined_tables.isLeftTableSubquery()) { diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index c0c726b1e9b..86ec067b00c 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -185,9 +185,9 @@ StoragePtr JoinedTables::getLeftTableStorage() return DatabaseCatalog::instance().getTable(table_id, context); } -bool JoinedTables::resolveTables(bool with_materialized) +bool JoinedTables::resolveTables(bool include_all_columns) { - tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, with_materialized); + tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, include_all_columns); if (tables_with_columns.size() != table_expressions.size()) throw Exception("Unexpected tables count", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h index 6cbbb7c1400..52581c19999 100644 --- a/src/Interpreters/JoinedTables.h +++ b/src/Interpreters/JoinedTables.h @@ -30,7 +30,7 @@ public: } StoragePtr getLeftTableStorage(); - bool resolveTables(bool with_materialized); + bool resolveTables(bool include_all_columns); /// Make fake tables_with_columns[0] in case we have predefined input in InterpreterSelectQuery void makeFakeTable(StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const Block & source_header); diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index 8050e184852..1a1f0267ab0 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -42,7 +42,7 @@ struct SelectQueryOptions bool ignore_alias = false; bool is_internal = false; bool is_subquery = false; // non-subquery can also have subquery_depth > 0, e.g. insert select - bool with_materialized = false; /// asterisk include materialized columns + bool with_all_cols = false; /// asterisk include materialized and aliased columns SelectQueryOptions( QueryProcessingStage::Enum stage = QueryProcessingStage::Complete, @@ -118,9 +118,9 @@ struct SelectQueryOptions return *this; } - SelectQueryOptions & setWithMaterialized(bool value = true) + SelectQueryOptions & setWithAllColumns(bool value = true) { - with_materialized = value; + with_all_cols = value; return *this; } }; diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index e2e7b68e757..b997e53f745 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -897,9 +897,9 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( const auto & right_table = tables_with_columns[1]; auto & cols_from_joined = result.analyzed_join->columns_from_joined_table; cols_from_joined = right_table.columns; - /// query can use materialized columns from right joined table, add it to columns_from_joined_table - cols_from_joined.insert( - cols_from_joined.end(), right_table.hidden_columns.begin(), right_table.hidden_columns.end()); + /// query can use materialized or aliased columns from right joined table, + /// we want to request it for right table + cols_from_joined.insert(cols_from_joined.end(), right_table.hidden_columns.begin(), right_table.hidden_columns.end()); result.analyzed_join->deduplicateAndQualifyColumnNames( source_columns_set, right_table.table.getQualifiedNamePrefix()); diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 43f7030d06e..2d9391f4673 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -116,13 +116,13 @@ static NamesAndTypesList getColumnsFromTableExpression( TablesWithColumns getDatabaseAndTablesWithColumns( const ASTTableExprConstPtrs & table_expressions, ContextPtr context, - bool add_materialized) + bool include_all) { TablesWithColumns tables_with_columns; String current_database = context->getCurrentDatabase(); - bool include_alias_cols = context->getSettingsRef().asterisk_include_alias_columns; - bool include_materialized_cols = add_materialized || context->getSettingsRef().asterisk_include_materialized_columns; + bool include_alias_cols = include_all || context->getSettingsRef().asterisk_include_alias_columns; + bool include_materialized_cols = include_all || context->getSettingsRef().asterisk_include_materialized_columns; for (const ASTTableExpression * table_expression : table_expressions) { diff --git a/src/Interpreters/getTableExpressions.h b/src/Interpreters/getTableExpressions.h index 19c27057c2f..6a999729a2f 100644 --- a/src/Interpreters/getTableExpressions.h +++ b/src/Interpreters/getTableExpressions.h @@ -23,6 +23,6 @@ ASTPtr extractTableExpression(const ASTSelectQuery & select, size_t table_number TablesWithColumns getDatabaseAndTablesWithColumns( const ASTTableExprConstPtrs & table_expressions, ContextPtr context, - bool add_materialized = false); + bool include_all = false); } diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference index 8d93af00109..fe00b746e57 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.reference +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -10,3 +10,13 @@ - 2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 +- +2020-01-01 12:00:00 fact1 t1_val1 2019-01-01 12:00:00 fact4 t2_val2 +2020-01-01 12:00:00 fact1 t1_val1 2020-01-01 12:00:00 fact1 t2_val2 +2020-01-01 13:00:00 fact3 t1_val3 2019-01-01 12:00:00 fact4 t2_val2 +2020-01-01 13:00:00 fact3 t1_val3 2020-01-01 12:00:00 fact1 t2_val2 +- +2020-02-02 13:00:00 fact2 t1_val2 2020-02-05 13:00:00 fact2 t1_val2 +- +fact1t1_val1 fact1t2_val2 +fact2t1_val2 fact2t1_val2 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql index 91106a25436..16fe00beb63 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.sql +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -1,8 +1,19 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; -CREATE TABLE t1 (time DateTime, foo String, dimension_1 String, dt Date MATERIALIZED toDate(time)) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, foo); -CREATE TABLE t2 (time DateTime, bar String, dimension_2 String, dt Date MATERIALIZED toDate(time)) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, bar); +CREATE TABLE t1 ( + time DateTime, foo String, dimension_1 String, + dt Date MATERIALIZED toDate(time), + dt1 Date MATERIALIZED toDayOfYear(time), + aliascol1 ALIAS foo || dimension_1 +) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, foo); + +CREATE TABLE t2 ( + time DateTime, bar String, dimension_2 String, + dt Date MATERIALIZED toDate(time), + dt2 Date MATERIALIZED toDayOfYear(time), + aliascol2 ALIAS bar || dimension_2 +) ENGINE = MergeTree() PARTITION BY toYYYYMM(dt) ORDER BY (dt, bar); INSERT INTO t1 VALUES ('2020-01-01 12:00:00', 'fact1', 't1_val1'), ('2020-02-02 13:00:00', 'fact2', 't1_val2'), ('2020-01-01 13:00:00', 'fact3', 't1_val3'); INSERT INTO t2 VALUES ('2020-01-01 12:00:00', 'fact1', 't2_val2'), ('2020-02-05 13:00:00', 'fact2', 't1_val2'), ('2019-01-01 12:00:00', 'fact4', 't2_val2'); @@ -16,3 +27,12 @@ SELECT '-'; SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time; SELECT '-'; SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time; +SELECT '-'; +SELECT * FROM t1 JOIN t2 ON t1.dt1 = t2.dt2 ORDER BY t1.time, t2.time; +SELECT '-'; +SELECT * FROM t1 JOIN t2 ON t1.foo = t2.bar WHERE t2.aliascol2 == 'fact2t1_val2'; +SELECT '-'; +SELECT t1.aliascol1, t2.aliascol2 FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.time, t2.time; +-- SELECT '-'; +-- SELECT * FROM t1 JOIN t2 ON t1.aliascol1 = t2.aliascol2 ORDER BY t1.time, t2.time; + From 2949cd1e6f7cfe5a26619681a454bf4776dc5d41 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 25 Jun 2021 13:46:19 +0300 Subject: [PATCH 074/290] Support ALIASed columns in JOIN ON expression --- src/Interpreters/DatabaseAndTableWithAlias.h | 2 +- src/Interpreters/ExpressionAnalyzer.cpp | 3 ++- src/Interpreters/InterpreterSelectQuery.cpp | 1 - src/Interpreters/TreeRewriter.cpp | 19 ++++++++++++------- .../01925_join_materialized_columns.reference | 2 ++ .../01925_join_materialized_columns.sql | 5 ++--- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index e60674d93c6..e43481025a0 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -61,7 +61,7 @@ struct TableWithColumnNamesAndTypes names.insert(col.name); } - bool hasColumn(const String & name) const { return names.count(name); } + bool hasColumn(const String & name) const { return names.contains(name); } void addHiddenColumns(const NamesAndTypesList & addition) { diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 00ffd540da0..326b4ac6705 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -806,7 +806,8 @@ JoinPtr SelectQueryExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain } ExpressionActionsChain::Step & step = chain.lastStep(columns_after_array_join); - chain.steps.push_back(std::make_unique(syntax->analyzed_join, table_join, step.getResultColumns())); + chain.steps.push_back(std::make_unique( + syntax->analyzed_join, table_join, step.getResultColumns())); chain.addStep(); return table_join; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 71181a84e1a..f95750ed5e2 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index b997e53f745..679ab4ea354 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -508,14 +508,10 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul } /// Find the columns that are obtained by JOIN. -void collectJoinedColumns(TableJoin & analyzed_join, const ASTSelectQuery & select_query, +void collectJoinedColumns(TableJoin & analyzed_join, const ASTTableJoin & table_join, const TablesWithColumns & tables, const Aliases & aliases) { - const ASTTablesInSelectQueryElement * node = select_query.join(); - if (!node || tables.size() < 2) - return; - - const auto & table_join = node->table_join->as(); + assert(tables.size() >= 2); if (table_join.using_expression_list) { @@ -936,7 +932,16 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( setJoinStrictness( *select_query, settings.join_default_strictness, settings.any_join_distinct_right_table_keys, result.analyzed_join->table_join); - collectJoinedColumns(*result.analyzed_join, *select_query, tables_with_columns, result.aliases); + if (const auto * join_ast = select_query->join(); join_ast && tables_with_columns.size() >= 2) + { + auto & table_join_ast = join_ast->table_join->as(); + if (table_join_ast.using_expression_list && result.metadata_snapshot) + replaceAliasColumnsInQuery(table_join_ast.using_expression_list, result.metadata_snapshot->getColumns(), result.array_join_result_to_source, getContext()); + if (table_join_ast.on_expression && result.metadata_snapshot) + replaceAliasColumnsInQuery(table_join_ast.on_expression, result.metadata_snapshot->getColumns(), result.array_join_result_to_source, getContext()); + + collectJoinedColumns(*result.analyzed_join, table_join_ast, tables_with_columns, result.aliases); + } result.aggregates = getAggregates(query, *select_query); result.window_function_asts = getWindowFunctions(query, *select_query); diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.reference b/tests/queries/0_stateless/01925_join_materialized_columns.reference index fe00b746e57..1dfda3c769b 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.reference +++ b/tests/queries/0_stateless/01925_join_materialized_columns.reference @@ -20,3 +20,5 @@ - fact1t1_val1 fact1t2_val2 fact2t1_val2 fact2t1_val2 +- +2020-02-02 13:00:00 2020-02-05 13:00:00 diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql index 16fe00beb63..6a34fef96ab 100644 --- a/tests/queries/0_stateless/01925_join_materialized_columns.sql +++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql @@ -33,6 +33,5 @@ SELECT '-'; SELECT * FROM t1 JOIN t2 ON t1.foo = t2.bar WHERE t2.aliascol2 == 'fact2t1_val2'; SELECT '-'; SELECT t1.aliascol1, t2.aliascol2 FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.time, t2.time; --- SELECT '-'; --- SELECT * FROM t1 JOIN t2 ON t1.aliascol1 = t2.aliascol2 ORDER BY t1.time, t2.time; - +SELECT '-'; +SELECT t1.time, t2.time FROM t1 JOIN t2 ON t1.aliascol1 = t2.aliascol2 ORDER BY t1.time, t2.time; From 06bad997201af908bf016b78161a9e1b4fe0592c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 25 Jun 2021 17:07:46 +0300 Subject: [PATCH 075/290] Update src/Functions/geoToH3.cpp Co-authored-by: Bharat Nallan --- src/Functions/geoToH3.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 90e29248d32..57973ab94fe 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -86,7 +86,8 @@ public: H3Index hindex; H3Error err = latLngToCell(&coord, res, &hindex); - if (err) { + if (err) + { throw Exception( "Incorrect coordinates lat:" + std::to_string(coord.lat) + " lng:" + std::to_string(coord.lng) + " err:" + std::to_string(err), ErrorCodes::INCORRECT_DATA); From f8b1a6d185ddfedfd0d8b54bbc2c1d6eaebe38a9 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 18:33:31 +0300 Subject: [PATCH 076/290] =?UTF-8?q?WriteBuffer=20is=20canceled=20?= =?UTF-8?q?=F0=9F=98=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Functions/DummyJSONParser.h | 6 ++---- src/Functions/FunctionSQLJSON.h | 27 ++++++++++++--------------- src/Functions/SimdJSONParser.h | 7 +------ 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 128ee88e0ca..01fdab1abb6 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -2,8 +2,6 @@ #include #include -#include -#include namespace DB { @@ -42,7 +40,7 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - ALWAYS_INLINE Element getUnderlyingElement() const { return {}; } + Element getElement() { return {}; } }; /// References an array in a JSON document. @@ -101,7 +99,7 @@ struct DummyJSONParser #endif }; -inline ALWAYS_INLINE WriteBufferFromString& operator<<(WriteBufferFromString& out, const DB::DummyJSONParser::Element &) +inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 1e9b25ee508..dc31ee2d3ff 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include @@ -30,10 +28,10 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; +extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -281,11 +279,11 @@ public: return false; } - String result; - WriteBufferFromString out(result); - out << current_element.getUnderlyingElement(); + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out << current_element.getElement(); + auto output_str = out.str(); ColumnString & col_str = assert_cast(dest); - col_str.insertData(result.data(), result.size()); + col_str.insertData(output_str.data(), output_str.size()); return true; } }; @@ -309,9 +307,7 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - String result; - WriteBufferFromString out(result); - + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM /// Create json array of results: [res1, res2, ...] out << "["; bool success = false; @@ -324,7 +320,7 @@ public: out << ", "; } success = true; - out << current_element.getUnderlyingElement(); + out << current_element.getElement(); } else if (status == VisitorStatus::Error) { @@ -338,7 +334,8 @@ public: return false; } ColumnString & col_str = assert_cast(dest); - col_str.insertData(reinterpret_cast(result.data()), result.size()); + auto output_str = out.str(); + col_str.insertData(output_str.data(), output_str.size()); return true; } }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index c11fca3272c..c5793088baf 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -50,7 +50,7 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; - ALWAYS_INLINE simdjson::dom::element getUnderlyingElement() const { return element; } + ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } private: simdjson::dom::element element; @@ -165,11 +165,6 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } -inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) -{ - return out << element.getUnderlyingElement(); -} - } #endif From 6981eb64ac9e641070c167fff598a4172e75948c Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 19:24:22 +0300 Subject: [PATCH 077/290] Fixes --- src/Functions/FunctionSQLJSON.h | 30 +++++++------------ .../JSONPath/ASTs/ASTJSONPathRange.h | 4 +-- .../JSONPath/Generators/GeneratorJSONPath.h | 9 +++--- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index dc31ee2d3ff..9e13d447b7d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -41,7 +41,7 @@ public: class Executor { public: - static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) + static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth) { MutableColumnPtr to{result_type->createColumn()}; to->reserve(input_rows_count); @@ -76,23 +76,10 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - /// If argument is successfully cast to (ColumnConst *) then it is quoted string - /// Example: - /// SomeFunction('some string argument') - /// - /// Otherwise it is a column - /// Example: - /// SomeFunction(database.table.column) - const ColumnPtr & arg_jsonpath = first_column.column; const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); - if (!arg_jsonpath_string) - { - throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; - } - const ColumnPtr & arg_json = second_column.column; const auto * col_json_const = typeid_cast(arg_json.get()); const auto * col_json_string @@ -102,14 +89,14 @@ public: const ColumnString::Chars & chars_path = arg_jsonpath_string->getChars(); const ColumnString::Offsets & offsets_path = arg_jsonpath_string->getOffsets(); - /// Get data and offsets for 1 argument (JSON) + /// Prepare to parse 1 argument (JSONPath) const char * query_begin = reinterpret_cast(&chars_path[0]); const char * query_end = query_begin + offsets_path[0] - 1; /// Tokenize query Tokens tokens(query_begin, query_end); /// Max depth 0 indicates that depth is not limited - IParser::Pos token_iterator(tokens, 0); + IParser::Pos token_iterator(tokens, parse_depth); /// Parse query and create AST tree Expected expected; @@ -121,7 +108,7 @@ public: throw Exception{"Unable to parse JSONPath", ErrorCodes::BAD_ARGUMENTS}; } - /// Get data and offsets for 1 argument (JSON) + /// Get data and offsets for 2 argument (JSON) const ColumnString::Chars & chars_json = col_json_string->getChars(); const ColumnString::Offsets & offsets_json = col_json_string->getOffsets(); @@ -179,12 +166,13 @@ public: /// 1. Lexer(path) -> Tokens /// 2. Create ASTPtr /// 3. Parser(Tokens, ASTPtr) -> complete AST - /// 4. Execute functions, call interpreter for each json (in function) + /// 4. Execute functions: call getNextItem on generator and handle each item + uint32_t parse_depth = getContext()->getSettingsRef().max_parser_depth; #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count, parse_depth); #endif - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count, parse_depth); } }; @@ -325,6 +313,8 @@ public: else if (status == VisitorStatus::Error) { /// ON ERROR + /// Here it is possible to handle errors with ON ERROR (as described in ISO/IEC TR 19075-6), + /// however this functionality is not implemented yet } current_element = root; } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h index 8a963d7fc6b..746c6211f29 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -14,8 +14,8 @@ public: public: /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) - /// Range is represented as - /// Single index is represented as + /// Range is represented as + /// Single index is represented as std::vector> ranges; bool is_star = false; }; diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 071a7ac3089..b918ceac003 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -73,7 +73,8 @@ public: { while (true) { - auto root = element; + /// element passed to us actually is root, so here we assign current to root + auto current = element; if (current_visitor < 0) { return VisitorStatus::Exhausted; @@ -81,13 +82,13 @@ public: for (int i = 0; i < current_visitor; ++i) { - visitors[i]->apply(root); + visitors[i]->apply(current); } VisitorStatus status = VisitorStatus::Error; for (size_t i = current_visitor; i < visitors.size(); ++i) { - status = visitors[i]->visit(root); + status = visitors[i]->visit(current); current_visitor = i; if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { @@ -98,7 +99,7 @@ public: if (status != VisitorStatus::Ignore) { - element = root; + element = current; return status; } } From 7892e44467294d6f02b0bd73c8b8a6b450f17694 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 21:14:08 +0300 Subject: [PATCH 078/290] =?UTF-8?q?Fix=20style=20yet=20again=20?= =?UTF-8?q?=F0=9F=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Functions/FunctionSQLJSON.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 9e13d447b7d..a6024a27e95 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -28,7 +28,6 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; From ffe49589a1d8fc2d97acdf3524b7679052206b94 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 26 Jun 2021 16:08:23 +0000 Subject: [PATCH 079/290] Description draft --- .../reference/quantilebfloat16.md | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md new file mode 100644 index 00000000000..00bec8e00db --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -0,0 +1,68 @@ +--- +toc_priority: 209 +--- + +# quantileBFloat16 {#quantilebfloat16} + + + +# Is there a BFloat16 data type in ClickHouse? + # How conversion to BFloat16 is made? +# Does quantile calculations have general implementation? And some methods are implemented to support BFloat16? +# Is quantile calculation is really table based as stated somewhere in PR + +# Perhaps add quantilesBFloat16 to page quantiles.md +# Add alias to page median.md + +Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. +bfloat16 is a floating point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. The function converts input values to 32-bit floats and then take the most significant 16 bits. Then it calculates the histogram of these values. Calculated bfloat16 value is converted to 64-bit float data type by appending zero bits. +The function is a fast quantile estimator with a relative error no more than 0.390625%. + +**Syntax** + +``` sql +quantileBFloat16[(level)](expr) +``` + +Alias: `medianBFloat16` + +**Arguments** + +- `expr` — sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md). + +**Parameters** + +- `level` — Level of quantile. Optional. Possible values are in a range from 0 to 1. Default value: 0.5. [Float](../../../sql-reference/data-types/float.md). + +**Returned value** + +- Approximate quantile of the specified level. + +Type: [Float64](../../../sql-reference/data-types/float.md#float32-float64). + +**Example** + +Input table has an integer and a float columns: + +``` text +┌─a─┬─────b─┐ +│ 1 │ 1.001 │ +│ 2 │ 1.002 │ +│ 3 │ 1.003 │ +└───┴───────┘ +``` + +Query: + +``` sql +SELECT quantilesBFloat16(0.75)(a), quantilesBFloat16(0.75)(b) FROM example_table; +``` + +Result: + +``` text +┌─quantilesBFloat16(0.75)(a)─┬─quantilesBFloat16(0.75)(b)─┐ +│ [3] │ [1] │ +└────────────────────────────┴────────────────────────────┘ +``` +Note that all floating point values were truncated to 1.0 when converting to bfloat16. From 27ba48ebe76482ed11691db0c24d5836af6b5252 Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 26 Jun 2021 16:13:55 +0000 Subject: [PATCH 080/290] Aliases added --- .../aggregate-functions/reference/median.md | 1 + .../aggregate-functions/reference/quantiles.md | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/median.md b/docs/en/sql-reference/aggregate-functions/reference/median.md index b4f38a9b562..b15a97f0992 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/median.md +++ b/docs/en/sql-reference/aggregate-functions/reference/median.md @@ -12,6 +12,7 @@ Functions: - `medianTimingWeighted` — Alias for [quantileTimingWeighted](#quantiletimingweighted). - `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest). - `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted). +- `medianBFloat16` — Alias for [quantileBFloat16](#quantilebfloat16). **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index abce6a9e7f0..49a2df57bf5 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -6,4 +6,12 @@ toc_priority: 201 Syntax: `quantiles(level1, level2, …)(x)` -All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. +All the quantile functions also have corresponding quantiles functions. They calculate quantiles of all listed levels in one pass, and return them as an array. + +- `quantiles`; +- `quantilesDeterministic`; +- `quantilesTiming`; +- `quantilesTimingWeighted`; +- `quantilesExact`; +- `quantilesExactWeighted`; +- `quantilesTDigest`. From ccddb60f453a9020e533b457710b1e77414d6e7c Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 26 Jun 2021 16:15:37 +0000 Subject: [PATCH 081/290] Removed comments --- .../aggregate-functions/reference/quantilebfloat16.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 00bec8e00db..9e18dff423e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,16 +4,6 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} - - -# Is there a BFloat16 data type in ClickHouse? - # How conversion to BFloat16 is made? -# Does quantile calculations have general implementation? And some methods are implemented to support BFloat16? -# Is quantile calculation is really table based as stated somewhere in PR - -# Perhaps add quantilesBFloat16 to page quantiles.md -# Add alias to page median.md - Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. The function converts input values to 32-bit floats and then take the most significant 16 bits. Then it calculates the histogram of these values. Calculated bfloat16 value is converted to 64-bit float data type by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. From d48f5227ea3182957dae190bb58177933d7a408d Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 26 Jun 2021 20:04:29 +0000 Subject: [PATCH 082/290] Fixes and updates --- .../reference/quantilebfloat16.md | 15 ++++++++++----- .../aggregate-functions/reference/quantiles.md | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 9e18dff423e..87b7e96dd7e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,8 +4,8 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} -Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. -bfloat16 is a floating point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. The function converts input values to 32-bit floats and then take the most significant 16 bits. Then it calculates the histogram of these values. Calculated bfloat16 value is converted to 64-bit float data type by appending zero bits. +Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. +The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates the histogram of these values. Resulting value is converted to 64-bit float by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. **Syntax** @@ -18,11 +18,11 @@ Alias: `medianBFloat16` **Arguments** -- `expr` — sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md). +- `expr` — Column with numeric data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md). **Parameters** -- `level` — Level of quantile. Optional. Possible values are in a range from 0 to 1. Default value: 0.5. [Float](../../../sql-reference/data-types/float.md). +- `level` — Level of quantile. Optional. Possible values are in the range from 0 to 1. Default value: 0.5. [Float](../../../sql-reference/data-types/float.md). **Returned value** @@ -55,4 +55,9 @@ Result: │ [3] │ [1] │ └────────────────────────────┴────────────────────────────┘ ``` -Note that all floating point values were truncated to 1.0 when converting to bfloat16. +Note that all floating point values in the example are truncated to 1.0 when converting to bfloat16. + +**See Also** + +- [median](../../../sql-reference/aggregate-functions/reference/median.md#median) +- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index 49a2df57bf5..766766d2f94 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -14,4 +14,5 @@ All the quantile functions also have corresponding quantiles functions. They cal - `quantilesTimingWeighted`; - `quantilesExact`; - `quantilesExactWeighted`; -- `quantilesTDigest`. +- `quantilesTDigest`; +- `quantilesBFloat16`; From 024cf55252da50dc7cba93370f0185089871a55e Mon Sep 17 00:00:00 2001 From: Alexey Date: Sat, 26 Jun 2021 20:10:24 +0000 Subject: [PATCH 083/290] toc_priority set on median page to place it after quantiles --- docs/en/sql-reference/aggregate-functions/reference/median.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/sql-reference/aggregate-functions/reference/median.md b/docs/en/sql-reference/aggregate-functions/reference/median.md index b15a97f0992..b309b20fd5f 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/median.md +++ b/docs/en/sql-reference/aggregate-functions/reference/median.md @@ -1,3 +1,7 @@ +--- +toc_priority: 212 +--- + # median {#median} The `median*` functions are the aliases for the corresponding `quantile*` functions. They calculate median of a numeric data sample. From c977c33d6d60f042d3c1b7452cc982be17b01d14 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 27 Jun 2021 19:18:15 +0300 Subject: [PATCH 084/290] Fix bug in execution of TTL GROUP BY --- src/DataStreams/TTLAggregationAlgorithm.cpp | 154 ++++++++++-------- src/DataStreams/TTLColumnAlgorithm.cpp | 2 + src/DataStreams/TTLDeleteAlgorithm.cpp | 2 + .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 51 ++++++ .../MergeTree/MergeTreeDataPartTTLInfo.h | 9 + src/Storages/MergeTree/TTLMergeSelector.cpp | 3 + src/Storages/StorageReplicatedMergeTree.cpp | 4 + tests/integration/test_ttl_replicated/test.py | 13 +- .../test.py | 5 +- 9 files changed, 174 insertions(+), 69 deletions(-) diff --git a/src/DataStreams/TTLAggregationAlgorithm.cpp b/src/DataStreams/TTLAggregationAlgorithm.cpp index 9a1cf45772f..6d5c234a074 100644 --- a/src/DataStreams/TTLAggregationAlgorithm.cpp +++ b/src/DataStreams/TTLAggregationAlgorithm.cpp @@ -36,88 +36,110 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( storage_.getContext()->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); aggregator = std::make_unique(params); + + if (isMinTTLExpired()) + new_ttl_info.finished = true; } void TTLAggregationAlgorithm::execute(Block & block) { - if (!block) - { - if (!aggregation_result.empty()) - { - MutableColumns result_columns = header.cloneEmptyColumns(); - finalizeAggregates(result_columns); - block = header.cloneWithColumns(std::move(result_columns)); - } - return; - } - - const auto & column_names = header.getNames(); + bool some_rows_were_aggregated = false; MutableColumns result_columns = header.cloneEmptyColumns(); - MutableColumns aggregate_columns = header.cloneEmptyColumns(); - auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); - auto where_column = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); - - size_t rows_aggregated = 0; - size_t current_key_start = 0; - size_t rows_with_current_key = 0; - - for (size_t i = 0; i < block.rows(); ++i) + if (!block) /// Empty block -- no more data, but we may still have some accumulated rows { - UInt32 cur_ttl = getTimestampByIndex(ttl_column.get(), i); - bool where_filter_passed = !where_column || where_column->getBool(i); - bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed; - - bool same_as_current = true; - for (size_t j = 0; j < description.group_by_keys.size(); ++j) + if (!aggregation_result.empty()) /// Still have some aggregated data, let's update TTL { - const String & key_column = description.group_by_keys[j]; - const IColumn * values_column = block.getByName(key_column).column.get(); - if (!same_as_current || (*values_column)[i] != current_key_value[j]) - { - values_column->get(i, current_key_value[j]); - same_as_current = false; - } - } - - if (!same_as_current) - { - if (rows_with_current_key) - calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key); finalizeAggregates(result_columns); - - current_key_start = rows_aggregated; - rows_with_current_key = 0; + some_rows_were_aggregated = true; } - - if (ttl_expired) + else /// No block, all aggregated, just finish { - ++rows_with_current_key; - ++rows_aggregated; - for (const auto & name : column_names) - { - const IColumn * values_column = block.getByName(name).column.get(); - auto & column = aggregate_columns[header.getPositionByName(name)]; - column->insertFrom(*values_column, i); - } - } - else - { - new_ttl_info.update(cur_ttl); - for (const auto & name : column_names) - { - const IColumn * values_column = block.getByName(name).column.get(); - auto & column = result_columns[header.getPositionByName(name)]; - column->insertFrom(*values_column, i); - } + return; } } + else + { + const auto & column_names = header.getNames(); + MutableColumns aggregate_columns = header.cloneEmptyColumns(); - if (rows_with_current_key) - calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key); + auto ttl_column = executeExpressionAndGetColumn(description.expression, block, description.result_column); + auto where_column = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); + + size_t rows_aggregated = 0; + size_t current_key_start = 0; + size_t rows_with_current_key = 0; + + for (size_t i = 0; i < block.rows(); ++i) + { + UInt32 cur_ttl = getTimestampByIndex(ttl_column.get(), i); + bool where_filter_passed = !where_column || where_column->getBool(i); + bool ttl_expired = isTTLExpired(cur_ttl) && where_filter_passed; + + bool same_as_current = true; + for (size_t j = 0; j < description.group_by_keys.size(); ++j) + { + const String & key_column = description.group_by_keys[j]; + const IColumn * values_column = block.getByName(key_column).column.get(); + if (!same_as_current || (*values_column)[i] != current_key_value[j]) + { + values_column->get(i, current_key_value[j]); + same_as_current = false; + } + } + + if (!same_as_current) + { + if (rows_with_current_key) + { + some_rows_were_aggregated = true; + calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key); + } + finalizeAggregates(result_columns); + + current_key_start = rows_aggregated; + rows_with_current_key = 0; + } + + if (ttl_expired) + { + ++rows_with_current_key; + ++rows_aggregated; + for (const auto & name : column_names) + { + const IColumn * values_column = block.getByName(name).column.get(); + auto & column = aggregate_columns[header.getPositionByName(name)]; + column->insertFrom(*values_column, i); + } + } + else + { + for (const auto & name : column_names) + { + const IColumn * values_column = block.getByName(name).column.get(); + auto & column = result_columns[header.getPositionByName(name)]; + column->insertFrom(*values_column, i); + } + } + } + + if (rows_with_current_key) + { + some_rows_were_aggregated = true; + calculateAggregates(aggregate_columns, current_key_start, rows_with_current_key); + } + } block = header.cloneWithColumns(std::move(result_columns)); + + /// If some rows were aggregated we have to recalculate ttl info's + if (some_rows_were_aggregated) + { + auto ttl_column_after_aggregation = executeExpressionAndGetColumn(description.expression, block, description.result_column); + for (size_t i = 0; i < block.rows(); ++i) + new_ttl_info.update(getTimestampByIndex(ttl_column_after_aggregation.get(), i)); + } } void TTLAggregationAlgorithm::calculateAggregates(const MutableColumns & aggregate_columns, size_t start_pos, size_t length) @@ -133,6 +155,7 @@ void TTLAggregationAlgorithm::calculateAggregates(const MutableColumns & aggrega aggregator->executeOnBlock(aggregate_chunk, length, aggregation_result, key_columns, columns_for_aggregator, no_more_keys); + } void TTLAggregationAlgorithm::finalizeAggregates(MutableColumns & result_columns) @@ -140,6 +163,7 @@ void TTLAggregationAlgorithm::finalizeAggregates(MutableColumns & result_columns if (!aggregation_result.empty()) { auto aggregated_res = aggregator->convertToBlocks(aggregation_result, true, 1); + for (auto & agg_block : aggregated_res) { for (const auto & it : description.set_parts) diff --git a/src/DataStreams/TTLColumnAlgorithm.cpp b/src/DataStreams/TTLColumnAlgorithm.cpp index 140631ac0bf..5c0a5e1ae83 100644 --- a/src/DataStreams/TTLColumnAlgorithm.cpp +++ b/src/DataStreams/TTLColumnAlgorithm.cpp @@ -21,6 +21,8 @@ TTLColumnAlgorithm::TTLColumnAlgorithm( new_ttl_info = old_ttl_info; is_fully_empty = false; } + else + new_ttl_info.finished = true; } void TTLColumnAlgorithm::execute(Block & block) diff --git a/src/DataStreams/TTLDeleteAlgorithm.cpp b/src/DataStreams/TTLDeleteAlgorithm.cpp index c364bb06f3e..f1bbe6d4b7d 100644 --- a/src/DataStreams/TTLDeleteAlgorithm.cpp +++ b/src/DataStreams/TTLDeleteAlgorithm.cpp @@ -9,6 +9,8 @@ TTLDeleteAlgorithm::TTLDeleteAlgorithm( { if (!isMinTTLExpired()) new_ttl_info = old_ttl_info; + else + new_ttl_info.finished = true; } void TTLDeleteAlgorithm::execute(Block & block) diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp index e130fbc1798..f1beb09c482 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.cpp @@ -55,6 +55,10 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) MergeTreeDataPartTTLInfo ttl_info; ttl_info.min = col["min"].getUInt(); ttl_info.max = col["max"].getUInt(); + + if (col.has("finished")) + ttl_info.finished = col["finished"].getUInt(); + String name = col["name"].getString(); columns_ttl.emplace(name, ttl_info); @@ -67,6 +71,9 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) table_ttl.min = table["min"].getUInt(); table_ttl.max = table["max"].getUInt(); + if (table.has("finished")) + table_ttl.finished = table["finished"].getUInt(); + updatePartMinMaxTTL(table_ttl.min, table_ttl.max); } @@ -77,6 +84,10 @@ void MergeTreeDataPartTTLInfos::read(ReadBuffer & in) MergeTreeDataPartTTLInfo ttl_info; ttl_info.min = elem["min"].getUInt(); ttl_info.max = elem["max"].getUInt(); + + if (elem.has("finished")) + ttl_info.finished = elem["finished"].getUInt(); + String expression = elem["expression"].getString(); ttl_info_map.emplace(expression, ttl_info); @@ -126,6 +137,8 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const writeIntText(it->second.min, out); writeString(",\"max\":", out); writeIntText(it->second.max, out); + writeString(R"(,"finished":)", out); + writeIntText(static_cast(it->second.finished), out); writeString("}", out); } writeString("]", out); @@ -138,6 +151,8 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const writeIntText(table_ttl.min, out); writeString(R"(,"max":)", out); writeIntText(table_ttl.max, out); + writeString(R"(,"finished":)", out); + writeIntText(static_cast(table_ttl.finished), out); writeString("}", out); } @@ -159,6 +174,8 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const writeIntText(it->second.min, out); writeString(R"(,"max":)", out); writeIntText(it->second.max, out); + writeString(R"(,"finished":)", out); + writeIntText(static_cast(it->second.finished), out); writeString("}", out); } writeString("]", out); @@ -202,6 +219,39 @@ time_t MergeTreeDataPartTTLInfos::getMinimalMaxRecompressionTTL() const return max; } +bool MergeTreeDataPartTTLInfos::hasAnyNonFinishedTTLs() const +{ + auto has_non_finished_ttl = [] (const TTLInfoMap & map) -> bool + { + for (const auto & [name, info] : map) + { + if (!info.finished) + return true; + } + return false; + }; + + if (!table_ttl.finished) + return true; + + if (has_non_finished_ttl(columns_ttl)) + return true; + + if (has_non_finished_ttl(rows_where_ttl)) + return true; + + if (has_non_finished_ttl(moves_ttl)) + return true; + + if (has_non_finished_ttl(recompression_ttl)) + return true; + + if (has_non_finished_ttl(group_by_ttl)) + return true; + + return false; +} + std::optional selectTTLDescriptionForTTLInfos(const TTLDescriptions & descriptions, const TTLInfoMap & ttl_info_map, time_t current_time, bool use_max) { time_t best_ttl_time = 0; @@ -232,4 +282,5 @@ std::optional selectTTLDescriptionForTTLInfos(const TTLDescripti return best_ttl_time ? *best_entry_it : std::optional(); } + } diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index 9d1606ee44a..2b79ad1aac5 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -14,6 +14,11 @@ struct MergeTreeDataPartTTLInfo time_t min = 0; time_t max = 0; + /// This TTL was computed on completely expired part. It doesn't make sense + /// to select such parts for TTL again. But make sense to recalcuate TTL + /// again for merge with multiple parts. + bool finished = false; + void update(time_t time) { if (time && (!min || time < min)) @@ -28,6 +33,7 @@ struct MergeTreeDataPartTTLInfo min = other_info.min; max = std::max(other_info.max, max); + finished &= other_info.finished; } }; @@ -60,6 +66,9 @@ struct MergeTreeDataPartTTLInfos void write(WriteBuffer & out) const; void update(const MergeTreeDataPartTTLInfos & other_infos); + /// Has any TTLs which are not calculated on completely expired parts. + bool hasAnyNonFinishedTTLs() const; + void updatePartMinMaxTTL(time_t time_min, time_t time_max) { if (time_min && (!part_min_ttl || time_min < part_min_ttl)) diff --git a/src/Storages/MergeTree/TTLMergeSelector.cpp b/src/Storages/MergeTree/TTLMergeSelector.cpp index fc7aa93e129..ab686c9952d 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.cpp +++ b/src/Storages/MergeTree/TTLMergeSelector.cpp @@ -111,6 +111,9 @@ bool TTLDeleteMergeSelector::isTTLAlreadySatisfied(const IMergeSelector::Part & if (only_drop_parts) return false; + if (!part.ttl_infos->hasAnyNonFinishedTTLs()) + return false; + return !part.shall_participate_in_merges; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e91f3d9554e..ea4376a56ec 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -551,6 +551,10 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( break; } + /// This replica inactive, don't check anything + if (!inactive_replicas.empty() && inactive_replicas.count(replica)) + break; + /// It maybe already removed from zk, but local in-memory mutations /// state was not updated. if (!getZooKeeper()->exists(fs::path(zookeeper_path) / "mutations" / mutation_id)) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index de5e5984082..f37c28b2a80 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -351,6 +351,7 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_delete_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND + SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 '''.format(suff=num_run, replica=node.name)) node.query( @@ -359,6 +360,7 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_group_by_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND GROUP BY id SET val = sum(val) + SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 '''.format(suff=num_run, replica=node.name)) node.query( @@ -367,6 +369,7 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_where_{suff}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND DELETE WHERE id % 2 = 1 + SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 '''.format(suff=num_run, replica=node.name)) node_left.query("INSERT INTO test_ttl_delete VALUES (now(), 1)") @@ -397,9 +400,9 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") - exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") - node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) - node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_left, "OPTIMIZE TABLE test_ttl_delete FINAL") + node_left.query("OPTIMIZE TABLE test_ttl_group_by FINAL", timeout=20) + node_left.query("OPTIMIZE TABLE test_ttl_where FINAL", timeout=20) # After OPTIMIZE TABLE, it is not guaranteed that everything is merged. # Possible scenario (for test_ttl_group_by): @@ -414,6 +417,10 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): node_right.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) node_right.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") + node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) + node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + assert node_left.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" assert node_right.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index dd8e1bc7a9e..03387b0be67 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -79,7 +79,10 @@ def test_upgrade_while_mutation(start_cluster): node3.restart_with_latest_version(signal=9) - exec_query_with_retry(node3, "ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) + # wait replica became active + exec_query_with_retry(node3, "SYSTEM RESTART REPLICA mt1") + + node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) # will delete nothing, but previous async mutation will finish with this query assert_eq_with_retry(node3, "SELECT COUNT() from mt1", "50000\n") From 1b56b0a02058054f8193002307af5835ed95320a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Jun 2021 11:10:38 +0300 Subject: [PATCH 085/290] Fix flaky test --- src/DataStreams/TTLAggregationAlgorithm.cpp | 7 ++++++- .../0_stateless/01280_ttl_where_group_by.reference | 8 ++++---- tests/queries/0_stateless/01280_ttl_where_group_by.sh | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/DataStreams/TTLAggregationAlgorithm.cpp b/src/DataStreams/TTLAggregationAlgorithm.cpp index 6d5c234a074..12d28ff4aea 100644 --- a/src/DataStreams/TTLAggregationAlgorithm.cpp +++ b/src/DataStreams/TTLAggregationAlgorithm.cpp @@ -137,8 +137,13 @@ void TTLAggregationAlgorithm::execute(Block & block) if (some_rows_were_aggregated) { auto ttl_column_after_aggregation = executeExpressionAndGetColumn(description.expression, block, description.result_column); + auto where_column_after_aggregation = executeExpressionAndGetColumn(description.where_expression, block, description.where_result_column); for (size_t i = 0; i < block.rows(); ++i) - new_ttl_info.update(getTimestampByIndex(ttl_column_after_aggregation.get(), i)); + { + bool where_filter_passed = !where_column_after_aggregation || where_column_after_aggregation->getBool(i); + if (where_filter_passed) + new_ttl_info.update(getTimestampByIndex(ttl_column_after_aggregation.get(), i)); + } } } diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.reference b/tests/queries/0_stateless/01280_ttl_where_group_by.reference index 7fe00709dee..65e7e5b158f 100644 --- a/tests/queries/0_stateless/01280_ttl_where_group_by.reference +++ b/tests/queries/0_stateless/01280_ttl_where_group_by.reference @@ -16,11 +16,11 @@ ttl_01280_3 2 1 0 3 3 1 8 2 ttl_01280_4 -1 1 0 4 -10 2 13 9 +0 4 +13 9 ttl_01280_5 1 2 7 5 2 3 6 5 ttl_01280_6 -1 5 3 5 -2 10 3 5 +1 3 5 +2 3 5 diff --git a/tests/queries/0_stateless/01280_ttl_where_group_by.sh b/tests/queries/0_stateless/01280_ttl_where_group_by.sh index 9f30c7c5872..c9936ce7afd 100755 --- a/tests/queries/0_stateless/01280_ttl_where_group_by.sh +++ b/tests/queries/0_stateless/01280_ttl_where_group_by.sh @@ -80,7 +80,7 @@ insert into ttl_01280_4 values (1, 5, 4, 9, now())" sleep 2 optimize "ttl_01280_4" -$CLICKHOUSE_CLIENT --query "select a, b, x, y from ttl_01280_4 ORDER BY a, b, x, y" +$CLICKHOUSE_CLIENT --query "select x, y from ttl_01280_4 ORDER BY a, b, x, y" $CLICKHOUSE_CLIENT --query "drop table if exists ttl_01280_5" @@ -107,7 +107,7 @@ insert into ttl_01280_6 values (1, 5, 3, 5, now())" sleep 2 optimize "ttl_01280_6" -$CLICKHOUSE_CLIENT --query "select a, b, x, y from ttl_01280_6 ORDER BY a, b, x, y" +$CLICKHOUSE_CLIENT --query "select a, x, y from ttl_01280_6 ORDER BY a, b, x, y" $CLICKHOUSE_CLIENT -q "DROP TABLE ttl_01280_1" $CLICKHOUSE_CLIENT -q "DROP TABLE ttl_01280_2" From 48d21bb03b3cd7c522735bdcfdc381f0185b9c13 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 28 Jun 2021 13:19:13 +0000 Subject: [PATCH 086/290] Fix inconsistency --- src/Interpreters/ActionsDAG.cpp | 3 +++ ...01925_test_const_column_group_by_consistency.reference | 6 ++++++ .../01925_test_const_column_group_by_consistency.sql | 8 ++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference create mode 100644 tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 9fa48f6ceab..55c863a6f8c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -219,6 +219,9 @@ const ActionsDAG::Node & ActionsDAG::addFunction( column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type); } + if (all_const && column && !isColumnConst(*column) && column->size() <= 1) + column = ColumnConst::create(std::move(column), column->size()); + /// If the result is not a constant, just in case, we will consider the result as unknown. if (column && isColumnConst(*column)) { diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference new file mode 100644 index 00000000000..f9bcdca26da --- /dev/null +++ b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference @@ -0,0 +1,6 @@ +1 0 +1 0 +1 0 +1 0 +1 1 0 +0 0 0 diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql new file mode 100644 index 00000000000..d288c7db023 --- /dev/null +++ b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql @@ -0,0 +1,8 @@ +SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; + +SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 ORDER BY a; + +SELECT isConstant(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT 1 as b, isConstant(b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT 0 as b, least(isConstant(materialize(1)), b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; From 54a7e2158c2f976bcc660c4ed53debd008ec3220 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 28 Jun 2021 17:09:26 +0300 Subject: [PATCH 087/290] Update ActionsDAG.cpp --- src/Interpreters/ActionsDAG.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 55c863a6f8c..f0c6e49b891 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -219,8 +219,8 @@ const ActionsDAG::Node & ActionsDAG::addFunction( column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type); } - if (all_const && column && !isColumnConst(*column) && column->size() <= 1) - column = ColumnConst::create(std::move(column), column->size()); + if (all_const && column && !isColumnConst(*column) && column->size() == 1) + column = ColumnConst::create(std::move(column), 1); /// If the result is not a constant, just in case, we will consider the result as unknown. if (column && isColumnConst(*column)) From 72b281987e59028d215c0ee77ec0bef99072d30f Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Jun 2021 17:14:26 +0300 Subject: [PATCH 088/290] Add more debug --- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 766d988500d..a348b07ba92 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -752,13 +752,16 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor bool force_ttl = false; for (const auto & part : parts) { - new_data_part->ttl_infos.update(part->ttl_infos); if (metadata_snapshot->hasAnyTTL() && !part->checkAllTTLCalculated(metadata_snapshot)) { LOG_INFO(log, "Some TTL values were not calculated for part {}. Will calculate them forcefully during merge.", part->name); need_remove_expired_values = true; force_ttl = true; } + else + { + new_data_part->ttl_infos.update(part->ttl_infos); + } } const auto & part_min_ttl = new_data_part->ttl_infos.part_min_ttl; @@ -939,7 +942,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merged_stream = std::make_shared(merged_stream, sort_description, SizeLimits(), 0 /*limit_hint*/, deduplicate_by_columns); if (need_remove_expired_values) + { + LOG_DEBUG(log, "Outdated rows found in source parts, TTLs processing enabled for merge"); merged_stream = std::make_shared(merged_stream, data, metadata_snapshot, new_data_part, time_of_merge, force_ttl); + } if (metadata_snapshot->hasSecondaryIndices()) { From c5e5ebcdf3e6b3db0fac6161b3fc4b5cfb6fa439 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 28 Jun 2021 21:25:52 +0300 Subject: [PATCH 089/290] First draft --- .../functions/logical-functions.md | 179 +++++++++++++++++- docs/en/sql-reference/operators/index.md | 10 +- 2 files changed, 182 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 6cce0e4fff5..6a41ac5bebf 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -5,15 +5,186 @@ toc_title: Logical # Logical Functions {#logical-functions} -Logical functions accept any numeric types, but return a UInt8 number equal to 0 or 1. +Logical functions accept any numeric types, but return a [UInt8](../../sql-reference/data-types/int-uint.md) number equal to 0, 1 or in some cases `NULL`. Zero as an argument is considered “false,” while any non-zero value is considered “true”. -## and, AND operator {#and-and-operator} +## and {#and-and-operator} -## or, OR operator {#or-or-operator} +Calculates the result of logical conjunction between two or more values. Corresponds to [Logical AND Operator](../../sql-reference/operators/index.md#logical-and-operator). -## not, NOT operator {#not-not-operator} +**Syntax** + +``` sql +and(val1, val2...) +``` + +**Arguments** + +- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `0`, if there is at least one zero value argument. +- `NULL`, if there are no zero values arguments and there is at least one `NULL` argument. +- `1`, otherwise. + +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). + +**Example** + +Query: + +``` sql +SELECT and(0, 1, -2); +``` + +Result: + +``` text +┌─and(0, 1, -2)─┐ +│ 0 │ +└───────────────┘ +``` + +With `NULL`: + +``` sql +SELECT and(NULL, 1, 10, -2); +``` + +Result: + +``` text +┌─and(NULL, 1, 10, -2)─┐ +│ ᴺᵁᴸᴸ │ +└──────────────────────┘ +``` + +## or {#or-or-operator} + +Calculates the result of logical disjunction between two or more values. Corresponds to [Logical OR Operator](../../sql-reference/operators/index.md#logical-or-operator). + +**Syntax** + +``` sql +and(val1, val2...) +``` + +**Arguments** + +- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `1`, if there is at least one non-zero value. +- `0`, if there are only zero values. +- `NULL`, if there is at least one `NULL` values. + +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). + +**Example** + +Query: + +``` sql +SELECT or(1, 0, 0, 2, NULL); +``` + +Result: + +``` text +┌─or(1, 0, 0, 2, NULL)─┐ +│ 1 │ +└──────────────────────┘ +``` + +With `NULL`: + +``` sql +SELECT or(0, NULL); +``` + +Result: + +``` text +┌─or(0, NULL)─┐ +│ ᴺᵁᴸᴸ │ +└─────────────┘ +``` + +## not {#not-not-operator} + +Calculates the result of logical negation of a value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). + +**Syntax** + +``` sql +not(val); +``` + +**Arguments** + +- `val` — value. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `1`, if the `val` is a `0`. +- `0`, if the `val` is a non-zero value. +- `NULL`, if the `val` is a `NULL` value. + +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). + +**Example** + +Query: + +``` sql +SELECT NOT(1); +``` + +Result: + +``` test +┌─not(1)─┐ +│ 0 │ +└────────┘ +``` ## xor {#xor} +Calculates the result of logical exclusive disjunction between two or more values. For more than two values the function calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). + +**Syntax** + +``` sql +xor(val1, val2...) +``` + +**Arguments** + +- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `1`, for two values: if one of the values is zero and other is not. +- `0`, for two values: if both values are zero or non-zero at the same. +- `NULL`, if there is at least one `NULL` values. + +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). + +**Example** + +Query: + +``` sql +SELECT xor(0, 1, 1); +``` + +Result + +``` text +┌─xor(0, 1, 1)─┐ +│ 0 │ +└──────────────┘ +``` diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 268e56a5034..dd519cb1454 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -213,15 +213,19 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 ## Logical Negation Operator {#logical-negation-operator} -`NOT a` – The `not(a)` function. +Syntax `NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). ## Logical AND Operator {#logical-and-operator} -`a AND b` – The`and(a, b)` function. +Syntax `a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#and-and-operator). ## Logical OR Operator {#logical-or-operator} -`a OR b` – The `or(a, b)` function. +Syntax `a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#or-or-operator). + +## Logical XOR operator (#logical-xor-operator) + +Syntax `a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#xor). ## Conditional Operator {#conditional-operator} From 985ca2cd5a1860233a98f4fb3fa79a0828c8d252 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 28 Jun 2021 21:44:59 +0300 Subject: [PATCH 090/290] some fixes --- .../functions/logical-functions.md | 24 +++++++++---------- docs/en/sql-reference/operators/index.md | 10 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 6a41ac5bebf..2896f3bdd33 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -11,7 +11,7 @@ Zero as an argument is considered “false,” while any non-zero value is consi ## and {#and-and-operator} -Calculates the result of logical conjunction between two or more values. Corresponds to [Logical AND Operator](../../sql-reference/operators/index.md#logical-and-operator). +Calculates the result of the logical conjunction between two or more values. Corresponds to [Logical AND Operator](../../sql-reference/operators/index.md#logical-and-operator). **Syntax** @@ -21,7 +21,7 @@ and(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -29,7 +29,7 @@ and(val1, val2...) - `NULL`, if there are no zero values arguments and there is at least one `NULL` argument. - `1`, otherwise. -Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). **Example** @@ -63,7 +63,7 @@ Result: ## or {#or-or-operator} -Calculates the result of logical disjunction between two or more values. Corresponds to [Logical OR Operator](../../sql-reference/operators/index.md#logical-or-operator). +Calculates the result of the logical disjunction between two or more values. Corresponds to [Logical OR Operator](../../sql-reference/operators/index.md#logical-or-operator). **Syntax** @@ -73,7 +73,7 @@ and(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -81,7 +81,7 @@ and(val1, val2...) - `0`, if there are only zero values. - `NULL`, if there is at least one `NULL` values. -Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). **Example** @@ -115,7 +115,7 @@ Result: ## not {#not-not-operator} -Calculates the result of logical negation of a value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). +Calculates the result of the logical negation of a value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). **Syntax** @@ -125,7 +125,7 @@ not(val); **Arguments** -- `val` — value. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val` — value. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -133,7 +133,7 @@ not(val); - `0`, if the `val` is a non-zero value. - `NULL`, if the `val` is a `NULL` value. -Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). **Example** @@ -153,7 +153,7 @@ Result: ## xor {#xor} -Calculates the result of logical exclusive disjunction between two or more values. For more than two values the function calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). +Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). **Syntax** @@ -163,7 +163,7 @@ xor(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt]](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -171,7 +171,7 @@ xor(val1, val2...) - `0`, for two values: if both values are zero or non-zero at the same. - `NULL`, if there is at least one `NULL` values. -Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([[UInt8](../../sql-reference/data-types/int-uint.md)]). +Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). **Example** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index dd519cb1454..0c58c8d0353 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -213,19 +213,19 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 ## Logical Negation Operator {#logical-negation-operator} -Syntax `NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). +Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). ## Logical AND Operator {#logical-and-operator} -Syntax `a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#and-and-operator). +Syntax `SELECT a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#and-and-operator). ## Logical OR Operator {#logical-or-operator} -Syntax `a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#or-or-operator). +Syntax `SELECT a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#or-or-operator). -## Logical XOR operator (#logical-xor-operator) +## Logical XOR operator {#logical-xor-operator} -Syntax `a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#xor). +Syntax `SELECT a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#xor). ## Conditional Operator {#conditional-operator} From 12798a2c471227e51fb7bcb4f0c6cd7391250305 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 28 Jun 2021 22:32:37 +0300 Subject: [PATCH 091/290] more fixes --- docs/en/sql-reference/functions/logical-functions.md | 2 +- docs/en/sql-reference/operators/index.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 2896f3bdd33..5b967b877c7 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -153,7 +153,7 @@ Result: ## xor {#xor} -Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). +Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function works as if it calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). **Syntax** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 0c58c8d0353..818edef6a90 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -211,10 +211,6 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type - [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions -## Logical Negation Operator {#logical-negation-operator} - -Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). - ## Logical AND Operator {#logical-and-operator} Syntax `SELECT a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#and-and-operator). @@ -223,6 +219,10 @@ Syntax `SELECT a AND b` — calculates logical conjunction of `a` and `b` with t Syntax `SELECT a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#or-or-operator). +## Logical Negation Operator {#logical-negation-operator} + +Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). + ## Logical XOR operator {#logical-xor-operator} Syntax `SELECT a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#xor). From c7def2a76d8a330acf076c322a6eb079f844d910 Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 28 Jun 2021 19:56:20 +0000 Subject: [PATCH 092/290] Example fixed --- .../aggregate-functions/reference/quantilebfloat16.md | 9 +++++---- .../aggregate-functions/reference/quantiles.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 87b7e96dd7e..763cb07bcb1 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -39,21 +39,22 @@ Input table has an integer and a float columns: │ 1 │ 1.001 │ │ 2 │ 1.002 │ │ 3 │ 1.003 │ +│ 4 │ 1.004 │ └───┴───────┘ ``` Query: ``` sql -SELECT quantilesBFloat16(0.75)(a), quantilesBFloat16(0.75)(b) FROM example_table; +SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; ``` Result: ``` text -┌─quantilesBFloat16(0.75)(a)─┬─quantilesBFloat16(0.75)(b)─┐ -│ [3] │ [1] │ -└────────────────────────────┴────────────────────────────┘ +┌─quantileBFloat16(0.75)(a)─┬─quantileBFloat16(0.75)(b)─┐ +│ 3 │ 1 │ +└───────────────────────────┴───────────────────────────┘ ``` Note that all floating point values in the example are truncated to 1.0 when converting to bfloat16. diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index 766766d2f94..8ed446863a9 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -6,7 +6,7 @@ toc_priority: 201 Syntax: `quantiles(level1, level2, …)(x)` -All the quantile functions also have corresponding quantiles functions. They calculate quantiles of all listed levels in one pass, and return them as an array. +All the quantile functions also have corresponding quantiles functions. They calculate quantiles of all listed levels in one pass and return them as an array. - `quantiles`; - `quantilesDeterministic`; @@ -15,4 +15,4 @@ All the quantile functions also have corresponding quantiles functions. They cal - `quantilesExact`; - `quantilesExactWeighted`; - `quantilesTDigest`; -- `quantilesBFloat16`; +- `quantilesBFloat16`. From f2c871f7c8368d977ac8ff5261060ea8a757f059 Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 28 Jun 2021 20:07:20 +0000 Subject: [PATCH 093/290] Removed implementation details --- .../aggregate-functions/reference/quantilebfloat16.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 763cb07bcb1..30dc145ff42 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -5,7 +5,7 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. -The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates the histogram of these values. Resulting value is converted to 64-bit float by appending zero bits. +The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates bfloat16 quantile value and converts the result to a 64-bit float by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. **Syntax** From 538558ccb55e67ad4dfa17d7926d4a3a18cb004e Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 28 Jun 2021 20:26:27 +0000 Subject: [PATCH 094/290] Fixed links in median.md --- .../aggregate-functions/reference/median.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/median.md b/docs/en/sql-reference/aggregate-functions/reference/median.md index b309b20fd5f..f2a6080eda3 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/median.md +++ b/docs/en/sql-reference/aggregate-functions/reference/median.md @@ -8,15 +8,15 @@ The `median*` functions are the aliases for the corresponding `quantile*` functi Functions: -- `median` — Alias for [quantile](#quantile). -- `medianDeterministic` — Alias for [quantileDeterministic](#quantiledeterministic). -- `medianExact` — Alias for [quantileExact](#quantileexact). -- `medianExactWeighted` — Alias for [quantileExactWeighted](#quantileexactweighted). -- `medianTiming` — Alias for [quantileTiming](#quantiletiming). -- `medianTimingWeighted` — Alias for [quantileTimingWeighted](#quantiletimingweighted). -- `medianTDigest` — Alias for [quantileTDigest](#quantiletdigest). -- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](#quantiletdigestweighted). -- `medianBFloat16` — Alias for [quantileBFloat16](#quantilebfloat16). +- `median` — Alias for [quantile](quantile.md). +- `medianDeterministic` — Alias for [quantileDeterministic](quantiledeterministic.md). +- `medianExact` — Alias for [quantileExact](quantileexact.md). +- `medianExactWeighted` — Alias for [quantileExactWeighted](quantileexactweighted.md). +- `medianTiming` — Alias for [quantileTiming](quantiletiming.md). +- `medianTimingWeighted` — Alias for [quantileTimingWeighted](quantiletimingweighted.md). +- `medianTDigest` — Alias for [quantileTDigest](quantiletdigest.md). +- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](quantiletdigestweighted.md). +- `medianBFloat16` — Alias for [quantileBFloat16](quantilebfloat16.md). **Example** From 8ac7e147b76d5044f431378acde0edfaae10d343 Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 28 Jun 2021 20:26:57 +0000 Subject: [PATCH 095/290] Text changed to match other quantile descriptions --- .../aggregate-functions/reference/quantilebfloat16.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 30dc145ff42..3e3824c3dfb 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,7 +4,7 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} -Calculates a [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. +Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates bfloat16 quantile value and converts the result to a 64-bit float by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. From 6816e8c5aa570638bad56baf1342e72b939b474d Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 29 Jun 2021 14:54:32 +0000 Subject: [PATCH 096/290] Revert change, fix injectiveness --- src/Functions/materialize.h | 5 +++++ src/Interpreters/ActionsDAG.cpp | 3 --- .../01925_test_const_column_group_by_consistency.reference | 4 ---- .../01925_test_const_column_group_by_consistency.sql | 6 ------ 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 5d36a4b2340..376fb34a672 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -29,6 +29,11 @@ public: return name; } + bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const override + { + return true; + } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } size_t getNumberOfArguments() const override diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index f0c6e49b891..9fa48f6ceab 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -219,9 +219,6 @@ const ActionsDAG::Node & ActionsDAG::addFunction( column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type); } - if (all_const && column && !isColumnConst(*column) && column->size() == 1) - column = ColumnConst::create(std::move(column), 1); - /// If the result is not a constant, just in case, we will consider the result as unknown. if (column && isColumnConst(*column)) { diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference index f9bcdca26da..2e50651146b 100644 --- a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference +++ b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference @@ -1,6 +1,2 @@ 1 0 1 0 -1 0 -1 0 -1 1 0 -0 0 0 diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql index d288c7db023..ed7a64d0877 100644 --- a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql +++ b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql @@ -1,8 +1,2 @@ SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; - SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 ORDER BY a; - -SELECT isConstant(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT 1 as b, isConstant(b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT 0 as b, least(isConstant(materialize(1)), b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; From 40c51889a7e75a34472854758f97d375c08205d8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 29 Jun 2021 16:05:53 +0000 Subject: [PATCH 097/290] Update test --- .../01925_test_const_column_group_by_consistency.reference | 2 -- .../01925_test_const_column_group_by_consistency.sql | 2 -- .../01925_test_group_by_const_consistency.reference | 5 +++++ .../0_stateless/01925_test_group_by_const_consistency.sql | 7 +++++++ 4 files changed, 12 insertions(+), 4 deletions(-) delete mode 100644 tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference delete mode 100644 tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql create mode 100644 tests/queries/0_stateless/01925_test_group_by_const_consistency.reference create mode 100644 tests/queries/0_stateless/01925_test_group_by_const_consistency.sql diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference deleted file mode 100644 index 2e50651146b..00000000000 --- a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.reference +++ /dev/null @@ -1,2 +0,0 @@ -1 0 -1 0 diff --git a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql b/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql deleted file mode 100644 index ed7a64d0877..00000000000 --- a/tests/queries/0_stateless/01925_test_const_column_group_by_consistency.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference new file mode 100644 index 00000000000..f2342900cb9 --- /dev/null +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference @@ -0,0 +1,5 @@ +1 0 +1 0 +1 0 +2 1 0 +D 0 diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql new file mode 100644 index 00000000000..f31d22a88ac --- /dev/null +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql @@ -0,0 +1,7 @@ +SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; + +SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 ORDER BY a; + +SELECT 2 as b, less(1, b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT upper('d') as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; From e6d423cca1c640d294d4761b5355be83c03d8b0b Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 29 Jun 2021 22:30:12 +0300 Subject: [PATCH 098/290] Apply suggestions from code review Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- .../functions/logical-functions.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 5b967b877c7..6db326a7092 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -5,9 +5,9 @@ toc_title: Logical # Logical Functions {#logical-functions} -Logical functions accept any numeric types, but return a [UInt8](../../sql-reference/data-types/int-uint.md) number equal to 0, 1 or in some cases `NULL`. +Performs logical operations on arguments of any numeric types, but returns a [UInt8](../../sql-reference/data-types/int-uint.md) number equal to 0, 1 or `NULL` in some cases. -Zero as an argument is considered “false,” while any non-zero value is considered “true”. +Zero as an argument is considered `false`, while any non-zero value is considered `true`. ## and {#and-and-operator} @@ -21,7 +21,7 @@ and(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. **Returned value** @@ -73,7 +73,7 @@ and(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. **Returned value** @@ -115,7 +115,7 @@ Result: ## not {#not-not-operator} -Calculates the result of the logical negation of a value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). +Calculates the result of the logical negation of the value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). **Syntax** @@ -125,11 +125,11 @@ not(val); **Arguments** -- `val` — value. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val` — The value. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. **Returned value** -- `1`, if the `val` is a `0`. +- `1`, if the `val` is `0`. - `0`, if the `val` is a non-zero value. - `NULL`, if the `val` is a `NULL` value. @@ -163,7 +163,7 @@ xor(val1, val2...) **Arguments** -- `val` — list of at least two values. Any [Int-UInt](../../sql-reference/data-types/int-uint.md) type, [float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. **Returned value** @@ -181,7 +181,7 @@ Query: SELECT xor(0, 1, 1); ``` -Result +Result: ``` text ┌─xor(0, 1, 1)─┐ From cdb75d7081df554c88f8c10664204ec76bf5b4a5 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 29 Jun 2021 22:42:34 +0300 Subject: [PATCH 099/290] small fixes --- docs/en/sql-reference/functions/logical-functions.md | 12 ++++++------ docs/en/sql-reference/operators/index.md | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 6db326a7092..100b6ce19af 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -9,7 +9,7 @@ Performs logical operations on arguments of any numeric types, but returns a [UI Zero as an argument is considered `false`, while any non-zero value is considered `true`. -## and {#and-and-operator} +## and {#logical-and-function} Calculates the result of the logical conjunction between two or more values. Corresponds to [Logical AND Operator](../../sql-reference/operators/index.md#logical-and-operator). @@ -61,7 +61,7 @@ Result: └──────────────────────┘ ``` -## or {#or-or-operator} +## or {#logical-or-function} Calculates the result of the logical disjunction between two or more values. Corresponds to [Logical OR Operator](../../sql-reference/operators/index.md#logical-or-operator). @@ -79,7 +79,7 @@ and(val1, val2...) - `1`, if there is at least one non-zero value. - `0`, if there are only zero values. -- `NULL`, if there is at least one `NULL` values. +- `NULL`, if there aren't any non-zero values besides `NULL`. Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). @@ -113,7 +113,7 @@ Result: └─────────────┘ ``` -## not {#not-not-operator} +## not {#logical-not-function} Calculates the result of the logical negation of the value. Corresponds to [Logical Negation Operator](../../sql-reference/operators/index.md#logical-negation-operator). @@ -151,7 +151,7 @@ Result: └────────┘ ``` -## xor {#xor} +## xor {#logical-xor-function} Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function works as if it calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). @@ -169,7 +169,7 @@ xor(val1, val2...) - `1`, for two values: if one of the values is zero and other is not. - `0`, for two values: if both values are zero or non-zero at the same. -- `NULL`, if there is at least one `NULL` values. +- `NULL`, if there is at least one `NULL` value. Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 818edef6a90..54239d48082 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -213,19 +213,19 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 ## Logical AND Operator {#logical-and-operator} -Syntax `SELECT a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#and-and-operator). +Syntax `SELECT a AND b` — calculates logical conjunction of `a` and `b` with the function [and](../../sql-reference/functions/logical-functions.md#logical-and-function). ## Logical OR Operator {#logical-or-operator} -Syntax `SELECT a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#or-or-operator). +Syntax `SELECT a OR b` — calculates logical disjunction of `a` and `b` with the function [or](../../sql-reference/functions/logical-functions.md#logical-or-function). ## Logical Negation Operator {#logical-negation-operator} -Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#not-not-operator). +Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#logical-not-function). ## Logical XOR operator {#logical-xor-operator} -Syntax `SELECT a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#xor). +Syntax `SELECT a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#logical-xor-function). ## Conditional Operator {#conditional-operator} From 94a4c7ff85c2cf784f8e9ef9a4da744e7226b5e2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 29 Jun 2021 19:56:05 +0000 Subject: [PATCH 100/290] Update ANTLR skip list --- tests/queries/skip_list.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 7c1f998e91d..76c0d0a4817 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -520,7 +520,8 @@ "01914_exchange_dictionaries", "01915_create_or_replace_dictionary", "01913_names_of_tuple_literal", - "01925_merge_prewhere_table" + "01925_merge_prewhere_table", + "01925_test_group_by_const_consistency" ], "parallel": [ From 6b8e2eee1961849353427773340587685cf45227 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 29 Jun 2021 23:47:19 +0300 Subject: [PATCH 101/290] Small fix --- docs/en/sql-reference/functions/logical-functions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 100b6ce19af..8dc82bb7f97 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -21,7 +21,7 @@ and(val1, val2...) **Arguments** -- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -73,7 +73,7 @@ and(val1, val2...) **Arguments** -- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -125,7 +125,7 @@ not(val); **Arguments** -- `val` — The value. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. +- `val` — The value. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** @@ -163,12 +163,12 @@ xor(val1, val2...) **Arguments** -- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md) type. +- `val1, val2, ...` — List of at least two values. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Nullable](../../sql-reference/data-types/nullable.md). **Returned value** - `1`, for two values: if one of the values is zero and other is not. -- `0`, for two values: if both values are zero or non-zero at the same. +- `0`, for two values: if both values are zero or non-zero at the same time. - `NULL`, if there is at least one `NULL` value. Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). From 283c2222b7b517e674b0b9fe934fbdd728610042 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 30 Jun 2021 00:02:45 +0300 Subject: [PATCH 102/290] Added translation --- .../functions/logical-functions.md | 184 +++++++++++++++++- docs/ru/sql-reference/operators/index.md | 16 +- 2 files changed, 188 insertions(+), 12 deletions(-) diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 8566657d2eb..10d15ffe377 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -5,15 +5,187 @@ toc_title: "Логические функции" # Логические функции {#logicheskie-funktsii} -Логические функции принимают любые числовые типы, а возвращают число типа UInt8, равное 0 или 1. +Логические функции производят логические операции над любыми числовыми типами, а возвращают число типа [UInt8](../../sql-reference/data-types/int-uint.md), равное 0, 1, а в некоторых случаях `NULL`. -Ноль в качестве аргумента считается «ложью», а любое ненулевое значение - «истиной». +Ноль в качестве аргумента считается `ложью`, а любое ненулевое значение — `истиной`. -## and, оператор AND {#and-operator-and} +## and {#logical-and-function} -## or, оператор OR {#or-operator-or} +Вычисляет результат логической конъюнкции между двумя и более значениями. Соответствует [оператору логического ‘И’](../../sql-reference/operators/index.md#logical-and-operator). -## not, оператор NOT {#not-operator-not} +**Синтаксис** -## xor {#xor} +``` sql +and(val1, val2...) +``` + +**Аргументы** + +- `val1, val2, ...` — список из как минимум двух значений. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Nullable](../../sql-reference/data-types/nullable.md). + +**Возвращаемое значение** + +- `0`, если среди аргументов есть хотя бы один нуль. +- `NULL`, если среди аргументов нет нулей, но есть хотя бы один `NULL`. +- `1`, в остальных случаях. + +Тип: [UInt8](../../sql-reference/data-types/int-uint.md) или [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Запрос: + +``` sql +SELECT and(0, 1, -2); +``` + +Результат: + +``` text +┌─and(0, 1, -2)─┐ +│ 0 │ +└───────────────┘ +``` + +Со значениями `NULL`: + +``` sql +SELECT and(NULL, 1, 10, -2); +``` + +Результат: + +``` text +┌─and(NULL, 1, 10, -2)─┐ +│ ᴺᵁᴸᴸ │ +└──────────────────────┘ +``` + +## or {#logical-or-function} + +Вычисляет результат логической дизъюнкции между двумя и более значениями. Соответствует [оператору логического ‘ИЛИ’](../../sql-reference/operators/index.md#logical-or-operator). + +**Синтаксис** + +``` sql +and(val1, val2...) +``` + +**Аргументы** + +- `val1, val2, ...` — список из как минимум двух значений. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `1`, если среди аргументов есть хотя бы одно ненулевое число. +- `0`, если среди аргументов только нули. +- `NULL`, если среди аргументов нет никаких других ненулевых значений кроме `NULL`. + +Тип: [UInt8](../../sql-reference/data-types/int-uint.md) или [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Запрос: + +``` sql +SELECT or(1, 0, 0, 2, NULL); +``` + +Результат: + +``` text +┌─or(1, 0, 0, 2, NULL)─┐ +│ 1 │ +└──────────────────────┘ +``` + +Со значениями `NULL`: + +``` sql +SELECT or(0, NULL); +``` + +Результат: + +``` text +┌─or(0, NULL)─┐ +│ ᴺᵁᴸᴸ │ +└─────────────┘ +``` + +## not {#logical-not-function} + +Вычисляет результат логического отрицания аргумента. Соответствует [оператору логического отрицания](../../sql-reference/operators/index.md#logical-negation-operator). + +**Синтаксис** + +``` sql +not(val); +``` + +**Аргументы** + +- `val` — значение. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Nullable](../../sql-reference/data-types/nullable.md). + +**Возвращаемое значение** + +- `1`, если `val` — это `0`. +- `0`, если `val` — это ненулевое число. +- `NULL`, если `val` — это `NULL`. + +Тип: [UInt8](../../sql-reference/data-types/int-uint.md) или [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Запрос: + +``` sql +SELECT NOT(1); +``` + +Результат: + +``` test +┌─not(1)─┐ +│ 0 │ +└────────┘ +``` + +## xor {#logical-xor-function} + +Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает, как-будто сначала вычисляет `XOR` для первых двух значений, а потом использует результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего ‘ИЛИ’](../../sql-reference/operators/index.md#logical-xor-operator). + +**Синтаксис** + +``` sql +xor(val1, val2...) +``` + +**Аргументы** + +- `val1, val2, ...` — список из как минимум двух значений. [Int](../../sql-reference/data-types/int-uint.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Nullable](../../sql-reference/data-types/nullable.md). + +**Returned value** + +- `1`, для двух значений: если одно из значений является нулем, а второе нет. +- `0`, для двух значений: если оба значения одновременно нули или ненулевые числа. +- `NULL`, если среди аргументов хотя бы один `NULL`. + +Тип: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). + +**Пример** + +Запрос: + +``` sql +SELECT xor(0, 1, 1); +``` + +Результат: + +``` text +┌─xor(0, 1, 1)─┐ +│ 0 │ +└──────────────┘ +``` diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 5cf21b64079..3eb6c07ec95 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -211,17 +211,21 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 - Тип данных [Interval](../../sql-reference/operators/index.md) - Функции преобразования типов [toInterval](../../sql-reference/operators/index.md#function-tointerval) -## Оператор логического отрицания {#operator-logicheskogo-otritsaniia} +## Оператор логического ‘И’ {#logical-and-operator} -`NOT a` - функция `not(a)` +Синтаксис `SELECT a AND b` — вычисляет логическую конъюнкцию между `a` и `b` функцией [and](../../sql-reference/functions/logical-functions.md#logical-and-function). -## Оператор логического ‘И’ {#operator-logicheskogo-i} +## Оператор логического ‘ИЛИ’ {#logical-or-operator} -`a AND b` - функция `and(a, b)` +Синтаксис `SELECT a OR b` — вычисляет логическую дизъюнкцию между `a` и `b` функцией [or](../../sql-reference/functions/logical-functions.md#logical-or-function). -## Оператор логического ‘ИЛИ’ {#operator-logicheskogo-ili} +## Оператор логического отрицания {#logical-negation-operator} -`a OR b` - функция `or(a, b)` +Синтаксис `SELECT NOT a` — вычисляет логическое отрицание `a` функцией [not](../../sql-reference/functions/logical-functions.md#logical-not-function). + +## Оператор логического исключающего ‘ИЛИ’ {#logical-xor-operator} + +Синтаксис `SELECT a XOR b` — вычисляет логическую исключающую дизъюнкцию между `a` и `b` функцией [xor](../../sql-reference/functions/logical-functions.md#logical-xor-function). ## Условный оператор {#uslovnyi-operator} From abba5021383b84a197da101e82b7d731ed0c6319 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 30 Jun 2021 00:54:03 +0300 Subject: [PATCH 103/290] Apply suggestions from code review Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/functions/logical-functions.md | 2 +- docs/ru/sql-reference/functions/logical-functions.md | 3 +-- docs/ru/sql-reference/operators/index.md | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 8dc82bb7f97..4b188184074 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -79,7 +79,7 @@ and(val1, val2...) - `1`, if there is at least one non-zero value. - `0`, if there are only zero values. -- `NULL`, if there aren't any non-zero values besides `NULL`. +- `NULL`, if there are only zero values and `NULL`. Type: [UInt8](../../sql-reference/data-types/int-uint.md) or [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 10d15ffe377..8c800f5ddae 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -79,7 +79,7 @@ and(val1, val2...) - `1`, если среди аргументов есть хотя бы одно ненулевое число. - `0`, если среди аргументов только нули. -- `NULL`, если среди аргументов нет никаких других ненулевых значений кроме `NULL`. +- `NULL`, если среди аргументов нет ненулевых значений, и есть `NULL`. Тип: [UInt8](../../sql-reference/data-types/int-uint.md) или [Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md)). @@ -188,4 +188,3 @@ SELECT xor(0, 1, 1); │ 0 │ └──────────────┘ ``` - diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 3eb6c07ec95..030de6a7574 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -211,11 +211,11 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 - Тип данных [Interval](../../sql-reference/operators/index.md) - Функции преобразования типов [toInterval](../../sql-reference/operators/index.md#function-tointerval) -## Оператор логического ‘И’ {#logical-and-operator} +## Оператор логического "И" {#logical-and-operator} Синтаксис `SELECT a AND b` — вычисляет логическую конъюнкцию между `a` и `b` функцией [and](../../sql-reference/functions/logical-functions.md#logical-and-function). -## Оператор логического ‘ИЛИ’ {#logical-or-operator} +## Оператор логического "ИЛИ" {#logical-or-operator} Синтаксис `SELECT a OR b` — вычисляет логическую дизъюнкцию между `a` и `b` функцией [or](../../sql-reference/functions/logical-functions.md#logical-or-function). @@ -223,7 +223,7 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 Синтаксис `SELECT NOT a` — вычисляет логическое отрицание `a` функцией [not](../../sql-reference/functions/logical-functions.md#logical-not-function). -## Оператор логического исключающего ‘ИЛИ’ {#logical-xor-operator} +## Оператор логического исключающего "ИЛИ" {#logical-xor-operator} Синтаксис `SELECT a XOR b` — вычисляет логическую исключающую дизъюнкцию между `a` и `b` функцией [xor](../../sql-reference/functions/logical-functions.md#logical-xor-function). From 6868c79c18f8159c205c3b0e2587ccb80a7a5269 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 30 Jun 2021 00:55:28 +0300 Subject: [PATCH 104/290] Small fix --- docs/ru/sql-reference/functions/logical-functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 8c800f5ddae..619752e7d0e 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -11,7 +11,7 @@ toc_title: "Логические функции" ## and {#logical-and-function} -Вычисляет результат логической конъюнкции между двумя и более значениями. Соответствует [оператору логического ‘И’](../../sql-reference/operators/index.md#logical-and-operator). +Вычисляет результат логической конъюнкции между двумя и более значениями. Соответствует [оператору логического "И"](../../sql-reference/operators/index.md#logical-and-operator). **Синтаксис** @@ -63,7 +63,7 @@ SELECT and(NULL, 1, 10, -2); ## or {#logical-or-function} -Вычисляет результат логической дизъюнкции между двумя и более значениями. Соответствует [оператору логического ‘ИЛИ’](../../sql-reference/operators/index.md#logical-or-operator). +Вычисляет результат логической дизъюнкции между двумя и более значениями. Соответствует [оператору логического "ИЛИ"](../../sql-reference/operators/index.md#logical-or-operator). **Синтаксис** @@ -153,7 +153,7 @@ SELECT NOT(1); ## xor {#logical-xor-function} -Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает, как-будто сначала вычисляет `XOR` для первых двух значений, а потом использует результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего ‘ИЛИ’](../../sql-reference/operators/index.md#logical-xor-operator). +Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает, как-будто сначала вычисляет `XOR` для первых двух значений, а потом использует результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего "ИЛИ"](../../sql-reference/operators/index.md#logical-xor-operator). **Синтаксис** From 245209d2bb738313ec505a3b3e1506bc35e5c521 Mon Sep 17 00:00:00 2001 From: George Date: Wed, 30 Jun 2021 01:27:36 +0300 Subject: [PATCH 105/290] Better wording --- docs/ru/sql-reference/functions/logical-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 619752e7d0e..e3fc75402ac 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -153,7 +153,7 @@ SELECT NOT(1); ## xor {#logical-xor-function} -Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает, как-будто сначала вычисляет `XOR` для первых двух значений, а потом использует результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего "ИЛИ"](../../sql-reference/operators/index.md#logical-xor-operator). +Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает так: сначала вычисляет `XOR` для первых двух значений, а потом использует полученный результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего "ИЛИ"](../../sql-reference/operators/index.md#logical-xor-operator). **Синтаксис** From 2eb27540b227fdb23d84fc7631156777a19b0094 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 30 Jun 2021 15:29:09 +0300 Subject: [PATCH 106/290] Some test version --- src/Storages/MergeTree/ActiveDataPartSet.cpp | 15 +++--------- src/Storages/MergeTree/ActiveDataPartSet.h | 2 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 23 +++++++++++++++++++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 10 ++++---- src/Storages/StorageReplicatedMergeTree.h | 18 +++++++++++++++ 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/Storages/MergeTree/ActiveDataPartSet.cpp b/src/Storages/MergeTree/ActiveDataPartSet.cpp index 28a4f9e2068..a3bbe1c4801 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.cpp +++ b/src/Storages/MergeTree/ActiveDataPartSet.cpp @@ -16,8 +16,7 @@ ActiveDataPartSet::ActiveDataPartSet(MergeTreeDataFormatVersion format_version_, add(name); } -/// FIXME replace warnings with logical errors -bool ActiveDataPartSet::add(const String & name, Strings * out_replaced_parts, Poco::Logger * log) +bool ActiveDataPartSet::add(const String & name, Strings * out_replaced_parts) { /// TODO make it exception safe (out_replaced_parts->push_back(...) may throw) auto part_info = MergeTreePartInfo::fromPartName(name, format_version); @@ -38,11 +37,7 @@ bool ActiveDataPartSet::add(const String & name, Strings * out_replaced_parts, P if (!part_info.contains(it->first)) { if (!part_info.isDisjoint(it->first)) - { - if (log) - LOG_ERROR(log, "Part {} intersects previous part {}. It is a bug.", name, it->first.getPartName()); - assert(false); - } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects previous part {}. It is a bug.", name, it->first.getPartName()); ++it; break; } @@ -65,11 +60,7 @@ bool ActiveDataPartSet::add(const String & name, Strings * out_replaced_parts, P } if (it != part_info_to_name.end() && !part_info.isDisjoint(it->first)) - { - if (log) - LOG_ERROR(log, "Part {} intersects next part {}. It is a bug.", name, it->first.getPartName()); - assert(false); - } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} intersects next part {}. It is a bug.", name, it->first.getPartName()); part_info_to_name.emplace(part_info, name); return true; diff --git a/src/Storages/MergeTree/ActiveDataPartSet.h b/src/Storages/MergeTree/ActiveDataPartSet.h index 188bba91795..3d0ac8f949d 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.h +++ b/src/Storages/MergeTree/ActiveDataPartSet.h @@ -50,7 +50,7 @@ public: /// Returns true if the part was actually added. If out_replaced_parts != nullptr, it will contain /// parts that were replaced from the set by the newly added part. - bool add(const String & name, Strings * out_replaced_parts = nullptr, Poco::Logger * log = nullptr); + bool add(const String & name, Strings * out_replaced_parts = nullptr); bool remove(const MergeTreePartInfo & part_info) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 0e6ab5e32f7..2bb285fcf34 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -439,7 +439,30 @@ Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormat /// DROP_RANGE does not add a real part, but we must disable merges in that range if (type == DROP_RANGE) + { + auto drop_range_part_info = MergeTreePartInfo::fromPartName(new_part_name, format_version); + + /// It's DROP PART and we don't want to add it into virtual parts + /// because it can lead to intersecting parts on stale replicas and this + /// problem is fundamental. So we have very weak guarantees for DROP + /// PART. If any concurrent merge will be assigned then DROP PART will + /// delete nothing and part will be successfuly merged into bigger part. + /// + /// dropPart used in the following cases: + /// 1) Remove empty parts after TTL. + /// 2) Remove parts after move between shards. + /// 3) User queries: ALTER TABLE DROP PART 'part_name'. + /// + /// In the first case merge of empty part is even better than DROP. In + /// the second case part UUIDs used to forbid merges for moding parts so + /// there is no problem with concurrent merges. The third case is quite + /// rare and we give very weak guarantee: there will be no active part + /// with this name, but possibly it was merged to some other part. + if (!drop_range_part_info.isFakeDropRangePart()) + return {}; + return {new_part_name}; + } if (type == REPLACE_RANGE) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index a14b6119f38..efa5d57fd1d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -52,8 +52,8 @@ void ReplicatedMergeTreeQueue::initialize(const MergeTreeData::DataParts & parts std::lock_guard lock(state_mutex); for (const auto & part : parts) { - current_parts.add(part->name, nullptr, log); - virtual_parts.add(part->name, nullptr, log); + current_parts.add(part->name, nullptr); + virtual_parts.add(part->name, nullptr); } } @@ -136,7 +136,7 @@ void ReplicatedMergeTreeQueue::insertUnlocked( { for (const String & virtual_part_name : entry->getVirtualPartNames(format_version)) { - virtual_parts.add(virtual_part_name, nullptr, log); + virtual_parts.add(virtual_part_name, nullptr); /// Don't add drop range parts to mutations /// they don't produce any useful parts if (entry->type != LogEntry::DROP_RANGE) @@ -230,7 +230,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( for (const String & virtual_part_name : entry->getVirtualPartNames(format_version)) { - current_parts.add(virtual_part_name, nullptr, log); + current_parts.add(virtual_part_name, nullptr); /// These parts are already covered by newer part, we don't have to /// mutate it. @@ -438,7 +438,7 @@ bool ReplicatedMergeTreeQueue::remove(zkutil::ZooKeeperPtr zookeeper, const Stri { auto part_in_current_parts = current_parts.getContainingPart(source_part); if (part_in_current_parts == source_part) - virtual_parts.add(source_part, nullptr, log); + virtual_parts.add(source_part, nullptr); } } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 205dc9687c7..1c7d23953da 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -686,6 +686,24 @@ private: bool fetch_part, ContextPtr query_context) override; + /// NOTE: there are no guarantees for concurrent merges. Dropping part can + /// be concurrently merged into some covering part and dropPart will do + /// nothing. There are some fundamental problems with it. But this is OK + /// because: + /// + /// dropPart used in the following cases: + /// 1) Remove empty parts after TTL. + /// 2) Remove parts after move between shards. + /// 3) User queries: ALTER TABLE DROP PART 'part_name'. + /// + /// In the first case merge of empty part is even better than DROP. In the + /// second case part UUIDs used to forbid merges for moving parts so there + /// is no problem with concurrent merges. The third case is quite rare and + /// we give very weak guarantee: there will be no active part with this + /// name, but possibly it was merged to some other part. + /// + /// NOTE: don't rely on dropPart if you 100% need to remove non-empty part + /// and don't use any explicit locking mechanism for merges. bool dropPartImpl(zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop); /// Check granularity of already existing replicated table in zookeeper if it exists From 414f48e1c05b3015d7a56bc71da643344c705246 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 30 Jun 2021 19:23:46 +0000 Subject: [PATCH 107/290] Function added to quantiles.md --- .../en/sql-reference/aggregate-functions/reference/quantiles.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index 73939f16db3..ca90872a8fc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -8,7 +8,7 @@ toc_priority: 201 Syntax: `quantiles(level1, level2, …)(x)` -All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. +All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`, `quantilesBFloat16`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. ## quantilesExactExclusive {#quantilesexactexclusive} From 8a3a142f75ee02dc5b4eb2fca1de993a703e0886 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 30 Jun 2021 19:28:57 +0000 Subject: [PATCH 108/290] Fixed links in quantiles.md --- .../sql-reference/aggregate-functions/reference/quantiles.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md index ca90872a8fc..67d1c1ca7e5 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantiles.md @@ -18,7 +18,7 @@ To get exact value, all the passed values ​​are combined into an array, whic This function is equivalent to [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba) Excel function, ([type R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)). -Works more efficiently with sets of levels than [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive). +Works more efficiently with sets of levels than [quantileExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive). **Syntax** @@ -70,7 +70,7 @@ To get exact value, all the passed values ​​are combined into an array, whic This function is equivalent to [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed) Excel function, ([type R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample)). -Works more efficiently with sets of levels than [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantilesexactinclusive). +Works more efficiently with sets of levels than [quantileExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactinclusive). **Syntax** From b852253153140d4c9421bf33938d360d46c26a29 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 30 Jun 2021 19:37:52 +0000 Subject: [PATCH 109/290] Fixed Wikipedia link --- .../aggregate-functions/reference/quantilebfloat16.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 3e3824c3dfb..f8ecd1d71ce 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,7 +4,7 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} -Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. +Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates bfloat16 quantile value and converts the result to a 64-bit float by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. From 46e6062c443dd96137984a56a1a5a7fcdca6ee53 Mon Sep 17 00:00:00 2001 From: Alexey Date: Wed, 30 Jun 2021 19:42:41 +0000 Subject: [PATCH 110/290] Fixed Syntax section of quantileExactLow --- .../aggregate-functions/reference/quantileexact.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md b/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md index 47164cec86d..069aadc225b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantileexact.md @@ -74,7 +74,7 @@ When using multiple `quantile*` functions with different levels in a query, the **Syntax** ``` sql -quantileExact(level)(expr) +quantileExactLow(level)(expr) ``` Alias: `medianExactLow`. From 1a4ccab8e6b851bdc0f4eaa84d2e4d852d04259a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 1 Jul 2021 15:12:27 +0300 Subject: [PATCH 111/290] Fix style --- src/Storages/MergeTree/ActiveDataPartSet.cpp | 5 +++++ src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ActiveDataPartSet.cpp b/src/Storages/MergeTree/ActiveDataPartSet.cpp index a3bbe1c4801..77fc2c2f0b9 100644 --- a/src/Storages/MergeTree/ActiveDataPartSet.cpp +++ b/src/Storages/MergeTree/ActiveDataPartSet.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + ActiveDataPartSet::ActiveDataPartSet(MergeTreeDataFormatVersion format_version_, const Strings & names) : format_version(format_version_) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 2bb285fcf34..d326ad10370 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -446,7 +446,7 @@ Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormat /// because it can lead to intersecting parts on stale replicas and this /// problem is fundamental. So we have very weak guarantees for DROP /// PART. If any concurrent merge will be assigned then DROP PART will - /// delete nothing and part will be successfuly merged into bigger part. + /// delete nothing and part will be successfully merged into bigger part. /// /// dropPart used in the following cases: /// 1) Remove empty parts after TTL. From aa377bcd4249221863e39056afe6cc3e86d06c9a Mon Sep 17 00:00:00 2001 From: George Date: Fri, 2 Jul 2021 13:23:00 +0300 Subject: [PATCH 112/290] Deleted wrong doc --- docs/en/sql-reference/functions/logical-functions.md | 2 +- docs/en/sql-reference/operators/index.md | 4 ---- docs/ru/sql-reference/functions/logical-functions.md | 2 +- docs/ru/sql-reference/operators/index.md | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 4b188184074..9d451dfe2b5 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -153,7 +153,7 @@ Result: ## xor {#logical-xor-function} -Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function works as if it calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. Corresponds to [Logical XOR Operator](../../sql-reference/operators/index.md#logical-xor-operator). +Calculates the result of the logical exclusive disjunction between two or more values. For more than two values the function works as if it calculates `XOR` of the first two values and then uses the result with the next value to calculate `XOR` and so on. **Syntax** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 54239d48082..55da4afd145 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -223,10 +223,6 @@ Syntax `SELECT a OR b` — calculates logical disjunction of `a` and `b` with th Syntax `SELECT NOT a` — calculates logical negation of `a` with the function [not](../../sql-reference/functions/logical-functions.md#logical-not-function). -## Logical XOR operator {#logical-xor-operator} - -Syntax `SELECT a XOR b` — calculates logical exclusive disjunction of `a` and `b` with the function [xor](../../sql-reference/functions/logical-functions.md#logical-xor-function). - ## Conditional Operator {#conditional-operator} `a ? b : c` – The `if(a, b, c)` function. diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index e3fc75402ac..f4dee477ee0 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -153,7 +153,7 @@ SELECT NOT(1); ## xor {#logical-xor-function} -Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает так: сначала вычисляет `XOR` для первых двух значений, а потом использует полученный результат при вычислении `XOR` со следующим значением и так далее. Соответствует [Оператору логического исключающего "ИЛИ"](../../sql-reference/operators/index.md#logical-xor-operator). +Вычисляет результат логической исключающей дизъюнкции между двумя и более значениями. При более чем двух значениях функция работает так: сначала вычисляет `XOR` для первых двух значений, а потом использует полученный результат при вычислении `XOR` со следующим значением и так далее. **Синтаксис** diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 030de6a7574..785c142cca7 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -223,10 +223,6 @@ SELECT toDateTime('2014-10-26 00:00:00', 'Europe/Moscow') AS time, time + 60 * 6 Синтаксис `SELECT NOT a` — вычисляет логическое отрицание `a` функцией [not](../../sql-reference/functions/logical-functions.md#logical-not-function). -## Оператор логического исключающего "ИЛИ" {#logical-xor-operator} - -Синтаксис `SELECT a XOR b` — вычисляет логическую исключающую дизъюнкцию между `a` и `b` функцией [xor](../../sql-reference/functions/logical-functions.md#logical-xor-function). - ## Условный оператор {#uslovnyi-operator} `a ? b : c` - функция `if(a, b, c)` From 12aea188b04db3a44c35d9b5099c578cf5b4f41d Mon Sep 17 00:00:00 2001 From: zxc111 Date: Thu, 24 Jun 2021 19:35:19 +0800 Subject: [PATCH 113/290] add bin/unbin support --- src/Common/hex.cpp | 34 ++ src/Common/hex.h | 11 + src/Functions/FunctionsCoding.cpp | 2 + src/Functions/FunctionsCoding.h | 389 ++++++++++++++++++ .../0_stateless/01926_bin_unbin.reference | 16 + tests/queries/0_stateless/01926_bin_unbin.sql | 17 + 6 files changed, 469 insertions(+) create mode 100644 tests/queries/0_stateless/01926_bin_unbin.reference create mode 100644 tests/queries/0_stateless/01926_bin_unbin.sql diff --git a/src/Common/hex.cpp b/src/Common/hex.cpp index bad1bf19b8d..e8f9b981062 100644 --- a/src/Common/hex.cpp +++ b/src/Common/hex.cpp @@ -56,3 +56,37 @@ const char * const hex_char_to_digit_table = "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + +const char * const bin_byte_to_char_table = + "0000000000000001000000100000001100000100000001010000011000000111" + "0000100000001001000010100000101100001100000011010000111000001111" + "0001000000010001000100100001001100010100000101010001011000010111" + "0001100000011001000110100001101100011100000111010001111000011111" + "0010000000100001001000100010001100100100001001010010011000100111" + "0010100000101001001010100010101100101100001011010010111000101111" + "0011000000110001001100100011001100110100001101010011011000110111" + "0011100000111001001110100011101100111100001111010011111000111111" + "0100000001000001010000100100001101000100010001010100011001000111" + "0100100001001001010010100100101101001100010011010100111001001111" + "0101000001010001010100100101001101010100010101010101011001010111" + "0101100001011001010110100101101101011100010111010101111001011111" + "0110000001100001011000100110001101100100011001010110011001100111" + "0110100001101001011010100110101101101100011011010110111001101111" + "0111000001110001011100100111001101110100011101010111011001110111" + "0111100001111001011110100111101101111100011111010111111001111111" + "1000000010000001100000101000001110000100100001011000011010000111" + "1000100010001001100010101000101110001100100011011000111010001111" + "1001000010010001100100101001001110010100100101011001011010010111" + "1001100010011001100110101001101110011100100111011001111010011111" + "1010000010100001101000101010001110100100101001011010011010100111" + "1010100010101001101010101010101110101100101011011010111010101111" + "1011000010110001101100101011001110110100101101011011011010110111" + "1011100010111001101110101011101110111100101111011011111010111111" + "1100000011000001110000101100001111000100110001011100011011000111" + "1100100011001001110010101100101111001100110011011100111011001111" + "1101000011010001110100101101001111010100110101011101011011010111" + "1101100011011001110110101101101111011100110111011101111011011111" + "1110000011100001111000101110001111100100111001011110011011100111" + "1110100011101001111010101110101111101100111011011110111011101111" + "1111000011110001111100101111001111110100111101011111011011110111" + "1111100011111001111110101111101111111100111111011111111011111111"; diff --git a/src/Common/hex.h b/src/Common/hex.h index a1fa7b32465..62867f99c48 100644 --- a/src/Common/hex.h +++ b/src/Common/hex.h @@ -39,6 +39,17 @@ inline void writeHexByteLowercase(UInt8 byte, void * out) memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2); } +extern const char * const bin_byte_to_char_table; + +inline void writeBinByte(UInt8 byte, void * out) +{ + memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); +} + +inline void writeSingleBinByte(UInt8 byte, void * out) +{ + memcpy(out, &hex_digit_to_char_uppercase_table[static_cast(byte)], 1); +} /// Produces hex representation of an unsigned int with leading zeros (for checksums) template diff --git a/src/Functions/FunctionsCoding.cpp b/src/Functions/FunctionsCoding.cpp index 150d792f63b..f1bbeb5c43f 100644 --- a/src/Functions/FunctionsCoding.cpp +++ b/src/Functions/FunctionsCoding.cpp @@ -21,6 +21,8 @@ void registerFunctionsCoding(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index da667bfc691..5004905863f 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1326,6 +1326,395 @@ public: } }; +class FunctionBin : public IFunction +{ +public: + static constexpr auto name = "bin"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + WhichDataType which(arguments[0]); + + if (!which.isStringOrFixedString() && + !which.isDate() && + !which.isDateTime() && + !which.isDateTime64() && + !which.isUInt() && + !which.isFloat() && + !which.isDecimal()) + throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + template + void executeOneUInt(T x, char *& out) const + { + UInt8 result[sizeof(x) * 8] = {0}; + int cnt = 0; + if (0 == x) + { + writeSingleBinByte(0, out); + ++out; + *out = '\0'; + ++out; + return; + } + for (; x != 0; x = x >> 1) + { + result[cnt] = (x & 1); + cnt += 1; + } + for (int i = cnt - 1; i >= 0; --i) + { + writeSingleBinByte(result[i], out); + out += 1; + } + + *out = '\0'; + ++out; + } + + template + bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnVector * col_vec = checkAndGetColumn>(col); + + static constexpr size_t MAX_UINT_HEX_LENGTH = sizeof(T) * 8 + 1; /// Including trailing zero byte. + + if (col_vec) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const typename ColumnVector::Container & in_vec = col_vec->getData(); + + size_t size = in_vec.size(); + out_offsets.resize(size); + out_vec.resize(MAX_UINT_HEX_LENGTH); + + size_t pos = 0; + for (size_t i = 0; i < size; ++i) + { + /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it). + if (pos + MAX_UINT_HEX_LENGTH > out_vec.size()) + out_vec.resize(out_vec.size() * 8 + MAX_UINT_HEX_LENGTH); + + char * begin = reinterpret_cast(&out_vec[pos]); + char * end = begin; + + executeOneUInt(in_vec[i], end); + + pos += end - begin; + out_offsets[i] = pos; + } + + out_vec.resize(pos); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + template + void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) const + { + const size_t hex_length = type_size_in_bytes * 8 + 1; /// Including trailing zero byte. + auto col_str = ColumnString::create(); + + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + size_t size = in_vec.size(); + out_offsets.resize(size); + out_vec.resize(size * hex_length); + + size_t pos = 0; + char * begin = reinterpret_cast(out_vec.data()); + char * out = begin; + for (size_t i = 0; i < size; ++i) + { + const UInt8 * in_pos = reinterpret_cast(&in_vec[i]); + executeOneString(in_pos, in_pos + type_size_in_bytes, out); + + pos += hex_length; + out_offsets[i] = pos; + } + col_res = std::move(col_str); + } + + template + bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnVector * col_vec = checkAndGetColumn>(col); + if (col_vec) + { + const typename ColumnVector::Container & in_vec = col_vec->getData(); + executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); + return true; + } + else + { + return false; + } + } + + template + bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnDecimal * col_dec = checkAndGetColumn>(col); + if (col_dec) + { + const typename ColumnDecimal::Container & in_vec = col_dec->getData(); + executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); + return true; + } + else + { + return false; + } + } + + + static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) + { + while (pos < end) + { + writeBinByte(*pos, out); + + ++pos; + out += 8; + } + *out = '\0'; + ++out; + } + + static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res) + { + const ColumnString * col_str_in = checkAndGetColumn(col); + + if (col_str_in) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const ColumnString::Chars & in_vec = col_str_in->getChars(); + const ColumnString::Offsets & in_offsets = col_str_in->getOffsets(); + + size_t size = in_offsets.size(); + + out_offsets.resize(size); + out_vec.resize((in_vec.size() - 1) * 8 + size); + + char * begin = reinterpret_cast(out_vec.data()); + char * pos = begin; + size_t prev_offset = 0; + + for (size_t i = 0; i < size; ++i) + { + size_t new_offset = in_offsets[i]; + executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos); + + out_offsets[i] = pos - begin; + + prev_offset = new_offset; + } + if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) + throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) + { + const ColumnFixedString * col_fstr_in = checkAndGetColumn(col); + + if (col_fstr_in) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const ColumnString::Chars & in_vec = col_fstr_in->getChars(); + + size_t size = col_fstr_in->size(); + + out_offsets.resize(size); + out_vec.resize(in_vec.size() * 8 + size); + + char * begin = reinterpret_cast(out_vec.data()); + char * pos = begin; + + size_t n = col_fstr_in->getN(); + + size_t prev_offset = 0; + + for (size_t i = 0; i < size; ++i) + { + size_t new_offset = prev_offset + n; + + executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos); + + out_offsets[i] = pos - begin; + prev_offset = new_offset; + } + + if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) + throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const IColumn * column = arguments[0].column.get(); + ColumnPtr res_column; + + if (tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) + || tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) + || tryExecuteString(column, res_column) || tryExecuteFixedString(column, res_column) + || tryExecuteFloat(column, res_column) || tryExecuteFloat(column, res_column) + || tryExecuteDecimal(column, res_column) || tryExecuteDecimal(column, res_column) + || tryExecuteDecimal(column, res_column)) + return res_column; + + throw Exception( + "Illegal column " + arguments[0].column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + } +}; + +class FunctionUnbin : public IFunction +{ +public: + static constexpr auto name = "unbin"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (!isString(arguments[0])) + throw Exception( + "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + static void unbinOne(const char * pos, const char * end, char *& out) + { + uint8_t left = 0; + for (int left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt) + { + left = left << 1; + if (*pos == '1') + { + left += 1; + } + ++pos; + } + if (0 != left) + { + *out = left; + ++out; + } + + while (end - pos != 0) + { + int c = 0; + for (int i = 0; i < 8; ++i) + { + c = c << 1; + if (*pos == '1') + { + c += 1; + } + ++pos; + } + *out = c; + ++out; + } + + *out = '\0'; + ++out; + } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const ColumnPtr & column = arguments[0].column; + + if (const ColumnString * col = checkAndGetColumn(column.get())) + { + auto col_res = ColumnString::create(); + + ColumnString::Chars & out_vec = col_res->getChars(); + ColumnString::Offsets & out_offsets = col_res->getOffsets(); + + const ColumnString::Chars & in_vec = col->getChars(); + const ColumnString::Offsets & in_offsets = col->getOffsets(); + + size_t size = in_offsets.size(); + out_offsets.resize(size); + out_vec.resize(in_vec.size() / 8 + size); + + char * begin = reinterpret_cast(out_vec.data()); + char * pos = begin; + size_t prev_offset = 0; + for (size_t i = 0; i < size; ++i) + { + size_t new_offset = in_offsets[i]; + + unbinOne(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); + + out_offsets[i] = pos - begin; + + prev_offset = new_offset; + } + + out_vec.resize(pos - begin); + + return col_res; + } + else + { + throw Exception("Illegal column " + arguments[0].column->getName() + + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + } +}; + class FunctionChar : public IFunction { public: diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference new file mode 100644 index 00000000000..b9ddf2f1db7 --- /dev/null +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -0,0 +1,16 @@ +0 +1 +1010 +1111111 +11111111 +00110000 +0011000100110000 +111001101011010110001011111010001010111110010101 +11100110101101011000101111101000101011111001010100000000000000000000000000000000 +10011010100110011001100100111111 +0011001100110011001100110011001100110011001100111111001100111111 +00000000000011100010011100000111 +0000000000000000000011000011110101011101010100111010101000000001 +0 +10 +测试 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql new file mode 100644 index 00000000000..fd7a77bd2fc --- /dev/null +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -0,0 +1,17 @@ +select bin(0); +select bin(1); +select bin(10); +select bin(127); +select bin(255); +select bin('0'); +select bin('10'); +select bin('测试'); +select bin(toFixedString('测试', 10)); +select bin(toFloat32(1.2)); +select bin(toFloat64(1.2)); +select bin(toDecimal32(1.2, 8)); +select bin(toDecimal64(1.2, 17)); + +select unbin('00110000'); -- 0 +select unbin('0011000100110000'); -- 10 +select unbin('111001101011010110001011111010001010111110010101'); -- 测试 From ace487278fb7ba7de852c68c993ac3055a18aae2 Mon Sep 17 00:00:00 2001 From: zxc111 Date: Fri, 2 Jul 2021 01:09:44 +0800 Subject: [PATCH 114/290] refactory hex/unhex/bin/unbin --- src/Common/hex.h | 5 - src/Functions/FunctionsCoding.h | 900 +++++++----------- .../0_stateless/01926_bin_unbin.reference | 4 +- tests/queries/0_stateless/01926_bin_unbin.sql | 2 + 4 files changed, 346 insertions(+), 565 deletions(-) diff --git a/src/Common/hex.h b/src/Common/hex.h index 62867f99c48..82eff776244 100644 --- a/src/Common/hex.h +++ b/src/Common/hex.h @@ -46,11 +46,6 @@ inline void writeBinByte(UInt8 byte, void * out) memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); } -inline void writeSingleBinByte(UInt8 byte, void * out) -{ - memcpy(out, &hex_digit_to_char_uppercase_table[static_cast(byte)], 1); -} - /// Produces hex representation of an unsigned int with leading zeros (for checksums) template inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 5004905863f..f2e340aaeef 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -65,6 +65,10 @@ namespace ErrorCodes constexpr size_t uuid_bytes_length = 16; constexpr size_t uuid_text_length = 36; +namespace ErrorCodes +{ +extern const int NOT_IMPLEMENTED; +} class FunctionIPv6NumToString : public IFunction { @@ -951,19 +955,20 @@ public: } }; - -class FunctionHex : public IFunction +template +class Conversion : public IFunction { public: - static constexpr auto name = "hex"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static constexpr auto name = Impl::name; + static constexpr size_t word_size = Impl::word_size; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } + + bool useDefaultImplementationForConstants() const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -983,235 +988,6 @@ public: return std::make_shared(); } - template - void executeOneUInt(T x, char *& out) const - { - bool was_nonzero = false; - for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) - { - UInt8 byte = x >> offset; - - /// Leading zeros. - if (byte == 0 && !was_nonzero && offset) // -V560 - continue; - - was_nonzero = true; - - writeHexByteUppercase(byte, out); - out += 2; - } - *out = '\0'; - ++out; - } - - template - bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnVector * col_vec = checkAndGetColumn>(col); - - static constexpr size_t MAX_UINT_HEX_LENGTH = sizeof(T) * 2 + 1; /// Including trailing zero byte. - - if (col_vec) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const typename ColumnVector::Container & in_vec = col_vec->getData(); - - size_t size = in_vec.size(); - out_offsets.resize(size); - out_vec.resize(size * 3 + MAX_UINT_HEX_LENGTH); /// 3 is length of one byte in hex plus zero byte. - - size_t pos = 0; - for (size_t i = 0; i < size; ++i) - { - /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it). - if (pos + MAX_UINT_HEX_LENGTH > out_vec.size()) - out_vec.resize(out_vec.size() * 2 + MAX_UINT_HEX_LENGTH); - - char * begin = reinterpret_cast(&out_vec[pos]); - char * end = begin; - executeOneUInt(in_vec[i], end); - - pos += end - begin; - out_offsets[i] = pos; - } - - out_vec.resize(pos); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - template - void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) const - { - const size_t hex_length = type_size_in_bytes * 2 + 1; /// Including trailing zero byte. - auto col_str = ColumnString::create(); - - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - size_t size = in_vec.size(); - out_offsets.resize(size); - out_vec.resize(size * hex_length); - - size_t pos = 0; - char * out = reinterpret_cast(&out_vec[0]); - for (size_t i = 0; i < size; ++i) - { - const UInt8 * in_pos = reinterpret_cast(&in_vec[i]); - executeOneString(in_pos, in_pos + type_size_in_bytes, out); - - pos += hex_length; - out_offsets[i] = pos; - } - col_res = std::move(col_str); - } - - template - bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnVector * col_vec = checkAndGetColumn>(col); - if (col_vec) - { - const typename ColumnVector::Container & in_vec = col_vec->getData(); - executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); - return true; - } - else - { - return false; - } - } - - template - bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnDecimal * col_dec = checkAndGetColumn>(col); - if (col_dec) - { - const typename ColumnDecimal::Container & in_vec = col_dec->getData(); - executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); - return true; - } - else - { - return false; - } - } - - - static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) - { - while (pos < end) - { - writeHexByteUppercase(*pos, out); - ++pos; - out += 2; - } - *out = '\0'; - ++out; - } - - static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res) - { - const ColumnString * col_str_in = checkAndGetColumn(col); - - if (col_str_in) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const ColumnString::Chars & in_vec = col_str_in->getChars(); - const ColumnString::Offsets & in_offsets = col_str_in->getOffsets(); - - size_t size = in_offsets.size(); - out_offsets.resize(size); - out_vec.resize(in_vec.size() * 2 - size); - - char * begin = reinterpret_cast(out_vec.data()); - char * pos = begin; - size_t prev_offset = 0; - - for (size_t i = 0; i < size; ++i) - { - size_t new_offset = in_offsets[i]; - - executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos); - - out_offsets[i] = pos - begin; - - prev_offset = new_offset; - } - - if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) - throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) - { - const ColumnFixedString * col_fstr_in = checkAndGetColumn(col); - - if (col_fstr_in) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const ColumnString::Chars & in_vec = col_fstr_in->getChars(); - - size_t size = col_fstr_in->size(); - - out_offsets.resize(size); - out_vec.resize(in_vec.size() * 2 + size); - - char * begin = reinterpret_cast(out_vec.data()); - char * pos = begin; - - size_t n = col_fstr_in->getN(); - - size_t prev_offset = 0; - - for (size_t i = 0; i < size; ++i) - { - size_t new_offset = prev_offset + n; - - executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos); - - out_offsets[i] = pos - begin; - prev_offset = new_offset; - } - - if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) - throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const IColumn * column = arguments[0].column.get(); @@ -1234,19 +1010,192 @@ public: + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } + + template + bool tryExecuteUInt(const IColumn *col, ColumnPtr &col_res) const + { + const ColumnVector * col_vec = checkAndGetColumn>(col); + + static constexpr size_t MAX_LENGTH = sizeof(T) * word_size + 1; /// Including trailing zero byte. + + if (col_vec) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const typename ColumnVector::Container & in_vec = col_vec->getData(); + + size_t size = in_vec.size(); + out_offsets.resize(size); + out_vec.resize(size * (word_size+1) + MAX_LENGTH); /// word_size+1 is length of one byte in hex/bin plus zero byte. + + size_t pos = 0; + for (size_t i = 0; i < size; ++i) + { + /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it). + if (pos + MAX_LENGTH > out_vec.size()) + out_vec.resize(out_vec.size() * word_size + MAX_LENGTH); + + char * begin = reinterpret_cast(&out_vec[pos]); + char * end = begin; + Impl::executeOneUInt(in_vec[i], end); + + pos += end - begin; + out_offsets[i] = pos; + } + out_vec.resize(pos); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + bool tryExecuteString(const IColumn *col, ColumnPtr &col_res) const + { + const ColumnString * col_str_in = checkAndGetColumn(col); + + if (col_str_in) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const ColumnString::Chars & in_vec = col_str_in->getChars(); + const ColumnString::Offsets & in_offsets = col_str_in->getOffsets(); + + size_t size = in_offsets.size(); + + out_offsets.resize(size); + if (getName() == "bin") + { + out_vec.resize((in_vec.size() - size) * word_size + size); + } else if (getName() == "hex") + { + out_vec.resize(in_vec.size() * word_size - size); + } else + { + throw Exception("new function is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + + char * begin = reinterpret_cast(out_vec.data()); + char * pos = begin; + size_t prev_offset = 0; + + for (size_t i = 0; i < size; ++i) + { + size_t new_offset = in_offsets[i]; + + Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos); + + out_offsets[i] = pos - begin; + + prev_offset = new_offset; + } + if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) + throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + template + bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnDecimal * col_dec = checkAndGetColumn>(col); + if (col_dec) + { + const typename ColumnDecimal::Container & in_vec = col_dec->getData(); + Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T)); + return true; + } + else + { + return false; + } + } + + static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) + { + const ColumnFixedString * col_fstr_in = checkAndGetColumn(col); + + if (col_fstr_in) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const ColumnString::Chars & in_vec = col_fstr_in->getChars(); + + size_t size = col_fstr_in->size(); + + out_offsets.resize(size); + out_vec.resize(in_vec.size() * word_size + size); + + char * begin = reinterpret_cast(out_vec.data()); + char * pos = begin; + + size_t n = col_fstr_in->getN(); + + size_t prev_offset = 0; + + for (size_t i = 0; i < size; ++i) + { + size_t new_offset = prev_offset + n; + + Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos); + + out_offsets[i] = pos - begin; + prev_offset = new_offset; + } + + if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) + throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } + + template + bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnVector * col_vec = checkAndGetColumn>(col); + if (col_vec) + { + const typename ColumnVector::Container & in_vec = col_vec->getData(); + Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T)); + return true; + } + else + { + return false; + } + } }; - -class FunctionUnhex : public IFunction +template +class UnConversion : public IFunction { public: - static constexpr auto name = "unhex"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static constexpr auto name = Impl::name; + static constexpr size_t word_size = Impl::word_size; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } @@ -1255,29 +1204,11 @@ public: { if (!isString(arguments[0])) throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); } - static void unhexOne(const char * pos, const char * end, char *& out) - { - if ((end - pos) & 1) - { - *out = unhex(*pos); - ++out; - ++pos; - } - while (pos < end) - { - *out = unhex2(pos); - pos += 2; - ++out; - } - *out = '\0'; - ++out; - } - bool useDefaultImplementationForConstants() const override { return true; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override @@ -1296,7 +1227,18 @@ public: size_t size = in_offsets.size(); out_offsets.resize(size); - out_vec.resize(in_vec.size() / 2 + size); + if (getName() == "unhex") + { + out_vec.resize(in_vec.size() / 2 + size); + } + else if (getName() == "unbin") + { + out_vec.resize(in_vec.size() / 8 + size); + } + else + { + throw Exception("new function is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } char * begin = reinterpret_cast(out_vec.data()); char * pos = begin; @@ -1306,7 +1248,7 @@ public: { size_t new_offset = in_offsets[i]; - unhexOne(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); + Impl::unConversion(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); out_offsets[i] = pos - begin; @@ -1326,56 +1268,130 @@ public: } }; -class FunctionBin : public IFunction +struct HexImpl { public: - static constexpr auto name = "bin"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + static constexpr auto name = "hex"; + static const size_t word_size = 2; + template + static void executeOneUInt(T x, char *& out) { - WhichDataType which(arguments[0]); + bool was_nonzero = false; + for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) + { + UInt8 byte = x >> offset; - if (!which.isStringOrFixedString() && - !which.isDate() && - !which.isDateTime() && - !which.isDateTime64() && - !which.isUInt() && - !which.isFloat() && - !which.isDecimal()) - throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + /// Leading zeros. + if (byte == 0 && !was_nonzero && offset) + continue; - return std::make_shared(); + was_nonzero = true; + writeHexByteUppercase(byte, out); + out += 2; + } + *out = '\0'; + ++out; + } + + static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) + { + while (pos < end) + { + writeHexByteUppercase(*pos, out); + ++pos; + out += 2; + } + *out = '\0'; + ++out; } template - void executeOneUInt(T x, char *& out) const + static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) { - UInt8 result[sizeof(x) * 8] = {0}; - int cnt = 0; - if (0 == x) + const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte. + auto col_str = ColumnString::create(); + + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + size_t size = in_vec.size(); + out_offsets.resize(size); + out_vec.resize(size * hex_length); + + size_t pos = 0; + char * out = reinterpret_cast(&out_vec[0]); + for (size_t i = 0; i < size; ++i) { - writeSingleBinByte(0, out); - ++out; - *out = '\0'; - ++out; - return; + const UInt8 * in_pos = reinterpret_cast(&in_vec[i]); + executeOneString(in_pos, in_pos + type_size_in_bytes, out); + + pos += hex_length; + out_offsets[i] = pos; } - for (; x != 0; x = x >> 1) + col_res = std::move(col_str); + } +}; + +struct UnhexImpl +{ +public: + static constexpr auto name = "unhex"; + + static String getName() + { + return name; + } + + static void unConversion(const char * pos, const char * end, char *& out) + { + if ((end - pos) & 1) { - result[cnt] = (x & 1); - cnt += 1; + *out = unhex(*pos); + ++out; + ++pos; } - for (int i = cnt - 1; i >= 0; --i) + while (pos < end) { - writeSingleBinByte(result[i], out); - out += 1; + *out = unhex2(pos); + pos += 2; + ++out; + } + *out = '\0'; + ++out; + } +}; + +struct BinImpl +{ +public: + static constexpr auto name = "bin"; + static constexpr size_t word_size = 8; + template + static void executeOneUInt(T x, char *& out) + { + bool was_nonzero = false; + T t = 1; + + for (int8_t offset = sizeof(x) * 8 - 1; offset >= 0; --offset) + { + t = t << offset; + if ((x & t) == t) + { + x = x - t; + was_nonzero = true; + *out = '1'; + t = 1; + } + else + { + t = 1; + if (!was_nonzero) + { + continue; + } + *out = '0'; + } + ++out; } *out = '\0'; @@ -1383,53 +1399,7 @@ public: } template - bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnVector * col_vec = checkAndGetColumn>(col); - - static constexpr size_t MAX_UINT_HEX_LENGTH = sizeof(T) * 8 + 1; /// Including trailing zero byte. - - if (col_vec) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const typename ColumnVector::Container & in_vec = col_vec->getData(); - - size_t size = in_vec.size(); - out_offsets.resize(size); - out_vec.resize(MAX_UINT_HEX_LENGTH); - - size_t pos = 0; - for (size_t i = 0; i < size; ++i) - { - /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it). - if (pos + MAX_UINT_HEX_LENGTH > out_vec.size()) - out_vec.resize(out_vec.size() * 8 + MAX_UINT_HEX_LENGTH); - - char * begin = reinterpret_cast(&out_vec[pos]); - char * end = begin; - - executeOneUInt(in_vec[i], end); - - pos += end - begin; - out_offsets[i] = pos; - } - - out_vec.resize(pos); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - template - void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) const + static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) { const size_t hex_length = type_size_in_bytes * 8 + 1; /// Including trailing zero byte. auto col_str = ColumnString::create(); @@ -1455,188 +1425,39 @@ public: col_res = std::move(col_str); } - template - bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnVector * col_vec = checkAndGetColumn>(col); - if (col_vec) - { - const typename ColumnVector::Container & in_vec = col_vec->getData(); - executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); - return true; - } - else - { - return false; - } - } - - template - bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const - { - const ColumnDecimal * col_dec = checkAndGetColumn>(col); - if (col_dec) - { - const typename ColumnDecimal::Container & in_vec = col_dec->getData(); - executeFloatAndDecimal::Container>(in_vec, col_res, sizeof(T)); - return true; - } - else - { - return false; - } - } - - static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) { while (pos < end) { writeBinByte(*pos, out); - ++pos; - out += 8; + out += word_size; } *out = '\0'; ++out; } - - static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res) - { - const ColumnString * col_str_in = checkAndGetColumn(col); - - if (col_str_in) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const ColumnString::Chars & in_vec = col_str_in->getChars(); - const ColumnString::Offsets & in_offsets = col_str_in->getOffsets(); - - size_t size = in_offsets.size(); - - out_offsets.resize(size); - out_vec.resize((in_vec.size() - 1) * 8 + size); - - char * begin = reinterpret_cast(out_vec.data()); - char * pos = begin; - size_t prev_offset = 0; - - for (size_t i = 0; i < size; ++i) - { - size_t new_offset = in_offsets[i]; - executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos); - - out_offsets[i] = pos - begin; - - prev_offset = new_offset; - } - if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) - throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res) - { - const ColumnFixedString * col_fstr_in = checkAndGetColumn(col); - - if (col_fstr_in) - { - auto col_str = ColumnString::create(); - ColumnString::Chars & out_vec = col_str->getChars(); - ColumnString::Offsets & out_offsets = col_str->getOffsets(); - - const ColumnString::Chars & in_vec = col_fstr_in->getChars(); - - size_t size = col_fstr_in->size(); - - out_offsets.resize(size); - out_vec.resize(in_vec.size() * 8 + size); - - char * begin = reinterpret_cast(out_vec.data()); - char * pos = begin; - - size_t n = col_fstr_in->getN(); - - size_t prev_offset = 0; - - for (size_t i = 0; i < size; ++i) - { - size_t new_offset = prev_offset + n; - - executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos); - - out_offsets[i] = pos - begin; - prev_offset = new_offset; - } - - if (!out_offsets.empty() && out_offsets.back() != out_vec.size()) - throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR); - - col_res = std::move(col_str); - return true; - } - else - { - return false; - } - } - - bool useDefaultImplementationForConstants() const override { return true; } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override - { - const IColumn * column = arguments[0].column.get(); - ColumnPtr res_column; - - if (tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) - || tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) - || tryExecuteString(column, res_column) || tryExecuteFixedString(column, res_column) - || tryExecuteFloat(column, res_column) || tryExecuteFloat(column, res_column) - || tryExecuteDecimal(column, res_column) || tryExecuteDecimal(column, res_column) - || tryExecuteDecimal(column, res_column)) - return res_column; - - throw Exception( - "Illegal column " + arguments[0].column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); - } }; -class FunctionUnbin : public IFunction +struct UnbinImpl { public: static constexpr auto name = "unbin"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override { return name; } + static String getName() { return name; } - size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + static void unConversion(const char * pos, const char * end, char *& out) { - if (!isString(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + UInt8 left = 0; - return std::make_shared(); - } - - static void unbinOne(const char * pos, const char * end, char *& out) - { - uint8_t left = 0; - for (int left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt) + /// end - pos is the length of input. + /// (length & 7) to make remain bits length mod 8 is zero to split. + /// e.g. the length is 9 and the input is "101000001", + /// first left_cnt is 1, left is 0, right shift, pos is 1, left = 1 + /// then, left_cnt is 0, remain input is '01000001'. + for (uint8_t left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt) { left = left << 1; - if (*pos == '1') + if (*pos != '0') { left += 1; } @@ -1648,13 +1469,15 @@ public: ++out; } + /// input character encoding is UTF-8. And + /// remain bits mod 8 is zero. while (end - pos != 0) { - int c = 0; - for (int i = 0; i < 8; ++i) + UInt8 c = 0; + for (uint8_t i = 0; i < 8; ++i) { c = c << 1; - if (*pos == '1') + if (*pos != '0') { c += 1; } @@ -1667,54 +1490,13 @@ public: *out = '\0'; ++out; } - - bool useDefaultImplementationForConstants() const override { return true; } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override - { - const ColumnPtr & column = arguments[0].column; - - if (const ColumnString * col = checkAndGetColumn(column.get())) - { - auto col_res = ColumnString::create(); - - ColumnString::Chars & out_vec = col_res->getChars(); - ColumnString::Offsets & out_offsets = col_res->getOffsets(); - - const ColumnString::Chars & in_vec = col->getChars(); - const ColumnString::Offsets & in_offsets = col->getOffsets(); - - size_t size = in_offsets.size(); - out_offsets.resize(size); - out_vec.resize(in_vec.size() / 8 + size); - - char * begin = reinterpret_cast(out_vec.data()); - char * pos = begin; - size_t prev_offset = 0; - for (size_t i = 0; i < size; ++i) - { - size_t new_offset = in_offsets[i]; - - unbinOne(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); - - out_offsets[i] = pos - begin; - - prev_offset = new_offset; - } - - out_vec.resize(pos - begin); - - return col_res; - } - else - { - throw Exception("Illegal column " + arguments[0].column->getName() - + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - } }; +using FunctionHex = Conversion; +using FunctionUnhex = UnConversion; +using FunctionBin = Conversion; +using FunctionUnbin = UnConversion; + class FunctionChar : public IFunction { public: diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index b9ddf2f1db7..54c01c5d145 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -1,4 +1,4 @@ -0 + 1 1010 1111111 @@ -11,6 +11,8 @@ 0011001100110011001100110011001100110011001100111111001100111111 00000000000011100010011100000111 0000000000000000000011000011110101011101010100111010101000000001 +0011000100110010001100110011001100110010001101000011001000110100 +0011000100110010001100110011001100110010001101000011001000110100 0 10 测试 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql index fd7a77bd2fc..40635091120 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.sql +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -11,6 +11,8 @@ select bin(toFloat32(1.2)); select bin(toFloat64(1.2)); select bin(toDecimal32(1.2, 8)); select bin(toDecimal64(1.2, 17)); +select bin('12332424'); +select bin(toLowCardinality(materialize('12332424'))); select unbin('00110000'); -- 0 select unbin('0011000100110000'); -- 10 From 55ce7de2484f8888ecd14e4e3a928cf450cc6354 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 2 Jul 2021 22:39:21 +0300 Subject: [PATCH 115/290] Remove trailing spaces -- style check fix --- src/Storages/StorageS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 4f6c55d1fe4..b4fec69e075 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -436,7 +436,7 @@ BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMet max_single_part_upload_size); } - + void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) { updateClientAndAuthSettings(local_context, client_auth); From 7e372a2e5349be67d04ca958c1b149978cf446e0 Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 2 Jul 2021 20:18:55 +0000 Subject: [PATCH 116/290] Ru translation + some updates + link fixes --- .../reference/quantilebfloat16.md | 4 +- .../aggregate-functions/reference/median.md | 22 ++++--- .../reference/quantilebfloat16.md | 64 +++++++++++++++++++ .../reference/quantiles.md | 6 +- 4 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index f8ecd1d71ce..cdbb60f2fe8 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -43,10 +43,10 @@ Input table has an integer and a float columns: └───┴───────┘ ``` -Query: +Query to calculate 0.75-quantile (third quartile): ``` sql -SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; +SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; ``` Result: diff --git a/docs/ru/sql-reference/aggregate-functions/reference/median.md b/docs/ru/sql-reference/aggregate-functions/reference/median.md index a208c21dd21..1472809e2e3 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/median.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/median.md @@ -1,17 +1,19 @@ # median {#median} -Функции `median*` — алиасы для соответствущих функций `quantile*`. Они вычисляют медиану числовой последовательности. +Функции `median*` — синонимы для соответствущих функций `quantile*`. Они вычисляют медиану числовой последовательности. -Functions: +Функции: -- `median` — алиас [quantile](#quantile). -- `medianDeterministic` — алиас [quantileDeterministic](#quantiledeterministic). -- `medianExact` — алиас [quantileExact](#quantileexact). -- `medianExactWeighted` — алиас [quantileExactWeighted](#quantileexactweighted). -- `medianTiming` — алиас [quantileTiming](#quantiletiming). -- `medianTimingWeighted` — алиас [quantileTimingWeighted](#quantiletimingweighted). -- `medianTDigest` — алиас [quantileTDigest](#quantiletdigest). -- `medianTDigestWeighted` — алиас [quantileTDigestWeighted](#quantiletdigestweighted). + +- `median` — синоним для [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile). +- `medianDeterministic` — синоним для [quantileDeterministic](../../../sql-reference/aggregate-functions/reference/quantiledeterministic.md#quantiledeterministic). +- `medianExact` — синоним для [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexact). +- `medianExactWeighted` — синоним для [quantileExactWeighted](../../../sql-reference/aggregate-functions/reference/quantileexactweighted.md#quantileexactweighted). +- `medianTiming` — синоним для [quantileTiming](../../../sql-reference/aggregate-functions/reference/quantiletiming.md#quantiletiming). +- `medianTimingWeighted` — синоним для [quantileTimingWeighted](../../../sql-reference/aggregate-functions/reference/quantiletimingweighted.md#quantiletimingweighted). +- `medianTDigest` — синоним для [quantileTDigest](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md#quantiletdigest). +- `medianTDigestWeighted` — синоним для [quantileTDigestWeighted](../../../sql-reference/aggregate-functions/reference/quantiletdigestweighted.md#quantiletdigestweighted). +- `medianBFloat16` — синоним для [quantileBFloat16](../../../sql-reference/aggregate-functions/reference/quantilebfloat16.md#quantilebfloat16). **Пример** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md new file mode 100644 index 00000000000..217da78d1d1 --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -0,0 +1,64 @@ +--- +toc_priority: 209 +--- + +# quantileBFloat16 {#quantilebfloat16} + +Приближенно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) выборки чисел в формате [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format). bfloat16 — это формат с плавающей точкой, в котором для представления числа используется 1 знаковый бит, 8 бит для порядка и 7 бит для мантиссы. +Функция преобразует входное число в 32-битное с плавающей точкой и обрабатывает его старшие 16 бит. Она вычисляет квантиль в формате bfloat16 и преобразует его в 64-битное число с плавающей точкой, добавляя нулевые биты. +Эта функция выполняет быстрые приближенные вычисления с относительной ошибкой не более 0.390625%. + +**Синтаксис** + +``` sql +quantileBFloat16[(level)](expr) +``` + +Синоним: `medianBFloat16` + +**Аргументы** + +- `expr` — столбец с числовыми данными. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md). + +**Параметры** + +- `level` — уровень квантиля. Необязательный. Допустимый диапазон значений от 0 до 1. Значение по умолчанию: 0.5. [Float](../../../sql-reference/data-types/float.md). + +**Возвращаемое значение** + +- Приближенное значение квантиля. + +Тип: [Float64](../../../sql-reference/data-types/float.md#float32-float64). + +**Пример** + +В таблице есть столбцы с целыми числами и с числами с плавающей точкой: + +``` text +┌─a─┬─────b─┐ +│ 1 │ 1.001 │ +│ 2 │ 1.002 │ +│ 3 │ 1.003 │ +│ 4 │ 1.004 │ +└───┴───────┘ +``` + +Запрос для вычисления 0.75-квантиля (верхнего квартиля): + +``` sql +SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; +``` + +Результат: + +``` text +┌─quantileBFloat16(0.75)(a)─┬─quantileBFloat16(0.75)(b)─┐ +│ 3 │ 1 │ +└───────────────────────────┴───────────────────────────┘ +``` +Обратите внимание, что все числа с плавающей точкой в примере были округлены до 1.0 при преобразовании к bfloat16. + +**See Also** + +- [median](../../../sql-reference/aggregate-functions/reference/median.md#median) +- [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md index d2e7003e4e7..2417d6de139 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md @@ -8,7 +8,7 @@ toc_priority: 201 Синтаксис: `quantiles(level1, level2, …)(x)` -Все функции для вычисления квантилей имеют соответствующие функции для вычисления нескольких квантилей: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. Эти функции вычисляют все квантили указанных уровней в один проход и возвращают массив с вычисленными значениями. +Все функции для вычисления квантилей имеют соответствующие функции для вычисления нескольких квантилей: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`, `quantilesBFloat16`. Эти функции вычисляют все квантили указанных уровней в один проход и возвращают массив с вычисленными значениями. ## quantilesExactExclusive {#quantilesexactexclusive} @@ -18,7 +18,7 @@ toc_priority: 201 Эта функция эквивалентна Excel функции [PERCENTILE.EXC](https://support.microsoft.com/en-us/office/percentile-exc-function-bbaa7204-e9e1-4010-85bf-c31dc5dce4ba), [тип R6](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample). -С наборами уровней работает эффективнее, чем [quantilesExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive). +С наборами уровней работает эффективнее, чем [quantileExactExclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactexclusive). **Синтаксис** @@ -70,7 +70,7 @@ SELECT quantilesExactExclusive(0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999)(x) FROM Эта функция эквивалентна Excel функции [PERCENTILE.INC](https://support.microsoft.com/en-us/office/percentile-inc-function-680f9539-45eb-410b-9a5e-c1355e5fe2ed), [тип R7](https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample). -С наборами уровней работает эффективнее, чем [quantilesExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantilesexactinclusive). +С наборами уровней работает эффективнее, чем [quantileExactInclusive](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexactinclusive). **Синтаксис** From 24803bd84689a4ec16b1deedcc872d71d218ca4c Mon Sep 17 00:00:00 2001 From: Alexey Date: Fri, 2 Jul 2021 20:21:38 +0000 Subject: [PATCH 117/290] bfloat16 wrapped into back ticks --- .../aggregate-functions/reference/quantilebfloat16.md | 6 +++--- .../aggregate-functions/reference/quantilebfloat16.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md index cdbb60f2fe8..b914e1feedf 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,8 +4,8 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} -Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. bfloat16 is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. -The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates bfloat16 quantile value and converts the result to a 64-bit float by appending zero bits. +Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a sample consisting of [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) numbers. `bfloat16` is a floating-point data type with 1 sign bit, 8 exponent bits and 7 fraction bits. +The function converts input values to 32-bit floats and takes the most significant 16 bits. Then it calculates `bfloat16` quantile value and converts the result to a 64-bit float by appending zero bits. The function is a fast quantile estimator with a relative error no more than 0.390625%. **Syntax** @@ -56,7 +56,7 @@ Result: │ 3 │ 1 │ └───────────────────────────┴───────────────────────────┘ ``` -Note that all floating point values in the example are truncated to 1.0 when converting to bfloat16. +Note that all floating point values in the example are truncated to 1.0 when converting to `bfloat16`. **See Also** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 217da78d1d1..1b882525c61 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -4,8 +4,8 @@ toc_priority: 209 # quantileBFloat16 {#quantilebfloat16} -Приближенно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) выборки чисел в формате [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format). bfloat16 — это формат с плавающей точкой, в котором для представления числа используется 1 знаковый бит, 8 бит для порядка и 7 бит для мантиссы. -Функция преобразует входное число в 32-битное с плавающей точкой и обрабатывает его старшие 16 бит. Она вычисляет квантиль в формате bfloat16 и преобразует его в 64-битное число с плавающей точкой, добавляя нулевые биты. +Приближенно вычисляет [квантиль](https://ru.wikipedia.org/wiki/Квантиль) выборки чисел в формате [bfloat16](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format). `bfloat16` — это формат с плавающей точкой, в котором для представления числа используется 1 знаковый бит, 8 бит для порядка и 7 бит для мантиссы. +Функция преобразует входное число в 32-битное с плавающей точкой и обрабатывает его старшие 16 бит. Она вычисляет квантиль в формате `bfloat16` и преобразует его в 64-битное число с плавающей точкой, добавляя нулевые биты. Эта функция выполняет быстрые приближенные вычисления с относительной ошибкой не более 0.390625%. **Синтаксис** @@ -56,7 +56,7 @@ SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; │ 3 │ 1 │ └───────────────────────────┴───────────────────────────┘ ``` -Обратите внимание, что все числа с плавающей точкой в примере были округлены до 1.0 при преобразовании к bfloat16. +Обратите внимание, что все числа с плавающей точкой в примере были округлены до 1.0 при преобразовании к `bfloat16`. **See Also** From 5bce3d35f8e042093a730aeb12ddaa57b4cd05b9 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 3 Jul 2021 07:57:57 +0300 Subject: [PATCH 118/290] Web UI + new adopter --- docs/en/interfaces/http.md | 28 ++++++++++------ docs/en/introduction/adopters.md | 1 + docs/ru/getting-started/playground.md | 2 +- docs/ru/interfaces/http.md | 48 +++++++++++++++------------ 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index dec3c839020..5f3eae34f92 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -7,16 +7,22 @@ toc_title: HTTP Interface The HTTP interface lets you use ClickHouse on any platform from any programming language. We use it for working from Java and Perl, as well as shell scripts. In other departments, the HTTP interface is used from Perl, Python, and Go. The HTTP interface is more limited than the native interface, but it has better compatibility. -By default, clickhouse-server listens for HTTP on port 8123 (this can be changed in the config). +By default, `clickhouse-server` listens for HTTP on port 8123 (this can be changed in the config). -If you make a GET / request without parameters, it returns 200 response code and the string which defined in [http_server_default_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) default value “Ok.” (with a line feed at the end) +If you make a `GET /` request without parameters, it returns 200 response code and the string which defined in [http_server_default_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) default value “Ok.” (with a line feed at the end) ``` bash $ curl 'http://localhost:8123/' Ok. ``` -Use GET /ping request in health-check scripts. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. +Web UI can be accessed here: + +``` bash +$ curl 'http://localhost:8123/play' +``` + +In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. ``` bash $ curl 'http://localhost:8123/ping' @@ -51,8 +57,8 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ 1 ``` -As you can see, curl is somewhat inconvenient in that spaces must be URL escaped. -Although wget escapes everything itself, we do not recommend using it because it does not work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. +As you can see, `curl` is somewhat inconvenient in that spaces must be URL escaped. +Although `wget` escapes everything itself, we do not recommend using it because it does not work well over HTTP 1.1 when using keep-alive and Transfer-Encoding: chunked. ``` bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -75,7 +81,7 @@ ECT 1 , expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception ``` -By default, data is returned in TabSeparated format (for more information, see the “Formats” section). +By default, data is returned in [TabSeparated](formats.md#tabseparated) format. You use the FORMAT clause of the query to request any other format. @@ -90,9 +96,11 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ └───┘ ``` -The POST method of transmitting data is necessary for INSERT queries. In this case, you can write the beginning of the query in the URL parameter, and use POST to pass the data to insert. The data to insert could be, for example, a tab-separated dump from MySQL. In this way, the INSERT query replaces LOAD DATA LOCAL INFILE from MySQL. +The POST method of transmitting data is necessary for `INSERT` queries. In this case, you can write the beginning of the query in the URL parameter, and use POST to pass the data to insert. The data to insert could be, for example, a tab-separated dump from MySQL. In this way, the `INSERT` query replaces `LOAD DATA LOCAL INFILE` from MySQL. -Examples: Creating a table: +**Examples** + +Creating a table: ``` bash $ echo 'CREATE TABLE t (a UInt8) ENGINE = Memory' | curl 'http://localhost:8123/' --data-binary @- @@ -632,6 +640,4 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' < Relative Path File * Connection #0 to host localhost left intact -``` - -[Original article](https://clickhouse.tech/docs/en/interfaces/http_interface/) +``` \ No newline at end of file diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 8d72e12f01b..34d3580c8ca 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -59,6 +59,7 @@ toc_title: Adopters | HUYA | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | | ICA | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) | | Idealista | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | +| Infobaleen | Marketing | Analytics | — | — | [Official site](https://infobaleen.com) | | Infovista | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | InnoGames | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | Instabug | APM Platform | Main product | — | — | [A quote from Co-Founder](https://altinity.com/) | diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index b51a9b2b436..d3101213b78 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -61,4 +61,4 @@ clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground Бэкэнд Playground - это кластер ClickHouse без дополнительных серверных приложений. Как упоминалось выше, способы подключения по HTTPS и TCP/TLS общедоступны как часть Playground. Они проксируются через [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) для добавления дополнительного уровня защиты и улучшенного глобального подключения. !!! warning "Предупреждение" -Открывать сервер ClickHouse для публичного доступа в любой другой ситуации **настоятельно не рекомендуется**. Убедитесь, что он настроен только на частную сеть и защищен брандмауэром. + Открывать сервер ClickHouse для публичного доступа в любой другой ситуации **настоятельно не рекомендуется**. Убедитесь, что он настроен только на частную сеть и защищен брандмауэром. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 9e553c12dc0..cf62045b61c 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -5,30 +5,36 @@ toc_title: "HTTP-интерфейс" # HTTP-интерфейс {#http-interface} -HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. +HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. -По умолчанию, clickhouse-server слушает HTTP на порту 8123 (это можно изменить в конфиге). -Если запросить GET / без параметров, то вернётся строка заданная с помощью настройки [http_server_default_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response). Значение по умолчанию «Ok.» (с переводом строки на конце). +По умолчанию `clickhouse-server` слушает HTTP на порту 8123 (это можно изменить в конфиге). +Если запросить `GET /` без параметров, то вернётся строка заданная с помощью настройки [http_server_default_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response). Значение по умолчанию «Ok.» (с переводом строки на конце). ``` bash $ curl 'http://localhost:8123/' Ok. ``` -В скриптах проверки доступности вы можете использовать GET /ping без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). +Веб-интерфейс доступен по адресу: + +``` bash +$ curl 'http://localhost:8123/play' +``` + +В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). ``` bash $ curl 'http://localhost:8123/ping' Ok. ``` -Запрос отправляется в виде URL параметра с именем query. Или как тело запроса при использовании метода POST. +Запрос отправляется в виде URL параметра с именем `query`. Или как тело запроса при использовании метода POST. Или начало запроса в URL параметре query, а продолжение POST-ом (зачем это нужно, будет объяснено ниже). Размер URL ограничен 16KB, это следует учитывать при отправке больших запросов. -В случае успеха, вам вернётся код ответа 200 и результат обработки запроса в теле ответа. -В случае ошибки, вам вернётся код ответа 500 и текст с описанием ошибки в теле ответа. +В случае успеха вам вернётся код ответа 200 и результат обработки запроса в теле ответа. +В случае ошибки вам вернётся код ответа 500 и текст с описанием ошибки в теле ответа. -При использовании метода GET, выставляется настройка readonly. То есть, для запросов, модифицирующие данные, можно использовать только метод POST. Сам запрос при этом можно отправлять как в теле POST-а, так и в параметре URL. +При использовании метода GET выставляется настройка readonly. То есть, для запросов, модифицирующих данные, можно использовать только метод POST. Сам запрос при этом можно отправлять как в теле POST запроса, так и в параметре URL. Примеры: @@ -51,8 +57,8 @@ X-ClickHouse-Summary: {"read_rows":"0","read_bytes":"0","written_rows":"0","writ 1 ``` -Как видно, curl немного неудобен тем, что надо URL-эскейпить пробелы. -Хотя wget сам всё эскейпит, но его не рекомендуется использовать, так как он плохо работает по HTTP 1.1 при использовании keep-alive и Transfer-Encoding: chunked. +Как видно, `curl` немного неудобен тем, что надо URL-эскейпить пробелы. +Хотя `wget` сам всё эскейпит, но его не рекомендуется использовать, так как он плохо работает по HTTP 1.1 при использовании `keep-alive` и `Transfer-Encoding: chunked`. ``` bash $ echo 'SELECT 1' | curl 'http://localhost:8123/' --data-binary @- @@ -65,7 +71,7 @@ $ echo '1' | curl 'http://localhost:8123/?query=SELECT' --data-binary @- 1 ``` -Если часть запроса отправляется в параметре, а часть POST-ом, то между этими двумя кусками данных ставится перевод строки. +Если часть запроса отправляется в параметре, а часть POST запросом, то между этими двумя кусками данных ставится перевод строки. Пример (так работать не будет): ``` bash @@ -75,9 +81,9 @@ ECT 1 , expected One of: SHOW TABLES, SHOW DATABASES, SELECT, INSERT, CREATE, ATTACH, RENAME, DROP, DETACH, USE, SET, OPTIMIZE., e.what() = DB::Exception ``` -По умолчанию, данные возвращаются в формате TabSeparated (подробнее смотри раздел «Форматы»). +По умолчанию данные возвращаются в формате [TabSeparated](formats.md#tabseparated). -Можно попросить любой другой формат - с помощью секции FORMAT запроса. +Можно попросить любой другой формат с помощью секции FORMAT запроса. Кроме того, вы можете использовать параметр URL-адреса `default_format` или заголовок `X-ClickHouse-Format`, чтобы указать формат по умолчанию, отличный от `TabSeparated`. @@ -90,9 +96,10 @@ $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @ └───┘ ``` -Возможность передавать данные POST-ом нужна для INSERT-запросов. В этом случае вы можете написать начало запроса в параметре URL, а вставляемые данные передать POST-ом. Вставляемыми данными может быть, например, tab-separated дамп, полученный из MySQL. Таким образом, запрос INSERT заменяет LOAD DATA LOCAL INFILE из MySQL. +Возможность передавать данные с помощью POST нужна для запросов `INSERT`. В этом случае вы можете написать начало запроса в параметре URL, а вставляемые данные передать POST запросом. Вставляемыми данными может быть, например, tab-separated дамп, полученный из MySQL. Таким образом, запрос `INSERT` заменяет `LOAD DATA LOCAL INFILE` из MySQL. + +**Примеры** -Примеры: Создаём таблицу: ``` bash @@ -147,7 +154,7 @@ $ curl 'http://localhost:8123/?query=SELECT%20a%20FROM%20t' $ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- ``` -Для запросов, которые не возвращают таблицу с данными, в случае успеха, выдаётся пустое тело ответа. +Для запросов, которые не возвращают таблицу с данными, в случае успеха выдаётся пустое тело ответа. ## Сжатие {#compression} @@ -165,7 +172,7 @@ $ echo 'DROP TABLE t' | curl 'http://localhost:8123/' --data-binary @- - `deflate` - `xz` -Для отправки сжатого запроса `POST`, добавьте заголовок `Content-Encoding: compression_method`. +Для отправки сжатого запроса `POST` добавьте заголовок `Content-Encoding: compression_method`. Чтобы ClickHouse сжимал ответ, разрешите сжатие настройкой [enable_http_compression](../operations/settings/settings.md#settings-enable_http_compression) и добавьте заголовок `Accept-Encoding: compression_method`. Уровень сжатия данных для всех методов сжатия можно задать с помощью настройки [http_zlib_compression_level](../operations/settings/settings.md#settings-http_zlib_compression_level). !!! note "Примечание" @@ -281,13 +288,13 @@ X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_ro HTTP интерфейс позволяет передать внешние данные (внешние временные таблицы) для использования запроса. Подробнее смотрите раздел «Внешние данные для обработки запроса» -## Буферизация ответа {#buferizatsiia-otveta} +## Буферизация ответа {#response-buffering} Существует возможность включить буферизацию ответа на стороне сервера. Для этого предусмотрены параметры URL `buffer_size` и `wait_end_of_query`. `buffer_size` определяет количество байт результата которые будут буферизованы в памяти сервера. Если тело результата больше этого порога, то буфер будет переписан в HTTP канал, а оставшиеся данные будут отправляться в HTTP-канал напрямую. -Чтобы гарантировать буферизацию всего ответа необходимо выставить `wait_end_of_query=1`. В этом случае данные, не поместившиеся в памяти, будут буферизованы во временном файле сервера. +Чтобы гарантировать буферизацию всего ответа, необходимо выставить `wait_end_of_query=1`. В этом случае данные, не поместившиеся в памяти, будут буферизованы во временном файле сервера. Пример: @@ -295,7 +302,7 @@ HTTP интерфейс позволяет передать внешние да $ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wait_end_of_query=1' -d 'SELECT toUInt8(number) FROM system.numbers LIMIT 9000000 FORMAT RowBinary' ``` -Буферизация позволяет избежать ситуации когда код ответа и HTTP-заголовки были отправлены клиенту, после чего возникла ошибка выполнения запроса. В такой ситуации сообщение об ошибке записывается в конце тела ответа, и на стороне клиента ошибка может быть обнаружена только на этапе парсинга. +Буферизация позволяет избежать ситуации, когда код ответа и HTTP-заголовки были отправлены клиенту, после чего возникла ошибка выполнения запроса. В такой ситуации сообщение об ошибке записывается в конце тела ответа, и на стороне клиента ошибка может быть обнаружена только на этапе парсинга. ### Запросы с параметрами {#cli-queries-with-parameters} @@ -634,4 +641,3 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' Relative Path File * Connection #0 to host localhost left intact ``` - From 5ae0d19cb4f8602bbd9a16da384dbb0c4feca0d8 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 3 Jul 2021 08:10:10 +0300 Subject: [PATCH 119/290] Update adopters.md --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 34d3580c8ca..2c7496197bc 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -59,7 +59,7 @@ toc_title: Adopters | HUYA | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) | | ICA | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) | | Idealista | Real Estate | Analytics | — | — | [Blog Post in English, April 2019](https://clickhouse.tech/blog/en/clickhouse-meetup-in-madrid-on-april-2-2019) | -| Infobaleen | Marketing | Analytics | — | — | [Official site](https://infobaleen.com) | +| Infobaleen | AI markting tool | Analytics | — | — | [Official site](https://infobaleen.com) | | Infovista | Networks | Analytics | — | — | [Slides in English, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup30/infovista.pdf) | | InnoGames | Games | Metrics, Logging | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/graphite_and_clickHouse.pdf) | | Instabug | APM Platform | Main product | — | — | [A quote from Co-Founder](https://altinity.com/) | From 26c4f3047bedbfd6020f406f32b4b944ec928859 Mon Sep 17 00:00:00 2001 From: lehasm Date: Sat, 3 Jul 2021 10:06:17 +0300 Subject: [PATCH 120/290] Update docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- .../aggregate-functions/reference/quantilebfloat16.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 1b882525c61..47038d279df 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -22,7 +22,7 @@ quantileBFloat16[(level)](expr) **Параметры** -- `level` — уровень квантиля. Необязательный. Допустимый диапазон значений от 0 до 1. Значение по умолчанию: 0.5. [Float](../../../sql-reference/data-types/float.md). +- `level` — уровень квантиля. Необязательный параметр. Допустимый диапазон значений от 0 до 1. Значение по умолчанию: 0.5. [Float](../../../sql-reference/data-types/float.md). **Возвращаемое значение** From 5268f64b144949090913b4003373e03ac8629d2e Mon Sep 17 00:00:00 2001 From: lehasm Date: Sat, 3 Jul 2021 10:06:32 +0300 Subject: [PATCH 121/290] Update docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- .../aggregate-functions/reference/quantilebfloat16.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md index 47038d279df..ba4a762dff7 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantilebfloat16.md @@ -58,7 +58,7 @@ SELECT quantileBFloat16(0.75)(a), quantileBFloat16(0.75)(b) FROM example_table; ``` Обратите внимание, что все числа с плавающей точкой в примере были округлены до 1.0 при преобразовании к `bfloat16`. -**See Also** +**См. также** - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) From 02681019f86743bc8d5d063623e9877588a6f6c2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 3 Jul 2021 07:45:37 +0000 Subject: [PATCH 122/290] Fix --- src/Interpreters/ExpressionAnalyzer.cpp | 7 ++++++- src/Interpreters/ExpressionAnalyzer.h | 2 ++ src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Parsers/ASTSelectQuery.h | 1 + .../01925_test_group_by_const_consistency.reference | 6 +----- .../0_stateless/01925_test_group_by_const_consistency.sql | 7 +------ 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index e693d4ba988..d4d0c0d0a9b 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -231,7 +231,6 @@ void ExpressionAnalyzer::analyzeAggregation() if (has_aggregation) { - /// Find out aggregation keys. if (select_query) { @@ -252,6 +251,8 @@ void ExpressionAnalyzer::analyzeAggregation() /// Constant expressions have non-null column pointer at this stage. if (node->column && isColumnConst(*node->column)) { + select_query->group_by_with_constant_keys = true; + /// But don't remove last key column if no aggregate functions, otherwise aggregation will not work. if (!aggregate_descriptions.empty() || size > 1) { @@ -288,6 +289,10 @@ void ExpressionAnalyzer::analyzeAggregation() else aggregated_columns = temp_actions->getNamesAndTypesList(); + /// Constant expressions are already removed during first 'analyze' run. + /// So for second `analyze` information is taken from select_query. + has_const_aggregation_keys = select_query->group_by_with_constant_keys; + for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index fe00b3e9f88..ac5d281f337 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -65,6 +65,7 @@ struct ExpressionAnalyzerData bool has_aggregation = false; NamesAndTypesList aggregation_keys; + bool has_const_aggregation_keys = false; AggregateDescriptions aggregate_descriptions; WindowDescriptions window_descriptions; @@ -309,6 +310,7 @@ public: bool hasTableJoin() const { return syntax->ast_join; } const NamesAndTypesList & aggregationKeys() const { return aggregation_keys; } + bool hasConstAggregationKeys() const { return has_const_aggregation_keys; } const AggregateDescriptions & aggregates() const { return aggregate_descriptions; } const PreparedSets & getPreparedSets() const { return prepared_sets; } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 181b60b7bf3..324340e4635 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2035,7 +2035,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes, settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, + settings.empty_result_for_aggregation_by_empty_set || (keys.empty() && query_analyzer->hasConstAggregationKeys()), context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data); diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index e9aaa4ab83b..3fc8efb5311 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -44,6 +44,7 @@ public: bool group_by_with_totals = false; bool group_by_with_rollup = false; bool group_by_with_cube = false; + bool group_by_with_constant_keys = false; bool limit_with_ties = false; ASTPtr & refSelect() { return getExpression(Expression::SELECT); } diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference index f2342900cb9..573541ac970 100644 --- a/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference @@ -1,5 +1 @@ -1 0 -1 0 -1 0 -2 1 0 -D 0 +0 diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql index f31d22a88ac..8a5de0e7c4f 100644 --- a/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql @@ -1,7 +1,2 @@ SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; - -SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT materialize(1) as a, count() FROM numbers(10)　WHERE 0 ORDER BY a; - -SELECT 2 as b, less(1, b) as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT upper('d') as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; +SELECT count() FROM numbers(10) WHERE 0 From 8a8e72b77f0fc3683477c9e43eb4d611a6e2555a Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 3 Jul 2021 18:44:17 +0000 Subject: [PATCH 123/290] Update .reference --- tests/queries/0_stateless/01414_optimize_any_bug.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/01414_optimize_any_bug.reference b/tests/queries/0_stateless/01414_optimize_any_bug.reference index 573541ac970..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01414_optimize_any_bug.reference +++ b/tests/queries/0_stateless/01414_optimize_any_bug.reference @@ -1 +0,0 @@ -0 From c762e2247d3248556996b8c262e5d790f9123c65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 4 Jul 2021 00:00:50 +0300 Subject: [PATCH 124/290] More instrumentation for network interaction: add counters for recv/send bytes; add gauges for recvs/sends. --- src/Common/CurrentMetrics.cpp | 2 ++ src/Common/ProfileEvents.cpp | 6 ++++-- src/IO/ReadBufferFromPocoSocket.cpp | 11 +++++++++++ src/IO/WriteBufferFromPocoSocket.cpp | 10 ++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 1e482361f85..e9fa13e11e6 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -30,6 +30,8 @@ M(OpenFileForWrite, "Number of files open for writing") \ M(Read, "Number of read (read, pread, io_getevents, etc.) syscalls in fly") \ M(Write, "Number of write (write, pwrite, io_getevents, etc.) syscalls in fly") \ + M(NetworkReceive, "Number of threads receiving data from network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ + M(NetworkSend, "Number of threads sending data to network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ M(SendScalars, "Number of connections that are sending data for scalars to remote servers.") \ M(SendExternalTables, "Number of connections that are sending data for external tables to remote servers. External tables are used to implement GLOBAL IN and GLOBAL JOIN operators with distributed subqueries.") \ M(QueryThread, "Number of query processing threads") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 86f06f27455..dffe2239e62 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -49,8 +49,10 @@ M(CreatedReadBufferMMapFailed, "") \ M(DiskReadElapsedMicroseconds, "Total time spent waiting for read syscall. This include reads from page cache.") \ M(DiskWriteElapsedMicroseconds, "Total time spent waiting for write syscall. This include writes to page cache.") \ - M(NetworkReceiveElapsedMicroseconds, "") \ - M(NetworkSendElapsedMicroseconds, "") \ + M(NetworkReceiveElapsedMicroseconds, "Total time spent waiting for data to receive or receiving data from network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ + M(NetworkSendElapsedMicroseconds, "Total time spent waiting for data to send to network or sending data to network. Only ClickHouse-related network interaction is included, not by 3rd party libraries..") \ + M(NetworkReceiveBytes, "Total number of bytes received from network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ + M(NetworkSendBytes, "Total number of bytes send to network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ M(ThrottlerSleepMicroseconds, "Total time a query was sleeping to conform the 'max_network_bandwidth' setting.") \ \ M(QueryMaskingRulesMatch, "Number of times query masking rules was successfully matched.") \ diff --git a/src/IO/ReadBufferFromPocoSocket.cpp b/src/IO/ReadBufferFromPocoSocket.cpp index e043764d280..5e8e41d0c3e 100644 --- a/src/IO/ReadBufferFromPocoSocket.cpp +++ b/src/IO/ReadBufferFromPocoSocket.cpp @@ -5,11 +5,19 @@ #include #include #include +#include +#include namespace ProfileEvents { extern const Event NetworkReceiveElapsedMicroseconds; + extern const Event NetworkReceiveBytes; +} + +namespace CurrentMetrics +{ + extern const Metric NetworkReceive; } @@ -31,6 +39,8 @@ bool ReadBufferFromPocoSocket::nextImpl() /// Add more details to exceptions. try { + CurrentMetrics::Increment metric_increment(CurrentMetrics::NetworkReceive); + /// If async_callback is specified, and read will block, run async_callback and try again later. /// It is expected that file descriptor may be polled externally. /// Note that receive timeout is not checked here. External code should check it while polling. @@ -57,6 +67,7 @@ bool ReadBufferFromPocoSocket::nextImpl() /// NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); if (bytes_read) working_buffer.resize(bytes_read); diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index 78705857ec4..a0e4de4c831 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -6,11 +6,19 @@ #include #include #include +#include +#include namespace ProfileEvents { extern const Event NetworkSendElapsedMicroseconds; + extern const Event NetworkSendBytes; +} + +namespace CurrentMetrics +{ + extern const Metric NetworkSend; } @@ -40,6 +48,7 @@ void WriteBufferFromPocoSocket::nextImpl() /// Add more details to exceptions. try { + CurrentMetrics::Increment metric_increment(CurrentMetrics::NetworkSend); res = socket.impl()->sendBytes(working_buffer.begin() + bytes_written, offset() - bytes_written); } catch (const Poco::Net::NetException & e) @@ -62,6 +71,7 @@ void WriteBufferFromPocoSocket::nextImpl() } ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); + ProfileEvents::increment(ProfileEvents::NetworkSendBytes, bytes_written); } WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) From 1960c717ed9bfe9288cfae0180f32112e900fe48 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 4 Jul 2021 01:59:28 +0300 Subject: [PATCH 125/290] A couple of tests --- ...01939_network_receive_bytes_metrics.reference | 1 + .../01939_network_receive_bytes_metrics.sh | 16 ++++++++++++++++ .../01939_network_send_bytes_metrics.reference | 1 + .../01939_network_send_bytes_metrics.sh | 16 ++++++++++++++++ 4 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/01939_network_receive_bytes_metrics.reference create mode 100755 tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh create mode 100644 tests/queries/0_stateless/01939_network_send_bytes_metrics.reference create mode 100755 tests/queries/0_stateless/01939_network_send_bytes_metrics.sh diff --git a/tests/queries/0_stateless/01939_network_receive_bytes_metrics.reference b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh new file mode 100755 index 00000000000..03babad40f3 --- /dev/null +++ b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;" + +seq 1 1000 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV" + +${CLICKHOUSE_CLIENT} --multiquery --query "SYSTEM FLUSH LOGS; + WITH ProfileEvents['NetworkReceiveBytes'] AS bytes + SELECT bytes >= 8000 AND bytes < 9000 ? 1 : bytes FROM system.query_log + WHERE current_database = currentDatabase() AND query_kind = 'Insert' AND event_date >= yesterday() AND type = 2 ORDER BY event_time DESC LIMIT 1;" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t" diff --git a/tests/queries/0_stateless/01939_network_send_bytes_metrics.reference b/tests/queries/0_stateless/01939_network_send_bytes_metrics.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01939_network_send_bytes_metrics.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01939_network_send_bytes_metrics.sh b/tests/queries/0_stateless/01939_network_send_bytes_metrics.sh new file mode 100755 index 00000000000..e862a273de4 --- /dev/null +++ b/tests/queries/0_stateless/01939_network_send_bytes_metrics.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;" + +${CLICKHOUSE_CLIENT} --query "SELECT number FROM numbers(1000)" > /dev/null + +${CLICKHOUSE_CLIENT} --multiquery --query "SYSTEM FLUSH LOGS; + WITH ProfileEvents['NetworkSendBytes'] AS bytes + SELECT bytes >= 8000 AND bytes < 9000 ? 1 : bytes FROM system.query_log + WHERE current_database = currentDatabase() AND query_kind = 'Select' AND event_date >= yesterday() AND type = 2 ORDER BY event_time DESC LIMIT 1;" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE t" From 7054010cac2e0b18452fd5ae765d85b48e007df5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 4 Jul 2021 02:04:43 +0300 Subject: [PATCH 126/290] Update DataTypeMap.cpp --- src/DataTypes/DataTypeMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 5ee5c90c59e..8fd375aa86e 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -71,7 +71,7 @@ void DataTypeMap::assertKeyType() const if (type_error) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Type of Map key must be a type, that can be represented by integer or String (possibly LowCardinality(String)) or UUID," + "Type of Map key must be a type, that can be represented by integer or String or FixedString (possibly LowCardinality) or UUID," " but {} given", key_type->getName()); } From e866faa975013f70eb9c455a0497d66c4c73c56b Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 4 Jul 2021 02:20:36 +0300 Subject: [PATCH 127/290] Fix --- src/Interpreters/ExpressionAnalyzer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index d4d0c0d0a9b..0897efe08fb 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -291,7 +291,8 @@ void ExpressionAnalyzer::analyzeAggregation() /// Constant expressions are already removed during first 'analyze' run. /// So for second `analyze` information is taken from select_query. - has_const_aggregation_keys = select_query->group_by_with_constant_keys; + if (select_query) + has_const_aggregation_keys = select_query->group_by_with_constant_keys; for (const auto & desc : aggregate_descriptions) aggregated_columns.emplace_back(desc.column_name, desc.function->getReturnType()); From 07693664413311f1a635cc8dd5298c53bce0fd8e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 4 Jul 2021 04:19:19 +0300 Subject: [PATCH 128/290] Remove part of trash --- src/Common/DiskStatisticsOS.cpp | 67 ++++++++---------------- src/Common/DiskStatisticsOS.h | 16 +++--- src/Common/MemoryInfoOS.cpp | 33 ++++++------ src/Common/MemoryInfoOS.h | 6 --- src/Interpreters/AsynchronousMetrics.cpp | 12 +++-- 5 files changed, 50 insertions(+), 84 deletions(-) diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp index 69f15b30a9e..1b404be07fe 100644 --- a/src/Common/DiskStatisticsOS.cpp +++ b/src/Common/DiskStatisticsOS.cpp @@ -1,14 +1,12 @@ #if defined(OS_LINUX) -#include "DiskStatisticsOS.h" +#include +#include -#include - -#include - -#include +#include #include + namespace DB { @@ -17,61 +15,38 @@ namespace ErrorCodes extern const int CANNOT_STATVFS; } -namespace -{ - void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); - } -} - -static constexpr auto mounts_filename = "/proc/mounts"; - -static constexpr std::size_t READ_BUFFER_BUF_SIZE = (64 << 10); - -DiskStatisticsOS::DiskStatisticsOS() {} - -DiskStatisticsOS::~DiskStatisticsOS() {} DiskStatisticsOS::Data DiskStatisticsOS::get() { - ReadBufferFromFile mounts_in(mounts_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); + ReadBufferFromFile mounts_in("/proc/mounts", 4096 /* arbitrary small buffer */); - DiskStatisticsOS::Data data = {0, 0}; + Data data{}; + + std::string fs_device; + std::string fs_path; while (!mounts_in.eof()) { - String filesystem = readNextFilesystem(mounts_in); + readStringUntilWhitespace(fs_device, mounts_in); + skipWhitespaceIfAny(mounts_in); + readStringUntilWhitespace(fs_path, mounts_in); + skipWhitespaceIfAny(mounts_in); - struct statvfs stat; + /// Only real devices + if (!fs_device.starts_with("/dev/") || fs_device.starts_with("/dev/loop")) + continue; - if (statvfs(filesystem.c_str(), &stat)) - throwFromErrno("Cannot statvfs", ErrorCodes::CANNOT_STATVFS); + struct statvfs stat = getStatVFS(fs_path); - uint64_t total_blocks = static_cast(stat.f_blocks); - uint64_t free_blocks = static_cast(stat.f_bfree); - uint64_t used_blocks = total_blocks - free_blocks; - uint64_t block_size = static_cast(stat.f_bsize); - - data.total += total_blocks * block_size; - data.used += used_blocks * block_size; + data.total_bytes += (stat.f_blocks) * stat.f_bsize; + data.used_bytes += (stat.f_blocks - stat.f_bfree) * stat.f_bsize; + data.total_inodes += stat.f_files; + data.used_inodes += stat.f_files - stat.f_ffree; } return data; } -String DiskStatisticsOS::readNextFilesystem(ReadBuffer & mounts_in) -{ - String filesystem, unused; - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(unused, mounts_in); - readStringUntilWhitespace(filesystem, mounts_in); - skipToNextLineOrEOF(mounts_in); - - return filesystem; -} - } #endif diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h index d4ec2417924..390846e4b6c 100644 --- a/src/Common/DiskStatisticsOS.h +++ b/src/Common/DiskStatisticsOS.h @@ -5,11 +5,13 @@ #include -#include namespace DB { +class ReadBuffer; + + /** Opens file /proc/mounts, reads all mounted filesystems and * calculates disk usage. */ @@ -19,17 +21,13 @@ public: // In bytes struct Data { - uint64_t total; - uint64_t used; + uint64_t total_bytes; + uint64_t used_bytes; + uint64_t total_inodes; + uint64_t used_inodes; }; - DiskStatisticsOS(); - ~DiskStatisticsOS(); - Data get(); - -private: - String readNextFilesystem(ReadBuffer & mounts_in); }; } diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp index 301fcb6ad15..7b712a0bb06 100644 --- a/src/Common/MemoryInfoOS.cpp +++ b/src/Common/MemoryInfoOS.cpp @@ -28,15 +28,27 @@ namespace readStringUntilWhitespace(s, buf); skipWhitespaceIfAny(buf); } + + std::pair readField(ReadBuffer & meminfo_in) + { + String key; + uint64_t val; + + readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); + readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); + skipToNextLineOrEOF(meminfo_in); + + // Delete the read ":" from the end + key.pop_back(); + + return std::make_pair(key, val); + } } static constexpr auto meminfo_filename = "/proc/meminfo"; static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); -MemoryInfoOS::MemoryInfoOS() {} - -MemoryInfoOS::~MemoryInfoOS() {} MemoryInfoOS::Data MemoryInfoOS::get() { @@ -63,21 +75,6 @@ MemoryInfoOS::Data MemoryInfoOS::get() return data; } -std::pair MemoryInfoOS::readField(ReadBuffer & meminfo_in) -{ - String key; - uint64_t val; - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); - readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); - skipToNextLineOrEOF(meminfo_in); - - // Delete the read ":" from the end - key.pop_back(); - - return std::make_pair(key, val); -} - } #endif diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h index 63cda5b5c37..4390c9d5697 100644 --- a/src/Common/MemoryInfoOS.h +++ b/src/Common/MemoryInfoOS.h @@ -33,13 +33,7 @@ public: uint64_t swap_cached; }; - MemoryInfoOS(); - ~MemoryInfoOS(); - Data get(); - -private: - std::pair readField(ReadBuffer & meminfo_in); }; } diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 8a4cc508328..9d869899d6f 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -245,17 +245,17 @@ void AsynchronousMetrics::update() MemoryInfoOS::Data data = memory_info.get(); new_values["MemoryTotal"] = data.total; - new_values["MemoryFree"] = data.free; + new_values["MemoryFreeWithoutCached"] = data.free; new_values["MemoryBuffers"] = data.buffers; new_values["MemoryCached"] = data.cached; - new_values["MemoryFreeAndCached"] = data.free_and_cached; + new_values["MemoryFreeOrCached"] = data.free_and_cached; new_values["MemorySwapTotal"] = data.swap_total; new_values["MemorySwapFree"] = data.swap_free; new_values["MemorySwapCached"] = data.swap_cached; } #endif - /// Process processor usage according to OS + /// Process CPU usage according to OS #if defined(OS_LINUX) { ProcessorStatisticsOS::Data data = proc_stat.get(); @@ -288,8 +288,10 @@ void AsynchronousMetrics::update() { DiskStatisticsOS::Data data = disk_stat.get(); - new_values["DiskTotal"] = data.total; - new_values["DiskUsed"] = data.used; + new_values["FilesystemsTotalBytes"] = data.total_bytes; + new_values["FilesystemsUsedBytes"] = data.used_bytes; + new_values["FilesystemsTotalINodes"] = data.total_inodes; + new_values["FilesystemsUsedINodes"] = data.used_inodes; } #endif From 935e0327a52ab32440e9fa52ed196e9dec979065 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 4 Jul 2021 06:03:49 +0300 Subject: [PATCH 129/290] Development --- src/IO/ReadBufferFromFileDescriptor.cpp | 16 ++++- src/IO/ReadBufferFromFileDescriptor.h | 3 + src/Interpreters/AsynchronousMetrics.cpp | 86 +++++++++++++++++++++--- src/Interpreters/AsynchronousMetrics.h | 62 +++++++++-------- 4 files changed, 123 insertions(+), 44 deletions(-) diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index babdc953514..893c2bcb5d8 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -149,7 +149,7 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) off_t res = ::lseek(fd, new_pos, SEEK_SET); if (-1 == res) throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), - ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + ErrorCodes::CANNOT_SEEK_THROUGH_FILE); file_offset_of_buffer_end = new_pos; watch.stop(); @@ -160,6 +160,20 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) } +void ReadBufferFromFileDescriptor::rewind() +{ + ProfileEvents::increment(ProfileEvents::Seek); + off_t res = ::lseek(fd, 0, SEEK_SET); + if (-1 == res) + throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), + ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + /// Clearing the buffer with existing data. New data will be read on subsequent call to 'next'. + working_buffer.resize(0); + pos = working_buffer.begin(); +} + + /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) { diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index bf22bb3d4a3..1883c6802bc 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -39,6 +39,9 @@ public: /// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen. off_t seek(off_t off, int whence) override; + /// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read. + void rewind(); + off_t size(); void setProgressCallback(ContextPtr context); diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 9d869899d6f..4e46cdc27f2 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,49 @@ namespace CurrentMetrics namespace DB { +static void openFileIfExists(const char * filename, std::optional & out) +{ + static constexpr size_t small_buffer_size = 4096; + + /// Ignoring time of check is not time of use cases, as procfs/sysfs files are fairly persistent. + + std::error_code ec; + if (std::filesystem::is_regular_file(filename, ec)) + out.emplace(filename, small_buffer_size); +} + + +AsynchronousMetrics::AsynchronousMetrics( + ContextPtr global_context_, + int update_period_seconds, + std::shared_ptr> servers_to_start_before_tables_, + std::shared_ptr> servers_) + : WithContext(global_context_) + , update_period(update_period_seconds) + , servers_to_start_before_tables(servers_to_start_before_tables_) + , servers(servers_) +{ +#if defined(OS_LINUX) + openFileIfExists("/proc/meminfo", meminfo); + openFileIfExists("/proc/mounts", mounts); + openFileIfExists("/proc/loadavg", loadavg); + openFileIfExists("/proc/stat", proc_stat); + openFileIfExists("/proc/cpuinfo", cpuinfo); + openFileIfExists("/proc/schedstat", schedstat); + openFileIfExists("/proc/sockstat", sockstat); + openFileIfExists("/proc/netstat", netstat); + openFileIfExists("/proc/sys/fs/file-nr", file_nr); +#endif +} + +void AsynchronousMetrics::start() +{ + /// Update once right now, to make metrics available just after server start + /// (without waiting for asynchronous_metrics_update_period_s). + update(); + thread = std::make_unique([this] { run(); }); +} + AsynchronousMetrics::~AsynchronousMetrics() { try @@ -206,7 +250,7 @@ void AsynchronousMetrics::update() new_values["Uptime"] = getContext()->getUptimeSeconds(); - /// Process memory usage according to OS + /// Process process memory usage according to OS #if defined(OS_LINUX) { MemoryStatisticsOS::Data data = memory_stat.get(); @@ -239,19 +283,39 @@ void AsynchronousMetrics::update() } #endif - /// Process memory information according to OS #if defined(OS_LINUX) + if (loadavg) { - MemoryInfoOS::Data data = memory_info.get(); + loadavg->rewind(); + + Float64 loadavg1 = 0; + Float64 loadavg5 = 0; + Float64 loadavg15 = 0; + UInt64 threads_runnable = 0; + UInt64 threads_total = 0; + + readText(loadavg1, *loadavg); + skipWhitespaceIfAny(*loadavg); + readText(loadavg5, *loadavg); + skipWhitespaceIfAny(*loadavg); + readText(loadavg15, *loadavg); + skipWhitespaceIfAny(*loadavg); + readText(threads_runnable, *loadavg); + assertChar('/', *loadavg); + readText(threads_total, *loadavg); + + new_values["LoadAverage1"] = loadavg1; + new_values["LoadAverage5"] = loadavg5; + new_values["LoadAverage15"] = loadavg15; + new_values["OSThreadsRunnable"] = threads_runnable; + new_values["OSThreadsTotal"] = threads_total; + } + + if (meminfo) + { + meminfo->rewind(); + - new_values["MemoryTotal"] = data.total; - new_values["MemoryFreeWithoutCached"] = data.free; - new_values["MemoryBuffers"] = data.buffers; - new_values["MemoryCached"] = data.cached; - new_values["MemoryFreeOrCached"] = data.free_and_cached; - new_values["MemorySwapTotal"] = data.swap_total; - new_values["MemorySwapFree"] = data.swap_free; - new_values["MemorySwapCached"] = data.swap_cached; } #endif diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 36e0fabd8a9..7bb281842dd 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -6,13 +6,16 @@ #include #include #include +#include #include #include #include #include +#include #include + namespace DB { @@ -29,6 +32,23 @@ using AsynchronousMetricValues = std::unordered_map> servers_to_start_before_tables_, + std::shared_ptr> servers_); + + ~AsynchronousMetrics(); + + /// Separate method allows to initialize the `servers` variable beforehand. + void start(); + + /// Returns copy of all values. + AsynchronousMetricValues getValues() const; + #if defined(ARCADIA_BUILD) /// This constructor needs only to provide backward compatibility with some other projects (hello, Arcadia). /// Never use this in the ClickHouse codebase. @@ -41,35 +61,6 @@ public: } #endif - /// The default value of update_period_seconds is for ClickHouse-over-YT - /// in Arcadia -- it uses its own server implementation that also uses these - /// metrics. - AsynchronousMetrics( - ContextPtr global_context_, - int update_period_seconds, - std::shared_ptr> servers_to_start_before_tables_, - std::shared_ptr> servers_) - : WithContext(global_context_) - , update_period(update_period_seconds) - , servers_to_start_before_tables(servers_to_start_before_tables_) - , servers(servers_) - { - } - - ~AsynchronousMetrics(); - - /// Separate method allows to initialize the `servers` variable beforehand. - void start() - { - /// Update once right now, to make metrics available just after server start - /// (without waiting for asynchronous_metrics_update_period_s). - update(); - thread = std::make_unique([this] { run(); }); - } - - /// Returns copy of all values. - AsynchronousMetricValues getValues() const; - private: const std::chrono::seconds update_period; std::shared_ptr> servers_to_start_before_tables{nullptr}; @@ -82,9 +73,16 @@ private: #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; - MemoryInfoOS memory_info; - ProcessorStatisticsOS proc_stat; - DiskStatisticsOS disk_stat; + + std::optional meminfo; + std::optional mounts; + std::optional loadavg; + std::optional proc_stat; + std::optional cpuinfo; + std::optional schedstat; + std::optional sockstat; + std::optional netstat; + std::optional file_nr; #endif std::unique_ptr thread; From 520c4a8f8a0ed65649fce49d9eea99af657a584c Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 12:10:16 +0300 Subject: [PATCH 130/290] Fix according to review --- src/Functions/FunctionSQLJSON.h | 4 +++- src/Functions/JSONPath/CMakeLists.txt | 4 ++-- src/Functions/JSONPath/Generator/CMakeLists.txt | 8 ++++++++ .../{Generators => Generator}/GeneratorJSONPath.h | 12 ++++++------ .../JSONPath/{Generators => Generator}/IGenerator.h | 4 ++-- .../{Generators => Generator}/IGenerator_fwd.h | 2 +- .../JSONPath/{Generators => Generator}/IVisitor.h | 2 +- .../VisitorJSONPathMemberAccess.h | 8 +++----- .../{Generators => Generator}/VisitorJSONPathRange.h | 4 ++-- .../{Generators => Generator}/VisitorJSONPathRoot.h | 4 ++-- .../{Generators => Generator}/VisitorJSONPathStar.h | 4 ++-- .../{Generators => Generator}/VisitorStatus.h | 0 src/Functions/JSONPath/Generators/CMakeLists.txt | 8 -------- .../JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 6 +----- src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h | 5 +---- .../0_stateless/01889_sql_json_functions.reference | 1 + .../queries/0_stateless/01889_sql_json_functions.sql | 1 + 17 files changed, 36 insertions(+), 41 deletions(-) create mode 100644 src/Functions/JSONPath/Generator/CMakeLists.txt rename src/Functions/JSONPath/{Generators => Generator}/GeneratorJSONPath.h (90%) rename src/Functions/JSONPath/{Generators => Generator}/IGenerator.h (81%) rename src/Functions/JSONPath/{Generators => Generator}/IGenerator_fwd.h (83%) rename src/Functions/JSONPath/{Generators => Generator}/IVisitor.h (94%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathMemberAccess.h (87%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathRange.h (94%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathRoot.h (87%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathStar.h (92%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorStatus.h (100%) delete mode 100644 src/Functions/JSONPath/Generators/CMakeLists.txt diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index a6024a27e95..9e469c4ebac 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -257,6 +257,8 @@ public: else if (status == VisitorStatus::Error) { /// ON ERROR + /// Here it is possible to handle errors with ON ERROR (as described in ISO/IEC TR 19075-6), + /// however this functionality is not implemented yet } current_element = root; } diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index 8e65f7c8c6d..a1f5bf9bf2c 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -1,8 +1,8 @@ add_subdirectory(ASTs) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_asts) -add_subdirectory(Generators) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) +add_subdirectory(Generator) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generator) add_subdirectory(Parsers) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) diff --git a/src/Functions/JSONPath/Generator/CMakeLists.txt b/src/Functions/JSONPath/Generator/CMakeLists.txt new file mode 100644 index 00000000000..11215ba1078 --- /dev/null +++ b/src/Functions/JSONPath/Generator/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_generator .) +add_library(clickhouse_functions_jsonpath_generator ${clickhouse_functions_jsonpath_generator_sources} ${clickhouse_functions_jsonpath_generator_headers}) +target_link_libraries(clickhouse_functions_jsonpath_generator PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_generator PRIVATE "-g0") +endif() diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h similarity index 90% rename from src/Functions/JSONPath/Generators/GeneratorJSONPath.h rename to src/Functions/JSONPath/Generator/GeneratorJSONPath.h index b918ceac003..291150f6df4 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h @@ -1,11 +1,11 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generator/IGenerator.h similarity index 81% rename from src/Functions/JSONPath/Generators/IGenerator.h rename to src/Functions/JSONPath/Generator/IGenerator.h index d2cef9fe27b..323145e07e1 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generator/IGenerator.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generator/IGenerator_fwd.h similarity index 83% rename from src/Functions/JSONPath/Generators/IGenerator_fwd.h rename to src/Functions/JSONPath/Generator/IGenerator_fwd.h index 57ed04d0f6f..bb5f64cd6f9 100644 --- a/src/Functions/JSONPath/Generators/IGenerator_fwd.h +++ b/src/Functions/JSONPath/Generator/IGenerator_fwd.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generator/IVisitor.h similarity index 94% rename from src/Functions/JSONPath/Generators/IVisitor.h rename to src/Functions/JSONPath/Generator/IVisitor.h index 1461b842829..1a94106a435 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generator/IVisitor.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h similarity index 87% rename from src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h index b0c601458b6..5fe35e75a84 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { @@ -25,19 +25,17 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { + this->setExhausted(true); if (!element.isObject()) { - this->setExhausted(true); return VisitorStatus::Error; } typename JSONParser::Element result; if (!element.getObject().find(std::string_view(member_access_ptr->member_name), result)) { - this->setExhausted(true); return VisitorStatus::Error; } apply(element); - this->setExhausted(true); return VisitorStatus::Ok; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h similarity index 94% rename from src/Functions/JSONPath/Generators/VisitorJSONPathRange.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathRange.h index 57e208271d0..40d4f6ad95e 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h similarity index 87% rename from src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h index d8b88ce0255..5c48c12782f 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h similarity index 92% rename from src/Functions/JSONPath/Generators/VisitorJSONPathStar.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathStar.h index bc840597f2a..4a54a76c199 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generator/VisitorStatus.h similarity index 100% rename from src/Functions/JSONPath/Generators/VisitorStatus.h rename to src/Functions/JSONPath/Generator/VisitorStatus.h diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt deleted file mode 100644 index 76a116132fd..00000000000 --- a/src/Functions/JSONPath/Generators/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") -add_headers_and_sources(clickhouse_functions_jsonpath_generators .) -add_library(clickhouse_functions_jsonpath_generators ${clickhouse_functions_jsonpath_generators_sources} ${clickhouse_functions_jsonpath_generators_headers}) -target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") -endif() diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 85b43217867..c7f047eb8fb 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -36,11 +36,7 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & auto member_access = std::make_shared(); node = member_access; - if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) - { - return false; - } - return true; + return tryGetIdentifierNameInto(member_name, member_access->member_name); } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h index cffec125c70..fbe7321562e 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h @@ -9,9 +9,6 @@ class ParserJSONPathQuery : public IParserBase { protected: const char * getName() const override { return "ParserJSONPathQuery"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - -private: - /// backlog: strict or lax mode + bool parseImpl(Pos & pos, ASTPtr & query, Expected & expected) override; }; } diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index e38058ffc50..8ab94781237 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -19,6 +19,7 @@ null [["world","world2"]] [{"world":"!"}] +[0, 1, 4, 0, -1, -4] --JSON_EXISTS-- 1 diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index a1749b3be24..98378a0090c 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -21,6 +21,7 @@ SELECT JSON_QUERY('$.hello', '{"hello":["world","world2"]}'); SELECT JSON_QUERY('$.hello', '{"hello":{"world":"!"}}'); SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => default value (empty string) SELECT JSON_QUERY('$.hello', ''); +SELECT JSON_QUERY('$.array[*][0 to 2, 4]', '{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}'); SELECT '--JSON_EXISTS--'; SELECT JSON_EXISTS('$', '{"hello":1}'); From ae5cb2c8bfb12e42536f3832cc5c99f72331b17e Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 13:43:22 +0300 Subject: [PATCH 131/290] Fix test (empty string) --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 8ab94781237..1ae1fccdd56 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -19,8 +19,8 @@ null [["world","world2"]] [{"world":"!"}] -[0, 1, 4, 0, -1, -4] +[0, 1, 4, 0, -1, -4] --JSON_EXISTS-- 1 0 From 0955d001cdca38473d32330444e2dbf284ab66f4 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Sun, 4 Jul 2021 16:11:29 +0300 Subject: [PATCH 132/290] added array to types --- .../external-dictionaries/external-dicts-dict-structure.md | 2 +- .../external-dictionaries/external-dicts-dict-structure.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index a7ab23da7cb..bee77a382d7 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -159,7 +159,7 @@ Configuration fields: | Tag | Description | Required | |------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| | `name` | Column name. | Yes | -| `type` | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported. | Yes | +| `type` | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported. | Yes | | `null_value` | Default value for a non-existing element.
In the example, it is an empty string. [NULL](../../syntax.md#null-literal) value can be used only for the `Nullable` types (see the previous line with types description). | Yes | | `expression` | [Expression](../../../sql-reference/syntax.md#syntax-expressions) that ClickHouse executes on the value.
The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.

Default value: no expression. | No | | `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md).

Default value: `false`. | No | diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 2feb088b4d9..197fde71279 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -159,7 +159,7 @@ CREATE DICTIONARY somename ( | Тег | Описание | Обязательный | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | `name` | Имя столбца. | Да | -| `type` | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`.
[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да | +| `type` | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`.
[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да | | `null_value` | Значение по умолчанию для несуществующего элемента.
В примере это пустая строка. Значение [NULL](../../syntax.md#null-literal) можно указывать только для типов `Nullable` (см. предыдущую строку с описанием типов). | Да | | `expression` | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.
Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.

Значение по умолчанию: нет выражения. | Нет | | `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).

Значение по умолчанию: `false`. | Нет | From 7e17290acddc9710a8b4daba5e4e79a0fd055773 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 29 Jun 2021 21:03:39 +0300 Subject: [PATCH 133/290] Fix sharding_key from column w/o function for remote() P.S. that code looks redundant, and I'm not even sure that it was required when it was first added in 325cc47ca54142b7d4018c740286b71ea7c0b278. --- src/TableFunctions/TableFunctionRemote.cpp | 5 ----- .../01932_remote_sharding_key_column.reference | 0 .../01932_remote_sharding_key_column.sql | 15 +++++++++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01932_remote_sharding_key_column.reference create mode 100644 tests/queries/0_stateless/01932_remote_sharding_key_column.sql diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 4d3524c7563..40bfa2cbb6b 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -153,11 +153,6 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (arg_num < args.size()) throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - /// ExpressionAnalyzer will be created in InterpreterSelectQuery that will meet these `Identifier` when processing the request. - /// We need to mark them as the name of the database or table, because the default value is column. - for (auto ast : args) - setIdentifierSpecial(ast); - if (!cluster_name.empty()) { /// Use an existing cluster from the main config diff --git a/tests/queries/0_stateless/01932_remote_sharding_key_column.reference b/tests/queries/0_stateless/01932_remote_sharding_key_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01932_remote_sharding_key_column.sql b/tests/queries/0_stateless/01932_remote_sharding_key_column.sql new file mode 100644 index 00000000000..ded2f187821 --- /dev/null +++ b/tests/queries/0_stateless/01932_remote_sharding_key_column.sql @@ -0,0 +1,15 @@ +-- regression test for the following query: +-- +-- select * from remote('127.1', system.one, dummy) +-- +-- that produce the following error before: +-- +-- Unknown column: dummy, there are only columns . +-- +-- NOTE: that wrapping column into any function works before. +select * from remote('127.1', system.one, dummy) format Null; +select * from remote('127.1', system.one, identity(dummy)) format Null; +select * from remote('127.1', view(select * from system.one), identity(dummy)) format Null; +select * from remote('127.{1,2}', view(select * from system.one), identity(dummy)) format Null; +select * from remote('127.1', view(select * from system.one), dummy) format Null; +select * from remote('127.{1,2}', view(select * from system.one), dummy) format Null; From 7bcb57afe1f8f011d3a2e1bf6e3b6526ae958ff4 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:10:01 +0300 Subject: [PATCH 134/290] Fix failing special builds (probably) --- src/Functions/JSONPath/Parsers/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt index ecabe5cc13b..e9cdbbfea32 100644 --- a/src/Functions/JSONPath/Parsers/CMakeLists.txt +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -2,6 +2,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_jsonpath_parsers .) add_library(clickhouse_functions_jsonpath_parsers ${clickhouse_functions_jsonpath_parsers_sources} ${clickhouse_functions_jsonpath_parsers_headers}) target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) +target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE clickhouse_parsers) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") From 618a77fafaa52912c37d652bbe3ade7f1d6ac306 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 1 Jul 2021 17:41:59 +0300 Subject: [PATCH 135/290] Improve logging in integration tests. --- tests/integration/ci-runner.py | 103 ++++++++++++------ tests/integration/helpers/cluster.py | 37 +++++-- .../pytest_xdist_logging_to_separate_files.py | 28 +++++ tests/integration/pytest.ini | 12 +- tests/integration/runner | 6 +- 5 files changed, 134 insertions(+), 52 deletions(-) create mode 100644 tests/integration/helpers/pytest_xdist_logging_to_separate_files.py diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 0af76fe2648..97d076f698e 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -3,6 +3,7 @@ import logging import subprocess import os +import glob import time import shutil from collections import defaultdict @@ -17,7 +18,6 @@ SLEEP_BETWEEN_RETRIES = 5 PARALLEL_GROUP_SIZE = 100 CLICKHOUSE_BINARY_PATH = "/usr/bin/clickhouse" CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "/usr/bin/clickhouse-odbc-bridge" -DOCKERD_LOGS_PATH = "/ClickHouse/tests/integration/dockerd.log" CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "/usr/bin/clickhouse-library-bridge" TRIES_COUNT = 10 @@ -256,8 +256,8 @@ class ClickhouseIntegrationTestsRunner: shutil.copy(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, result_path_library_bridge) return None, None - def _compress_logs(self, path, result_path): - subprocess.check_call("tar czf {} -C {} .".format(result_path, path), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + def _compress_logs(self, dir, relpaths, result_path): + subprocess.check_call("tar czf {} -C {} {}".format(result_path, dir, ' '.join(relpaths)), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL def _get_all_tests(self, repo_path): image_cmd = self._get_runner_image_cmd(repo_path) @@ -336,6 +336,27 @@ class ClickhouseIntegrationTestsRunner: logging.info("Cannot run with custom docker image version :(") return image_cmd + def _find_test_data_dirs(self, repo_path, test_names): + relpaths = {} + for test_name in test_names: + if '/' in test_name: + test_dir = test_name[:test_name.find('/')] + else: + test_dir = test_name + if os.path.isdir(os.path.join(repo_path, "tests/integration", test_dir)): + for name in os.listdir(os.path.join(repo_path, "tests/integration", test_dir)): + relpath = os.path.join(os.path.join(test_dir, name)) + mtime = os.path.getmtime(os.path.join(repo_path, "tests/integration", relpath)) + relpaths[relpath] = mtime + return relpaths + + def _get_test_data_dirs_difference(self, new_snapshot, old_snapshot): + res = set() + for path in new_snapshot: + if (not path in old_snapshot) or (old_snapshot[path] != new_snapshot[path]): + res.add(path) + return res + def run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): counters = { "ERROR": [], @@ -355,18 +376,14 @@ class ClickhouseIntegrationTestsRunner: image_cmd = self._get_runner_image_cmd(repo_path) test_group_str = test_group.replace('/', '_').replace('.', '_') + log_paths = [] + test_data_dirs = {} for i in range(num_tries): logging.info("Running test group %s for the %s retry", test_group, i) clear_ip_tables_and_restart_daemons() - output_path = os.path.join(str(self.path()), "test_output_" + test_group_str + "_" + str(i) + ".log") - log_name = "integration_run_" + test_group_str + "_" + str(i) + ".txt" - log_path = os.path.join(str(self.path()), log_name) - log_paths.append(log_path) - logging.info("Will wait output inside %s", output_path) - test_names = set([]) for test_name in tests_in_group: if test_name not in counters["PASSED"]: @@ -375,11 +392,19 @@ class ClickhouseIntegrationTestsRunner: else: test_names.add(test_name) + if i == 0: + test_data_dirs = self._find_test_data_dirs(repo_path, test_names) + + info_basename = test_group_str + "_" + str(i) + ".nfo" + info_path = os.path.join(repo_path, "tests/integration", info_basename) + test_cmd = ' '.join([test for test in sorted(test_names)]) parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else "" - cmd = "cd {}/tests/integration && ./runner --tmpfs {} -t {} {} '-ss -rfEp --run-id={} --color=no --durations=0 {}' | tee {}".format( - repo_path, image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), output_path) + cmd = "cd {}/tests/integration && ./runner --tmpfs {} -t {} {} '-rfEp --run-id={} --color=no --durations=0 {}' | tee {}".format( + repo_path, image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), info_path) + log_basename = test_group_str + "_" + str(i) + ".log" + log_path = os.path.join(repo_path, "tests/integration", log_basename) with open(log_path, 'w') as log: logging.info("Executing cmd: %s", cmd) retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() @@ -388,15 +413,41 @@ class ClickhouseIntegrationTestsRunner: else: logging.info("Some tests failed") - if os.path.exists(output_path): - lines = parse_test_results_output(output_path) + extra_logs_names = [log_basename] + log_result_path = os.path.join(str(self.path()), 'integration_run_' + log_basename) + shutil.copy(log_path, log_result_path) + log_paths.append(log_result_path) + + for pytest_log_path in glob.glob(os.path.join(repo_path, "tests/integration/pytest*.log")): + new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(pytest_log_path) + os.rename(pytest_log_path, os.path.join(repo_path, "tests/integration", new_name)) + extra_logs_names.append(new_name) + + dockerd_log_path = os.path.join(repo_path, "tests/integration/dockerd.log") + if os.path.exists(dockerd_log_path): + new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(dockerd_log_path) + os.rename(dockerd_log_path, os.path.join(repo_path, "tests/integration", new_name)) + extra_logs_names.append(new_name) + + if os.path.exists(info_path): + extra_logs_names.append(info_basename) + lines = parse_test_results_output(info_path) new_counters = get_counters(lines) - times_lines = parse_test_times(output_path) + times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) self._update_counters(counters, new_counters) for test_name, test_time in new_tests_times.items(): tests_times[test_name] = test_time - os.remove(output_path) + + test_data_dirs_new = self._find_test_data_dirs(repo_path, test_names) + test_data_dirs_diff = self._get_test_data_dirs_difference(test_data_dirs_new, test_data_dirs) + test_data_dirs = test_data_dirs_new + + if extra_logs_names or test_data_dirs_diff: + extras_result_path = os.path.join(str(self.path()), "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz") + self._compress_logs(os.path.join(repo_path, "tests/integration"), extra_logs_names + list(test_data_dirs_diff), extras_result_path) + log_paths.append(extras_result_path) + if len(counters["PASSED"]) + len(counters["FLAKY"]) == len(tests_in_group): logging.info("All tests from group %s passed", test_group) break @@ -459,15 +510,6 @@ class ClickhouseIntegrationTestsRunner: break time.sleep(5) - logging.info("Finally all tests done, going to compress test dir") - test_logs = os.path.join(str(self.path()), "./test_dir.tar.gz") - self._compress_logs("{}/tests/integration".format(repo_path), test_logs) - logging.info("Compression finished") - - result_path_dockerd_logs = os.path.join(str(self.path()), "dockerd.log") - if os.path.exists(result_path_dockerd_logs): - shutil.copy(DOCKERD_LOGS_PATH, result_path_dockerd_logs) - test_result = [] for state in ("ERROR", "FAILED", "PASSED", "SKIPPED", "FLAKY"): if state == "PASSED": @@ -479,7 +521,7 @@ class ClickhouseIntegrationTestsRunner: test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, "{:.2f}".format(tests_times[c])) for c in counters[state]] status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) - return result_state, status_text, test_result, [test_logs] + logs + return result_state, status_text, test_result, logs def run_impl(self, repo_path, build_path): if self.flaky_check: @@ -539,15 +581,6 @@ class ClickhouseIntegrationTestsRunner: logging.info("Collected more than 20 failed/error tests, stopping") break - logging.info("Finally all tests done, going to compress test dir") - test_logs = os.path.join(str(self.path()), "./test_dir.tar.gz") - self._compress_logs("{}/tests/integration".format(repo_path), test_logs) - logging.info("Compression finished") - - result_path_dockerd_logs = os.path.join(str(self.path()), "dockerd.log") - if os.path.exists(result_path_dockerd_logs): - shutil.copy(DOCKERD_LOGS_PATH, result_path_dockerd_logs) - if counters["FAILED"] or counters["ERROR"]: logging.info("Overall status failure, because we have tests in FAILED or ERROR state") result_state = "failure" @@ -580,7 +613,7 @@ class ClickhouseIntegrationTestsRunner: if '(memory)' in self.params['context_name']: result_state = "success" - return result_state, status_text, test_result, [test_logs] + return result_state, status_text, test_result, [] def write_results(results_file, status_file, results, status): with open(results_file, 'w') as f: diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 8863492dd12..1db9e07a69e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -30,6 +30,7 @@ from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio from helpers.test_tools import assert_eq_with_retry +from helpers import pytest_xdist_logging_to_separate_files import docker @@ -56,22 +57,22 @@ def run_and_check(args, env=None, shell=False, stdout=subprocess.PIPE, stderr=su subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, shell=shell) return + logging.debug(f"Command:{args}") res = subprocess.run(args, stdout=stdout, stderr=stderr, env=env, shell=shell, timeout=timeout) out = res.stdout.decode('utf-8') err = res.stderr.decode('utf-8') - if res.returncode != 0: - # check_call(...) from subprocess does not print stderr, so we do it manually - logging.debug(f"Command:{args}") - logging.debug(f"Stderr:{err}") + # check_call(...) from subprocess does not print stderr, so we do it manually + if out: logging.debug(f"Stdout:{out}") - logging.debug(f"Env: {env}") + if err: + logging.debug(f"Stderr:{err}") + if res.returncode != 0: + logging.debug(f"Exitcode:{res.returncode}") + if env: + logging.debug(f"Env:{env}") if not nothrow: raise Exception(f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}") - else: - logging.debug(f"Command:{args}") - logging.debug(f"Stderr: {err}") - logging.debug(f"Stdout: {out}") - return out + return out # Based on https://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python/2838309#2838309 def get_free_port(): @@ -192,6 +193,7 @@ class ClickHouseCluster: zookeeper_keyfile=None, zookeeper_certfile=None): for param in list(os.environ.keys()): logging.debug("ENV %40s %s" % (param, os.environ[param])) + self.base_path = base_path self.base_dir = p.dirname(base_path) self.name = name if name is not None else '' @@ -1290,6 +1292,9 @@ class ClickHouseCluster: raise Exception("Can't wait Cassandra to start") def start(self, destroy_dirs=True): + pytest_xdist_logging_to_separate_files.setup() + logging.info("Running tests in {}".format(self.base_path)) + logging.debug("Cluster start called. is_up={}, destroy_dirs={}".format(self.is_up, destroy_dirs)) if self.is_up: return @@ -1771,12 +1776,14 @@ class ClickHouseInstance: # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer def query(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False): + logging.debug(f"Executing query {sql} on {self.name}") return self.client.query(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database, ignore_error=ignore_error) def query_with_retry(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None, ignore_error=False, retry_count=20, sleep_time=0.5, check_callback=lambda x: True): + logging.debug(f"Executing query {sql} on {self.name}") result = None for i in range(retry_count): try: @@ -1794,23 +1801,27 @@ class ClickHouseInstance: raise Exception("Can't execute query {}".format(sql)) # As query() but doesn't wait response and returns response handler - def get_query_request(self, *args, **kwargs): - return self.client.get_query_request(*args, **kwargs) + def get_query_request(self, sql, *args, **kwargs): + logging.debug(f"Executing query {sql} on {self.name}") + return self.client.get_query_request(sql, *args, **kwargs) # Connects to the instance via clickhouse-client, sends a query (1st argument), expects an error and return its code def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): + logging.debug(f"Executing query {sql} on {self.name}") return self.client.query_and_get_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) # The same as query_and_get_error but ignores successful query. def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None, password=None, database=None): + logging.debug(f"Executing query {sql} on {self.name}") return self.client.query_and_get_answer_with_error(sql, stdin=stdin, timeout=timeout, settings=settings, user=user, password=password, database=database) # Connects to the instance via HTTP interface, sends a query and returns the answer def http_query(self, sql, data=None, params=None, user=None, password=None, expect_fail_and_get_error=False): + logging.debug(f"Executing query {sql} on {self.name} via HTTP interface") if params is None: params = {} else: @@ -1845,11 +1856,13 @@ class ClickHouseInstance: # Connects to the instance via HTTP interface, sends a query and returns the answer def http_request(self, url, method='GET', params=None, data=None, headers=None): + logging.debug(f"Sending HTTP request {url} to {self.name}") url = "http://" + self.ip_address + ":8123/" + url return requests.request(method=method, url=url, params=params, data=data, headers=headers) # Connects to the instance via HTTP interface, sends a query, expects an error and return the error message def http_query_and_get_error(self, sql, data=None, params=None, user=None, password=None): + logging.debug(f"Executing query {sql} on {self.name} via HTTP interface") return self.http_query(sql=sql, data=data, params=params, user=user, password=password, expect_fail_and_get_error=True) diff --git a/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py new file mode 100644 index 00000000000..ee9a52e042c --- /dev/null +++ b/tests/integration/helpers/pytest_xdist_logging_to_separate_files.py @@ -0,0 +1,28 @@ +import logging +import os.path + +# Makes the parallel workers of pytest-xdist to log to separate files. +# Without this function all workers will log to the same log file +# and mix everything together making it much more difficult for troubleshooting. +def setup(): + worker_name = os.environ.get('PYTEST_XDIST_WORKER', 'master') + if worker_name == 'master': + return + logger = logging.getLogger('') + new_handlers = [] + handlers_to_remove = [] + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + filename, ext = os.path.splitext(handler.baseFilename) + if not filename.endswith('-' + worker_name): + new_filename = filename + '-' + worker_name + new_handler = logging.FileHandler(new_filename + ext) + new_handler.setFormatter(handler.formatter) + new_handler.setLevel(handler.level) + new_handlers.append(new_handler) + handlers_to_remove.append(handler) + for new_handler in new_handlers: + logger.addHandler(new_handler) + for handler in handlers_to_remove: + handler.flush() + logger.removeHandler(handler) diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 6d451adf7eb..9a6ddd1be4b 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -4,10 +4,14 @@ norecursedirs = _instances* timeout = 1800 junit_duration_report = call junit_suite_name = integration -log_cli = 1 +log_level = DEBUG +log_format = %(asctime)s %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s) +log_date_format=%Y-%m-%d %H:%M:%S +log_cli = true log_cli_level = CRITICAL -log_cli_format = %%(asctime)s [%(levelname)8s] %(funcName)s %(message)s (%(filename)s:%(lineno)s) +log_cli_format = %(asctime)s %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s) +log_cli_date_format=%Y-%m-%d %H:%M:%S log_file = pytest.log log_file_level = DEBUG -log_file_format = %(asctime)s [%(levelname)8s] %(funcName)s %(message)s (%(filename)s:%(lineno)s) -log_file_date_format=%Y-%m-%d %H:%M:%S +log_file_format = %(asctime)s %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s) +log_file_date_format = %Y-%m-%d %H:%M:%S diff --git a/tests/integration/runner b/tests/integration/runner index 160c4a23652..1bef8f60db9 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -3,6 +3,7 @@ import subprocess import os import getpass +import glob import argparse import logging import signal @@ -99,7 +100,7 @@ signal.signal(signal.SIGINT, docker_kill_handler_handler) # 2) path of runner script is used to determine paths for trivial case, when we run it from repository if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s : %(message)s (%(filename)s:%(lineno)s, %(funcName)s)') parser = argparse.ArgumentParser(description="ClickHouse integration tests runner") parser.add_argument( @@ -257,6 +258,9 @@ if __name__ == "__main__": if sys.stdout.isatty() and sys.stdin.isatty(): tty = "-it" + # Remove old logs. + for old_log_path in glob.glob(args.cases_dir + "/pytest*.log"): + os.remove(old_log_path) cmd = "docker run {net} {tty} --rm --name {name} --privileged \ --volume={odbc_bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \ From a48f500956b769e3b6fd70f7d658e6484d927d94 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:28:05 +0300 Subject: [PATCH 136/290] Add tests with multiple rows --- .../0_stateless/01889_sql_json_functions.reference | 4 ++++ .../queries/0_stateless/01889_sql_json_functions.sql | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 1ae1fccdd56..fae7418b818 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -37,3 +37,7 @@ null 1 0 1 +--MANY ROWS-- +["Vasily", "Kostya"] +["Katya", "Anatoliy"] +["Tihon", "Ernest"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 98378a0090c..9ee4ee1b95c 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -38,4 +38,13 @@ SELECT JSON_EXISTS('$.hello[0]', '{"hello":["world"]}'); SELECT JSON_EXISTS('$.hello[1]', '{"hello":["world"]}'); SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); -SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); \ No newline at end of file +SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); + +SELECT '--MANY ROWS--' +DROP TABLE IF EXISTS 01889_sql_json; +CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}') +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}') +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}') +SELECT JSON_QUERY('$.friends[0 to 2]', json) +DROP TABLE 01889_sql_json From 9f05b387e541f03301fb240728a75064bbbf8747 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:33:13 +0300 Subject: [PATCH 137/290] Add colons --- .../queries/0_stateless/01889_sql_json_functions.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 9ee4ee1b95c..9b978c2223d 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -40,11 +40,11 @@ SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); -SELECT '--MANY ROWS--' +SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}') -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}') -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}') -SELECT JSON_QUERY('$.friends[0 to 2]', json) -DROP TABLE 01889_sql_json +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +SELECT JSON_QUERY('$.friends[0 to 2]', json); +DROP TABLE 01889_sql_json; From 3f8a22c35d23e4c1f7b7b1eea6fc95b8d797a16f Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:34:34 +0300 Subject: [PATCH 138/290] Fix syntax --- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 9b978c2223d..0589a330280 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -46,5 +46,5 @@ CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -SELECT JSON_QUERY('$.friends[0 to 2]', json); +SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From 4ef27cfc2d20c8ab32f6024e6e536bb51fd6cca9 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:36:35 +0300 Subject: [PATCH 139/290] Fix ans order --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index fae7418b818..d361dea6b18 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] ["Katya", "Anatoliy"] +["Vasily", "Kostya"] ["Tihon", "Ernest"] From 3d1e2fe55078b807f214cbea06e982dcc86a3d22 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:37:44 +0300 Subject: [PATCH 140/290] Fix order again --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index d361dea6b18..bf70961b1f8 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Katya", "Anatoliy"] ["Vasily", "Kostya"] ["Tihon", "Ernest"] +["Katya", "Anatoliy"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 0589a330280..fe37a3dc53e 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -43,8 +43,8 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From a0b6281790e5e72b3ed839aa1b0e25da0a22c3a7 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:38:43 +0300 Subject: [PATCH 141/290] Final fix order --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index bf70961b1f8..8fb2f9430e0 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] ["Tihon", "Ernest"] ["Katya", "Anatoliy"] +["Vasily", "Kostya"] From 7965c638d2ba2d2db783a88ef63f6bb9975f825a Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:44:18 +0300 Subject: [PATCH 142/290] Another try --- .../0_stateless/01889_sql_json_functions.reference | 2 +- tests/queries/0_stateless/01889_sql_json_functions.sql | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 8fb2f9430e0..bf70961b1f8 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- +["Vasily", "Kostya"] ["Tihon", "Ernest"] ["Katya", "Anatoliy"] -["Vasily", "Kostya"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index fe37a3dc53e..79af60a15a4 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -42,9 +42,9 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; -CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY id; +INSERT INTO 01889_sql_json(json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From fd07dbe1f7379f7e7511576d8c339005b551e523 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:46:45 +0300 Subject: [PATCH 143/290] Please just work --- tests/queries/0_stateless/01889_sql_json_functions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 79af60a15a4..77cb32352cf 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -43,8 +43,8 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY id; -INSERT INTO 01889_sql_json(json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); -INSERT INTO 01889_sql_json(json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -INSERT INTO 01889_sql_json(json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From 57972410c794a504fe407a5ec293a8484007ee6e Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:54:43 +0300 Subject: [PATCH 144/290] =?UTF-8?q?=F0=9F=98=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queries/0_stateless/01889_sql_json_functions.reference | 6 +++--- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index bf70961b1f8..7d400b4f6e7 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] -["Tihon", "Ernest"] -["Katya", "Anatoliy"] +0 ["Vasily", "Kostya"] +1 ["Tihon", "Ernest"] +2 ["Katya", "Anatoliy"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 77cb32352cf..1c5069ccfde 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -46,5 +46,5 @@ CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY INSERT INTO 01889_sql_json(id, json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(id, json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); INSERT INTO 01889_sql_json(id, json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); -SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; +SELECT id, JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json ORDER BY id; DROP TABLE 01889_sql_json; From 7d1c561a7b37634f70ba36c97d14dde51d79d0f4 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:57:16 +0300 Subject: [PATCH 145/290] =?UTF-8?q?=F0=9F=98=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queries/0_stateless/01889_sql_json_functions.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 7d400b4f6e7..593f2fb2d20 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -0 ["Vasily", "Kostya"] -1 ["Tihon", "Ernest"] -2 ["Katya", "Anatoliy"] +0 ["Vasily", "Kostya"] +1 ["Tihon", "Ernest"] +2 ["Katya", "Anatoliy"] From c4675285bffd547a9cf328fdcfb99fff19681ba4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 4 Jul 2021 23:49:36 +0300 Subject: [PATCH 146/290] Development --- src/Common/ErrorCodes.cpp | 1 + src/Interpreters/AsynchronousMetrics.cpp | 368 ++++++++++++++++++----- src/Interpreters/AsynchronousMetrics.h | 40 ++- 3 files changed, 340 insertions(+), 69 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f4ceef2896a..8301ea656bf 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -557,6 +557,7 @@ M(587, CONCURRENT_ACCESS_NOT_SUPPORTED) \ M(588, DISTRIBUTED_BROKEN_BATCH_INFO) \ M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \ + M(590, CANNOT_SYSCONF) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 4e46cdc27f2..89196b5e25f 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -36,10 +36,16 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int CANNOT_SYSCONF; +} + +static constexpr size_t small_buffer_size = 4096; + static void openFileIfExists(const char * filename, std::optional & out) { - static constexpr size_t small_buffer_size = 4096; - /// Ignoring time of check is not time of use cases, as procfs/sysfs files are fairly persistent. std::error_code ec; @@ -47,6 +53,15 @@ static void openFileIfExists(const char * filename, std::optional openFileIfExists(const std::string & filename) +{ + std::error_code ec; + if (std::filesystem::is_regular_file(filename, ec)) + return std::make_unique(filename, small_buffer_size); + return {}; +} + + AsynchronousMetrics::AsynchronousMetrics( ContextPtr global_context_, @@ -60,7 +75,6 @@ AsynchronousMetrics::AsynchronousMetrics( { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); - openFileIfExists("/proc/mounts", mounts); openFileIfExists("/proc/loadavg", loadavg); openFileIfExists("/proc/stat", proc_stat); openFileIfExists("/proc/cpuinfo", cpuinfo); @@ -68,6 +82,17 @@ AsynchronousMetrics::AsynchronousMetrics( openFileIfExists("/proc/sockstat", sockstat); openFileIfExists("/proc/netstat", netstat); openFileIfExists("/proc/sys/fs/file-nr", file_nr); + openFileIfExists("/proc/uptime", uptime); + + size_t thermal_device_index = 0; + while (true) + { + std::unique_ptr file = openFileIfExists(fmt::format("/sys/class/thermal/thermal_zone{}/temp", thermal_device_index)); + if (!file) + break; + thermal.emplace_back(std::move(file)); + ++thermal_device_index; + } #endif } @@ -211,6 +236,63 @@ static void saveAllArenasMetric(AsynchronousMetricValues & values, } #endif + +#if defined(OS_LINUX) + +void AsynchronousMetrics::ProcStatValuesCPU::read(ReadBuffer & in) +{ + readText(user, in); + skipWhitespaceIfAny(in); + readText(nice, in); + skipWhitespaceIfAny(in); + readText(system, in); + skipWhitespaceIfAny(in); + readText(idle, in); + skipWhitespaceIfAny(in); + readText(iowait, in); + skipWhitespaceIfAny(in); + readText(irq, in); + skipWhitespaceIfAny(in); + readText(softirq, in); + skipWhitespaceIfAny(in); + readText(steal, in); + skipWhitespaceIfAny(in); + readText(guest, in); + skipWhitespaceIfAny(in); + readText(guest_nice, in); + skipToNextLineOrEOF(in); +} + +AsynchronousMetrics::ProcStatValuesCPU +AsynchronousMetrics::ProcStatValuesCPU::operator-(const AsynchronousMetrics::ProcStatValuesCPU & other) const +{ + ProcStatValuesCPU res{}; + res.user = user - other.user; + res.nice = nice - other.nice; + res.system = system - other.system; + res.idle = idle - other.idle; + res.iowait = iowait - other.iowait; + res.irq = irq - other.irq; + res.softirq = softirq - other.softirq; + res.steal = steal - other.steal; + res.guest = guest - other.guest; + res.guest_nice = guest_nice - other.guest_nice; + return res; +} + +AsynchronousMetrics::ProcStatValuesOther +AsynchronousMetrics::ProcStatValuesOther::operator-(const AsynchronousMetrics::ProcStatValuesOther & other) const +{ + ProcStatValuesOther res{}; + res.interrupts = interrupts - other.interrupts; + res.context_switches = context_switches - other.context_switches; + res.processes_created = processes_created - other.processes_created; + return res; +} + +#endif + + void AsynchronousMetrics::update() { AsynchronousMetricValues new_values; @@ -311,42 +393,234 @@ void AsynchronousMetrics::update() new_values["OSThreadsTotal"] = threads_total; } + if (uptime) + { + uptime->rewind(); + + Float64 uptime_seconds = 0; + readText(uptime_seconds, *uptime); + + new_values["OSUptime"] = uptime_seconds; + } + + if (proc_stat) + { + proc_stat->rewind(); + + int64_t hz = sysconf(_SC_CLK_TCK); + if (-1 == hz) + throwFromErrno("Cannot call 'sysconf' to obtain system HZ", ErrorCodes::CANNOT_SYSCONF); + + double multiplier = 1.0 / hz / update_period.count(); + + ProcStatValuesOther current_other_values{}; + + while (!proc_stat->eof()) + { + String name; + readStringUntilWhitespace(name, *proc_stat); + skipWhitespaceIfAny(*proc_stat); + + if (name.starts_with("cpu")) + { + String cpu_num_str = name.substr(strlen("cpu")); + UInt64 cpu_num = 0; + if (!cpu_num_str.empty()) + { + cpu_num = parse(cpu_num_str); + + if (cpu_num > 1000000) /// Safety check, arbitrary large number, suitable for supercomputing applications. + throw Exception(ErrorCodes::CORRUPTED_DATA, "Too many CPUs (at least {}) in '/proc/stat' file", cpu_num); + + if (proc_stat_values_per_cpu.size() <= cpu_num) + proc_stat_values_per_cpu.resize(cpu_num + 1); + } + + ProcStatValuesCPU current_values{}; + current_values.read(*proc_stat); + + ProcStatValuesCPU & prev_values = !cpu_num_str.empty() ? proc_stat_values_per_cpu[cpu_num] : proc_stat_values_all_cpus; + + if (!first_run) + { + ProcStatValuesCPU delta_values = current_values - prev_values; + + String cpu_suffix; + if (!cpu_num_str.empty()) + cpu_suffix = "CPU" + cpu_num_str; + + new_values["OSUserTime" + cpu_suffix] = delta_values.user * multiplier; + new_values["OSNiceTime" + cpu_suffix] = delta_values.nice * multiplier; + new_values["OSSystemTime" + cpu_suffix] = delta_values.system * multiplier; + new_values["OSIdleTime" + cpu_suffix] = delta_values.idle * multiplier; + new_values["OSIOWaitTime" + cpu_suffix] = delta_values.iowait * multiplier; + new_values["OSIrqTime" + cpu_suffix] = delta_values.irq * multiplier; + new_values["OSSoftIrqTime" + cpu_suffix] = delta_values.softirq * multiplier; + new_values["OSStealTime" + cpu_suffix] = delta_values.steal * multiplier; + new_values["OSGuestTime" + cpu_suffix] = delta_values.guest * multiplier; + new_values["OSGuestNiceTime" + cpu_suffix] = delta_values.guest_nice * multiplier; + } + + prev_values = current_values; + } + else if (name == "intr") + { + readText(current_other_values.interrupts, *proc_stat); + skipToNextLineOrEOF(*proc_stat); + } + else if (name == "ctxt") + { + readText(current_other_values.context_switches, *proc_stat); + skipToNextLineOrEOF(*proc_stat); + } + else if (name == "processes") + { + readText(current_other_values.processes_created, *proc_stat); + skipToNextLineOrEOF(*proc_stat); + } + else if (name == "procs_running") + { + UInt64 processes_running = 0; + readText(processes_running, *proc_stat); + skipToNextLineOrEOF(*proc_stat); + new_values["OSProcessesRunning"] = processes_running; + } + else if (name == "procs_blocked") + { + UInt64 processes_blocked = 0; + readText(processes_blocked, *proc_stat); + skipToNextLineOrEOF(*proc_stat); + new_values["OSProcessesBlocked"] = processes_blocked; + } + else + skipToNextLineOrEOF(*proc_stat); + } + + if (!first_run) + { + ProcStatValuesOther delta_values = current_other_values - proc_stat_values_other; + + new_values["OSInterrupts"] = delta_values.interrupts * multiplier; + new_values["OSContextSwitches"] = delta_values.context_switches * multiplier; + new_values["OSProcessesCreated"] = delta_values.processes_created * multiplier; + } + + proc_stat_values_other = current_other_values; + } + if (meminfo) { meminfo->rewind(); + uint64_t free_plus_cached_bytes = 0; + while (!meminfo->eof()) + { + String name; + readStringUntilWhitespace(name, *meminfo); + skipWhitespaceIfAny(*meminfo); + + uint64_t kb = 0; + readText(kb, *meminfo); + if (kb) + { + skipWhitespaceIfAny(*meminfo); + assertString("kB", *meminfo); + + uint64_t bytes = kb * 1024; + + if (name == "MemTotal:") + { + new_values["OSMemoryTotal"] = bytes; + } + else if (name == "MemFree:") + { + free_plus_cached_bytes += bytes; + new_values["OSMemoryFreeWithoutCached"] = bytes; + } + else if (name == "MemAvailable:") + { + new_values["OSMemoryAvailable"] = bytes; + } + else if (name == "Buffers:") + { + new_values["OSMemoryBuffers"] = bytes; + } + else if (name == "Cached:") + { + free_plus_cached_bytes += bytes; + new_values["OSMemoryCached"] = bytes; + } + else if (name == "SwapCached:") + { + new_values["OSMemorySwapCached"] = bytes; + } + } + + skipToNextLineOrEOF(*meminfo); + } + + new_values["OSMemoryFreePlusCached"] = free_plus_cached_bytes; } -#endif - /// Process CPU usage according to OS -#if defined(OS_LINUX) + // Try to add processor frequencies, ignoring errors. + if (cpuinfo) { - ProcessorStatisticsOS::Data data = proc_stat.get(); + try + { + cpuinfo->rewind(); - new_values["LoadAvg1"] = data.loadavg.avg1; - new_values["LoadAvg5"] = data.loadavg.avg5; - new_values["LoadAvg15"] = data.loadavg.avg15; + // We need the following lines: + // processor : 4 + // cpu MHz : 4052.941 + // They contain tabs and are interspersed with other info. - new_values["FreqMin"] = data.freq.min; - new_values["FreqMax"] = data.freq.max; - new_values["FreqAvg"] = data.freq.avg; + int core_id = 0; + while (!cpuinfo->eof()) + { + std::string s; + // We don't have any backslash escape sequences in /proc/cpuinfo, so + // this function will read the line until EOL, which is exactly what + // we need. + readEscapedStringUntilEOL(s, *cpuinfo); + // It doesn't read the EOL itself. + ++cpuinfo->position(); - new_values["TimeLoadUser"] = data.stload.user_time; - new_values["TimeLoadNice"] = data.stload.nice_time; - new_values["TimeLoadSystem"] = data.stload.system_time; - new_values["TimeLoadIDLE"] = data.stload.idle_time; - new_values["TimeLoadIowait"] = data.stload.iowait_time; - new_values["TimeLoadSteal"] = data.stload.steal_time; - new_values["TimeLoadGuest"] = data.stload.guest_time; - new_values["TimeLoadGuestNice"] = data.stload.guest_nice_time; + if (s.rfind("processor", 0) == 0) + { + if (auto colon = s.find_first_of(':')) + { + core_id = std::stoi(s.substr(colon + 2)); + } + } + else if (s.rfind("cpu MHz", 0) == 0) + { + if (auto colon = s.find_first_of(':')) + { + auto mhz = std::stod(s.substr(colon + 2)); + new_values[fmt::format("CPUFrequencyMHz_{}", core_id)] = mhz; + } + } + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } - new_values["Processes"] = data.stload.processes; - new_values["ProcessesRunning"] = data.stload.procs_running; - new_values["ProcessesBlocked"] = data.stload.procs_blocked; + if (file_nr) + { + file_nr->rewind(); + + uint64_t open_files = 0; + readText(open_files, *file_nr); + new_values["OSOpenFiles"] = open_files; } #endif + + /// Process disk usage according to OS #if defined(OS_LINUX) { @@ -530,50 +804,6 @@ void AsynchronousMetrics::update() saveAllArenasMetric(new_values, "muzzy_purged"); #endif -#if defined(OS_LINUX) - // Try to add processor frequencies, ignoring errors. - try - { - ReadBufferFromFile buf("/proc/cpuinfo", 32768 /* buf_size */); - - // We need the following lines: - // processor : 4 - // cpu MHz : 4052.941 - // They contain tabs and are interspersed with other info. - int core_id = 0; - while (!buf.eof()) - { - std::string s; - // We don't have any backslash escape sequences in /proc/cpuinfo, so - // this function will read the line until EOL, which is exactly what - // we need. - readEscapedStringUntilEOL(s, buf); - // It doesn't read the EOL itself. - ++buf.position(); - - if (s.rfind("processor", 0) == 0) - { - if (auto colon = s.find_first_of(':')) - { - core_id = std::stoi(s.substr(colon + 2)); - } - } - else if (s.rfind("cpu MHz", 0) == 0) - { - if (auto colon = s.find_first_of(':')) - { - auto mhz = std::stod(s.substr(colon + 2)); - new_values[fmt::format("CPUFrequencyMHz_{}", core_id)] = mhz; - } - } - } - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -#endif - /// Add more metrics as you wish. // Log the new metrics. @@ -582,6 +812,8 @@ void AsynchronousMetrics::update() log->addValues(new_values); } + first_run = false; + // Finally, update the current metrics. std::lock_guard lock(mutex); values = new_values; diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 7bb281842dd..9f6e63f6ce6 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -20,6 +21,7 @@ namespace DB { class ProtocolServerAdapter; +class ReadBuffer; using AsynchronousMetricValue = double; using AsynchronousMetricValues = std::unordered_map; @@ -71,11 +73,14 @@ private: bool quit {false}; AsynchronousMetricValues values; + /// Some values are incremental and we have to calculate the difference. + /// On first run we will only collect the values to subtract later. + bool first_run = true; + #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; std::optional meminfo; - std::optional mounts; std::optional loadavg; std::optional proc_stat; std::optional cpuinfo; @@ -83,6 +88,39 @@ private: std::optional sockstat; std::optional netstat; std::optional file_nr; + std::optional uptime; + std::vector> thermal; + + struct ProcStatValuesCPU + { + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; + uint64_t iowait; + uint64_t irq; + uint64_t softirq; + uint64_t steal; + uint64_t guest; + uint64_t guest_nice; + + void read(ReadBuffer & in); + ProcStatValuesCPU operator-(const ProcStatValuesCPU & other) const; + }; + + struct ProcStatValuesOther + { + uint64_t interrupts; + uint64_t context_switches; + uint64_t processes_created; + + ProcStatValuesOther operator-(const ProcStatValuesOther & other) const; + }; + + ProcStatValuesCPU proc_stat_values_all_cpus{}; + ProcStatValuesOther proc_stat_values_other{}; + std::vector proc_stat_values_per_cpu; + #endif std::unique_ptr thread; From 08aca329bd111cea866ba8bba26504dc9e7cff34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 00:22:58 +0300 Subject: [PATCH 147/290] Development --- src/Common/DiskStatisticsOS.cpp | 52 ------ src/Common/DiskStatisticsOS.h | 35 ---- src/Common/MemoryInfoOS.cpp | 80 ---------- src/Common/MemoryInfoOS.h | 41 ----- src/Common/ProcessorStatisticsOS.cpp | 194 ----------------------- src/Common/ProcessorStatisticsOS.h | 90 ----------- src/Interpreters/AsynchronousMetrics.cpp | 59 +++++-- src/Interpreters/AsynchronousMetrics.h | 5 +- 8 files changed, 48 insertions(+), 508 deletions(-) delete mode 100644 src/Common/DiskStatisticsOS.cpp delete mode 100644 src/Common/DiskStatisticsOS.h delete mode 100644 src/Common/MemoryInfoOS.cpp delete mode 100644 src/Common/MemoryInfoOS.h delete mode 100644 src/Common/ProcessorStatisticsOS.cpp delete mode 100644 src/Common/ProcessorStatisticsOS.h diff --git a/src/Common/DiskStatisticsOS.cpp b/src/Common/DiskStatisticsOS.cpp deleted file mode 100644 index 1b404be07fe..00000000000 --- a/src/Common/DiskStatisticsOS.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#if defined(OS_LINUX) - -#include -#include - -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_STATVFS; -} - - -DiskStatisticsOS::Data DiskStatisticsOS::get() -{ - ReadBufferFromFile mounts_in("/proc/mounts", 4096 /* arbitrary small buffer */); - - Data data{}; - - std::string fs_device; - std::string fs_path; - - while (!mounts_in.eof()) - { - readStringUntilWhitespace(fs_device, mounts_in); - skipWhitespaceIfAny(mounts_in); - readStringUntilWhitespace(fs_path, mounts_in); - skipWhitespaceIfAny(mounts_in); - - /// Only real devices - if (!fs_device.starts_with("/dev/") || fs_device.starts_with("/dev/loop")) - continue; - - struct statvfs stat = getStatVFS(fs_path); - - data.total_bytes += (stat.f_blocks) * stat.f_bsize; - data.used_bytes += (stat.f_blocks - stat.f_bfree) * stat.f_bsize; - data.total_inodes += stat.f_files; - data.used_inodes += stat.f_files - stat.f_ffree; - } - - return data; -} - -} - -#endif diff --git a/src/Common/DiskStatisticsOS.h b/src/Common/DiskStatisticsOS.h deleted file mode 100644 index 390846e4b6c..00000000000 --- a/src/Common/DiskStatisticsOS.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once -#if defined (OS_LINUX) - -#include - -#include - - -namespace DB -{ - -class ReadBuffer; - - -/** Opens file /proc/mounts, reads all mounted filesystems and - * calculates disk usage. - */ -class DiskStatisticsOS -{ -public: - // In bytes - struct Data - { - uint64_t total_bytes; - uint64_t used_bytes; - uint64_t total_inodes; - uint64_t used_inodes; - }; - - Data get(); -}; - -} - -#endif diff --git a/src/Common/MemoryInfoOS.cpp b/src/Common/MemoryInfoOS.cpp deleted file mode 100644 index 7b712a0bb06..00000000000 --- a/src/Common/MemoryInfoOS.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#if defined(OS_LINUX) - -#include -#include -#include - -#include "MemoryInfoOS.h" - -#include - -#include -#include - -namespace DB -{ - -namespace -{ - template - void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) - { - readIntText(x, buf); - skipWhitespaceIfAny(buf); - } - - void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); - } - - std::pair readField(ReadBuffer & meminfo_in) - { - String key; - uint64_t val; - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(key, meminfo_in); - readIntTextAndSkipWhitespaceIfAny(val, meminfo_in); - skipToNextLineOrEOF(meminfo_in); - - // Delete the read ":" from the end - key.pop_back(); - - return std::make_pair(key, val); - } -} - -static constexpr auto meminfo_filename = "/proc/meminfo"; - -static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); - - -MemoryInfoOS::Data MemoryInfoOS::get() -{ - ReadBufferFromFile meminfo_in(meminfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - - MemoryInfoOS::Data data; - String field_name; - - std::unordered_map meminfo; - - while (!meminfo_in.eof()) - meminfo.insert(readField(meminfo_in)); - - data.total = meminfo["MemTotal"]; - data.free = meminfo["MemFree"]; - data.buffers = meminfo["Buffers"]; - data.cached = meminfo["Cached"]; - data.swap_total = meminfo["SwapTotal"]; - data.swap_cached = meminfo["SwapCached"]; - data.swap_free = meminfo["SwapFree"]; - - data.free_and_cached = data.free + data.cached; - - return data; -} - -} - -#endif diff --git a/src/Common/MemoryInfoOS.h b/src/Common/MemoryInfoOS.h deleted file mode 100644 index 4390c9d5697..00000000000 --- a/src/Common/MemoryInfoOS.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once -#if defined(OS_LINUX) - -#include -#include -#include - -#include - -#include - -namespace DB -{ - -/** Opens file /proc/meminfo and reads statistics about memory usage. - * This is Linux specific. - * See: man procfs - */ -class MemoryInfoOS -{ -public: - // In kB - struct Data - { - uint64_t total; - uint64_t free; - uint64_t buffers; - uint64_t cached; - uint64_t free_and_cached; - - uint64_t swap_total; - uint64_t swap_free; - uint64_t swap_cached; - }; - - Data get(); -}; - -} - -#endif diff --git a/src/Common/ProcessorStatisticsOS.cpp b/src/Common/ProcessorStatisticsOS.cpp deleted file mode 100644 index 9b43fa428a9..00000000000 --- a/src/Common/ProcessorStatisticsOS.cpp +++ /dev/null @@ -1,194 +0,0 @@ -#if defined(OS_LINUX) - -#include -#include -#include -#include - -#include "ProcessorStatisticsOS.h" - -#include "Poco/String.h" - -#include - -#include - -#include - -#include -#include - -namespace DB -{ - -namespace -{ - template - void readIntTextAndSkipWhitespaceIfAny(T & x, ReadBuffer & buf) - { - readIntText(x, buf); - skipWhitespaceIfAny(buf); - } - - void readStringAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readString(s, buf); - skipWhitespaceIfAny(buf); - } - - void readStringUntilWhitespaceAndSkipWhitespaceIfAny(String & s, ReadBuffer & buf) - { - readStringUntilWhitespace(s, buf); - skipWhitespaceIfAny(buf); - } - - void readCharAndSkipWhitespaceIfAny(char & c, ReadBuffer & buf) - { - readChar(c, buf); - skipWhitespaceIfAny(buf); - } - - void readFloatAndSkipWhitespaceIfAny(float & f, ReadBuffer & buf) - { - readFloatText(f, buf); - skipWhitespaceIfAny(buf); - } -} - -static constexpr auto loadavg_filename = "/proc/loadavg"; -static constexpr auto procst_filename = "/proc/stat"; -static constexpr auto cpuinfo_filename = "/proc/cpuinfo"; - -static const uint64_t USER_HZ = static_cast(sysconf(_SC_CLK_TCK)); - -static constexpr size_t READ_BUFFER_BUF_SIZE = (64 << 10); - -ProcessorStatisticsOS::ProcessorStatisticsOS() -{ - ProcStLoad unused; - calcStLoad(unused); -} - -ProcessorStatisticsOS::~ProcessorStatisticsOS() {} - -ProcessorStatisticsOS::Data ProcessorStatisticsOS::ProcessorStatisticsOS::get() -{ - Data data; - readLoadavg(data.loadavg); - calcStLoad(data.stload); - readFreq(data.freq); - return data; -} - -void ProcessorStatisticsOS::readLoadavg(ProcLoadavg& loadavg) -{ - ReadBufferFromFile loadavg_in(loadavg_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - - readFloatAndSkipWhitespaceIfAny(loadavg.avg1, loadavg_in); - readFloatAndSkipWhitespaceIfAny(loadavg.avg5, loadavg_in); - readFloatAndSkipWhitespaceIfAny(loadavg.avg15, loadavg_in); -} - -void ProcessorStatisticsOS::calcStLoad(ProcStLoad & stload) -{ - ProcTime cur_proc_time; - readProcTimeAndProcesses(cur_proc_time, stload); - - std::time_t cur_time = std::time(nullptr); - float time_dif = static_cast(cur_time - last_stload_call_time); - - stload.user_time = (cur_proc_time.user - last_proc_time.user) / time_dif; - stload.nice_time = (cur_proc_time.nice - last_proc_time.nice) / time_dif; - stload.system_time = (cur_proc_time.system - last_proc_time.system) / time_dif; - stload.idle_time = (cur_proc_time.idle - last_proc_time.idle) / time_dif; - stload.iowait_time = (cur_proc_time.iowait - last_proc_time.iowait) / time_dif; - stload.steal_time = (cur_proc_time.steal - last_proc_time.steal) / time_dif; - stload.guest_time = (cur_proc_time.guest - last_proc_time.guest) / time_dif; - stload.guest_nice_time = (cur_proc_time.guest_nice - last_proc_time.guest_nice) / time_dif; - - last_stload_call_time = cur_time; - last_proc_time = cur_proc_time; -} - -void ProcessorStatisticsOS::readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad & stload) -{ - ReadBufferFromFile procst_in(procst_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - - String field_name, field_val; - uint64_t unused; - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - - readIntTextAndSkipWhitespaceIfAny(proc_time.user, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.nice, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.system, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.idle, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.iowait, procst_in); - proc_time.user /= USER_HZ; - proc_time.nice /= USER_HZ; - proc_time.system /= USER_HZ; - proc_time.idle /= USER_HZ; - proc_time.iowait /= USER_HZ; - - readIntTextAndSkipWhitespaceIfAny(unused, procst_in); - readIntTextAndSkipWhitespaceIfAny(unused, procst_in); - - readIntTextAndSkipWhitespaceIfAny(proc_time.steal, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.guest, procst_in); - readIntTextAndSkipWhitespaceIfAny(proc_time.guest_nice, procst_in); - proc_time.steal /= USER_HZ; - proc_time.guest /= USER_HZ; - proc_time.guest_nice /= USER_HZ; - - do - { - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readStringAndSkipWhitespaceIfAny(field_val, procst_in); - } while (field_name != String("processes")); - - stload.processes = static_cast(std::stoul(field_val)); - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readIntTextAndSkipWhitespaceIfAny(stload.procs_running, procst_in); - - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_name, procst_in); - readIntTextAndSkipWhitespaceIfAny(stload.procs_blocked, procst_in); -} - -void ProcessorStatisticsOS::readFreq(ProcFreq & freq) -{ - ReadBufferFromFile cpuinfo_in(cpuinfo_filename, READ_BUFFER_BUF_SIZE, O_RDONLY | O_CLOEXEC); - - String field_name, field_val; - char unused; - int cpu_count = 0; - freq.max = freq.min = freq.avg = 0; - - do - { - do - { - readStringAndSkipWhitespaceIfAny(field_name, cpuinfo_in); - } while (!cpuinfo_in.eof() && field_name != String("cpu MHz")); - - if (cpuinfo_in.eof()) - break; - - readCharAndSkipWhitespaceIfAny(unused, cpuinfo_in); - readStringUntilWhitespaceAndSkipWhitespaceIfAny(field_val, cpuinfo_in); - - cpu_count++; - - float cur_cpu_freq = stof(field_val); - - freq.avg += cur_cpu_freq; - freq.max = (cpu_count == 1 ? cur_cpu_freq : std::max(freq.max, cur_cpu_freq)); - freq.min = (cpu_count == 1 ? cur_cpu_freq : std::min(freq.min, cur_cpu_freq)); - } while (true); - - freq.avg /= static_cast(cpu_count); -} - -} - -#endif diff --git a/src/Common/ProcessorStatisticsOS.h b/src/Common/ProcessorStatisticsOS.h deleted file mode 100644 index 10b6d050b8c..00000000000 --- a/src/Common/ProcessorStatisticsOS.h +++ /dev/null @@ -1,90 +0,0 @@ -#pragma once -#if defined(OS_LINUX) - -#include -#include - -#include - -#include - -namespace DB -{ - -/** Opens files: /proc/loadavg, /proc/stat, /proc/cpuinfo and reads processor statistics in get() method. - * This is Linux specific. - * See: man procfs - */ -class ProcessorStatisticsOS -{ -public: - struct ProcLoadavg - { - float avg1; - float avg5; - float avg15; - }; - - struct ProcStLoad - { - float user_time; - float nice_time; - float system_time; - float idle_time; - float iowait_time; - float steal_time; - float guest_time; - float guest_nice_time; - - uint32_t processes; - uint32_t procs_running; - uint32_t procs_blocked; - }; - - struct ProcFreq - { - float max; - float min; - float avg; - }; - - struct Data - { - ProcLoadavg loadavg; - ProcStLoad stload; - ProcFreq freq; - }; - - ProcessorStatisticsOS(); - ~ProcessorStatisticsOS(); - - Data get(); - -private: - struct ProcTime - { - // The amount of time, measured in seconds - uint64_t user; - uint64_t nice; - uint64_t system; - uint64_t idle; - uint64_t iowait; - uint64_t steal; - uint64_t guest; - uint64_t guest_nice; - }; - - void readLoadavg(ProcLoadavg & loadavg); - void calcStLoad(ProcStLoad & stload); - void readFreq(ProcFreq & freq); - - void readProcTimeAndProcesses(ProcTime & proc_time, ProcStLoad & stload); - -private: - std::time_t last_stload_call_time; - ProcTime last_proc_time; -}; - -} - -#endif diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 89196b5e25f..f7e54c661b4 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -78,9 +79,6 @@ AsynchronousMetrics::AsynchronousMetrics( openFileIfExists("/proc/loadavg", loadavg); openFileIfExists("/proc/stat", proc_stat); openFileIfExists("/proc/cpuinfo", cpuinfo); - openFileIfExists("/proc/schedstat", schedstat); - openFileIfExists("/proc/sockstat", sockstat); - openFileIfExists("/proc/netstat", netstat); openFileIfExists("/proc/sys/fs/file-nr", file_nr); openFileIfExists("/proc/uptime", uptime); @@ -617,22 +615,57 @@ void AsynchronousMetrics::update() readText(open_files, *file_nr); new_values["OSOpenFiles"] = open_files; } -#endif - - - /// Process disk usage according to OS -#if defined(OS_LINUX) + for (size_t i = 0, size = thermal.size(); i < size; ++i) { - DiskStatisticsOS::Data data = disk_stat.get(); + ReadBufferFromFile & in = *thermal[i]; - new_values["FilesystemsTotalBytes"] = data.total_bytes; - new_values["FilesystemsUsedBytes"] = data.used_bytes; - new_values["FilesystemsTotalINodes"] = data.total_inodes; - new_values["FilesystemsUsedINodes"] = data.used_inodes; + in.rewind(); + uint64_t temperature = 0; + readText(temperature, in); + new_values[fmt::format("Temperature{}", i)] = temperature * 0.001; } #endif + /// Free space in filesystems at data path and logs path. + { + auto stat = getStatVFS(getContext()->getPath()); + + new_values["FilesystemMainPathTotalBytes"] = stat.f_blocks * stat.f_bsize; + new_values["FilesystemMainPathAvailableBytes"] = stat.f_bavail * stat.f_bsize; + new_values["FilesystemMainPathUsedBytes"] = (stat.f_blocks - stat.f_bavail) * stat.f_bsize; + new_values["FilesystemMainPathTotalINodes"] = stat.f_files; + new_values["FilesystemMainPathAvailableINodes"] = stat.f_favail; + new_values["FilesystemMainPathUsedINodes"] = stat.f_files - stat.f_favail; + } + + { + auto stat = getStatVFS("."); + + new_values["FilesystemLogsPathTotalBytes"] = stat.f_blocks * stat.f_bsize; + new_values["FilesystemLogsPathAvailableBytes"] = stat.f_bavail * stat.f_bsize; + new_values["FilesystemLogsPathUsedBytes"] = (stat.f_blocks - stat.f_bavail) * stat.f_bsize; + new_values["FilesystemLogsPathTotalINodes"] = stat.f_files; + new_values["FilesystemLogsPathAvailableINodes"] = stat.f_favail; + new_values["FilesystemLogsPathUsedINodes"] = stat.f_files - stat.f_favail; + } + + /// Free and total space on every configured disk. + { + DisksMap disks_map = getContext()->getDisksMap(); + for (const auto & [name, disk] : disks_map) + { + auto total = disk->getTotalSpace(); + auto available = disk->getAvailableSpace(); + auto unreserved = disk->getUnreservedSpace(); + + new_values[fmt::format("DiskTotal_{}", name)] = total; + new_values[fmt::format("DiskUsed_{}", name)] = total - available; + new_values[fmt::format("DiskAvailable_{}", name)] = available; + new_values[fmt::format("DiskUnreserved_{}", name)] = unreserved; + } + } + { auto databases = DatabaseCatalog::instance().getDatabases(); diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 9f6e63f6ce6..247c9858129 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -84,13 +84,12 @@ private: std::optional loadavg; std::optional proc_stat; std::optional cpuinfo; - std::optional schedstat; - std::optional sockstat; - std::optional netstat; std::optional file_nr; std::optional uptime; std::vector> thermal; + /// TODO: IO load, Network rx/tx, sockets, EDAC. + struct ProcStatValuesCPU { uint64_t user; From 3a10d3802b1faf5c60a584ac602e206237093e65 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 00:33:00 +0300 Subject: [PATCH 148/290] Development --- src/Interpreters/AsynchronousMetrics.cpp | 4 ++++ src/Interpreters/AsynchronousMetrics.h | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index f7e54c661b4..add448b129b 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -533,6 +533,10 @@ void AsynchronousMetrics::update() } else if (name == "MemFree:") { + /// We cannot simply name this metric "Free", because it confuses users. + /// See https://www.linuxatemyram.com/ + /// For convenience we also provide OSMemoryFreePlusCached, that should be somewhat similar to OSMemoryAvailable. + free_plus_cached_bytes += bytes; new_values["OSMemoryFreeWithoutCached"] = bytes; } diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 247c9858129..2a2d434c007 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -2,9 +2,6 @@ #include #include -#include -#include -#include #include #include From b5840210c1c2def061ddec2201ca77d9b0007a8c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 00:54:46 +0300 Subject: [PATCH 149/290] Adjustments --- src/Interpreters/AsynchronousMetrics.cpp | 69 +++++++++++++++++++----- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index add448b129b..ec814f96da1 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -240,24 +240,39 @@ static void saveAllArenasMetric(AsynchronousMetricValues & values, void AsynchronousMetrics::ProcStatValuesCPU::read(ReadBuffer & in) { readText(user, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(nice, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(system, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(idle, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(iowait, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(irq, in); - skipWhitespaceIfAny(in); + skipWhitespaceIfAny(in, true); readText(softirq, in); - skipWhitespaceIfAny(in); - readText(steal, in); - skipWhitespaceIfAny(in); - readText(guest, in); - skipWhitespaceIfAny(in); - readText(guest_nice, in); + + /// Just in case for old Linux kernels, we check if these values present. + + if (!checkChar('\n', in)) + { + skipWhitespaceIfAny(in, true); + readText(steal, in); + } + + if (!checkChar('\n', in)) + { + skipWhitespaceIfAny(in, true); + readText(guest, in); + } + + if (!checkChar('\n', in)) + { + skipWhitespaceIfAny(in, true); + readText(guest_nice, in); + } + skipToNextLineOrEOF(in); } @@ -410,8 +425,10 @@ void AsynchronousMetrics::update() throwFromErrno("Cannot call 'sysconf' to obtain system HZ", ErrorCodes::CANNOT_SYSCONF); double multiplier = 1.0 / hz / update_period.count(); + size_t num_cpus = 0; ProcStatValuesOther current_other_values{}; + ProcStatValuesCPU delta_values_all_cpus{}; while (!proc_stat->eof()) { @@ -445,7 +462,12 @@ void AsynchronousMetrics::update() String cpu_suffix; if (!cpu_num_str.empty()) + { cpu_suffix = "CPU" + cpu_num_str; + ++num_cpus; + } + else + delta_values_all_cpus = delta_values; new_values["OSUserTime" + cpu_suffix] = delta_values.user * multiplier; new_values["OSNiceTime" + cpu_suffix] = delta_values.nice * multiplier; @@ -501,6 +523,20 @@ void AsynchronousMetrics::update() new_values["OSInterrupts"] = delta_values.interrupts * multiplier; new_values["OSContextSwitches"] = delta_values.context_switches * multiplier; new_values["OSProcessesCreated"] = delta_values.processes_created * multiplier; + + /// Also write values normalized to 0..1 by diving to the number of CPUs. + /// These values are good to be averaged across the cluster of non-uniform servers. + + new_values["OSUserTimeNormalized"] = delta_values_all_cpus.user * multiplier / num_cpus; + new_values["OSNiceTimeNormalized"] = delta_values_all_cpus.nice * multiplier / num_cpus; + new_values["OSSystemTimeNormalized"] = delta_values_all_cpus.system * multiplier / num_cpus; + new_values["OSIdleTimeNormalized"] = delta_values_all_cpus.idle * multiplier / num_cpus; + new_values["OSIOWaitTimeNormalized"] = delta_values_all_cpus.iowait * multiplier / num_cpus; + new_values["OSIrqTimeNormalized"] = delta_values_all_cpus.irq * multiplier / num_cpus; + new_values["OSSoftIrqTimeNormalized"] = delta_values_all_cpus.softirq * multiplier / num_cpus; + new_values["OSStealTimeNormalized"] = delta_values_all_cpus.steal * multiplier / num_cpus; + new_values["OSGuestTimeNormalized"] = delta_values_all_cpus.guest * multiplier / num_cpus; + new_values["OSGuestNiceTimeNormalized"] = delta_values_all_cpus.guest_nice * multiplier / num_cpus; } proc_stat_values_other = current_other_values; @@ -516,13 +552,13 @@ void AsynchronousMetrics::update() { String name; readStringUntilWhitespace(name, *meminfo); - skipWhitespaceIfAny(*meminfo); + skipWhitespaceIfAny(*meminfo, true); uint64_t kb = 0; readText(kb, *meminfo); if (kb) { - skipWhitespaceIfAny(*meminfo); + skipWhitespaceIfAny(*meminfo, true); assertString("kB", *meminfo); uint64_t bytes = kb * 1024; @@ -660,6 +696,11 @@ void AsynchronousMetrics::update() for (const auto & [name, disk] : disks_map) { auto total = disk->getTotalSpace(); + + /// Some disks don't support information about the space. + if (!total) + continue; + auto available = disk->getAvailableSpace(); auto unreserved = disk->getUnreservedSpace(); From 0f8ea9b8f6dbf4170352e4ac4feed1afd1ea0488 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 01:33:32 +0300 Subject: [PATCH 150/290] More metrics --- src/Interpreters/AsynchronousMetricLog.cpp | 2 +- src/Interpreters/AsynchronousMetrics.cpp | 25 ++++++++++++++++------ src/Interpreters/AsynchronousMetrics.h | 3 ++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/AsynchronousMetricLog.cpp b/src/Interpreters/AsynchronousMetricLog.cpp index 79e2d513d5c..c7003cff169 100644 --- a/src/Interpreters/AsynchronousMetricLog.cpp +++ b/src/Interpreters/AsynchronousMetricLog.cpp @@ -18,7 +18,7 @@ NamesAndTypesList AsynchronousMetricLogElement::getNamesAndTypes() {"event_date", std::make_shared()}, {"event_time", std::make_shared()}, {"event_time_microseconds", std::make_shared(6)}, - {"name", std::make_shared(std::make_shared())}, + {"metric", std::make_shared(std::make_shared())}, {"value", std::make_shared(),} }; } diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index ec814f96da1..26c9a2ad65c 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -98,7 +98,7 @@ void AsynchronousMetrics::start() { /// Update once right now, to make metrics available just after server start /// (without waiting for asynchronous_metrics_update_period_s). - update(); + update(std::chrono::system_clock::now()); thread = std::make_unique([this] { run(); }); } @@ -158,10 +158,12 @@ void AsynchronousMetrics::run() while (true) { + auto next_update_time = get_next_update_time(update_period); + { // Wait first, so that the first metric collection is also on even time. std::unique_lock lock{mutex}; - if (wait_cond.wait_until(lock, get_next_update_time(update_period), + if (wait_cond.wait_until(lock, next_update_time, [this] { return quit; })) { break; @@ -170,7 +172,7 @@ void AsynchronousMetrics::run() try { - update(); + update(next_update_time); } catch (...) { @@ -306,10 +308,19 @@ AsynchronousMetrics::ProcStatValuesOther::operator-(const AsynchronousMetrics::P #endif -void AsynchronousMetrics::update() +void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_time) { + Stopwatch watch; + AsynchronousMetricValues new_values; + auto current_time = std::chrono::system_clock::now(); + auto time_after_previous_update = current_time - previous_update_time; + previous_update_time = update_time; + + /// This is also a good indicator of system responsiveness. + new_values["Jitter"] = std::chrono::duration_cast(current_time - update_time).count() / 1e9; + { if (auto mark_cache = getContext()->getMarkCache()) { @@ -424,7 +435,7 @@ void AsynchronousMetrics::update() if (-1 == hz) throwFromErrno("Cannot call 'sysconf' to obtain system HZ", ErrorCodes::CANNOT_SYSCONF); - double multiplier = 1.0 / hz / update_period.count(); + double multiplier = 1.0 / hz / (std::chrono::duration_cast(time_after_previous_update).count() / 1e9); size_t num_cpus = 0; ProcStatValuesOther current_other_values{}; @@ -884,7 +895,9 @@ void AsynchronousMetrics::update() /// Add more metrics as you wish. - // Log the new metrics. + new_values["AsynchronousMetricsCalculationTimeSpent"] = watch.elapsedSeconds(); + + /// Log the new metrics. if (auto log = getContext()->getAsynchronousMetricLog()) { log->addValues(new_values); diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 2a2d434c007..95ba5492d86 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -73,6 +73,7 @@ private: /// Some values are incremental and we have to calculate the difference. /// On first run we will only collect the values to subtract later. bool first_run = true; + std::chrono::system_clock::time_point previous_update_time; #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; @@ -122,7 +123,7 @@ private: std::unique_ptr thread; void run(); - void update(); + void update(std::chrono::system_clock::time_point update_time); }; } From c059d0a0ee1e13c73cdefb821cb40aa01f6981c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 01:41:09 +0300 Subject: [PATCH 151/290] More metrics --- programs/server/Server.cpp | 2 +- programs/server/config.xml | 4 +-- src/Interpreters/AsynchronousMetrics.cpp | 35 +++++++++--------------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 88f7564a7f2..28cf085e699 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1159,7 +1159,7 @@ int Server::main(const std::vector & /*args*/) { /// This object will periodically calculate some metrics. AsynchronousMetrics async_metrics( - global_context, config().getUInt("asynchronous_metrics_update_period_s", 60), servers_to_start_before_tables, servers); + global_context, config().getUInt("asynchronous_metrics_update_period_s", 1), servers_to_start_before_tables, servers); attachSystemTablesAsync(*DatabaseCatalog::instance().getSystemDatabase(), async_metrics); for (const auto & listen_host : listen_hosts) diff --git a/programs/server/config.xml b/programs/server/config.xml index dd50a693403..6f0b228dda7 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -583,7 +583,7 @@ 9019 --> - + @@ -917,7 +917,7 @@ Asynchronous metrics are updated once a minute, so there is no need to flush more often. --> - 60000 + 7000 0 + 0 diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index 3835fd58c77..ad5c67d88d4 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -185,8 +185,8 @@ public: auto * denominator_type = toNativeType(b); static constexpr size_t denominator_offset = offsetof(Fraction, denominator); - auto * denominator_dst_ptr = b.CreatePointerCast(b.CreateConstGEP1_32(nullptr, aggregate_data_dst_ptr, denominator_offset), denominator_type->getPointerTo()); - auto * denominator_src_ptr = b.CreatePointerCast(b.CreateConstGEP1_32(nullptr, aggregate_data_src_ptr, denominator_offset), denominator_type->getPointerTo()); + auto * denominator_dst_ptr = b.CreatePointerCast(b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_dst_ptr, denominator_offset), denominator_type->getPointerTo()); + auto * denominator_src_ptr = b.CreatePointerCast(b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_src_ptr, denominator_offset), denominator_type->getPointerTo()); auto * denominator_dst_value = b.CreateLoad(denominator_type, denominator_dst_ptr); auto * denominator_src_value = b.CreateLoad(denominator_type, denominator_src_ptr); diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h index 80e18f1a141..68d48803718 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h @@ -74,7 +74,7 @@ public: auto * denominator_type = toNativeType(b); static constexpr size_t denominator_offset = offsetof(Fraction, denominator); - auto * denominator_offset_ptr = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, denominator_offset); + auto * denominator_offset_ptr = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, denominator_offset); auto * denominator_ptr = b.CreatePointerCast(denominator_offset_ptr, denominator_type->getPointerTo()); auto * weight_cast_to_denominator = nativeCast(b, arguments_types[1], argument_values[1], denominator_type); diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index e7c48c8988c..c074daf45be 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -139,7 +139,7 @@ public: if constexpr (result_is_nullable) b.CreateStore(llvm::ConstantInt::get(b.getInt8Ty(), 1), aggregate_data_ptr); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileAdd(b, aggregate_data_ptr_with_prefix_size_offset, { removeNullable(nullable_type) }, { wrapped_value }); b.CreateBr(join_block); @@ -290,7 +290,7 @@ public: if constexpr (result_is_nullable) b.CreateStore(llvm::ConstantInt::get(b.getInt8Ty(), 1), aggregate_data_ptr); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileAdd(b, aggregate_data_ptr_with_prefix_size_offset, non_nullable_types, wrapped_values); b.CreateBr(join_block); diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index 23dad0c097c..cc670fdf823 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -199,7 +199,7 @@ public: static constexpr size_t value_offset_from_structure = offsetof(SingleValueDataFixed, value); auto * type = toNativeType(builder); - auto * value_ptr_with_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, value_offset_from_structure); + auto * value_ptr_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, value_offset_from_structure); auto * value_ptr = b.CreatePointerCast(value_ptr_with_offset, type->getPointerTo()); return value_ptr; diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index b7a67f2cc1b..7890e96ef66 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -207,7 +207,7 @@ public: if constexpr (result_is_nullable) b.CreateMemSet(aggregate_data_ptr, llvm::ConstantInt::get(b.getInt8Ty(), 0), this->prefix_size, llvm::assumeAligned(this->alignOfData())); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileCreate(b, aggregate_data_ptr_with_prefix_size_offset); } @@ -225,8 +225,8 @@ public: b.CreateStore(is_null_result_value, aggregate_data_dst_ptr); } - auto * aggregate_data_dst_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_dst_ptr, this->prefix_size); - auto * aggregate_data_src_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_src_ptr, this->prefix_size); + auto * aggregate_data_dst_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_dst_ptr, this->prefix_size); + auto * aggregate_data_src_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_src_ptr, this->prefix_size); this->nested_function->compileMerge(b, aggregate_data_dst_ptr_with_prefix_size_offset, aggregate_data_src_ptr_with_prefix_size_offset); } @@ -260,7 +260,7 @@ public: b.CreateBr(join_block); b.SetInsertPoint(if_not_null); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); auto * nested_result = this->nested_function->compileGetResult(builder, aggregate_data_ptr_with_prefix_size_offset); b.CreateStore(b.CreateInsertValue(nullable_value, nested_result, {0}), nullable_value_ptr); b.CreateBr(join_block); @@ -351,7 +351,7 @@ public: if constexpr (result_is_nullable) b.CreateStore(llvm::ConstantInt::get(b.getInt8Ty(), 1), aggregate_data_ptr); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileAdd(b, aggregate_data_ptr_with_prefix_size_offset, { removeNullable(nullable_type) }, { wrapped_value }); b.CreateBr(join_block); @@ -479,7 +479,7 @@ public: if constexpr (result_is_nullable) b.CreateStore(llvm::ConstantInt::get(b.getInt8Ty(), 1), aggregate_data_ptr); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileAdd(b, aggregate_data_ptr_with_prefix_size_offset, arguments_types, wrapped_values); b.CreateBr(join_block); @@ -488,7 +488,7 @@ public: else { b.CreateStore(llvm::ConstantInt::get(b.getInt8Ty(), 1), aggregate_data_ptr); - auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstGEP1_32(nullptr, aggregate_data_ptr, this->prefix_size); + auto * aggregate_data_ptr_with_prefix_size_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_ptr, this->prefix_size); this->nested_function->compileAdd(b, aggregate_data_ptr_with_prefix_size_offset, non_nullable_types, wrapped_values); } } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 31eaeaadbeb..28e46160a98 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -108,7 +108,7 @@ class IColumn; M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \ M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ - M(UInt64, min_count_to_compile_aggregate_expression, 0, "The number of identical aggreagte expressions before they are JIT-compiled", 0) \ + M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 766c2290e42..18a2400d22f 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -168,7 +168,7 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio for (size_t i = 0; i <= arg_types.size(); ++i) { const auto & type = i == arg_types.size() ? function.getResultType() : arg_types[i]; - auto * data = b.CreateLoad(data_type, b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i)); + auto * data = b.CreateLoad(data_type, b.CreateConstInBoundsGEP1_64(data_type, columns_arg, i)); columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(type))->getPointerTo()); columns[i].null_init = type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr; } @@ -236,9 +236,9 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio auto * cur_block = b.GetInsertBlock(); for (auto & col : columns) { - col.data->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.data, 1), cur_block); + col.data->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.data, 1), cur_block); if (col.null) - col.null->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1), cur_block); + col.null->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.null, 1), cur_block); } auto * value = b.CreateAdd(counter_phi, llvm::ConstantInt::get(size_type, 1)); @@ -295,7 +295,7 @@ static void compileCreateAggregateStatesFunctions(llvm::Module & module, const s { size_t aggregate_function_offset = function_to_compile.aggregate_data_offset; const auto * aggregate_function = function_to_compile.function; - auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_32(nullptr, aggregate_data_place_arg, aggregate_function_offset); + auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_place_arg, aggregate_function_offset); aggregate_function->compileCreate(b, aggregation_place_with_offset); } @@ -338,7 +338,7 @@ static void compileAddIntoAggregateStatesFunctions(llvm::Module & module, const for (size_t column_argument_index = 0; column_argument_index < function_arguments_size; ++column_argument_index) { const auto & argument_type = argument_types[column_argument_index]; - auto * data = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_32(column_data_type, columns_arg, previous_columns_size + column_argument_index)); + auto * data = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_64(column_data_type, columns_arg, previous_columns_size + column_argument_index)); data_placeholder.data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(argument_type))->getPointerTo()); data_placeholder.null_init = argument_type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr; columns.emplace_back(data_placeholder); @@ -408,7 +408,7 @@ static void compileAddIntoAggregateStatesFunctions(llvm::Module & module, const arguments_values[column_argument_index] = nullable_value; } - auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_32(nullptr, aggregation_place, aggregate_function_offset); + auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregation_place, aggregate_function_offset); aggregate_function_ptr->compileAdd(b, aggregation_place_with_offset, arguments_types, arguments_values); previous_columns_size += function_arguments_size; @@ -419,13 +419,13 @@ static void compileAddIntoAggregateStatesFunctions(llvm::Module & module, const auto * cur_block = b.GetInsertBlock(); for (auto & col : columns) { - col.data->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.data, 1), cur_block); + col.data->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.data, 1), cur_block); if (col.null) - col.null->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1), cur_block); + col.null->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.null, 1), cur_block); } - places_phi->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, places_phi, 1), cur_block); + places_phi->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, places_phi, 1), cur_block); auto * value = b.CreateAdd(counter_phi, llvm::ConstantInt::get(size_type, 1)); counter_phi->addIncoming(value, cur_block); @@ -457,8 +457,8 @@ static void compileMergeAggregatesStates(llvm::Module & module, const std::vecto size_t aggregate_function_offset = function_to_compile.aggregate_data_offset; const auto * aggregate_function_ptr = function_to_compile.function; - auto * aggregate_data_place_merge_dst_with_offset = b.CreateConstInBoundsGEP1_32(nullptr, aggregate_data_place_dst_arg, aggregate_function_offset); - auto * aggregate_data_place_merge_src_with_offset = b.CreateConstInBoundsGEP1_32(nullptr, aggregate_data_place_src_arg, aggregate_function_offset); + auto * aggregate_data_place_merge_dst_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_place_dst_arg, aggregate_function_offset); + auto * aggregate_data_place_merge_src_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_place_src_arg, aggregate_function_offset); aggregate_function_ptr->compileMerge(b, aggregate_data_place_merge_dst_with_offset, aggregate_data_place_merge_src_with_offset); } @@ -490,7 +490,7 @@ static void compileInsertAggregatesIntoResultColumns(llvm::Module & module, cons for (size_t i = 0; i < functions.size(); ++i) { auto return_type = functions[i].function->getReturnType(); - auto * data = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_32(column_data_type, columns_arg, i)); + auto * data = b.CreateLoad(column_data_type, b.CreateConstInBoundsGEP1_64(column_data_type, columns_arg, i)); columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(return_type))->getPointerTo()); columns[i].null_init = return_type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr; } @@ -526,7 +526,7 @@ static void compileInsertAggregatesIntoResultColumns(llvm::Module & module, cons const auto * aggregate_function_ptr = functions[i].function; auto * aggregate_data_place = b.CreateLoad(b.getInt8Ty()->getPointerTo(), aggregate_data_place_phi); - auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_32(nullptr, aggregate_data_place, aggregate_function_offset); + auto * aggregation_place_with_offset = b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_place, aggregate_function_offset); auto * final_value = aggregate_function_ptr->compileGetResult(b, aggregation_place_with_offset); @@ -546,16 +546,16 @@ static void compileInsertAggregatesIntoResultColumns(llvm::Module & module, cons auto * cur_block = b.GetInsertBlock(); for (auto & col : columns) { - col.data->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.data, 1), cur_block); + col.data->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.data, 1), cur_block); if (col.null) - col.null->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1), cur_block); + col.null->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, col.null, 1), cur_block); } auto * value = b.CreateAdd(counter_phi, llvm::ConstantInt::get(size_type, 1), "", true, true); counter_phi->addIncoming(value, cur_block); - aggregate_data_place_phi->addIncoming(b.CreateConstInBoundsGEP1_32(nullptr, aggregate_data_place_phi, 1), cur_block); + aggregate_data_place_phi->addIncoming(b.CreateConstInBoundsGEP1_64(nullptr, aggregate_data_place_phi, 1), cur_block); b.CreateCondBr(b.CreateICmpEQ(value, rows_count_arg), end, loop); diff --git a/tests/performance/jit_aggregate_functions.xml b/tests/performance/jit_aggregate_functions.xml index 31b621f7258..21683ef2004 100644 --- a/tests/performance/jit_aggregate_functions.xml +++ b/tests/performance/jit_aggregate_functions.xml @@ -3,6 +3,11 @@ hits_100m_single + + 1 + 0 + + CREATE TABLE jit_test_memory ( key UInt64, From cc137878882e5b42ea8e8bbb226416a9169d7563 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 5 Jul 2021 12:50:33 +0300 Subject: [PATCH 164/290] Minor style changes for (un)bin/hex --- src/Functions/FunctionsCoding.h | 95 ++++++++++++--------------------- 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index f2e340aaeef..e9ec013d6eb 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -65,11 +65,6 @@ namespace ErrorCodes constexpr size_t uuid_bytes_length = 16; constexpr size_t uuid_text_length = 36; -namespace ErrorCodes -{ -extern const int NOT_IMPLEMENTED; -} - class FunctionIPv6NumToString : public IFunction { public: @@ -955,13 +950,15 @@ public: } }; +/// Encode number or string to string with binary or hexadecimal representation template -class Conversion : public IFunction +class EncodeToBinaryRepr : public IFunction { public: static constexpr auto name = Impl::name; static constexpr size_t word_size = Impl::word_size; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -1012,7 +1009,7 @@ public: } template - bool tryExecuteUInt(const IColumn *col, ColumnPtr &col_res) const + bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const { const ColumnVector * col_vec = checkAndGetColumn>(col); @@ -1071,16 +1068,8 @@ public: size_t size = in_offsets.size(); out_offsets.resize(size); - if (getName() == "bin") - { - out_vec.resize((in_vec.size() - size) * word_size + size); - } else if (getName() == "hex") - { - out_vec.resize(in_vec.size() * word_size - size); - } else - { - throw Exception("new function is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } + /// reserve `word_size` bytes for each non trailing zero byte from input + `size` bytes for trailing zeros + out_vec.resize((in_vec.size() - size) * word_size + size); char * begin = reinterpret_cast(out_vec.data()); char * pos = begin; @@ -1187,13 +1176,14 @@ public: } }; +/// Decode number or string from string with binary or hexadecimal representation template -class UnConversion : public IFunction +class DecodeFromBinaryRepr : public IFunction { public: static constexpr auto name = Impl::name; static constexpr size_t word_size = Impl::word_size; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override { return name; } @@ -1227,18 +1217,7 @@ public: size_t size = in_offsets.size(); out_offsets.resize(size); - if (getName() == "unhex") - { - out_vec.resize(in_vec.size() / 2 + size); - } - else if (getName() == "unbin") - { - out_vec.resize(in_vec.size() / 8 + size); - } - else - { - throw Exception("new function is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } + out_vec.resize(in_vec.size() / word_size + size); char * begin = reinterpret_cast(out_vec.data()); char * pos = begin; @@ -1248,7 +1227,7 @@ public: { size_t new_offset = in_offsets[i]; - Impl::unConversion(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); + Impl::decode(reinterpret_cast(&in_vec[prev_offset]), reinterpret_cast(&in_vec[new_offset - 1]), pos); out_offsets[i] = pos - begin; @@ -1270,9 +1249,9 @@ public: struct HexImpl { -public: static constexpr auto name = "hex"; - static const size_t word_size = 2; + static constexpr size_t word_size = 2; + template static void executeOneUInt(T x, char *& out) { @@ -1287,7 +1266,7 @@ public: was_nonzero = true; writeHexByteUppercase(byte, out); - out += 2; + out += word_size; } *out = '\0'; ++out; @@ -1299,7 +1278,7 @@ public: { writeHexByteUppercase(*pos, out); ++pos; - out += 2; + out += word_size; } *out = '\0'; ++out; @@ -1334,15 +1313,10 @@ public: struct UnhexImpl { -public: static constexpr auto name = "unhex"; + static constexpr size_t word_size = 2; - static String getName() - { - return name; - } - - static void unConversion(const char * pos, const char * end, char *& out) + static void decode(const char * pos, const char * end, char *& out) { if ((end - pos) & 1) { @@ -1353,7 +1327,7 @@ public: while (pos < end) { *out = unhex2(pos); - pos += 2; + pos += word_size; ++out; } *out = '\0'; @@ -1363,16 +1337,16 @@ public: struct BinImpl { -public: static constexpr auto name = "bin"; static constexpr size_t word_size = 8; + template static void executeOneUInt(T x, char *& out) { bool was_nonzero = false; T t = 1; - for (int8_t offset = sizeof(x) * 8 - 1; offset >= 0; --offset) + for (Int8 offset = sizeof(x) * 8 - 1; offset >= 0; --offset) { t = t << offset; if ((x & t) == t) @@ -1401,7 +1375,7 @@ public: template static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) { - const size_t hex_length = type_size_in_bytes * 8 + 1; /// Including trailing zero byte. + const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte. auto col_str = ColumnString::create(); ColumnString::Chars & out_vec = col_str->getChars(); @@ -1412,8 +1386,7 @@ public: out_vec.resize(size * hex_length); size_t pos = 0; - char * begin = reinterpret_cast(out_vec.data()); - char * out = begin; + char * out = reinterpret_cast(out_vec.data()); for (size_t i = 0; i < size; ++i) { const UInt8 * in_pos = reinterpret_cast(&in_vec[i]); @@ -1440,12 +1413,10 @@ public: struct UnbinImpl { -public: static constexpr auto name = "unbin"; + static constexpr size_t word_size = 8; - static String getName() { return name; } - - static void unConversion(const char * pos, const char * end, char *& out) + static void decode(const char * pos, const char * end, char *& out) { UInt8 left = 0; @@ -1454,7 +1425,7 @@ public: /// e.g. the length is 9 and the input is "101000001", /// first left_cnt is 1, left is 0, right shift, pos is 1, left = 1 /// then, left_cnt is 0, remain input is '01000001'. - for (uint8_t left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt) + for (UInt8 left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt) { left = left << 1; if (*pos != '0') @@ -1469,12 +1440,12 @@ public: ++out; } - /// input character encoding is UTF-8. And - /// remain bits mod 8 is zero. + assert((end - pos) % 8 == 0); + while (end - pos != 0) { UInt8 c = 0; - for (uint8_t i = 0; i < 8; ++i) + for (UInt8 i = 0; i < 8; ++i) { c = c << 1; if (*pos != '0') @@ -1492,10 +1463,10 @@ public: } }; -using FunctionHex = Conversion; -using FunctionUnhex = UnConversion; -using FunctionBin = Conversion; -using FunctionUnbin = UnConversion; +using FunctionHex = EncodeToBinaryRepr; +using FunctionUnhex = DecodeFromBinaryRepr; +using FunctionBin = EncodeToBinaryRepr; +using FunctionUnbin = DecodeFromBinaryRepr; class FunctionChar : public IFunction { From 231740f2d65e3bdb2ebb383d9061a814860514ee Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 5 Jul 2021 14:44:50 +0300 Subject: [PATCH 165/290] Function bin for uint uses writeBinByte, correct for single zero --- src/Common/hex.h | 15 ++++++++ src/Functions/FunctionsCoding.h | 34 ++++++++++--------- .../0_stateless/01926_bin_unbin.reference | 9 +++++ tests/queries/0_stateless/01926_bin_unbin.sql | 10 ++++++ 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/Common/hex.h b/src/Common/hex.h index 82eff776244..69bc6f4f79f 100644 --- a/src/Common/hex.h +++ b/src/Common/hex.h @@ -1,5 +1,6 @@ #pragma once #include +#include /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. @@ -46,6 +47,20 @@ inline void writeBinByte(UInt8 byte, void * out) memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); } +inline size_t writeBinByteNoLeadZeros(UInt8 byte, char * out) +{ + if (byte == 0) + return 0; + + int clz = std::countl_zero(byte); + for (Int8 offset = sizeof(UInt8) * 8 - clz - 1; offset >= 0; --offset) + { + *out = ((byte >> offset) & 1) ? '1' : '0'; + ++out; + } + return sizeof(UInt8) * 8 - clz; +} + /// Produces hex representation of an unsigned int with leading zeros (for checksums) template inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index e9ec013d6eb..71cce3193ba 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1344,30 +1344,32 @@ struct BinImpl static void executeOneUInt(T x, char *& out) { bool was_nonzero = false; - T t = 1; - - for (Int8 offset = sizeof(x) * 8 - 1; offset >= 0; --offset) + for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) { - t = t << offset; - if ((x & t) == t) + UInt8 byte = x >> offset; + + /// Skip leading zeros + if (byte == 0 && !was_nonzero) + continue; + + /// First non-zero byte without leading zeros + if (was_nonzero) { - x = x - t; - was_nonzero = true; - *out = '1'; - t = 1; + writeBinByte(byte, out); + out += word_size; } else { - t = 1; - if (!was_nonzero) - { - continue; - } - *out = '0'; + size_t written = writeBinByteNoLeadZeros(byte, out); + out += written; } + was_nonzero = true; + } + if (!was_nonzero) + { + *out = '0'; ++out; } - *out = '\0'; ++out; } diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index 54c01c5d145..595b7389a5d 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -1,4 +1,5 @@ +0 1 1010 1111111 @@ -13,6 +14,14 @@ 0000000000000000000011000011110101011101010100111010101000000001 0011000100110010001100110011001100110010001101000011001000110100 0011000100110010001100110011001100110010001101000011001000110100 +0011000100110010001100110011001100110010001101000011001000110100 +0011000100110010001100110011001100110010001101000011001000110100 + 0 10 测试 +0 +0 +0 +1 +1 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql index 40635091120..fadf236ce9a 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.sql +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -1,3 +1,4 @@ +select bin(''); select bin(0); select bin(1); select bin(10); @@ -12,8 +13,17 @@ select bin(toFloat64(1.2)); select bin(toDecimal32(1.2, 8)); select bin(toDecimal64(1.2, 17)); select bin('12332424'); +select bin(materialize('12332424')); +select bin(toNullable(materialize('12332424'))); select bin(toLowCardinality(materialize('12332424'))); +select unbin(''); select unbin('00110000'); -- 0 select unbin('0011000100110000'); -- 10 select unbin('111001101011010110001011111010001010111110010101'); -- 测试 +select unbin(materialize('00110000')); +select unbin(toNullable(materialize('00110000'))); +select unbin(toLowCardinality(materialize('00110000'))); + +select unbin(bin('')) == ''; +select bin(unbin('')) == ''; From dd06866fa8e7b7788d34def117c893c9a33f9601 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 5 Jul 2021 14:56:39 +0300 Subject: [PATCH 166/290] Fix unbin for corner cases --- src/Functions/FunctionsCoding.h | 14 +++++++++----- .../queries/0_stateless/01926_bin_unbin.reference | 5 +++++ tests/queries/0_stateless/01926_bin_unbin.sql | 7 +++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 71cce3193ba..33b26afc8dc 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1420,6 +1420,13 @@ struct UnbinImpl static void decode(const char * pos, const char * end, char *& out) { + if (pos == end) + { + *out = '\0'; + ++out; + return; + } + UInt8 left = 0; /// end - pos is the length of input. @@ -1431,12 +1438,11 @@ struct UnbinImpl { left = left << 1; if (*pos != '0') - { left += 1; - } ++pos; } - if (0 != left) + + if (left != 0 || end - pos == 0) { *out = left; ++out; @@ -1451,9 +1457,7 @@ struct UnbinImpl { c = c << 1; if (*pos != '0') - { c += 1; - } ++pos; } *out = c; diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index 595b7389a5d..ace28af5211 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -17,6 +17,7 @@ 0011000100110010001100110011001100110010001101000011001000110100 0011000100110010001100110011001100110010001101000011001000110100 +1 0 10 测试 @@ -25,3 +26,7 @@ 0 1 1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql index fadf236ce9a..3593448d407 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.sql +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -18,6 +18,7 @@ select bin(toNullable(materialize('12332424'))); select bin(toLowCardinality(materialize('12332424'))); select unbin(''); +select unbin('0') == '\0'; select unbin('00110000'); -- 0 select unbin('0011000100110000'); -- 10 select unbin('111001101011010110001011111010001010111110010101'); -- 测试 @@ -27,3 +28,9 @@ select unbin(toLowCardinality(materialize('00110000'))); select unbin(bin('')) == ''; select bin(unbin('')) == ''; +select bin(unbin('0')) == '00000000'; + +-- hex and bin consistent for corner cases +select hex('') == bin(''); +select unhex('') == unbin(''); +select unhex('0') == unbin('0'); From 9071ecd428929ead37a6e218ecf5f1f7d82bf071 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 5 Jul 2021 15:44:58 +0300 Subject: [PATCH 167/290] fix alter of settings in MergeTree --- src/Storages/MergeTree/MergeTreeData.cpp | 19 ++++++++----------- src/Storages/MergeTree/MergeTreeData.h | 3 +++ src/Storages/StorageMergeTree.cpp | 5 +++++ src/Storages/StorageMergeTree.h | 2 ++ src/Storages/StorageReplicatedMergeTree.cpp | 5 +++++ src/Storages/StorageReplicatedMergeTree.h | 2 ++ 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 40b37f5afc4..ae3d2220936 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1818,11 +1818,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context if (MergeTreeSettings::isPartFormatSetting(setting_name) && !new_value) { /// Use default settings + new and check if doesn't affect part format settings - MergeTreeSettings copy = *getSettings(); - copy.resetToDefault(); - copy.applyChanges(new_changes); + auto copy = getDefaultSettings(); + copy->applyChanges(new_changes); String reason; - if (!canUsePolymorphicParts(copy, &reason) && !reason.empty()) + if (!canUsePolymorphicParts(*copy, &reason) && !reason.empty()) throw Exception("Can't change settings. Reason: " + reason, ErrorCodes::NOT_IMPLEMENTED); } @@ -1984,14 +1983,12 @@ void MergeTreeData::changeSettings( } } - MergeTreeSettings copy = *getSettings(); - /// reset to default settings before applying existing - copy.resetToDefault(); - copy.applyChanges(new_changes); + /// Reset to default settings before applying existing. + auto copy = getDefaultSettings(); + copy->applyChanges(new_changes); + copy->sanityCheck(getContext()->getSettingsRef()); - copy.sanityCheck(getContext()->getSettingsRef()); - - storage_settings.set(std::make_unique(copy)); + storage_settings.set(std::move(copy)); StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); new_metadata.setSettingsChanges(new_settings); setInMemoryMetadata(new_metadata); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index da8c7fcbb65..a6ece4a7a98 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1087,6 +1087,9 @@ private: // Get partition matcher for FREEZE / UNFREEZE queries. MatcherFn getPartitionMatcher(const ASTPtr & partition, ContextPtr context) const; + + /// Returns default settings for storage with possible changes from global config. + virtual std::unique_ptr getDefaultSettings() const = 0; }; /// RAII struct to record big parts that are submerging or emerging. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 6f8b69ba419..8f387187074 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1577,4 +1577,9 @@ void StorageMergeTree::startBackgroundMovesIfNeeded() background_moves_executor.start(); } +std::unique_ptr StorageMergeTree::getDefaultSettings() const +{ + return std::make_unique(getContext()->getMergeTreeSettings()); +} + } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 6678ae06b53..a359de07c07 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -235,6 +235,8 @@ private: void startBackgroundMovesIfNeeded() override; + std::unique_ptr getDefaultSettings() const override; + friend class MergeTreeProjectionBlockOutputStream; friend class MergeTreeBlockOutputStream; friend class MergeTreeData; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b51b39f7d68..b6a08afa2eb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7174,6 +7174,11 @@ void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() background_moves_executor.start(); } +std::unique_ptr StorageReplicatedMergeTree::getDefaultSettings() const +{ + return std::make_unique(getContext()->getReplicatedMergeTreeSettings()); +} + void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 6f717b7c450..e03255ccd07 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -702,6 +702,8 @@ private: void startBackgroundMovesIfNeeded() override; + std::unique_ptr getDefaultSettings() const override; + std::set getPartitionIdsAffectedByCommands(const MutationCommands & commands, ContextPtr query_context) const; PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions( const MutationCommands & commands, ContextPtr query_context, const zkutil::ZooKeeperPtr & zookeeper) const; From 7ae15fee31645f760e417e770a3a7c674d31649d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 22:08:29 +0300 Subject: [PATCH 168/290] Change performance test after adjusted the name of column --- docker/test/performance-comparison/compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index c3447c17d35..9a8ffff7cd9 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1178,11 +1178,11 @@ create view right_async_metric_log as -- Use the right log as time reference because it may have higher precision. create table metrics engine File(TSV, 'metrics/metrics.tsv') as with (select min(event_time) from right_async_metric_log) as min_time - select name metric, r.event_time - min_time event_time, l.value as left, r.value as right + select metric, r.event_time - min_time event_time, l.value as left, r.value as right from right_async_metric_log r asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes, '$(cat left-async-metric-log.tsv.columns)') l - on l.name = r.name and r.event_time <= l.event_time + on l.metric = r.metric and r.event_time <= l.event_time order by metric, event_time ; From 22ba93789b69fbc2fb649ba5b2542ec780473550 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 22:29:36 +0300 Subject: [PATCH 169/290] Fix warning --- src/Interpreters/AsynchronousMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index c4a084e2c9b..edf7dd69234 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -481,7 +481,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti AsynchronousMetricValues new_values; auto current_time = std::chrono::system_clock::now(); - auto time_after_previous_update = current_time - previous_update_time; + auto time_after_previous_update [[maybe_unused]] = current_time - previous_update_time; previous_update_time = update_time; /// This is also a good indicator of system responsiveness. From 1fde0e13ccd4d3a38ec889a535071ef955c9f383 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 22:32:33 +0300 Subject: [PATCH 170/290] A check just in case --- src/Interpreters/AsynchronousMetrics.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index edf7dd69234..7f843852fc6 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -718,16 +718,19 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti /// Also write values normalized to 0..1 by diving to the number of CPUs. /// These values are good to be averaged across the cluster of non-uniform servers. - new_values["OSUserTimeNormalized"] = delta_values_all_cpus.user * multiplier / num_cpus; - new_values["OSNiceTimeNormalized"] = delta_values_all_cpus.nice * multiplier / num_cpus; - new_values["OSSystemTimeNormalized"] = delta_values_all_cpus.system * multiplier / num_cpus; - new_values["OSIdleTimeNormalized"] = delta_values_all_cpus.idle * multiplier / num_cpus; - new_values["OSIOWaitTimeNormalized"] = delta_values_all_cpus.iowait * multiplier / num_cpus; - new_values["OSIrqTimeNormalized"] = delta_values_all_cpus.irq * multiplier / num_cpus; - new_values["OSSoftIrqTimeNormalized"] = delta_values_all_cpus.softirq * multiplier / num_cpus; - new_values["OSStealTimeNormalized"] = delta_values_all_cpus.steal * multiplier / num_cpus; - new_values["OSGuestTimeNormalized"] = delta_values_all_cpus.guest * multiplier / num_cpus; - new_values["OSGuestNiceTimeNormalized"] = delta_values_all_cpus.guest_nice * multiplier / num_cpus; + if (num_cpus) + { + new_values["OSUserTimeNormalized"] = delta_values_all_cpus.user * multiplier / num_cpus; + new_values["OSNiceTimeNormalized"] = delta_values_all_cpus.nice * multiplier / num_cpus; + new_values["OSSystemTimeNormalized"] = delta_values_all_cpus.system * multiplier / num_cpus; + new_values["OSIdleTimeNormalized"] = delta_values_all_cpus.idle * multiplier / num_cpus; + new_values["OSIOWaitTimeNormalized"] = delta_values_all_cpus.iowait * multiplier / num_cpus; + new_values["OSIrqTimeNormalized"] = delta_values_all_cpus.irq * multiplier / num_cpus; + new_values["OSSoftIrqTimeNormalized"] = delta_values_all_cpus.softirq * multiplier / num_cpus; + new_values["OSStealTimeNormalized"] = delta_values_all_cpus.steal * multiplier / num_cpus; + new_values["OSGuestTimeNormalized"] = delta_values_all_cpus.guest * multiplier / num_cpus; + new_values["OSGuestNiceTimeNormalized"] = delta_values_all_cpus.guest_nice * multiplier / num_cpus; + } } proc_stat_values_other = current_other_values; From ac1baaf6d43cac3146f89e0b0a6c7331c8d0d2de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 22:37:12 +0300 Subject: [PATCH 171/290] Comments --- src/Interpreters/AsynchronousMetrics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 7f843852fc6..db0fd1f7c43 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1054,6 +1054,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti for (size_t i = 0, size = edac.size(); i < size; ++i) { /// NOTE maybe we need to take difference with previous values. + /// But these metrics should be exceptionally rare, so it's ok to keep them accumulated. try { From 945b54441d0b20efee12b3093f8514491efb7a44 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 5 Jul 2021 22:41:50 +0300 Subject: [PATCH 172/290] Comments --- src/Interpreters/AsynchronousMetrics.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index f012dda267c..606d117e605 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -28,6 +28,9 @@ using AsynchronousMetricValues = std::unordered_map Date: Mon, 5 Jul 2021 22:58:36 +0300 Subject: [PATCH 173/290] Some partially working code --- src/Storages/MergeTree/DropPartsRanges.cpp | 60 +++++++++++++++++++ src/Storages/MergeTree/DropPartsRanges.h | 33 ++++++++++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 34 +++++++++++ .../MergeTree/ReplicatedMergeTreeQueue.h | 7 +++ src/Storages/StorageReplicatedMergeTree.cpp | 12 ++++ src/Storages/ya.make | 1 + 6 files changed, 147 insertions(+) create mode 100644 src/Storages/MergeTree/DropPartsRanges.cpp create mode 100644 src/Storages/MergeTree/DropPartsRanges.h diff --git a/src/Storages/MergeTree/DropPartsRanges.cpp b/src/Storages/MergeTree/DropPartsRanges.cpp new file mode 100644 index 00000000000..e9cf07fb51f --- /dev/null +++ b/src/Storages/MergeTree/DropPartsRanges.cpp @@ -0,0 +1,60 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +bool DropPartsRanges::isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const +{ + if (entry.new_part_name.empty()) + return false; + + MergeTreePartInfo entry_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); + for (const auto & [znode, drop_range] : drop_ranges) + { + if (!drop_range.isDisjoint(entry_info)) + { + postpone_reason = fmt::format("Has DROP RANGE with entry. Will postpone it's execution.", drop_range.getPartName()); + return true; + } + } + + return false; +} + +void DropPartsRanges::addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * /*log*/) +{ + if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to add entry of type {} to drop ranges, expected DROP_RANGE", entry->typeToString()); + + //LOG_DEBUG(log, "ADD DROP RANGE {}", *entry->getDropRange(format_version)); + MergeTreePartInfo entry_info = MergeTreePartInfo::fromPartName(*entry->getDropRange(format_version), format_version); + drop_ranges.emplace(entry->znode_name, entry_info); +} + +void DropPartsRanges::removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * /*log*/) +{ + if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove entry of type {} from drop ranges, expected DROP_RANGE", entry->typeToString()); + + //LOG_DEBUG(log, "REMOVE DROP RANGE {}", *entry->getDropRange(format_version)); + drop_ranges.erase(entry->znode_name); +} + +bool DropPartsRanges::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const +{ + for (const auto & [znode_name, drop_range] : drop_ranges) + { + if (drop_range.contains(new_drop_range_info)) + return true; + } + + return false; +} + +} diff --git a/src/Storages/MergeTree/DropPartsRanges.h b/src/Storages/MergeTree/DropPartsRanges.h new file mode 100644 index 00000000000..23f38b70420 --- /dev/null +++ b/src/Storages/MergeTree/DropPartsRanges.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class DropPartsRanges +{ +private: + MergeTreeDataFormatVersion format_version; + + std::map drop_ranges; +public: + + explicit DropPartsRanges(MergeTreeDataFormatVersion format_version_) + : format_version(format_version_) + {} + + bool isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const; + + bool hasDropRange(const MergeTreePartInfo & new_drop_range_info) const; + + void addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * log); + + void removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * log); + +}; + +} diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index e5e4787da14..8fa69bb2c36 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -26,6 +26,7 @@ ReplicatedMergeTreeQueue::ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & , format_version(storage.format_version) , current_parts(format_version) , virtual_parts(format_version) + , drop_ranges(format_version) { zookeeper_path = storage.zookeeper_path; replica_path = storage.replica_path; @@ -168,6 +169,13 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } else { + drop_ranges.addDropRange(entry, log); + auto drop_range = *entry->getDropRange(format_version); + /// DROP PARTS removes parts from virtual parts + MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(drop_range, format_version); + if (!drop_range_info.isFakeDropRangePart() && virtual_parts.getContainingPart(drop_range_info) == drop_range) + virtual_parts.removePartAndCoveredParts(drop_range); + queue.push_front(entry); } @@ -261,6 +269,11 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( virtual_parts.remove(*drop_range_part_name); } + if (entry->type == LogEntry::DROP_RANGE) + { + drop_ranges.removeDropRange(entry, log); + } + if (entry->type == LogEntry::ALTER_METADATA) { LOG_TRACE(log, "Finishing metadata alter with version {}", entry->alter_version); @@ -269,6 +282,11 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( } else { + if (entry->type == LogEntry::DROP_RANGE) + { + drop_ranges.removeDropRange(entry, log); + } + for (const String & virtual_part_name : entry->getVirtualPartNames(format_version)) { /// Because execution of the entry is unsuccessful, @@ -1003,6 +1021,16 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( return false; } + if (entry.type != LogEntry::DROP_RANGE && drop_ranges.isAffectedByDropRange(entry, out_postpone_reason)) + { + //LOG_DEBUG(log, "POSTPONE ENTRY {} ({}) PRODUCING PART {} BECAUSE OF DROP RANGE {}", entry.znode_name, entry.typeToString(), entry.new_part_name); + return false; + } + else + { + //LOG_DEBUG(log, "NO DROP RANGE FOUND FOR PART {} OF TYPE {}", entry.new_part_name, entry.typeToString()); + } + /// Check that fetches pool is not overloaded if ((entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART) && !storage.canExecuteFetch(entry, out_postpone_reason)) @@ -2074,6 +2102,12 @@ bool ReplicatedMergeTreeMergePredicate::isMutationFinished(const ReplicatedMerge return true; } +bool ReplicatedMergeTreeMergePredicate::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const +{ + std::lock_guard lock(queue.state_mutex); + return queue.drop_ranges.hasDropRange(new_drop_range_info); +} + ReplicatedMergeTreeQueue::SubscriberHandler ReplicatedMergeTreeQueue::addSubscriber(ReplicatedMergeTreeQueue::SubscriberCallBack && callback) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 820d2794a31..f97ab74bd28 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -100,6 +101,10 @@ private: */ ActiveDataPartSet virtual_parts; + + /// + DropPartsRanges drop_ranges; + /// A set of mutations loaded from ZooKeeper. /// mutations_by_partition is an index partition ID -> block ID -> mutation into this set. /// Note that mutations are updated in such a way that they are always more recent than @@ -475,6 +480,8 @@ public: /// The version of "log" node that is used to check that no new merges have appeared. int32_t getVersion() const { return merges_version; } + bool hasDropRange(const MergeTreePartInfo & new_drop_range_info) const; + private: const ReplicatedMergeTreeQueue & queue; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b51b39f7d68..6945dbb82ae 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2104,6 +2104,10 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; + + if (!entry.actual_new_part_name.empty()) + LOG_DEBUG(log, "Will fetch part {} instead of {}", entry.actual_new_part_name, entry.new_part_name); + if (!fetchPart(part_name, metadata_snapshot, fs::path(zookeeper_path) / "replicas" / replica, false, entry.quorum)) return false; } @@ -6986,6 +6990,14 @@ bool StorageReplicatedMergeTree::dropPartImpl( return false; } + if (merge_pred.hasDropRange(part->info)) + { + if (throw_if_noop) + throw Exception("Already has DROP RANGE for part " + part_name + " in queue.", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); + + return false; + } + /// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already /// finished the merge. if (partIsAssignedToBackgroundOperation(part)) diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 6e412cddba7..495ec9c4fd6 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -30,6 +30,7 @@ SRCS( MergeTree/BackgroundJobsExecutor.cpp MergeTree/BoolMask.cpp MergeTree/DataPartsExchange.cpp + MergeTree/DropPartsRanges.cpp MergeTree/EphemeralLockInZooKeeper.cpp MergeTree/IMergeTreeDataPart.cpp MergeTree/IMergeTreeDataPartWriter.cpp From 87f59ba67096dc9ee4a4805434f59f8e25414454 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 5 Jul 2021 20:42:37 +0300 Subject: [PATCH 174/290] Fix parallel execution of integration tests. --- tests/integration/helpers/cluster.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 1db9e07a69e..07d503dfe1a 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -394,11 +394,13 @@ class ClickHouseCluster: def cleanup(self): # Just in case kill unstopped containers from previous launch try: - result = run_and_check(f'docker container list --all --filter name={self.project_name} | wc -l', shell=True) + # We need to have "^/" and "$" in the "--filter name" option below to filter by exact name of the container, see + # https://stackoverflow.com/questions/48767760/how-to-make-docker-container-ls-f-name-filter-by-exact-name + result = run_and_check(f'docker container list --all --filter name=^/{self.project_name}$ | wc -l', shell=True) if int(result) > 1: - logging.debug(f"Trying to kill unstopped containers for project{self.project_name}...") - run_and_check(f'docker kill $(docker container list --all --quiet --filter name={self.project_name})', shell=True) - run_and_check(f'docker rm $(docker container list --all --quiet --filter name={self.project_name})', shell=True) + logging.debug(f"Trying to kill unstopped containers for project {self.project_name}...") + run_and_check(f'docker kill $(docker container list --all --quiet --filter name=^/{self.project_name}$)', shell=True) + run_and_check(f'docker rm $(docker container list --all --quiet --filter name=^/{self.project_name}$)', shell=True) logging.debug("Unstopped containers killed") run_and_check(['docker-compose', 'ps', '--services', '--all']) else: From 0241253a8e93bb5a4cd5041e11011d9a72e96e74 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Jul 2021 01:05:12 +0400 Subject: [PATCH 175/290] Guidelines for adding new third-party libraries --- docs/en/development/contrib.md | 12 ++++++++++++ docs/en/development/developer-instruction.md | 2 ++ docs/en/development/style.md | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 64ca2387029..d370a9ea78a 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -89,3 +89,15 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li | xz | Public Domain | /contrib/xz/COPYING | | zlib-ng | zLib | /contrib/zlib-ng/LICENSE.md | | zstd | BSD | /contrib/zstd/LICENSE | + +## Guidelines for adding new third-party libraries and maintaining custom changes in them {#adding-third-party-libraries} + +1. All external third-party code should reside in the dedicated directories under `contrib` directory of ClickHouse repo. Prefer Git submodules, when available. +2. Fork/mirror the official repo in [Clickhouse-extras](https://github.com/ClickHouse-Extras). Prefer official GitHub repos, when available. +3. Branch from the branch you want to integrate, e.g., `master` -> `clickhouse/master`, or `release/vX.Y.Z` -> `clickhouse/release/vX.Y.Z`. +4. All forks in [Clickhouse-extras](https://github.com/ClickHouse-Extras) should be automatically synchronized with upstreams. `clickhouse/...` branches will remain unaffected, since virtually nobody is going to use that naming pattern in their upstream repos. +5. Add submodules under `contrib` of ClickHouse repo that refer the above forks/mirrors. Set the submodules to track the corresponding `clickhouse/...` branches. +6. Every time the custom changes have to be made in the library code, a dedicated branch should be created, like `clickhouse/my-fix`. Then this branch should be merged into the branch, that is tracked by the submodule, e.g., `clickhouse/master` or `clickhouse/release/vX.Y.Z`. +7. No code should be pushed in any branch of the forks in [Clickhouse-extras](https://github.com/ClickHouse-Extras), whose names do not follow `clickhouse/...` pattern. +8. Always write the custom changes with the official repo in mind. Once the PR is merged from (a feature/fix branch in) your personal fork into the fork in [Clickhouse-extras](https://github.com/ClickHouse-Extras), and the submodule is bumped in ClickHouse repo, consider opening another PR from (a feature/fix branch in) the fork in [Clickhouse-extras](https://github.com/ClickHouse-Extras) to the official repo of the library. This will make sure, that 1) the contribution has more than a single use case and importance, 2) others will also benefit from it, 3) the change will not remain a maintenance burden solely on ClickHouse developers. +9. When a submodule needs to start using a newer code from the original branch (e.g., `master`), and since the custom changes might be merged in the branch it is tracking (e.g., `clickhouse/master`) and so it may diverge from its original counterpart (i.e., `master`), a careful merge should be carried out first, i.e., `master` -> `clickhouse/master`, and only then the submodule can be bumped in ClickHouse. diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index ac6d4a2b563..90f406f3ba8 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -237,6 +237,8 @@ The description of ClickHouse architecture can be found here: https://clickhouse The Code Style Guide: https://clickhouse.tech/docs/en/development/style/ +Adding third-party libraries: https://clickhouse.tech/docs/en/development/contrib/#adding-third-party-libraries + Writing tests: https://clickhouse.tech/docs/en/development/tests/ List of tasks: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22 diff --git a/docs/en/development/style.md b/docs/en/development/style.md index 2151735c2f4..bee567be468 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -757,7 +757,7 @@ If there is a good solution already available, then use it, even if it means you **3.** You can install a library that isn’t in the packages, if the packages do not have what you need or have an outdated version or the wrong type of compilation. -**4.** If the library is small and does not have its own complex build system, put the source files in the `contrib` folder. +**4.** If the library is small and does not have its own complex build system, put the source files in the `contrib` folder. See [Guidelines for adding new third-party libraries](https://clickhouse.tech/docs/en/development/contrib/#adding-third-party-libraries) for details. **5.** Preference is always given to libraries that are already in use. From f4fc1d0807be569b022b063d6289445c5f81d742 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Jul 2021 01:05:30 +0400 Subject: [PATCH 176/290] Minor fixes --- docs/en/development/contrib.md | 4 ++-- docs/en/development/style.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index d370a9ea78a..f372da8859f 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -7,13 +7,13 @@ toc_title: Third-Party Libraries Used The list of third-party libraries can be obtained by the following query: -``` +``` sql SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en' ``` [Example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==) -| library_name | license_type | license_path | +| library_name | license_type | license_path | |:-|:-|:-| | abseil-cpp | Apache | /contrib/abseil-cpp/LICENSE | | AMQP-CPP | Apache | /contrib/AMQP-CPP/LICENSE | diff --git a/docs/en/development/style.md b/docs/en/development/style.md index bee567be468..78e3c8fc966 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -628,7 +628,7 @@ If the class is not intended for polymorphic use, you do not need to make functi **18.** Encodings. -Use UTF-8 everywhere. Use `std::string`and`char *`. Do not use `std::wstring`and`wchar_t`. +Use UTF-8 everywhere. Use `std::string` and `char *`. Do not use `std::wstring` and `wchar_t`. **19.** Logging. From 0811b74ad66d737f64534cd708d7e8a55c821c16 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 6 Jul 2021 00:19:29 +0300 Subject: [PATCH 177/290] 1 try --- docs/en/images/play.png | Bin 0 -> 37317 bytes docs/en/interfaces/http.md | 7 +++---- docs/ru/images/play.png | Bin 0 -> 37317 bytes docs/ru/interfaces/http.md | 6 ++---- 4 files changed, 5 insertions(+), 8 deletions(-) create mode 100644 docs/en/images/play.png create mode 100644 docs/ru/images/play.png diff --git a/docs/en/images/play.png b/docs/en/images/play.png new file mode 100644 index 0000000000000000000000000000000000000000..583bd8a7ad9cccacefe859b0144911438b771946 GIT binary patch literal 37317 zcmY(q19V+o*9Mx#PGj4)(Z;rICylKJjg!W<)7Z9ctFfKNxVvB9f82Y=IQt|yJ7=xA z*P49hv%(bQ#1UX|U_n4Y5F{l;lt4hhIY2-@`9VVhpVYQX$^$=O>?JgvKtK@t|6ZRG zX%TUO522hTWksR(p&-x^KyrSgnu353fk=u7s<>yIZTP9osb@WYP|8hnVpx@k#1klw z)KCAWgFPDtsi2Ec-V0_t4CyvR?>or? z#$SxFQbjkLq=bi!sP?<8Ehb4fY*w!NnoY5~0A)fuo_gzxkP+yoUdEbppNu~L_S?x9 zpLm@(Fa67<4o;DzeP(dMl-cE!M@(dJ!Oy@8LtR2*SP4_`b7wu8lSjG(n@W=C?Q;>DPkGG)UU=m3b>&5bIe<%VXHa2HLIC|&^Y6T);aL#`N1r4B}ps4O{6=RaVS+th^Fpf$nS^(1>K5MW<*N)H`C^!F7pFn4eV&dTSHh`jze=ywn} zE84)e{nd;(p~IgO)y_W^cDZV0fs?)VJee95YVW6A?-A_Qi&w*>KR;t&a9jq!^$OwQ z;u_o7M6Na2IX&H&e%ovxIbUl^Zq@iVLGMyOAd+T_Bl+oKh4ItPeo#E&gx(Ai6O?v~o3C2L}g|X5w|$&UybR>0}0GCZ?}iS~z8D6Z_g0iMMUEvlz%QG7(0Le^m`?hDvWbEU8Hv?{cqW;%6fVP&s=%8UJp1m z8*Q+O4%R;>Ep`wrYj7;!Co|~o{N0MSt+$H+2zC%)hJnQj?Wemr`4;En*#La?&j)>| z>(y)L7Hnzc3{n4E>3}Pjq8ST&F1+t4x3ZrDkeR9-_D9hLo{vbCl$9|}MdPQZ6*}L} z?QVg!zg{uVrhmMKFk~1U9DJP1e(&e(ypC$~++;PMTwl0_`S<7#jp(#$_AmFg$n&2u z*3X$A^J#-V99#X~k6^d&)pOEhVQskZ5TBo)G3ZOSYaI8bfQgQekFUi@Aq-1H84>=c zso>jsWUHmULM{A|V(QSvRo*>4tW8wBPU)<|##mjTJyDYWQ9lKqLqiCC-33FkWlR@- zsq(#E^?BdTh)4J`2txNiL z?*9C8puP3HSRS0S-s}*Vtmg^y9aGQUa1jC;0bw7wgYj3*!otGpd|yQlCo@UO$N=pG zM}r+0M@lKi<@|Q_vh|Ubl|>-%w1*0^v$G>1A@RFrh+wz#CNQ?Iqi&-5uNQK&|E)53-1!RRd0s!E#{r~zund6U}t9sRCeOH#?sOf4FiKgg1fb~l?e-e zvr8G!n7!tJu7NXii2RzGSU@e9`1tVV-xU=V`Q9IQ&z5V9d&W>o?4GBEkzx}P`rW!- zbfFPKBtJ#gRP~4RZf11>UI>`24wk_^0Ozkh=Y|lZIT6oWH9Tbd5=TZy8!=&F@HfxQ z%?Z1>uyfjN-mPtJZOH>BNGX?*)7+d=P(Wr!>74rn$YYuFvYGYOR+lHT^6{ zgK5G?Z4>4bV~Ob{6a&938IYPLs@>2?Noejl`FU_*KUYWiisBvPs<58%jy}1i9WCu$ znJDe)Qg85&85R}y1tnxm29tYJ2Ybp*TZI`O-XjFj>oS92vAfI~&cvh^X~)!Zg$NX! zmq$dc#cRYg6H=@>zT%Tkq#vH<%x9;;#?Qc1|@99E;aO6)sWNe*Hlxjl(rj z*&nZmf)8PYo=3TwUFw^9ALiLUgR2eJfs>0VSo7}pE9T}dM*6wAx#hZTNwn&f)vo8O z+=Ng>%*?Tyo13CY-!cSz(zf33n?T6O$f`}pFyi9kngMgdWYGD&HxiE(CH8)t{T^nX zo=k&z3MH;h_14sFO5`osQjmG^!E0ttwUJtxueT1&|lW}sZ zpb#872YRiOl1CUc)!R5m-6mol^5cZxhMe#5%#GtDwyNh%k--gf@wgpL)Lt$(+T403 zHx?EK8myNzIy~8bUyO&BM-3d}c3+)~Y_)sD14=mqD7=o&dO&7XLj(Hvn;S{0Fmz(h14QP6y+ufRg-OdCo2G_&_M6THiO%XafFGJKtMe5CXqf2jACS zLZ{9DrlWg92P)v!JdnKFF8nnV-wGrV$T-H){I{*dwsnkI8;d zn3EaYyA>*M)CvlHw*+{6?h4?X7XiR7fiUQ`4?_`pf3>orsi~>y9~=a|ZPWUFdN7eb zzqBL|Yvms7%m1+M^qsdG*xgThK7Uj>Y*)#G3yJ3R(q#R00vpJ2_Sw96#3$!g=cTeO zu{3i<@Z$&X*V*b(Jp(6m>*m!tNzL(dxTt&0Up!W|){fW_jNK$fEO&>;Ck*px<`3Ty zpJmi+7$8PVu;Q8yV=5yINDjX;f-&H8Gsj7Un;?y#nWf+Gck#1xxSyFFRKx_@KCM;CT} zJzxSGfufSjI({P@@NP_8ob(;t3Gxv*`vH}8}1?bbG(i_7m!H9BpVdNX^(;?3}cp83^aE56?oiKlnvq>n9% zqSkH8wLX_Z?zJ|_sqxxAUPuVLP&7F~PAx4u=Y1}KW(v`{?llDa|+tUg;^g|Y) zzz*lD?3;kw0_psCy9B$~X142k;{cTF{v8_5?pNe@KxgCtuawGaj=@&Ft-0a!_WbAd z%>2tOWS#HRn7-T-mOKdAaxN|F^_{FwAP%K`*4^f1I{wYs)_bJ>YnA?u-^UL6JKw^F zIy9WW%NCaL6N$OaYu}gGzC41hwKX>`yr3Tw>v&p`Ignp9G?oMCUs{)YqT(s#9d8e0 z?TQ9f{%06~kdb1$0U1litvQg*fwUye4sQQ*xr~(q;oaFhF5&a@^En5nSL00%C-c;o zn59f$#i@%Y`FEwY%PpMIp^B)sva<7wA$48$`E7H%SUcY0v=zE<@NM0Gp8JZ=+UdvY z{qYT~5%$T6m5XX8TQLfW--N+}XiRiQqfGYgNCS{3v%eMniK56uJTna38mf%<@BF0G zinnuNwe^)m30K@q2IChrW>PdN8ylW*{tLCG{zAF;7>HSNbVhm9Q4$@>?`}2&Vx8n4 zumG+FxLeYWy zJwLWI7SrsUNEvs={0?1xE(oGYTSUo3n+FRus19QA_~DU7#X;yk)aKAc7oU@NMFE!M zy-D65+q@ug*_Y3zI}G{Fdt2NlnIqvwzCnR@nUfL{5 z>$T`pQ&Zzd$?4iDv)ls!9hqHQYqK~^VMwV-`KzSHKlu+3QrdyiCjS1~B&1xwD{BHK z{jub{0|Fn?iSf!es#S*}iBVtRd=fFlSTO(xWGtC+Xy0+7-8cq=^=~2r#$;-mGVz$Z zlE$TY)SnX^LfL!`q}D3noSl0DkG})I8}sw0;-K1Yv?i{uu6jUlm>A<1lJC#E=zmQj zATP|x_%#Z8x9P!DUay{{q_z^U9T!9O+eMK!zNzK7W7R>ukhiRVi1kC2P6%@j^N!8T&AHj!!1Po+`dEXYVp`+%1jM} z9L3o5&Azb;Fb#Vs4l&Pk3mj-lp9i_|KeEejSzp0%A6_G)HrHw_bivw}?XhpEEJ(f& zlQr5stXBzRn&o~BgP7sI`y&j2bbv89w>SiS66PPWJYh4$ZX|9)3lcvL9UCV$-x@qx-kW^Icx9FXj6KkPuJ%Ud!K6rR7m6SD#yK;uV9$k&~5&E2DNZ zi9%8*i`=-ZYmMT`^NePfJ6SjIW$e?AZQMiZiVSUBv3p%i&@rJn@l5VWF4-Gqc?||C z6>8;34YRM%ck1t^=hu6_uLRRkU$$d_FkmNJe-FmY&j0ikc?xON4cYoQ=Q#>-oC7N z4RuD=cuQ`e#tGQ>EqNV3$G~+Sh*r!H;=!FD4h!*Xf9Ewd0F%2nODPT-xSw z&POeN@4`_63q$?)_=q9>0~cwY7r(UV(U+?&`x51`&075CzG~QBcCC6E5c*aP1cqDU zRY*Fncv8v*!$SqiDniS?R6QAoR!Cbj#A_dMOIvo-CShr+{JVNrrt%4+pNE8$Mqn+x zyWDr4V6xXQD}^V;r7J-!E{vL5AFL8M2&>n`f?B$CIO60XZIjF~xMPC~Aei@?oR9n5 zyd22^_AcXxh5Om$$%Pf_M1quFNGSbtBJht1Y-n^UeC z`GA3JTR3Y`1Xt2usyaU21(kV(ySpY|8VLtMR!k9#O%d)Q#0T_mcu6cZXE#y!V5S}- z&3U%Rr^HPRS5R_M4#jq{9?K<-mf-VBC2X6muFCXZPJ3)&9u9Euly8er+b)ZnfMP#g zVJ#p~u(f!t)Lob1K%isv4n$y&^4{wj}@&Y9Sa_B2PI&_R=sY0Q$ z1tX4hAkDw4$1FaiS&oeQu?Q4O4i4A#FOP`l?mJ017bmT!ZQ%&`c4X zpEe>?9&>#&#`ppuTLh=_xOoWkjc&oFN1wcNMO0C7PG1#{0?vxUtd84YBb5tnpzk)9 z-+Yk=Rw*lwcb`a=-lL=PJ`xg~GZ4S|@gqtNBkulX6QJYd;PHn6d zKbEx=^~B`h;)bj!C|eSSf+c-UQ6XZ-?BMU5Jyf($?>pqJzit#j<5+MWNg0yIc%eY0 z5MrH~nGA@P^NG=*%;FIw)6dS2#Ea0ClOXM5cLm#nT-(5U*-JoWLQn7-rk7TJ^3y~k zp#J)4oP!xJE&eUCZF6}7EX4%Br7c?o{lws?4^iv%C)3%g_%&Mmk6`P$;gxu~An^Pm z#I&?i%S8f%B}n{Ms?<&Yv1-HiU2AqjSJBwBHDRNKTYR78rNY>9h+s4KT47y0J1im+ z+UyQyWG0><(plw3Vjk-W$9Q^W2WlSkaXrA#m7{3B4=Hw317t*H+6Q% zKt}j=B(eLFbm$F{Q6jx1tc6*Jv>IM*hWG@!Tr|%nqJ%-w;!uQ$Ke`2V8uhHoyh5oN zC}d&!@E7LO;#b-z2@1h~_uuHzH|R3#)VkKn;s!hQ3u@#n4Ks)9s^!efG%L@>WYl5H zV!{8ZpIyI#j4Mj8ZRM(GV>NJ%<0Cl2VMk!1J04a7BQ@vN+8B@d2FL67Svtr%6p5Fw z%ART!3qQQ`lT7|`$-1R%3RG~cbE_d5nJi(~ z7aK5cO=dhR9zpFJy;<=zOLpp!_2jc;-p?56W0U=T?VI*!$+H2HsUw@?AcrYm!(+Pl zd-p-7CSg_iG_{R3QdMmy*g9q+Z@T0AgCP|OmOdv#1&58VPQ2Ik(DMgg?2HpwQ! zLQMjQu2fYWct7_@PCmh?s+YZi-bgDet7g}89FT^FhW5vu2&Yv6OrZ3gpH~4O8Ta#s zrN{KHCtMIa1=8j68mOj z)8)u)2ze|(9nn6Q(k`?S?y=q$7fca(d9%WxC(M&6EPM-Ur!IohMV56{%<@W1hy>re z7t6A;8Pd`UT3rOb32}%1HlNUplomS^I`LUt)B|omidv(BC64DuWF+`}cYg!{ZGpv| zXU7yCF7C3bny-0Cb=2O)S#jewbO_|mjHKCf$j&gM9PGEVH*h&=s3G37{yh^p3E8GA zy?sMDq}{BN+Uaxq%$#j?q)w%#`>>`PN>z&%b7o zd3|AY^G8Tp4w=kl`>F-Ug0aYpU#5)3LNKtH)2nG+j@q$N@-x2Z$@VGWd48qRR(8j0*TRd#7NCd=6;?+&aCa%Lmw7IPb>Wb zL!#~$G>0>WE#dJ2xk=5$f{Qib)yHG!SPswUsZEcDDOaY?S)-klLzJ!ad(|83@_$0! zA`1#zc=KQ(q#9roFJC?v^>`U7LvvAZm=~Cz=ETxx%h=eGUJx;^HD^TYd5rr5IV_Id`m4M;?h95vb1_#9+EXO9amn& ztE@FstHM@&3L5ilpk@)g#?RVbBgj0zH}F^;&597t@|Xb(`Vq(p_W8xFtbT1}SY{1P zCQ-&HDl6|IXvnaOtZ1T{l_0o(PnLjA z#Uu;?@UnDrCUez@@MYJBufWHv(?!?EJGGndMayBkCN9vwIlO$oxWp(I8h8Ho6UaiK zd}NZxe)Jd^I6Eg?U0rF?oATmMh3pqfC!1w9FzaXKj3D_v3U{4%X6neR$>t2f+#bT% z00{Em0>2I@t1eX`kfv@1;E(DvZwn{))_MCd=Fw#@@9CYVXc+Sjdcx_(RAhw zo?41e&O>q!>Pcde8pA#2KqQg2B3q-EfXNu-ON~3SV5eQJSO#feo^H7_F)2U1#93pG z6BpD(PW1R9l07Y1x+^tL$nHejLNt4 zh=3Hp|5_>tZ>tf2ME%DZXvNE~Dh}$bl1c%AdK>XF2Ll`|v02*(JwvxFyP{lu(Tn%< zo+;j3X;Rjr?4lE4joo-j*|l-V3HQM;Q|Pu+mWe6>@w% zDiR`}?hWB;3q(0h_Xhk2c;NerxnI{UKiU#VW%#wfdr8hM9w0u`bDS|ips+9$$_Gz@ z;$Y(lseQvkDqqGTaLkpueHy~$FpN#?w=Bj@-$fK#NM}Ez+Y}(y;1Dc9k!=p}N5-B; zX>!1?!1qbXsOfKoJYuB|9r+2QKS_qT7Dhz^dK!3PtrR%VkoIJ@WaB*FR|+X6?0i3y zr@LXchG4Xqh|wJ$TiNampH^2b$L|x(EjgnllVt65;DWU?#*!VH%{VRNH?O1@B0PRM zX8L1PORGR-bC=B?TbjvdH@IJVYtMPNIssIo5(H+kziVT zgq2uqB-kKv6#vLG-og|#@o%}8c(^E1cDWfEJkc{YGck2`YeCvUX8xZDvGg{=2VQjBEJS&W-l%Whpqe?03w+(VJ4T!v+vVxXAgD-XZ77H zRej3%(8$Q1l#b43_A_c=4^aS)^bvq}k`fbt3he2Ax$K&1v|Yn^#gj&!p0;Y(^gYfM z4rz8d#Q<6+n{XQ)9UbjAqqGunA`TABeSLiqp4TokZnifHKZJUKCS~ykHyvFhfUzg( zzwTh_v^s|YEh5XF0Nf_KExqYPX;27ujFU&84f$8F46Yvik3ag1!^}Szk*4-CHa0f? z#(4l&y%bUS-3dCqv2niP)(Ie{BI@eu(}Z7vPKeQ5k*t$|el>CE_ek`;eN`bvbQe|X z5@sdMD|NQXozJu70Vp}+mAw}9xs|^H!e_e*etBcMY`E;na+eN)A85SDm>pYxH6LhlI;B7kr9idfUPX_TNjcPG`Bv01 z{I`x2G#=N%@p-|N11Fo3K+b4aK_iAXR|^>P&`D$QV*mn>B%%HlQO9rAXdcC4pq4YH ze|VnKf+pph?()bujB+5bo51gdKJo)9l=;u>%8cOKHsZYI1vg4JL^zh?!~SFO6fIdV zssBv3IDCZ41XF6hQ7h<14MRkS{NU4oJX*MYZ}IAuWl=+8*5)w%*?JvvBKSe077hjc zQ0jxX!X%Q%?mi@>2_{219*tdx(|VCQwOw53mhNCY>pi+Sx8E!D9}0Tj2sh+;G$;W( zGZP(@lSPha8M%5#Mhfca*FjQl(A{m!_qP?Wf$5}r8>hXP)!dGFt08Rg%!DnXGdmcP#1<93ZubWb3h{2 zNBT+#D-(uKd@<0MVO$W$wFqCEV)#h)P%M7`P8m3wZPsR)ZL*69ax?`I38J6RKAcn}yDO>L z!aBn22o2&fZkTnXt_sDm;CS7gdYw)lem#UC+eUw>+*D% zRm{=dJ;m>B$&Z|aW6>@v;`!MV=$yU{DoqXn+4J$)Sq9r(KPz)0W8F$*k3q^%a&25t zka*9uP?J3E;^L+d=Zp!NM2@^**AB}0M!fnt@`5YyB3inFCV^i>U-T1+KUl?d~B};Rq#FpmMw4ie(peDRRo1X#L(G1 zrDdj6FnMQ|`6oyOu(aQSIH;n7vE}i{m*<5|OCs(%H;-4POl&7qOh^W42ewENlB9Yp z4vka&-A{&j9D~er?Ntmb5LC^#iRE9hyW%_AP;)P-?3^{sylvm2ZVZ)C-@+&OQqd!< zgc(|{rGN1PnOm%W+-LK@OwOZ4*HwA23o$bs* zgmOVNv+#byM49YXcAUutZRRhuv+9~62esJ%&9=?P=_=npv|Su3zP|4TIl<=cqkn6- z4*0U-FZ;(!?n!=sHX0RZ1!3!&fG%rqZ|~z?ygU|8nk7jMW5{0s`(ev38^FW=Hh?Wg z-(1g@KQ&Y0`T)Sth1&H6j@_KmL>jyI^Kr4DF8V#tIcIVDOAg`#P-rLL_8}uE_U^aW zY|j&F-v`J3 z_-33^lyljSRva*Fau~9Nq`LeQ>hPPqQo36i0#A6$p>(w4Pd8MI9trcce6V|t;YxY2 zHi5&>e6@c(^Gfr6wMQS3@Jp3r*)|pNEk#PAkMfO50Wdh-PmS7pa&t=0}=}T9TTwY@#zPfgJo0Yn{LGvsD#IoS92>ZR^?p zd`>DNUpLJyZoo`u$yz=>4E}?|vm{06aJF93ZB}Cd_SL^Y1sNMM|B&}MG0}pQw3|Di zuqmq2ko=l^*8@Hu>@E%M=*A_U=e2U1V3nsYY)RyIt@Et+WHb+}EgzT=i4bUdX-EXA zX8*4|1;^+*TSAo70wJS>To_>g@-psS!u1MSsppZmcs-hf80uKZ%=QD zXLCoZ*n-pOqJ_W9X?Z1(iMC=nl!R{OkdUnVZ!qMBEudh-AlBixIpXuk?qw1~?njVp;Fn`aJCMi;kv1gWr9%j2*UCfBak7}O^b<&shri4iU#30=bzck*Ga zU?4AjxqvU5E-*z|(v}yl*zUn&|k{h}rnV~-@M`c*UyByCt9(!QSX8ll0I-H9TeLKPSKr8VoxcJwvaO~}& zxII!$OoSbQGVJ4o)7WBizlB)xYz_|kteH)=Yk-oPr(4v~NRH2wAR$CSS)d1%k<_?l z{$8(+>mO$dmZi8T&&>Op8!3cqS$yh_*>^&B@1ex6a(pK77DH+~bR0=bEW0n_>Tkqn z3`WG_z^i#cfHrcyIgTv}3!zD2&q)`sjQ``To-p@4D!=3&Bi`FBjW>WP|Mk@y;JZnx zt79^@Ux5NS=U>{zw+$W;2K~;n4o9R5n}{D4Z5{zs09T`+v2oGswZ5LM^P%JN>70}f zK|Z)|B%X4m!|S0vdS4PTWdaQXVB&zLxP6}Jo8$C(+E3S!!kb$HmzP z*&k2Y9nzs5K|g-{&C?l=mf7W7cgm_drHqV7fNtI9Zu$g(#jhTnpJzH!Tv(bWE4GxS z&(^YN!D>+{VJa*k0PV8`lPLuIdwfCv*fUKXV4DE(1!ualjFXcw9zTdTg*653Ziu6fbh6N#-Pf zZt2{&!m5fuvvj`q^_}63MDESEg%~}Z$oW@9{0VVx`nDLLBUI}=2hI_I9iB(x;l>=k zXvU-UbJo-?H>OvyGD7gw<5#TYYYtXgIW+Y%vwL^C2NFeB*m z$5MwXrngV=G+~obTJ?nBMjXe&wyaqGK12PGzlwr8qTio_XY-h}lPqeuAAc_;dT#NK zry~dzJzIDj5C*tLR{C0dRW@3siEiYi zT}dbfeP&7}q|AxT&Kt#yN-peWe$_T~2M8>TX#%9}$u|~OPVfvQp~MXNZy&@bKbvYD zKB7}UJ~GT!wKkjtO$MIlQ$i8;OisAw=VipT=+;0d2T&v(zwwwOHC<1(kV(y&aV_Z{ zcF)`OLS*oIhB|GYpAK_ZLk)P8y>vcXWxuQ!^c&ru@*GvuNNyw_B#Y*ctjihcv)fh> zqI8KLPUgUczX$sgfT^n0+5)`MS(C6}B(D&T59ZoDSfDB>5K6 z#$AnwnvoI7{iELl+2^TJg+)KB^LUK0Yy7W4#wRBqy3M>jbbauD09ka8vOIqS6>TqF zeTxhM4#`;UgCZ~pKs(iR83u?#+xTFdONA)_KMNzmR1g&RaI&Rq2a~mA`xMC2Gx$8> z0CHnjA`_7K|7}`f(Bs?#h7OfE=-RZT^v;5fi+lU-eLaBbebocScfV{b_~jj7qy3MZ zky>wTY}|nw4tYAB$|{(B3y_F10ICL(z+-n+r|oKeScP`eq&0%8VQHjeftu~1J7Pwj(cHij^C^k2N&1l_bC;+pf;YXt`~E5`Gk=BB0+?e1 z0|Wg70~>{LJ}u*%)is~ZP?tY<28c~5y;;$2w;|QiE`B(<34-B96Vq`R6l5P1$@Mxt z)K~|moXMxaQIaMZ>I|pw%1_%!<~~_iYG(74h`+6c0-Amx0&W zNl>>k?li^ExeGy!y!`os{K3`4vjt(LJ-b8dk25qu1jEVTV_x4NwKC8`hS*mjfJ_W7 z2=dU5Ggoo&mkj886Q8|4fCAII6rw)}*=&By4=5@S>dLhmtQ6$|T6Ol^%F17QMxz03 z9(1owXypG(EN0;d@*3&Ls;RNr39hbY+S%PbJ~`RBKVOS*H45ny1mY$Du8@=v(@|eT z5dWhPuaAj%seUfvq^|x1up6}qJob=V+u9s>Fny!kLi&1IoDSXhV#VEm`$J*8J_dZI z+hPohiTMn$qwTJWzq}n+CJCBY|3^md{I=0L0Fciq8&n|#q3H|^zzl$<>)#~%uciX@ zV(%Wg$G`G#ZcYXw31Ho6bp=!}^Xa2)Z*LpfI{ZgCmei!fuv2S;wt(&(FVaPF0fqqZ zBy)30q%W^bJel7n%L@O0lywqi;pRhtM*KGqh!i6oOQJUhHb)|^etrPuAGy4f!bI}` zNcREOI?yr!U}&Kj3Rwc2(7x^8`9B4AfIF`%*>BhYfeV4Bd)t|(#c>}Nu>6maG?lLAi;QJgvbdps) zUldLO4r?!izMkSrHLB7zWGA!gHyRM$9rADHD|Sv@aXa9c#^>xVY`cD_g^@ek-+tGk z-8Co5naYozpIziNq*xb7yS$3AlUQ`^jYo5-I=jFi*v!)0j9btDZ*mt@3sAnT^Ufw= z=)Cyjy7P!@x&3~QXO%K_Xnn@K3m9q3U(cAKZa-mMtcre%b*n0Z0#7os`584F_TLaf z_7tvjm5Z$E!>5WX1o+GTS*1G<07XZC9g^&{(B`c(tjNtV4lPWsi_ z5c+`6r$Ag^C!4RoWuk_Mhehd|{@Vy|T?p3LuFI`#pL=Yg)uZ0muvSX%)X2z48SYrB zZk80CSkW|q2L3;x>7q)5eIqzptCs;ixNt!|%|E;}Mdf@vb8wTQ_)jK%;T~rUFxd+# z=v(rWtgAvc*-RaZJcs|*t-8MGR3sc2Aq5zr(+1SkRt6`W#rJ1P_kaIjpDf#SlXGMO z8zDK*zpcl~!NFm=7`&x`rl<+=PaU^6i2*T{kX_`PlV!`Z$3V}Rg~^jF$Kw?+8t9ae zFCkMeW-Ss{Tr`T;ha^M;;u1y$FO%+3*a5r0$WHiWNznGVH^@gOuUWu%vIoP9H8w3x z{fMUjPwgI2pyv?@(`%v#zE=dpQ>V>daH9!vC@Cih)hprH+0js2WGw&A$i`l)>(y;C zaFPFv;on36SrQQsh6^{WmMre9_)`B;k6Aj>ww7%a$eyrBMO{kum4AR?v6Y3!20|r4 z#H-0vMHAbJ>xk!(iuOWEu$W?>orD*UYm?q&0VNsYdk+oEohjCV1GKdlIRy0#6*-Kg zQhbcA)ULtrR>Sh~uW>9THL~sEX4N$bHXS(BWZ;sel=%ga`Im-!f;iH3lA&rOKP0H~ zQHRSc^k=^&{%_}cJ4~v7g~o|dm9QGrXjH@Fl#VB1+~M^!nkN@@5kdDRkJb)Ewqvg$ z$b2B16s#Gf8)pA7%P%a?=s9y7MJZYqy(r3S&rG;+WTEi&l}3Ski4{PK3nFunPY~y* zpQ0`4F+hh{L0^W&!A>wW(dIQB6@#ty2_$C$xb9X`tix1D-p7I$45YhNCpiU(fdc`# zHE5>0NFxlkgjMD&8>ZN3Y*GpH&a4*9vwEq$1|A%u>?HO7i-Nr!hNV@}xCmkr)P;Q7 zKeZ}cKj_5rod)mElhzkf> z`V&{X?hv(dKBEk6`PfH1WWnBE?DF1$31fRw2aYC5-P~Q2ot^}N{(}D` zunJ!&5)u@nV3^Ul91J_El}BAREX+N?5Xf#iuOK8kqw<8O9FM@BVDe~!zn3_}lG|lt zOHIm1j8ljfP!LH0>_%6y$!XEA?)YcQT)E-=JRc=1p_u*BJR-sKNtx+G#T9z_X1}7F zQLIy`;c41fG8jp0S*ch*4{cbo-uDx%Y<>4s5qX~6+R%}}M4Ne2=^zORHo5k(Fq{n6qnzF_ae+v4M4HMG52w-QTTJ#TGPq}@z5bu!Ibm4j4L->&Q{h@uNhy*_qFDKG zn4;q5ezz&lCj(XHjVhC7E;cq!}75(6zYla_XYSrg%q`NPXGrBnE zetFuEG>D?1)CV%nG@7hm;jTob2Q6}mM%&I~|2{ut^ zrpeQp3O{-%NVT=8a)aB?VEMMd9Nn<7JzOLTqp)ld%Qq5_)})6StnMIvR>y80CEY+( z#c+*{)o|)65-<~HS=JQ((37eriou>lE{nC6hJHKrOHlC~Ly_^oF3&9~Ne`kSt)(MH zT8!n?5R+WU1W|I$M3}4U>I?t#a*X3wHDtqXGF4Pg4Y60+R{HDAwF{Ah)P?;#Mp0zY zy)ca7SS$(ZxRREfm*>hbFXF_a>Kfn3#Mpis*6PnA2Rn8ldV)J*ID)Y*x9Xt94XoK) zG~7R@+**^fdL%{zCuO0iZcsm>(MB6uS(f*bG{w+}Ayq5QM4>Wcn;2Vho*?DoYugnB zOKOY=_0ebzLF@XO-h_XN#j2D`_(jH;+G9V^gi{MBb&RViAcQSH?cYjmArLLU2UJ@G zlz9c%yFD%p!2EHTbh25WO;*y33($&FFi%!0NM^=?^0~j-OOU*veN*}l&N{_0Qa9EA zveCsNHN?u1;2Ay|?moXNwA+s)=fi#e_QsZUR7_X5t|#Z=2D`tHtz6WGT{c=0!H~~q z_scuFbOoo(BH@;*YM4R`Gyx_d0TbD_Ja-TTID4TwU?9e=Dhg9lcuf~R2t0CL$IwU# z8;em1nKu0=DW&k6={5Lqf!wbo`Rzup4bnL!Wa;RG-!8pTS(7oVv1>?j)qZvnjG~RM zRyZf0yiEoMO}%g}l$DWH4Swy)b#$B-PL)+P=2kMLi6tfu5*bu98850V3I%Wq*rbM< zpjk0?@_mSS7CtT5grFc16m=t_!ph9i1c2D`^Piu^gKwO+P*GM;b1v3ZPdenx&ZsDi z<7xRG9v*q^bBCsy6#IMaK9EEVZ`CVqF!X6oQwxFc=k-IJAplsV;GG79Ckf)Vs)mDv1lk zi7HI{(5Tk)hHB3smvhtear3==wfM7as!e7O4#R(Z-dRMcg; z16&V`^pX|gRqFDHbegAB!2RlNgbYb%?^`v z?Z$Y0@k^-n!7rpClDTn5(Y4(<=CVZE;~a_cIx~!#U~Ul%_uO=4PgQWFtwY6x3aoRJ zcCqqsrs34XdCpDaOCII!)0Tt1_HjFl$iG4*fO&%tO1S~Q*jAbcwCN2Gb=7ywI6HSu z1KD7>!7qJuqX=%^EYWhSD~hnO;A$<&H~da&{r1$<;3~;F$tTcq&BchSUr76^l@j_E z4LB@)X$+oO!~77=KQ(-AODk43#au218Fv3u3~ZPKGD-{*OwXB9L2*&SSrbAznh9`W z^2Y6IX!-siuwM6d6F~I-*Gzhdac-Yv}INl#6?Xbyd0&qInTwy&StN3 zAc=4h;5W03a}QKEuWv?JMvu^Lq-aVknP=bN*v}JP+zz&h9uqZ9%8>JCaC9nS4wbxY z>XXf4)G=ifb2GyuVeFdMg<`cx*6bh>u5;OHna2m#wwKUa(N&Yg4nBg43#2{ow1E&zeR�}ivy$=#|&X-VE%FzVuB($rXus7$lMEZ16CTU>~C2Rny&IW$DP~nYsJ;0?Iz%c0 zR?6XV5M&5yjPh%!{1IRZ2);J|OK#;lE44T8KKi-uxs|CNHN=VQsPxG?W}^vd6gl>hvXmuKD|V?! z1!P$dldt0BS5-{}HS*x=Q$><9cy|7;O}y`4Jv=z6r@l<-m#2`R%xep3=BK?e zo*OKn#pYGcS|nBI{5@nW>9tSNH1&?K1vCW{X_IsPO1C<;e&N4fY%EC}F$h-a1S@5W zN8J#8+}n9+e(}vf-+E%d?9}{TK?3$?)U-r-l&R=7`b&G`gwF-~n)`bL;aAgeFX~wo zFkU^c_`^CAs~Z%mw>~W6ON<eV^s0 zcX~?$inv%vYLq|_W-$nG7!fiGwFu*5EqaRkRaGV4hr$+7lZ=U&H@?L#ebDR5{ zYjx!39mRL`9g3VQ2wrn!^G}65oQMuqRS7BOq33xi;NWy3uS^6ApNQ!^AjzCRrjg9V zSf6_?!?vn`W1-`$Q+!ho9tVQs53E~+V`ACj=^wPDq?nJJlZRYhPcH9gOH$PIvaLTj zZyTD#v9PJPx2HbTJ7>{~U7Y9>6BF6kE-7YTZ95-|FVh`vdF?jaJ^sS1O=~XcClT)r zYcG!v`Y*URMr$Rs2&k6Jf+OYv(tYb(*Virqf4twl^o^8la9y^&Z~72y&Mp5R_TDNg zt|xdGB@hViZi8EJcSvw2K!QUE?(Q-Hf?M$5Zo%DM1HlRI?m9Tk+zI)e`*!bIXRY&a z9{z7Ld-rasuKK#V`|IlUtF_07Uu*GPK8=9(#Pc-Z1%XYtZ2=uA%)hDMj_yd=at)5 z(>31<2U_ld-rb{JsXyw9_i=#RCl={;t<>}7ssfUsFYH@mnPJKK!#S9C&Zu+_q znOmCkw>`%+%6*~UIio(kosaY13as=D7o6f&imWg$1b4`GR!q&2jMLY^Q^Xf~ldb6) z7#PS`v9CnihowTg#^2aI+tJ9y<+n;fpU<-0^BL<%GLk^QwVK{R+?vzA^g5VZNBr)` zgAk{wye8*GsagE8OcH6FoEy8g=lhKC7nw18o-4o)ouO!W}Z3r!7q19O!%~ueA@PaCS zvr@q{8yB=By9LiOX6vk8&gx)OdnSPOti?SUtc(|OL_%4xa zhGcT>Z#WCxsOdbRnW;%L25uBw)2jHTCyO;OLl)iLDqxLcWH^`27e768fW~&}i-Z{M znZL^>Wz{i@v5D>Y`8>Bh`CJZGls`Q^&CC*edd>~Z58>J||LTN|zY+3t#HV3-D{Baq z$6tvHM!qrX#knn#-Yxt_{3~a%kC0?bUzx5as=9GJOAOKg2>6>!4!a8)$zUi@E@!}_ zG~1_XR>yerg6wU@%eyI)tqp{HH}7E;ByKYfosxeKPm)BF6gtXzUjC`E4$W8@t7+oC zl$zhFEw7ny5v1Hn{qo=Gol`J}C-wsB+ff6d(OAqhx9gQIUb_OjhPmM4l}YENTT*LV z3XR&tilq+oe_u>%5J~wpurjzQ{VMZS_*STYA*7G`@@kc%aG3RSdJp(VqvhcEuvbgs zZ$tBi%!LPKiG;a5r4ugp7W(JgU2%9~F>_Qy9qOV-g#=S(F7hkmPn3ARNqByvs<&BP z$iPPKYjvAi#-t@h#zK}Vwp|=Pou7|5(GC5UYB7>P;Z8|A4r;xlqK1#-_EcAhjH3k8D`Z=rh6(mQuCrSs>aRUSv|qp6l!P+fgh3Eg`J zwis1@dHpNa7J4#?^wSL8;)rjEK|M1zi^}GgzmAUbU7g7$9YWjMFk=_#8XAi-Ryy1o z`2Juc#n@4+E%nW=l*r(ntx8=LrdGR7jh*$CSeh!V%$s9It4>e!RFLP9{1wX73bjDd zkb-xnV`S8Ujsxh#%IvBFaY^VgbZ`rFY)98&jItycKPNxY+>WQ?Ha&ey2aPnan5sL^ zV^Do+Z9GzFCTOjDa2SCTb5fX+!EL*!%qOly_Q7laG^GvV;m#)#OKe5OCG;voH*J1V ze{d|5-?!SWp(b8`B`lw(Wudw`rQw{naS4536^;63yMmjGCk5g$|7bL-E>mjm5}{jl($8_ zaG0?Mom?=nLo3pQL6M&nmO`&Ks3FtNZKj}tV4*Fl9PL9-4x+1WYDr$iShbF;q3&DT z3%Ul|HA(K*->fZH8-grumEGK4LB-S~;&Kh+Qr(^tytvEz@b$F9E_C(-SKgFcbt#Lq z(iJbS6im-dy-fjSLYp$0u8^_I*&?S-k}8@~nsu1FJ6#ZrI0LsM06B;*(?Dq5%x%zX zenm}z5K$^^ei=qTgHf0A9Sh46pE?dYRyc2UdqGmiif!cHdo-bz=85`1rPv7z&qnSK z({ns-tT3B>UwkoidcSGiOk3z^H#&Zrtuh__WHmU>2E*}% z15v8y+pG6j#BMHFOYK2KX3{VCvGQRK&qk8Z-dMqc9wI^W{-lt_#@05^T1_m0VsbHm zpOTd7HBLFUS4%6Uj)IaCzpA>>tUNWkWJ$!p!NIA2h5qqz1m?rd63K$G>QK}tLys5aO)9KNyG zhUMpx<3-LC*71f(=1t5=*c_7%7@ydzsmZvMF>kckK5;kuBN_49+EOlpv2tT|h)LtZorf zy+BF2qkcC+tWup@zejC*;%+eJO8MxF)Y1H^D_3dX2P>-{kjw2vy)|FX=#pj0Nq%{I z+9`Q)NrtvJnrsZ7baMVSyGXgsYF$)j@TV59hyA0YyrTNd_553<@=9BjG#$Z`g;iyh zwS6}?-ch%=E5{a#iq<|Ttu0?b*VmJC%~7`nwsgx?8Dp7z2g?gRg|Zl}I>e!~4c~=v zWgK?Q7x-q+_7_>kDhe-n-p@1~7t+s4Ej7^P`nE1Klmw@a+CmTh{nhH7Z)M#BGl3xk zVS^Hqhp85;i}Jdf1m&*!bp}UU?By0pXdssY^swIHu1q`4%%{U#IdPd1znVp$%DKn|eV>y3g3z+m z8al^aThBPWZ`N{fn%R~pSr(F+4?{IO4Y$D%%RVtD=j|S<#4UvB)Yom-!`$5&6Q1gO>$w&ij}oQte~uqbM6Ct7r?@>WA3eHj5x zBTG2LBfAG&at2OWKIEq9(ld=-WU>l+zlS!0i&dkkB&AT5WU1kl4nM@BES^l)9lO-p z=t|2t(gLc16}*d^Xf_o=vb}UvC#@DSbL>|LB=y|3$XC;|Cw13-VYk;44fTm*+pO%d z43mBP8Wk|rE4mAlL{x-l)I4txanR}aS68In%VM<3(kf2pB_qDea>I(8xb(w3>@Rmj zHFzNPeX(SrSX`(s_jS>>;ah@Ql-;5tnK;9H8kgA9ib*1qO_5fMg$1P_qF!{!(^3jb z-;|aH&2z(Wj;xC4$xWZA5&7z?1g6FsF0{2bSL%59RMqF+KS~0#$+#sCEi3_I!J;if ztp^b^?97Z($3aJ=S$eMMjDcIHj zW}Q0!u{E*0{v9DeBCjEcEGukk7MGT^+{js%Wy#I9h;eyfHI^ zRTr)ko+ZnrRrbYU>bfx7@=iFR!%)fG8D96vnU)A!jKPHk4F!cq3xS44Tl&R@6blAP z0|rG~X9CyqRtq8wpZ92Wr>nmT=Xno%Jhy{L<`<{Mk50+v5R{{MYisMHD|Q#FE$#BS z>!s!=8v^SbKmYX@48Nf3@_@on!~Es)KyaI zOh-2`G(H}ZAI4c>f3EuSF2D6_cks&~GOUe2@`9L*=0rCCIwjq-it;6uQ_}w=`X3Se zj~2xJUnzpw{Lj!S+nI90%S3|4Qa*(fMfZQ1a3aTi)t`SB-jZfA!c@~R&%XTq#p2xq zOE{5RbN|0=z2(T~Ds63Zx%q0k!u&rVIAsFfx*Hlz=1L_bCMvhu!Bn92aKQ4L6u)HH zq7FSG+0j$^G+e2w=?>|uX=`f>Bl#8M?_)0ZGN#M92VSTeLnAx*oH_#s*4Nh)j`ntT z7VxBAfM>5sM$V)+P&o~b zV+BshzkDj!C|9Uze6cC@^vCcS5Arm=M^~PeX}BH3rl>pe{hwS`)E3@HvH4wyUHnoJ z?QLDtNBCY{FFnvLVO#$XK2EewtQd&Cr1E1lO*t_cW@WOBflGmn zx)_8GKch{7F76#715t9Uos=Ojm1~soA`{m2;P|r@ssCXBE^51iCWSmkY#gr^i3OGa zpW@V;31;tWR2H~M{lc}N5mT}6=J@!|NvgszdQOk|{Lxw}JDgYaEmHn^qiL8Jca`avKuFAL0(oG(&cXGvxZG_6HBY z>y$;}ON`>-r*tjzHBaMpZSvVvIyzTj*O2DYU`=)DURqZ%9(7ZJNK=`xI{c;3*k(JN z;YDFzD|LcZ- z+WB8;SHJ7VGN$lopB)=0pq9=?`@2a{smf7E$3t-mLg%_uwxVW+WB1RFkKDNq=ZRW# z^Vf>L!A$=Sjyra9(@wGaIqQOBZE|1`=a*%>pmBFJ|T9j$(4;2373$N{;8zc*n{Irz$3>d^PaI2xR{(R^a}ZN{ySC=NmcE z8=JlL46nZ{a0$KGAxL`RaH{7IGKigi9xnFKps?c892oDu5={M0RKchlcJ+G8o*i^l)Z}5mkKm5EH-jQ{e>k@^|2R$l zutgb8{o&K|fSKRX!Dv~;azftse7Q?>4AiLnMm0hrBL7&dPeE(0ArbD6WXIklmzP5% zt*_$A>DOYo&Y=c9wL(9ZK9}Y>w_X`f2nHeoMCG#E0-kc4w>kQT!~qiR=5l3sDyaBR z)ZINQPL3yU%sbPxgn&0oY4=Nj%wJDl4zw{~R}vuD1+DhzShSC1ZJ|3R2Ho6;2Y9e0 zd=vZeH?8tGA2@Cfc_3>qo3C@+83>I0sp1p9czNUj=|VzxcX8OhBifXp49Bnh;_55% zvB3XeZA9t|Tj$Wxn`lSISI+XuSSu4lixzT=UJ@X{VRq>iMf$f$}JhJ8N6#b>%y%lhlKX$$xhg19Ck+AkO;;dASLE3fmv-z@*%jfe&y}aYozPlx46>GPm*xc6_NmB(HW&#?gr`Ny8 z(&zIl_^r1$XIq179{wHo^<|zW0y+NFE@z(pH+~-bVXnAqJ?anXBsFayDt4W_?&cz9pBvWqklP`T`k}(b0(xh3CB50uv+-RPMh@ zjT4N{wYPnQiyC5N$&y2RHT1;7EBnrcB`1Iygwd&dPC!7Ub4eL&u7Pp5k=EI6dZ=pL zBh>=*kt3gp*WXDZgZR@0$wT0=JER2EdFkuo17{pSd_>o1?i-8lKAEWT@6% z8Vfgc&@k@nO1Vfeo{JsKk75X#(f~Zlh}uU%{__W{F@0HaBgUP#M_OBDKzQ~EcdxGzlByXD*qx-G zjdYb0?6!tnSd%I+?}33=lhqE^p?sMTYro}Ae;MA&qR^QW0~SPV<_Ze6TLyq$$|}5- zSG?F*_1CS!x_Jsj;3T@zhfqn1*L}tlEAT?@;zzd|pCY?_0$F&!%9ZmD7>*D+f!rs- ze>9QAE>PVfBN=tW zXl?nt#Y(x_z=j=)HrHG>dSXeI82U5~5L0vjfA{^>vtG{E&lj-6x%D&4jlnf3C!iKk z=Ot|i#0nnpFq5}D^BHVRD(Vt@e(0*PJhKIiz*z%RJw}KP z7Y_V1Jk-#LqgmP6h3K-1-o+$jc|`bpzMFZ4U4*b9yj`g+DT)*v4UQZkpgfbO44U+O z&4hpX38cV<%mafiyfh#-UZa(Kd%77?ZRThF$#6BeDK6Amz*PHdz-Nlb+wvSg)(`S^ z6-l1`?_|^I8_rA(9tU9VJ2PfQj2*-vpi8Dz$WUjY9oV844Cp-P@$2K_u6PR<%)fNO z?2WP3d-)Jk-y3%W-rT3a3`ur-|etH$cPXf^8 z`E(#OxM2^B$}-UEx*Uk9F%i`htO^+;U~{59vG~Gvye<>M)C^GvcI0$Hq{|2XA}TK%kb{O^HL02@A=DtvXf89yAIzvuEaS|Z?nw5Qc+!<$>~?u#d)?3 ze?F7aA`0+#mZ4)?`d?fC5ZZeqPwm6W*TW1M81&;=#$Ro&Go4+4drPDn?K5^ZG-4R|JEkxkmlLlf>fVoKKI?gsyKh1PC;3M~m5!GCM8va6ie=JU!!5W^~t(9BmH*hbx8a+ zC7%CRY;x9CABCQ#snt67-eZn~<3T=$DyP8Qy}s6p?8>hEdx>65(%iG1%1Q7cjOh{a z!h{y=9>(RB2;2D=`+x!ZL&P`l6Ql?n>aPg*f@iB3U_j(~?p0r(0;c=CLW*@rVa6~c zXc@{XauJ(+K%Rb}DNFe??;mJ?`hOECvpawZKy;g3qlmnA!D09~l*J{17E{(ah%~Ht#Dhhc{lmkC?0aB7t`mzAg_T`V!(d z@V0f#>3Yp4(gJf+q_PGGq%Gh==PiU5()XYcsu2@J3^(Xeexge%1ju#S@z$=jdzBMF zx!F$7?|Jvw=E5{aouvibdTghZK*f0X?sLmX^$*(XC}z2{1WnaA9{;hQ5Q-IVWt^zh zq`&pQY(muvcR-9fdgSmY$;)UxlS;O{jnu|3>3_62XZu@^mCqFFkeP+S z>RxRGlnnXY5V*k<{VI+-E~E{?;XfU57KK*`!_v75bHGOa*f9xW&WCL$ie*`O7!D3*Ox7 zid-x1+pjgTzXMQ}iS134jSz!n49DA2Cc8pEhYB22zLEHC6#6!dlAUo?~nZ#q% zzK5InQ`v+cK!bi6?`w`!?iY^nC+69rwJCfmm%p`h`qy~OTuqLkabY~+_=|k}nZ_BZLSf_~^NDPW-~yxjU}G*O~&pq zd?=Jm%F4Dm>`$A~^{hHcdln9l2`*TPvRfjFy#%AMJ)O2QsA$hl2&sB&(%^tM&^%|@d`DLKY{IhBM zN$)4-1hKxj1i^P-YYv$L9+feVLc*sS_eH$$hxxGEwOvC#)5azMYRq-;e@xc%yQHof zFqG(K3N~5YGcoO*?F^M6lkC|H&Ux-wyn@<5`XLc`Ua(To)-7$-I32+H<=5ei;OKIJ zbuprwbHg69hGzGZX#)N!845=@zxxwvj;zQ?gvzut=rtU3_WBG5E1rfL7d|Q;t3g+~ z@8kV16seiArsSF+lphH}qdhEk)v@`NBsZtkz z<#fkiN)B*6!O?P?5i}%GW~iBz^Mxe4*b81N@_KG@4kNwDcu)&j-v@xz{()bZ{1s2t747~a;8tKTo;@D@aZS@J-@MQ42~inJZlmFogjpY{>CQ@?u{}RJMI^QYXs3BvwXH5Hu6RB ze`j^f)HM~EJGCOsd=_nxrAiUC;a|yyw6ja&4pjj|Ez+}JKm zaYw|j$g@v3T*h)os1MR+&o+k)h2Nw~3C^d-d_l9CSj}+8)U;lWhA2G9r+XM;H4(s# z^XWL6_P}q=HE#3-0f3y%K8l3jWdrQkMH695Rh9RCp6>Mz9MpGIV~LZ&L~}HodX!!1>-#+d-I(`!(9!GH zQPgQ;w!<+o3PK8WXeSg|&Btp_hY3(8+k>6;+m_pPbDOH&6Nsh>{>4`IdGcYPn$Z*o z6%~fSVDp}g&C~;krHSA_^$G#|yP;S69(P~>$l9dPG-adyN2s$Y;d`kABYmuRd?m)Q zX3dVVm|HZ}jtj=asep!(-)M!idLcMS!AP7%?6&>lSUD7V<-Rbmc#`<|4o@>kn-a8r z%J2-24z&H)SGGsz*%huBo|gmV3u<+lSx25p0tG-qUZbbv2<_ z#Gw2Zn+Aqv1u{Dq4 zt##QYO$=c2@3W{ax=P{Yb%>m9my#Ti79r<%jZ?s__38g@d{=7UaJA*nlq7b}&y$|F zw}WnQKZfbMV=h3@7Q+MZgJC^3hK#UPnXACZM8{MKJA5Cr7gi~|Yo8JQq`*HhPLi=! zbmaqE{Nn;##M`$c=>x*}5*HAF3C{@phHN&dNJQ;P$VCqeXgk1bA5+WzhF36`;+G@y`FK=dXqvY^rIBMY}$V z>*z%5QVsrt1J^grbkiGX?-;aRUqRr{i7#@V#_Rsm zaB5nIbz$FUOgW~zuAnQuoXGR}nt&;_aDHh>5I+sX!VAou9E|cqQ*@TLxH!@?$f4zy zw3p_2NXR9+1BJb)#pX~@?Sj*amo*Qe$p`$p`k4&s1UKHv8F67$phkm#0!sGycv)34 zS_mmHDKrpOB|l(Rpeu4&?4qSuYMXhmAV~aXBt#ifTU3g4rIp7Z;3UpHORT)g6WL?% z-M8c|>!|kph8*w?Syu5i_naG-xG)E?TNLt#>N;#ezwQNfS`hlj%gmrx3{Acw0js>oxQaZDhf$D z_b-$wiDu2UfjTz4BJwm6aQ)E!)Dq|LYUSj9-P@~2E~}qOp5N<<{`_osiIJt!Uhlj2 z`4CvxEA`hkb1_144!l)uXoyXf4Vd@G$y-AG#=wO9UFEBQRNh`&vBhy)^lg-lPdM6< zCYH9Al!+@f;NjeRV}FG7RB7SE&3Iu`oF_aSs9j*mkll=JvNMgXy(vXZcmmDks^H9a zw>1#vofTi*#oix2PMcC1xnb=qzraSnX9c~;F?zpU+sj=x`lhmgCrN-rk$02N19NX! z^XsJc1zkffZw4URqT0~nTA5cja{O= zXT;0gP=6DKW~SwP_nJPYBe9M``FMh@Nli;mD)?(rc5!5QebfwdaNoUiACc{V;SYW# zXo&j$0#nPzxG=MuD|}6C#dL}rGb_(=G7(?0Ee4p^#l;m=oe*}BqByHzXpnXR9z`RU z`-H?k)w*D{F{e#&C%zxf>B0`g!^t>Al?n-L^lHM2Xa0exG|7)$MhGZBi*BI}8EWKD-r#Nu%Qq>St=@jhOYgucen3)g>rn$y+-%bK3e_T4Xfgn^aS z$nLB4QGd(+!DxcX3Mv=N2x?`&IPK03Jah^wYli>>M;~hX^igO9)Aup6z;|4r=US!O z$hI&Ou2!QXaw@>z`ns4SIp%E7hDuf&w1_dE)1Ua8M6^3rGX{&TJ^6BdpF@gDB7J2} z&Za8#(jrL=-`^WiX0F!k1eTCr2I2~GY~_jC=Bl?VUP1R=Qcrem>=`3c@)FTj;Ug7$ zEu>>)O@`@jfq6q`x7L*t7C{qiG}fjo05apii!hI?>klIn98KSc8tnU4ObH1Q zW^C#rNHac@p!UhoFMKg5pINu%;zO?GJEeCm|%rQ<#d1x_XWKSjsxjW@7DVf^*kWbYWo+0@+0eJ{|c)(8pdS!PYye) zc#-M%7?Dr6e6rh}$;~@EzwZ+->vr#Z^-e@&pBy6FZp&#M&lXe8sAswWz>VAa$Gf%5 z0n|@fx9jNOWB0sT!N^G}gZ$h5jTvNscva-krXpUGH-n+htBed%{`zk`5oa6nXMQaP z!b6FQh6pII=}T9(1`0Nx)VlOAZQ;{zX`m3|s zugq6Ypzx`-ZW4<5fuTxJN=?jSG^U#F0rB*T$)sGqOVc2bcvlylI`rQZeB=CL*?FkWk%$Fa$7A$_=Myt-QblEnLmF6y=_G!<<` z;G&)2e)3A9Oj~~-QN8l=0sKn-)H|IPM+yV8xia%IkL`T8u~Wt8+37vhdRC{=A^=}p zUBclG)&?^Pl-ce3k%#3YkT}HGazy&Dr8J_^1hv~5Vt3wT?$p~0E3Em}OUg@ZW1z8n zY`nHlXW9^n`GZBSYro)cz5qF?{c0)`Fe$#8To2BEEJykCGISQaF+RG!=>PX0g5G8g& zWBJ@I8eC@jIFR?zLNwd7NL~6*3;3X*cNI5T+xs2U+sR_wllnpGYXv7HZx|sCC?ubh z$@G^yIC!UA8{e-KnXS>X=^L(Dj3=pDl<6vL-i(HG3iq)SF%1XZVt)DI4}N9Wrj};3 zy`hNRx!zX-gBSBrIe}h7aB=TKP3!1H3!cSAjH zOPbw@Cr}T;hx!Y~9yvz6r>@(DCH3w{fOaxBg~ltXcsf3e52Pi;Bwx*u;SZ+$;0HwW zb5P60)kf>=w>>-yA8e43*ALIj63OS9_1r$zv=K=5mrWyXU8KC`X!= zfWFd!#U6S}-pTV$3=4dK1e=J5+xAw{0E|fu_l%k$(7&Vr4rpAv<#a}&mwGeJv^qb` zmlb553D!kdzEC)71#Yq3P0gj1jCIJng&9^sggSq;`GB9Fq?2ClN6Y~eTDs4st}{9} zhE%_Qw#1u!7}RdqJ0evu6lhAv@M#K&XA>EZfjTQy0aDj07vQjAJFJaA z8f^9=k5{oR^dyQ^9M0s$ar?+<|hFMAhW0B+JJ zEZ<|tS!jf;ZYC=}F{LJFj1WWL%)DnLMEnojopD2lw+2VmSY?}VX<1ZHPH$ucv$p2f z+3$l25~R-ZtlFx=ULmli@3oZnb1{!7R`xefhFE%(1DrL4+pM^Joa+iG- zgXw?(eCf@|=93s7g3eWSdZSq2)x1Zs2X|ZJcTpeX0-{<}INm4H@>uEBAMetF7mNLY z3(_M}gh-H5Xk-d5o{HOtKJh6q=<4x2TI$-@q})WMuVQLqv(tMVEh&^$toF6uLXxeW zhu$3B8;Z4MxVnFyP?L*uX+&}~JsF#1g>L1%(!}UHsJvsjyajs)JP9VB8uqt>ALWAI zX04xiL-6bk_Y8&s_B&~n_PeFPny5+8i2#ph4 zq|YV^3L7s(?SDhd{1d_-aX9@c$}dULR_#5?+=tsRn+%gv${H|VSXovDDb!QDF(TJ{ zJfA6rC#c?xHP?!5@c6K(kzbpNS5O>|7b7fUnZ^iP>T!OnY3h;lZ3qrPd)eh8grwS5 z6Z=M!S6Mf`dFSYE9jEr#1y`>xJPMHZKnN3n5;_Y^uy`;}GxGiu1t=0!aAX@)c3^Bu zzb-8TM;m^EFlPR4s3b}>Eu8!EHTO`OT#@a3aa`G3nTc3zG=3V$oB}_0{&^&onJt(> z_mKf@ns*zyYCje~(}%{^(GxBW8qK#{Ut?t`*aq;sX&*`?6c${G_OIsPy}Z1Xeltkg zn&siwUK8!;h^5mYjv@Y9$6e2QVq%5M3t;ti_tEHJcv4%%?hss*?DRC95%HPvYPWkm z$bV+X-*l#|^y@si!Gb%hL=Sgg*JN)xN$J(z8#s1umyAI8D7(treI%{6@C2gJ;kggB z+(S@z>FOaZW$Lz(-+RAro6fiP)BFC-FBLcB)b%J!0$wWDYv;4}VF57geShkfO$&Xg zi~azk{WoBD6W8Xyig01+EvIvD$syg^&;R0be@tkCV&F~blNf*J{UE4w54bVq+o`23yaj>j7MMI|Q2a$=i?|1#Ane35vv&{0Zu294 z(b=8RYSZ_Qw$Sq_Dcn&rvHnC{0w_VgGpkmEoew(GGmy3jhKiSK**jGsq1x-h_D2+r z>YzFKBk@j1OLkpYOl9=ygRgvC{xXhU_kLL-(b!IbWe{Au);8Ukp zR53opAq$WlOz0W2*6I>{t`xh-2;e;OiP8jS0E^J)$I70I_KAtt_&ciH(#?;QGQ7}8Dr>0{N_itK|O11RMxz=w#Wil!CR(|gs-)ovStc&qd@ocX06F1#j@DlIM-8&EOujg5xOQ=h81KQS+6%&=S=DnXs} z2GR7!K5Dz9fo0fO*A$)acb96ppcEHFDs?xm?+G}U=WCx&8HPLL&HLizn_* z(_+UtLjI~`K4fcc>*5zuSmUzf*7Jq@qtZc=Q>z(mbN^$OWfzXlg6ztYOw$wwQQ_2t zkbJs-z>d#)?BnSjD%)S)4@mjD`+oUbY8b|q57GoxOivX@ONV!a1nv)DLDFMy@6C2~ zR|z6u=0PZqe9`j(J($C5p0Idt-46kOHCRzDemPA49KC>4#rz{e6>i; zb1oYi^-zwEs3lw8cfFDaQ_(!n!uj?PmS|0k8lT|S?aDoV1 zD3tO-k27w;C$3#6dda-=cj3>owF<#2$%rV$4TSGCHhNAd=tIll)C>ApFeXr{T+6%n zCvN|ma!eaS-dH0_HiT)!4$@n)Gma<4r=;7&Qb z))SC^X^&mX4rqsk4Af?V*#e8xU#vLwPT zKi|EN2dKy!+cgn3{z82p9u{wv{i54b2{nJxx!_5;koYR-|I+nbS}GXgb<7keMQB%6 zRu7F@P8nWuY{yX|;LL9mlxj>^_(oEj2&LNHJ6`(APsd*D$dUZQM_5*(?AD+^uw(ky z)9w)^g;q*@)zUg$9Ptj+zFDdIlUCH$5l}KE^r&quKlJ`| zXeZ}eL&>AH=rpOvUb?p1F`WXQ5^hWXn5lf~UaRcWIjn{8SsmT^BFhuOFC88C6?$!z zVoziW0{MNz?7yUq5)Ge77Ba{C@+%mObvf7n%b*_`Z7ZRi*N!Go_8|PrkN>E!1!);# z+bTMo4R8{F&&WbOFD}JxRHB;TG+vK?WM8$4UyY{!5sPu`20)cKfg#a+NJQ0d!)fF? zpU|HL0H{HTEO+__{9saGl_Lv^{QCrE-|s47-omac^BDJ6MF5kX0POcU*hv+>kz!0~ z90xEkSln`w?=(D47X=49&na>yYbKx#FnryuF7Mvgkjjad1I})P=cN9Tyhd-nCyL*E4fv$e1dSu!7PIxV4tap}ezoa+pBF`P|I7ydx`jf%eG7zG&eHh} z8A719Gx_oImW0NEAdT*~>~`ty{rEm*Fr3er_ zf}9m8zae}f#c*&yrGP_8_h%3FKgWdI?PmPZ#EwHSSQ|0Xvgu(Z*~)QZX33ZHbzSkJ z0_B@E8`x0IISBqE6?MT z4cD0+%nu&hE-ycJ0PzV|H{DQs-3@m%vnsl$BHFy4Z4JCn&{lzG#Cy%7kCh!`^car- z3TU~A^nmNvQ2fqw@h4!-((XhZb239}!r>zB7l$#gyaX3%p7F$}RM*wPlwqxrzcO#| z3m;?delLz#g->tTv{R+rW?M)6G61yHSj14N*Nf05Gcgu z(?|S0q@q&)^?atiRh{Gyf$pYmoBIjV-0Ji9ar;VJZ|6rc=5*v!{o1OnxgQ>N7pp*n z9UG@TVY`camaflGQ&h5C`)n*sF0pB(3}vK?*x9vTQ?YeiU|B1yZ5aANa9CNoqcqvB z4x=o+k&${~w2U{V-nKmt(*}^SB#n)mjX}N83oh~)Ihq>`BJ{^+3h&eEjW8(0Ag#y8 zh-w>(Ivp^viMfZFvoSk(Ux_DpwA3n&8G`9hx)#~Z4yH_@p{RfFjF=-yQ-4v$3a-s; zhwd|{lWNYc@8-nOSQ?s-m!0KQyFK$5bht&8CIh=<6!x@X?Ajs3bAFJ83R=d;FoULn zkv)R-D?X3YDU_$XM8MT%fr1{m8>8vrq3D=t18VYu0s7s+f1pTeIlW+Qe-&|-WzGmr zXk7D3-Y77m6TraSiCf7q5OURqi@xY()i;eV$f!H&;)>QwUi zB{ye{fv?J)qBU?|8Uov66ZLp{Dy}d8;frzlYv@FAqdM~6sd2M|>hgI)-f&!Sy^My< zc?hKC*Jv7<&L8L%xkN0ZPVPa8i81#(?gZ#IM0{7@n(6Y4Q~ojLQwM&@i1fOnJt7>|N2a0q@?V@E%e{5w5VkLfp-m6zFQyQ0lrDm5~NrZAaEM@YA%eloLb2Y`J>f06Kj1@I;xE2{Zb7&!)LM+H$ct78%V7i zQm7hZTsJIA+4x)G;Qw@Fay0vkvABKJfSO=C%U4;`>Gaelz_k~uZ2XAwz$>f;z3_U2 zkky2|;OdIxtOQ#2T=oT$o@bG=Whrbvh78N>4<^YMk4-M76e={i zIAs=x);Z&{2m8KBPPQ|}HZA4*rk!j^ zV`4LxVa3oh9ljMhM zRP=F2z#nFa25x*cXq6X@r&2#vB*4q(%vV_{T(RJYG?s2h=-t@BbR zBYpd#6lNxLgYDA&2%|t{F0HYvD3>I)xBX7-;&qrR%rapN+^yVB9QE&0&6SIWl2Ts> z(PikkwS-4iiI;<^bKU%>4bkM1TehvytPGcZg5A z;hrD=$;wwA1bIrn@DMf=Chzt5z0&Hw&<1%3>#Tu7Q}N!vlbX9ThlfnSmPb77u-9Rd z0XX!^T?5yJiwwRJ~elu?F_M*eg0-ap7C4{Ju}-Lwlacq%sk?@3|`JW1fEAU#1USsKKi zP?rit>#7t%)M9hk0KLGC3W6h(04NUCT{E2WF`BANz6+4Vj8ohzsiR*bRU4^q}VMmr;B7lNIey6w&!DffDo%0vFN_`SkwIvq>SL~wn zIB7yTHCpNDUj`JB3O9I?Fv%|u1`j-!K?v_k1@GU%kmlhqt(B%ly3GwSx$RnKg5LNhpYP$AtCi^ffheF6c=4=j|Q&(JVvtk<+ z!wO?W%0)9nla%G-%ao9H5HiO;mU74`pP0inQX({2VUt6VecK$94my|)-`afd|G(e+ zKF|AH_w(NOdp-A$#ncZ{R7{+*7h5F#FsO~qNIut1Y4ot2tT?tzXp!b;G`+KNh_7iC zHFEIv>adk-Rj~z74v(oFF@q~(PqnC=1*(p?EpeQy2Ish-tbQv@29iY(jNJD)tw*<+ zV?z{7&5w{rIqE9q)NmSGyOtz(3Bs*yh-+IMtGOU|WQL5NRL0Q@ET64FMA3QCZCE`y zMUtvxPmL$C!&Smlv@Fex0bjRWzj&hYr<72Pf?MbP^2u8HrVMazitnIy4KFx;8(5iM z=?{1uSfM5wRT6(jPIIEBw{jjU4T^@ko|miL6kd6w-K<@&KGkLR!Tm3Ql2-(88EWcn zXNW^*J)QHr_Zew82I+Re?37nlPE#ruCT*C?{X20@*5yINafeTKZ=uF`AqdTw`VEaL z-o;4KIXh=2ws_EA;+=p82kgo-M%1>4o8I+54I3)7fmj7s)g~Yx9KEtI?wP!5Sglf_ zmXTBh;Oh5*Uw%6sitJ)-8xYUG_Su&@a)Td8Yyv}7(wt1BDoW*|aUm3qU4Sw}e-!n+ivZziehqGknoRLPiM{F; zMiE6jnCoU3rq|rf{2p_1NA^{pM+atRC8tzLNE~ZV9wD5!I{Yp+FiH_~6I~#N>5JF4 z=GiJxMCSNTTvup?v5xh88iO%H{@PXt?;+7#8gT0_UXMDLy?_pcJJ{mqbmIP$d`Q8e zQ|IYn`}N#(NHV%UAE^}+zf<^0scuCAw>S8=J?f8D>yn;UCEDYPY@#}L9YI1oEnp~= zp}n=}llHQ)xYw*VtSq+TP=!e@$D$pPMF(t8MDn6fbxhka5;EO8QyLG{VMT|$m_fH^ z7U6mt3$67+xD71A2PN$$GWOL3x_wb8>U^^g4LF9GW=Gz2U^ zlUf=<54}4*q&2>sM6Y%{kMw@H2Qq%b|5>6hk6eR58wE>oBZG+^*C1OWa- z0$E6m&ai(`ip#Y{sHl1)Tkip7bLkuw=GTi9eTC}-;r#+=OmfaZ_|HOwj;?MLT6iOA z51Yax4iD+uNi(vh7#tEJN(q`k4yRx3b-EZbPCaiTB$?+1?Xb@dMiX~0a2pD0)?kOJ zG?}R2#Ia1zB==lD2uA@$zCU)d&g$BUsCe9 zf^T|-o71z^eG=e44oPc>us(Booqgb$Z$jv5R6Q}gS)%{_e))pk%isA=BCApEP@xQR z;n1TEO)Hb2aW1ln309ERe7sS@oCXShWrGcAvvr#sdBtN$*F#n)CR_UPuGt};_mRAHVICQcTEkM3n$to&r6XIpf@GM0MyLe&9CnE3PnwnLKY{= zLj?{ZIS#Tp8hc!?{OD0HcAlNkxh}gQixM<@f90z}-QF*%h{#q*L_JYZ!`~XS!(sOQ zjb08`3L;7JD`%(JVI`M97PAg@c~m8Xyz#sM-3|3fCz+7oQ^s(7jPJ=zcgfmNar_c; z4K~TgE64;!38hLv2&pTks86-@9jhdMpU7P^;hC>d=y z$wp&+sw-%~0m{z7P{s=IjF#3<9P4GrntDsj&lp4#;e~?z0mT@4kJpF zv-n}}SnhrQhg9w0lFL0aW__V!2FzQRq^&%Og2qlj8$zft-SGi>c8;>$77^aqIHNv5 z=a-3MqXp|b8JlIhXgYAy_^p1-Tg3eWec9eMpg^-HsgL_B(=AC0n=#DC1@*-V2?Eoc zUK6SKN5AE41^P{!f3JC0v}wTbckrf513C78mj>47U+2IN?JgvKtK@t|6ZRG zX%TUO522hTWksR(p&-x^u=^w03PC`KKqN&3RopYqHvClP)UzHxDCMR(F|0~N;t7;T z>ZgCx!JZ9+RM15z?*+0BuilFO+M4Qycj{zu>7>neX{>#_<28M#S6-cRhIAXE_nl+` z<1fZosiK=rQo_SVRQp}l7L%kKHY-6$yloSN1ZNJ-413CW1 z^uZ13^B*wEZEC`7(3;@vdXl{q2(YiZ@y?ZbrH2k6`umC)m^-)wXXW!pL|*?K^g9Tg z6>VVK{%S^?(BaRCYUiH{yIi%hz{y^Fo=lAjwfEDm_Xu|D#j9b`pPw->I4%R=dWCRt zagA+kBG;PioStq>ziqaUoUb({w`%;Gpm(Vs5J|Jek^FSA!uaWCKPV_D^mnkmeDe}3 zhZ($$PK)D@&Q1XW-$z4do{fxXVlNoe({%%Fv*4sq@1Um>Y!@y#N_S4;*e2eq(YyiIc=Yu}f z_3E{A3$`?JhN%CobikEM(Ts&X7vA@jTiMS6$V}A^`=jUr&qt(6%F39gqVdzy3Y~A~ zcDKOVU$2;F(?4EA7%~hF4nEFhzxQ)?UPrZgZnBzBt}ooe{Co6=Ms!*=`*vgm`LsbFj;((0N3h%X>N#n$ur^$Hh|kZ@81yCEHIDmIz(mK#$Jb({5Qe3pj0peJ zRPb#*venXFp%#8fF?HzTD({{i)+Q=mr*u|fW2`RFo+wHGsGkDQp&^96?t&rNGNuc^ zRQX=7`n>OE#3Ote1flyMB80px|5K+2lo>U2jM@D!Kc>C@z-4iAJ-t+K#y>11fEvjJQ4DaCFck@uoNWha?{K~^kUrz_txH|V#WwyO` zcYl64(B67pEDz3EZ*~Yw*7Jnwy54ox2k^d~YMynSrLEy^R+sAU|0y^mq{aJ*#jFk+q(bdfmxhCa0(cF)toNU;eC{cc?^ zy3hzAlAj`Ls`|ruH?z6`F9gh12g~3dfb&Lv_C?K<=bk2Q(WarB~k{5Uq0q4X}^zq>@mdk{7 z-}(WE*=)ZZ@Nf^9#mLl@e$1__i;JSZ{#21{Iu#w=UFR0CILgIx*JpK8wbo0Ontqm} z!8GBcwh8lzvBdNeih*C23`k8A)oy5{BsBM&{5-g@pQ|H$Me&YtRann>N1xo%j+XYW zOqBL?sWYSt3YVyBzy2VU#^IW% z?2p$&!G|zH&!b$;F7-{l5A$rF!PN%qz{$lFta#YS;5s zZbB#`X6D$<%}r6HZy5qUX<+eh)Lx z@-XX}LW&Q>fkGkx|Gl5AkI@q)x(8!bP*haw`NxXD=S~sWIH>OZvE&x}ZP2Ak-Tbn$ z2w=bHB@F)lW;_(_@VnbzQj;e~)b9!RiB8}O|funpsi9qYuM&tgVl}a)-uLs_-K0;Uf$vC-H zPzVm41HINs$s-J!>TMjOZWA#N`EkN;L(cbj=EiXnTh;TX$l!*#c-)RAYA=@?ZEii2 z8w(2q4c1E<9iHsJFUG^mqXrIfyRXhgw%R@70i~P)6kbPXJs`8Hp#lB-&5ffSQO|D3 zmsjGhmyJsb{)5><>4fBDr-SiSKuP|tJm;2od>|B3t?!#>Gy(spo$oC!2!Y?LgYWAu zq0{j!x$kwCujJ^*6T&+&aqPzfDW$P~WGo@!=wOLy`O9DjD~2h9a8*11dzpGw-VFGX z3JltOirt}R{*3*flLZeb4!|+xK>;qYSx{FOALPsP%+KXo(})Ma8@2re>=D@P$7H`J z%*hPy-3k>rY6XS9TLL^jcLi|HivVDkKp6DehoK0)zgk(*)YR1U4-SIfwrTx7J(x(J zUs{rfwQ>*k<$qXr`p(-8?CvK$pFgS`wyR{pg+z0DX|n!0feqw1`)po3;*)c$^HSNC zSem&a`0<1H>umL?o`I9Ob@S?+q~`cJT-3ehFCME}Ye(z|#%_`#mb=5_6NdRT^M~(< z&oXK@3=pFwSaD5UG!9pjppT-Y|-oA*ihc59o?#pQRV8lARFy_r2?@n(2J&-`kz72ofP#M8TS(#IA> zQR}wlTAxcH_gb6e)Oc+lFC>IrD4LugrQDeSp2V;|6K|GI#VN`)maZmg}h0H=tx(sU`f?P-M^`XLKY zV2ATn_D#TTfpmVnU4mU~Guw5&aRAD7{|*gj_bc){pfhrSS4w3y$6%}8*4%Ju&Qh9slNR>pfEcwMzfS?_&r3oo``7 z9U9KxWedyriNxIIweQPoUmn5M+L{{|UeJ$;bv&)e9LTR48p{FnFRjZxQSp@Wj<*N0 zc143K|1%6g$Vjo>fQ%*M)*Q&@Kw6S!2e<#ZT*k_Q@a}9Lm+<-d`J98(tMMj>lX>b( z%u*(>;?%{H{JYZHay3%0;!4tr&&GZ^B?fG$uNuQ6~F#qyfm2+24x(L{a1+o*4#i4OPbbcYe}o z#oM{C+WJbOgez_)gYgR*GbtLCjSWvY|ApF8f1%ua48$xsI-@-5D2Wc`cQ=~>u}<<2 zScCcTZhJK5PoR478a&)HhQjGdV@XsHjqe;<+FLn-9^d(KFa4*3twV|1M7({8M`C|F zMj9Mn7F3j`8Qv>r@Cgqn*_yo^$VF#Z@AlpCyJyGkMXbeuu6;7X;CyEuv(i&4Yy+R0lD5{P4)4;vjS%YIA6!i_gisq5#YB z-X!mjZC;SL?8|4<9ftc89RY2;lr@qDU5lZupdX6TIqj6R zwWR^?0s{l%`;al?;l{EpA z{#bI}0f7(c#CYW!)vCjg#HcTDK8YA&tQdd;GM3CZwC_04ZXAQb`ZtjQV=^^OnRv`y zN#jyH>dy%dp=`beQfn1(&dxo7$KQe9jrsXgaZqhHS`$}SS3Mv&OpNgh$@k}7^uHz% zkQe4;{2B$l+w@>6uUAh}QdqxwRmhZWu^u~ zj$&;3X5Ux^n1($RhnQ!&1r9W&&x73fAKB%%tgqm>53i9?n`<=|x?t_g_Sm;n79`(? z$r|k*)~kdu&2m46LCkR9{Sk&hI=~p5TO5Ku3G)wGp0F8WHxjp@1&JSrj*SzW@6M;9 zk@DLqtnqbMp9vuo{9s@X$y_rE!#XZFS>PaXB{Z1|z82@5aylFEJZk|QuhcCf<8;B$ z%*@hQ48DDb_ujTKk(($jQpXl~KEy zL?Nk@MQ&WywMOyec}BC#ova)9GWKc5Htr#HMTR!6*u5?$=$KHPcqVrwm+TF*yaof6 z3bk^ihS^u>JN0+d^Xom|SAyxNFWWId7_gJAzXxMx=YRT&JcTsshHQPD^BjdZ&Vijw zg6AT`!|Uo<>n4*?QZg9l`yEBT-@G9HmoUBs?AS8%K!8!Dzo(veCBgTriGhUuNTnAe zez46Of^F_>aK^b@N4mDLm4;6SXv*fSmoA6~4K1v!vu=qol^cWdeg{r*%g}US;zyjC zhP#;o&B45-6&AQt{WB=psnT+BuuCBg?d!jR;K#hV7n6(V>vTl7+VRCW1Ox9tE^YHT z=c5+Cci||3g`xg?e8iCcfr~WHi(gvw=*!iXeTnkeW-WemUo~tmyH-672z{#t0>drw zDkL3OJSpXZ;h_R$6`^Hcs-BEPE2OO%;Z{$0H*Q~89^&qG2=Bd`|U zUG6(iFxl&umBN$a(v=_<7e-C34^{~rgw<aNRhAkZ;-2O=;_D=Hvzs;j5dH+a_0o&H8dct|kFKz}eOME9Sj9O+LB zRlWlg>6}lS51pJ<-Ggg(UFWthAwPcPE?9M7rt^h|hquRfd4UoEIrNnt9Xdv`RH0DW zf)PhLkmldjV-_FMEJsHDSOf|s2Z!tWmq)~N_nk1xlFG{W6G)Cq{jSMhs%1s^Q#ZYE zjPAVi+1@uW(4aRay`00o$Ou0iz>Xr_qH zPa6>`kGZ}XV|;;-ErL^d+&qN&Mz>(oqfg$sBC4o3r>_b}0cS;FR>y6yk;;WO(07~5 zZ@$O_tCW?;yHBJ_?@>|_m5?kT8jKDf1yK^qoM0NSOC7JSC@dO|;xQe}vifH4B1N4T zWm=MGU|W^5=ZB_{#v-A+sp9c&#Ue#ZHA`EA{fmm8Hiu|c!p)M6$e3!3s7)y^r#4oK zAIn;bdSY^LaYI%Vlr0HE!ID0ws1UJZcJO!39xB?W_Z{-qUpES%aV$8GqzuVpyilN0 z2(ix0Oa{ct`NU{YX7LD;>1XFh;zj7nNs#ujyMpaOu5Dnw>?NQwp(l6^(@QHq`DvmN zP=Eb2&cTeA7XKF6wz)h3mSTe6(v~fPeqwOchp2V>lj&?#{2DF(N3iwW@JhT~5O{tO zVp`g%5+r^rRqCeyShZpMt~I-%t7z=mny^vAExyn4QekX4M6j8At*|bh9TpJ@ zZFUDUG80b_>8x@iF^_ebr7>$un`=(5%+R&jW{G-houLWbn~z!n{S(@^P>>KfQ-wZ3 zx^{K11O|0@&8M{)DzTb7*XfhCF$pB`vUIGdF;n$uNIG+m;$td1k+Uw;gsssW zAS3)blGuGoI`oFfD3RV0*21hqS`Du@Lwo{VE}CZ(QNo~TaVSE>AKijFje1sPUZK@ev&1up=`Av|(a|^4TQr^i>s% z>2hQ?ggh3Yj%c4tX&2fE_gHU>3#N#?yjfw;6XwYj7QO|wQx`$$BFnleW_cwhM1t?# zi)C5a3~6Zvtu6xJgt$Y0n@?y)N{gKdo%k#+>H)VOMXgc6636o+G7@~gyFY?}w!q@f zvttSm7k61z&DT7nI%@CYthjL-Is|fOM$+s#WM`OB4))vG8@QY_)DZ7k|DK7QglyB5 z-oBw6(r#8s?Q}fF<+eSNaM+4uq&%MtawR#1nmAd`+(3?V$LOhTW=edcd}}Y<=U=nP zyuL8H`6DDPhfL1AsAT9>D9C@NA1`s`59mIWcw8Gyu#-K(}`vU z0`OrnJ>f^CI5e`({hr2F(mNsN1Ie*EjBSvcyObSd-_aue~Q zuHWJhBZ3k$1A|c8W9I%+-TFVN_TNIu(V)QI%+Q!ikA!+=SNtWV_MqCDT6q5)DU7|{ zaYQA5p+Ll!y+h)Xkb?zks84VfPW1wBw}x_rp3!px`!GL8OqYUKNc8tIS3PCYGS4=) zztEA9P$!pf3bPSb4bdPG*)mYLD2d5q)o{1;6MANLj!7v!6y>Ed{MyDdJ#r+h$V`rP z=Jg3Cp^jO!H8C!SoX(BFGWu=L_@)-R0I5mv1GJE5Q8&B6>g>&j4Ox6g?BRA+(##X_ z!LYU$a>|SP;pTbkrAVxb^zbW;Wo&FoFNhe|nlqyHJjVTj92UoJ{nciT8r>@1eIXR| zLCLH;OI?U?1sIi)YaFX#N$AD}2zB^yipgvq;@pfSv1K}BacQ7jSz5g=56K#tjw>(X zRo0rRRbi_>1&w(&P_qbL<7aKJ5oDg<8+fdaW<>~RdCUL?{Rrd)`~2coR=>6~EVG6t zlPF^pm6dl9G-TLCRya!(RaD3*P&*?2LOUlXCjcS~pv%UJEGvHZtZqE}N)X(?Crd!5 zViJY`cv(6*leua{__FK6SK#B->7wi7o!ZU!qUEq%6Bp>;9A3U(Tw;_9jXQt)31lHq zJ~GK;KY9!doShS{uC6rcO?mOBLiP)#lg%<4nDw)AMv(j-g}cr>Gj-(EWOIgKZVzE> z00jAOfnNudRhOy|NK-chaLCJ7^Gyan+0%U}>kOH6z6Y@=c{9Ayu+~cE#<~W|Xgc!- zPc6kK=OMWV^(3)Kjo}`1Ad*O1k*(27z+?>arN$juu+y$qEQ2&KPq*Ben3Nx0;;b>p zi3@5XCwhDl$)1)_w4^SCz!sUZI@W-49fPiAiLQ4?$Okj4MzF8+gVwz{qnL8m;_>Kc z1zS20x}jvh&zE3aAA(%_TvTUUud3d+i89OY0P?r$fHiU*lwsZcf~K2+AzFn!M&(<2 zL_iAQe=QY+x77$hqWef*sSe?o}pWoT~V&S=*9ba z&lGR2G%0IQcF~Ej#%{c%?Akcwg!^EaDRkQ@%S4rcc->au{PNk??g#3~=JmI;crbWb z6$z0~_l9t_1)`j$djtLhJn((R+^_4FA8iSwGW^=#y(H%r4-lW}InEd$P*@lW<%6d{ zaj%l`mrvIOa;-J`Le=7{(^{TNdM{?;?sVq_dyVZ3+-;a0r&5$TkP~BV$ja zG&$f`;QORx)bzJP9g8D}@vjcD|p< z)7>yzLoixQ#OMx>t!#IOPphkz7gJO7EcFkIqCBp=VqO=OpZNHDEE z!b&VQ5^Rt-ihtx8Z($0W__y3kJX{nhyW9*7p6D5ynV34ewV>}&s7|NMt!K<I(eQ%Ly=RQ9Qpvz!Tgob%EIiGs-fXuph{JmK+W@iXM71TT*I(37VQh82RzA>c>c zvcSG8NyAe-uSVZy6~)*}b1Ft%kzWHXvlo|x!&ZI`0Fg|YFq6yV+4pI;vj@9^v-<9p zsy=0WXk=thN=Iii`x!N`hbRC?`Ut=~Nr{O+1@?5mTy{-0+OA=|;z=VpelhWsm723L>%#~*#hVdfujV%}5p{L-X~M5SC&Xy3NY+U}znVDodnEeazN(NSx{E4x z3A2*sl{(wx&ga?k0F<2Z%3cfl+{#}8;j>)@zq~PBHeB{(xl4z@4>+P-S|qlb#x{P- z%aOL!rS(lr+zp`1rO4P>$pQG~f^Bt6%#N+Unh!KNol+o&QlMIKucAlFq#Ww8d@Jf0 z{#!>18jtJX_`G1sfs;*1AZN6zpb>92P|F$9 zKRi!qL6dS$cX?zSMmZ4JP2l%JANc_l%KT?`Wk&FA8*yIqf*Yk9A{@){VgIpsik7UG z)PJU196my2f+;oMs1cOMec1e2j0kH)UUX}w6D+Ac11OLs7y^&VZE+wT?n4+TAMgd6fa8kB&Y znTZa{$s)(Ij9k4VBL#Kz>@JVv|8W5}m~+=XY>U0J4Et@O1r@iXKaRxeDNyfAqWT{b z1^o+@a&ayt%Gk8(VQCqcvJ35~Qk`KGsHO)8m{*sE!iwC(vm-Mzs0+glh0)-xIUo`1 zBYh==l?lTqz8L7sFfNGWT7<7nF?^(YC>Fner;I187 zPj!c13R(*f8%j{HWdFokkded?-Gq=owlUQIq?UQ07M&d*Hnw%l+1_uCL}xo>iMtqA z#|icHh^+O~?X$9@xgMHklAux!NH0fG{IZs;HfyuYHrYi4Ihulq1kulDA5JQg-IdgA zVI5(1ga+{#H_SRxR|f8Hy-T7tHgsK&1B?e9{2JO#-}dKAm4VJ-jmxQ;9qzA^l40It zF<;+KH@~`-GX(c>a|;Ooqsz?9)M;~PAZq4|ucy(Fay4#iy-^*DY-`%wE=I_BM zT+RdC&utr>M7PM@JCN+%{M%9*8d!iS0xc5H(YH;nOaAw3!Via@N!&=F6LnCTb$L3> zD(2|!p5phmBB1c}ZYX{|gBVPR_7GVR+&Qw85QA8|&z()ucW)$6t4lYNY z*l>V+wGGOL1IN;?tN&LJe7gtUC^+mW5XSw)ga2S9MyY)jiX;2BVlxM6Ek>_q1Dm+D zj!2W4X1^5bbu(_jwSEiM%z>$d#Z-cv0#{s&cK48~aD#6gg$hw5=DB zYM2~dUry$XgH98L5w-WSQcNNs8`W`1cL?>&j4a$f<(^u-Y+?gm(V$VX-)3bHvH~(uDo~92DJZTt3?3@o zG^lj)Y#YZ<7Y!DIA>?c;79on%O`kYDAFX^%Y5e`|WhI&Lr&2Dd`K9mu<(70bV(4t1 z(lS#jn7lK~{1YSsSlaJE98}T4*z)+}%k#pfB@uU>o5!nCCbknQCM1Kj16!mBNm4x) zhsLS??kB@MjzQ+R_9})I2&(4W#PYA$UGW`lsJWL^cFr1R-nMU1H-^fnZ{ZVsspt__ z!VE3f(@XRHPcWwHB`OvwIBd91%khPStFDo1w`Y^CZU{3j?Fi`%hm$lE1ahnuLsP^< z`*C%|SAOwxvxC^SHuD$SS#?d3gW7C>X4_`tbd~QP+AaK$o?*xA$=`ULFf4&61>sG2}0R{jlYi4dCH_8^9K$ zZ?0#{pPDIgeE?wSLhbqj$8OGOB8}bq`M6k67yTaSoU=InB?s{VD72Gr`;ZY7d-vOG zw&w}8@5|cO&Dz#`(&nZfFvvE3k|WSZ1Y*&)uAGI11#}`!isk#^CBQmpp*?(R9h;gu zTu|39tgD;jO>ZP&(T2ryD9pkA(SJKG;3SaHYIh zo50~`zS=*Yd8K*3+M|z1_@&CRY@3SsmLet5NBKsj02rL^CrN2Q-t~y^SX4Sy#fX^D zpn`g!V&-=}YD3m~)?u2vi&V@R^CL|~ElEvTHqjZDKn{M0wa(x8*{X^p&depcw)O0P zJ|`8Cubbu;H(;i-WGx>b2LHj~S&|}jI9o62HmflJ`|4kyf{YEBf5>~Bm}o&t+RYtM z*c4T1NPf+|>j9q+c9({BbmJ1w^IEx0u*%aHwj}bq)_K-@GMb0gmJdvbLO2LjuCR(Dn>vMg3) zG<@zfIV!K}=dVh+Fv-VX-UtqbIZ>#4Vm92c!o!=RthCnBERURH%V7eMKW?SAx2Lzn zv$>;HY{6-C(Zb*5w7e3?L|d^ONhY5<#E^vlWSLN4C<4Ja!IL*#0Zy=gs$Ox27nq_fY0C>&?D@Wt81(Z6jB!>Gsfat~#r5G2RgWkZ@N-{#P5#)h8Ki5o z=f;Ci^*laf$qikR%+Mc{qcW`FU5;lRk3BGEvwo;09nM9FzMWuupp|$PT>R@-IQI5X z+#abWCc=(D8TN6)X>2jM-$E>THU|fN*32f`H9$$t(=F;~B**7TkPxDvEYO3>NNU_N zf3H`^^^Y?J%TipFXXbs)jTFMQEIxI|>^q^m_fXL6(;q}lSy)TKFGJysGFmXUr+&)kA&2joX?I&})UlC35-NIK?ROmF@fBG&? z=#xON-B=A&dU_uF=vkg8rR18_x0SL4&)r~1(`#!HbH`+C&IfjnUwfjC)%a7R|IvKl zSfIr0>=*%rc_sjdAS=~*%llT=)5{C2G7Jyk3<3nWT~P!pnFsm3r($G`EVZ)z%fc9B}Nf%$|`e}N-Iz}WE37Hbj+EZX_!t5`UoA(PRjTb72L~Ikx z1my=_TJn8z6T;^VZCbTzsW&QWPnX|L6B=A$3E{3<3FHY$hu6uwV;xCNDoRO2pVT7h zRTCC{x)ZgVlauKy@@u;@eUWulL`}Vr_sKt^v=(nH1j$#{*5T?rU-EuA!@BWWSlJfd zSK}v>WKh}iJ^kQDrTEAQ2dlHhI_qP@ySQyL*S+{86-OFt%6iDlV}aYat0Y0Sz@y}L z7=z8*i6iRAXeJ8(?L6FP#FpE-k6mjNZ-IN%6GJT9XvJvP{k2UpK+iWj-kBy*BK zw{-4XVO2$-SvueQ`p)o1BKPLoLX4hHevw2L~NftHSkH41^J-2wr z(-DMRLKt{%)Sq@d|l?63fmW6XEj4yP6zNnl6;G3 z$2l)R3+QG_&mOdshZ-a&^0UKa% z*9s&kZWnO!SB5}d25^?*l9Cz?J_Sc$GCohVC8edE0IYz&1U(Nv5JWsKrrnr*fDKt~ zIY*I1uPr7a5&9s0^>Ca0foCr-$GxyM$8Xk&gNy6&@%BJTPr;P9P^QkfDn%Z#!ubX~ zIi^X_fYCLajZT+urqkghP(Ur_)N)`10A3U*scbgexl2?P!JFRSeSejenZH750nD+1 zfr0*kfsMjApO$gX>YC4HsLP)_1H`73-mGZ1+mLE$7eAcb1i^5liRm~D3bGH1{V^B#)@8Ib6p1Fq0r0e+00`!)o?ig-e&#Jk-$5epl~c+|Vlj*@Cdrp4}n!#~GR+g5hNFF|TisS{Y~|L+q;%Kqdwk z1bJx3nX5SXO9u44iO*giK!NF93eg{gY&JjU2NV?ub>&(OR*LcftvY*dW#unDqtSpi z54zVTH1hu?7PD{!d5v^r)zsMR1Xoux?d8P(E zi2qTD*T+P>R6iGSQdfTh*o|5Q9(%~GZEX%bn7&bNA$>h9PKWM$vEuH({h=^k9|Jzq zZ83($#C!(W(RNqGU*3)@lLSqy|05%Je%ojr0LbT*4XO}=&~yd{U32>S5pCc zv3HN$<6rqVHzxy;1hDS3x&o?~`Sj7Yx3`UK9sVO6OKQ?#*r~NaTR``Y7wICo07C$H zlDRo0(wA2zp3HBPWrhDg$~uX%aPuKRBmSEQM2eA)CD9uLn^E$Hz=gomz3t4?;E$=I8GmEE8o<eaF1%>bOmnfn=MR9FH@$vm2C(Qr-6C#bK?N|C z>9h~zYB2>74XHOy<3k|Pm#>vSF&{na0fua%m5r3 zki)P>L!-@=9(dwvjak)Z^QX$_otNDf`TwTEh^MsQZUYV1KoD(K_bvp*C6FtQpAvY9 z1h{59_FVQYI)Fjpe+SSy!XH9{02lzAcrZL)ZGh9W5Y&KIhF}vh5dJrmhE1nG!jv7L zgBR%fBi+G7-l}0a!TFM(_s4xB*e~Mc)P7n=N-{m`@g09 zyLX0-6p-via2D?GYU60_DVA%G!4SSQ|5)JxSAUo}I3VW=G@(2`cNr5M@O=&-I?1Y@ zFAApshqaeMUr%wR8dd2UvXfc$8x08W4*56p6+5S{xE*jz<8yWwwp~Be!pNQNZ@+8N z?wS+jOyx(<&o1&BQmhN4U0%i5Ni4ed#-q7Zon2rMY-VY0#;xc7H@OR{1t{Ovd1sR_ zbYA>%-Fd{d+6qx1TUBRz<(Xx>XfHfhQT+{EQk7`)>## zdkWXbB3zt{wUO7}!|>)}W9}N|b2w#kP?Xo$Z0o}jkGyAoL`jPc5{i=cNB+FhNC;e(| z2z|ikQy{Ldlg-!PGEu|B!=m&}|80b~E(B|A*X35W&pkHL>QV1&SSzJ>YGh=j40kM5 zH%p36tY{iQ1OK1UbWx?jz7ZU))yse$T)3c~<{w^~qH;c-Ik-tt{3nyXaE~(vnCt}= z^ey>G)>R>!Y^DxHp2L6ZR$X6oDiRKikOGX*X#;9%D}xix;`_6t`@es%PnK=E$vHBC zjgXw@-`3;g;NUP_4Bk>eQ`Cg`r;gj3#DJJe$S(5D$+G3yW1wfu!sJPomr3_1?10^0WGDQxBxrlw8{{LC*DT;W*@NN58k?4; zeniv%r*@Ah(DR6d=`~RV-z$RQsncdJxY2|-l#~;M>XmTp>}aShGM0a5WMi+@_3Abm zxX6FT@NXi3EQyE*!-X4GOBQ!le5rq_$1ELbTgx^IWKUS6qAsQS%0Ixc*vdj<1EG>2 z;?-oTqKWOqb;R>XMSCG7SWL0cPQr`FwMlQXfRYUHy@!V7&J^pw0ovM&9D;g=iX28# zDLzJ5YS-X*t6};0*Ep7v8rgPnv+9}zn+_amGH}UK%KQSz{7b_~kQG^*ioO2?Bh?(ljV&65kdh@g9uM{5Tn+p*UW zWIm8h3f2tL4YPlkSc9Q!4MZw+Owy4 zpIQ~JA9P~*PJ{PnN_Uu@n@Oz~FugWUvIIOYur`G?bbXmw6s3~XmL}6)`kaL`#03N` z{fVnxcZgdtwWn8l2Apa!pHYUkeC#70vS4p7c6sl>gt0xT14omjZtgD1PEUeBf5Cqe zScNYX2?>f(FwE#&4u&1o%A+nD7Umvc2xK>%R}d1NQF+2sjz?fmFnKh=-%Fff$?dYS zr6y%0#wkP#D2Sv0cB8A<O@hc+x(@B0Z>w!V9+h&)ejZRki~qRqUibdUrDn_TTaYe@bKrgg#O%%ai%2A|}Xsc@~Sq!dXdQLKD8 zOi^)jzuT1OlL0G_-F$9NHHV6V((70DmZ5#gFPc(pi(%qNH8C{%n zzdUV78k$Hnc0?mwW`$h4LrM?RanlkajZ*5eUNr^(DQHx`MvT$1C=>MrW>uk~1e+){ z)8y$)g&#c>q}tk4xxwvcuzXu!j&9i49xf7vQCPN!cV~=qbM@y zUKqx3ES3ayTuDpL%X4Lz7ja@yb&YRiVr)MRYxU=mgB`mNJ;5C@9Kl$ZTXoRl2G;B? z8t$J{Zmr2#JrbjVld@1$H>e-cXrm3SEX#XInqp|gkgAntqEMN!O^mHLPmpr)we1Rm zB{fEb`e?Lyx$5W<$9_HU)O5QvuF1F9_o z%De*X-5wVPVE#BvI@zqxCM#*i1!%=7m?tY0Bs1ec`P^UaB}iV-zA1eNXPx30shjG5 z+2~@C8e-*0@C+Xfcb{Js+U-Y@^Wi>!dt*yFDyFMj*OT*bgWcc9RxWD8E*mY0V94jQ z`{f;7x`I3blv4Q3^cwuQK<-zP{C1<)2I-s1tb1RwB#1az+i3}>5j2BfFg#tJQY*Ir_ z(5x6c`94HE3!fHjLQs$hin<7d;e-CgDwc~sr0jdo`mReh6v95|x3c<@;7>o)^9NNXORF}9oi;k=#>RseOmBa<& zL=~oeXjE%_Q*;hk8mSZ>K9&Z-eB?aTD|&`!@I35(E-Em!NHOdNz00MMq`MWgJ#zP^ zkwsUUaHUW+T1A>fD&^6qZq8MLa`YK8kuB=TL+O!Qh6x=#qTGaVKHPl=t32f!D(bS_ z0j>u|ddUj$Ds_28I?Yq6;mYr|40O@VgJRF6(w4ggsfeYXQklR7HAuo+zTC-)GuwwNFHa?CTR27Tx5kw5FywFBJTqUD-l%V4n_YfHY10#s7`1D&qz=!zW`{|- zc4NH0_$Ac(;1|*m$=tZ3=-Tcab6FzoagM}zof$?=Ft-SXdv3b2rz$wo)}dlT1=hJq zyI6TR({SqHJm;qIC699VY0JT0`?#G&j+1~$wA86^e@rsvG5ptvaEtO=nU%>=lx zda*oHTLy;f{H+A=E&;-aPzUXIe*oabU;XS3Hi zkVLo$@S9o2xd*D7*Eb_9qeo~rQZyx&%(HKB?B|ItZU@^$kBOQlWytw6I64(Ehe}>H z^~q*2>X@>LxtZaSFm}!BLa|yTYjzL`*ST!9&6EF3DOshSfL+a7Ja}yITNj6kb3ohq z;TNO+;02y^j2Lv`ro@mNsi8oe?+c+lCukWbOsI0V@qv_O~nuO;`DxQN%?wo zy&fJZ>qx2^=A(x~r^8p4+9sPiW+#sxt@tP{eW84Fi|{i&N9ozqH_)$fN-%-v_0HNg zE9LMw2r>jUM)@^V{s=II@oF+N=H~f?<+iK2!jV4lw(DVAfCgKeSf7-vu=1_ns=>&DF28k?(MuZzxZaLZ#}VJc4~gFAOZU`YFeT^%2f0k{iVHe!sh~g&HcTB@T+OK7xgR( z7_XjJ{9&Dm)eQ>OTOXG3B}R_U0mIm6#!`!KTjB;%vmN7@J3_?NDe@8f%_d@AyEUBt z?gS{UDYX=tC3YN7I?esbj7CpuNs;UxiM7&FcQ=|6&H_v=8d9S{#xy}8} zwL0?ij^exe4n@ut1g|-=`KLl2PDBT*s)UsC(DS?$aBw=2S0)05PsDT{kYvsu(@17w ztj|4{VOv$evCwhWDZZ%(j|0K+2i7gZF|lm%^bcB6Qq0HA$wMx$CztoLB`Iop+14MN zw+&6=SlHCt+fyIvowI1gE>84`iHU4%mlU(Fww({fm+20-ymp)I9)DrhrZt!JlZf|* zwU@^S{TEyuqqP!R1XRmq!4Y!->ArQY>uZ;QKi=Aj_2EN6qY&`K^U7_j zabaj=B(QFos`7DaC-x!&1YN&k>=vg*+8xiza-v-Ov;v9#c(i!m9tc4*ca5Wwn_`RK zKAYW&neN;#0^W-O!NNg7L5ypY1q+vp>a4PME{+>L@X9#pgm*vMgfFkI=x33(cXpuR zDE^e?^t%`hu%HP^)1?wRv~uCe#nBvmO8x1S@^okR)Wb-7wK%;(r#>E=Vul4>H+@}* z%q`9N+n!?@<-SnwoKc_N&d2$01y*{73r=w>MOGLWf;(h8E2id1#_4O|DdG#g$=37? z3=HI}*jJ+M!%`t#<8SPq?P%oU@>`{#&u7{0`HXcW8A+huT21dDZp~?5dL7KIBYyYe zL5R~-UX$~pavy|6|68wQ#RkLC!Lw<@TTSN`W&`h*B}F_7MLl=Vlr=cx?twdglC3GW zkG*Jr)9w!6kO}D~e^snhnCrDxagVoQ2^vVIeM9)GoJU$(dT1PwvCs3J)GFVe{kcck zAoF*GF0>t~88&Mr<_{yyF70hME_@0YJe>OOzsd#F5HLn(t&D$~$sdf+LbD%TcF~6& zIOBSbV_iXn*8GHyr;WndIx4JNqPBi<$W83emSWo-G-duTC~`vg(+{*u-}He4g8$d@cto%AcN|W@d>!J?94Ihj8tfe|5sf-w63R;?uCal{JLQ zILvxEy$AfG(Q7Bcn()n}IDvu-f(Ns$8LUq?szuFhnW4xw#rn6Zm=4UI(^D;;hP ze1EW!V(h5ZmilH_N@VcPR;8{AQ>$I4#?JamEKL^<<`Y*U`{1>In$iaGaOV?=CAOmC5_*-Pn>N3w zKRA}j?_2HGP!q4e5|+=?vQXWe(s0h(@*qukaFidEVY5(kq@$${?as!|!Lx+do_7ia z9*-xDo{uM3g1WhLyg1gTBfA-WY3?{AL^+k#LX!I5{ehvvi_=v=yw~0o2qSi@g2w4b4F-IGz&eH%X(%*+Dm2=^nz;dkyTiqq>$P zYz}L7seC-s`b+ZW%*w*E$_f^9^;an31WUa31w|!MnI7pLWf8NBZaS&%O2~dM%G;t| zILugsPA-_(p%v-DpvX@OOQBaA)R1ZCHd9bRu+SD&j`pD^2hmkGwInZMtXjv_Q1`9v z1zm&fnk4t@Z`PKp4M7&S%5HA2pknF~ak++Zscz2+Ufkt<_57+E3Z`eK-ll*up-mZ0SIF4qY>`tZNfk{g%{t88oh}GQoPpaBfE+}ZX&|(2<~Hax zzoMoqX zfhg7U?bUlMVmBA8rS>2qGwB!nSotuAXCujHZ>(TJ50RjGe^SU|V{4mdttOU0F}aw( zPf1Gk8mAoFtEH7vM?uMnUsYXbR-PJNvLs^Q;NaB1LjU+U0&}rF;^uC@M%ET>BL$at zRe~8|QdaGw2yOu{wxTx6?1H?r1X`@@4+nllCFeI`cQ&mipvn5uASIp;R2%FY4&PX8 z!}9aU@gipm>v+Q?^CspbY>r6>j8AOV)MQ-Bm^WH%pSYX-k&Jk4Z7CPQSh=w}#Jr7F z#+vli`70oo>+k$mjZcSKKNFLaWmbDss`(}*RL1Xg(CTbT^IDsO9hBgdMwb>AR<{VL zUZ5o1QNJ4@R;kXd-=nrYaW@!qrF?Wo>S%t|m8-PxgOybe$mMpT-kL9Gbjh;hB)_~p z?UcN@BtzR9O*RHkIyrxvU8LM*wJs_%_*0A5!~W4xUQvDKdj73ad8I8%nvP(}!m6^$ z+P<3`@2K0`m1B!VMQb0F)|M}z>+8w6=BV2OTe{_{jIm6}UHdfCE?6vl_cfmBCH>pakPU9~fx&7y{k zggV*@6dV8GWuKUn^L7taVi}!@!lPSVky3du+jQK_ zsmA9LTHHUj%PX;`FE@VH&aKbeIRmFGA9B-l>6u0^GFb(^-$NV0#j4R%l2WKjvea-&hacim7Eh+@j$LYP zbfsk+X#v&13f{#{G@FVb*4=i_Ln=N z8a$BtzF4wQEG|@+`?_e`@GU_t%5G7SOq}68jZ5rl#Uzo*rbw&B!h+HdQ7<~=X(g=WZaU67M1|9V9^$# z)`N%{_UYxn4oO`85~c1ZM>72n?`;;VBk*-B!-Elbf)M9yX=m#q^mWZRwoRJCGFo*@ zPHYP4(q)*{%M$PDRq6hOZnH__5c;6v?l8GVAeYJYl!Sj?{uWX-zN6zuAM zvre7=*qT^g|BetKk=GDJmK8QNi%UydZse@XvgBr4#JD`Lo60h0Mx2wBMAj)|a*O|o zC0(Vi(4S?gQpuTQr6$nURo^BoI82hSnz}8qVU|TwNVj;UE}mRkSN64a!-U%OaG`9d zsteZ%&ywZRD*NIvbzPWkc_*CEVW?#846pm-OiP3<#^Az&hJwPQg+N22E&XCciUotD z0fVBgGl6S)s|68;&wI4G)74*v^Sp;Wp4&kr^NZ8sN2g?S2+C2swYBxp6}yYomUemE z^-}Yb4S{uzpZ|J{axHIn{`z{H9FU_SM5wQqK^SA?2*!iD^uVGKh8J~P&0*DR>MAL9 zrlT7e8XphI596$`KUaNum*4ueJNRV~8P-N1c|lA@b0V96osw=^MfsA-Dd~R_{f`L# zM+@TquN1*-{%7cv?Myl0Wgd!w5Z%H#5VXA4EXJ7vQV)5>Q zC7j5ux&L3b-g4w~mA1CI++9z;$Up|#k7yn7BWufDK)Lu#PAXT_f z+Nt=W+}<}LHnzH3thR0zW=Kewog0>&lauoz_+l`2H+0e#QP*YRm;^F;WL-_(9#8}RwMo#~RmiQN`oGGRZfD{32)TY{nvqVIva0euc z7<`|Z^9g~|%*ZxP|Jqlhq!GtBPr-IYLmwXy!{L7sv_&b|-~d84{--(njt&H8FKwo`-6wy zb;=^~B}Vb^Q@WPkGiQqq^V3;9sW{iY_lCs za?;54hx6r~i}P=~;~#{=+|8x^8qkvQiJRG1r3-6{#u4CBbjwT*Yrg!G`BYTvB$1*z z0tOi{opF6=MxQoWe|))VVU_KloHI%e*Zpn$`#rj153c39rO{uLEy({^wW1c7|8>JZ z?fkE_tKW5F8B=()&yEcgP)lc{{oN#}ROKk7y3STTwH^vHNGoNA6sQ^F*z= z`D;brV5WZu#~r)5X{T8IoOQvmHaRee^UJbbP`su8SA3NCWjv!q6r+ygvJzbQSm8%0 zr>@u%p>Vj67qfD4M=?2agDUoWCCBq z)>rZ5^lLF(=TL*5TA?3HpG)(cTd#~K1OpKPqH@`70Z+Nj+Z=sE;sA+ubGfoR6;%8u z>h2yDC&!aF=ACI;Lcp7)wEHDM=C3C&2ih30D+v(nf>wKUEZRr1w$L3DgKqA_13cIg zzKQ+#n^t+84;(j#Jdm}Q&DS~Z3k46uH zk)y*B1CP9%ofS3*1s!WBo{aH36`k4dV{$?l`21Gb|5}rKBPMlVzxD zd6A&qhtfHBcRFH;UVQi&Ug+hVUd!f} z2q619+#;i`i|+flbnm&O*?if@<@5QXUf%I(-`x_jinUu&Z0_req^SZ8GXag$)9YVk z>GSy&{MOr>v#r535C4w)`Z7-wfgFEomorcQ8$XZzFjw5Q9`y@l9tV)r4FC;G+Y?ZK zq`)AXzT4N~`!vm`%vE)7Ih!<2vp%dJ-;>vOWMCeSr+g=;%a;!gF42fe8`^D)(Qd z#tBB}+S@+DMGY~sWXYku8hT>km3`;Jk`q7;!st{!Cm*T6X3Nb77jJybRB zk!k@qg+Z>scHr%9ZE%2|>U+;?<_CJ@IE^c$7l27mnvld6soa_mC4b>eiBO`6EZ27 zNmE?94)MVbbmfXi!bz&HlwUHu_e}+Ef!jtu17JoN*ofEf&s-d`&Cy&Z4Nv87GF0m> zjfERJXc+f(rCg*KPaya3(d|wFx572YI_Dq-i_P$s5@YM-Z}>Rwz2IqG+rQW%ur-Cz zvxA})LXv9Tnfh;$s!Royw!ZbJ z1NYdw4L9&{2!Z*B;w_&m6}QV2g4E{BRs7mGkH9>(vx2=DOV(=)c10|yh-CFEaq7u@ z!^&}2pUh-*mM!oA1-%eZy38F0`K?x+AN84G;>|M>&fn7*vI5#!F=qmq@XfK73$1@z84>=l`D2U^v>wu0LW|HL`( zY>7VftaYmk;@pUMH?_9#{Y?VTf;~jv>j*@1PxrQ>v@4A&v+IET0#jyf{cdoy-IwPh zMpgtb4D2Je>@qunIpVLci?!MVdY&Ko9hsC}OcD7y+GW_2q6p%%#>|+qE>tr)WQ)qrklhIT0IA2zI|SgzYL#oWB5Le zmN9d+;Twhe=aIGDT>5!ELO-dXdgu!N#gfR$z`&Y9s)zegO$W6-5KhnkxLf**nAp?P zm0me8eE&ZAK-4Q$AxwT1@+(GNA9rv0h^+>=R&U)>_8@8nAUykoyVutUN!1Jn>`v0p zM!L!gc3VR(tVtD^_rSob$!Z7dP`*rvwcm25zYK3>QRvKx0Sh8Fa|MOkEdxL=Wfk7a zD_(4@`s-F<-8=;%a1veVL#QOh>ptU&6?h?c@uS;~Pmx_dfh@dV<;r;n3`Yo^K<<;^ zKblBl7pV1&1ZL@Ai44+t;J4a#0y&z`%0@^V{Uh299nn88Cu2nXtN3kcHuJOoWVjmK6c_3&V5j(L| ziX_kece3g94QHkXj{`9Gof)$t#tvc-&?VC;y4s1~i26UeD`1NsdSGt>(Fc>POlIVfD|&@*FAOpD^WhdZUFo&a(tAx1eSQ^fKfMa!Cjsd4 zd^!*s+^`2mWf^F7T@J+5n272LR)vfausPA5SbSkSUY7}BYKEu-@|dipo=AQ7LX7-O zb>R6Wo9YmiGYWvAWq9}Cc_{|1_x$BR*~us5U59TSSK=I~w^`<2si>~btTir4EpgbY1dFrbS?MtqdDBfKFY>!zwKBNqw-r%<_q_{t8Zy9DS_quhFpG`efalk$yY$IwXFZ z63_oDHaTmnk3vt=)M}l3?=i>0@gSc=l~Z8uUSDfPc4b%oy+ki2Y3|uhy0&2~q?O^;d*@!LwBiFd*_g_o}Z?0n>e6A;mhRFk_ez zv3dKJ)rbiqh8y%KKhY%>0_3{vcx%_%y~+uo z+-#@k_q=;-b72~z&e8&IJ+@Oypklmx_qk=H`UmZG6tmn}f~IO5kN?m+2P`Vj87rw(s=B!MKro6S!ho`;xWL0NlmRJd#J#_*vI=nB5?Z_oCi zO--Qe=N$x?%Qbzu?x0##EOHWaYT0*E@b{XP*dFd1`+~RISc_L!^4R5|%4Z67$jriE zb+0x8N``!H2;5+beig?ZmmlapOG!shXH9Xdg*h=h_u~w2g}hExXm~%Z-u?Kv)Aqiu zM!8Z&HbHg%h0)J}j&XuYpC8xNIBg(#`f8pSY$Vi)xux;+!;lhvU+7AqluLiTQd@I! zMK2P*jZZ3C>0m>9Rrpj7f9sOLuEQDAY*Md-3jInorUJQl+~VQnq1i%@{N)?n1#fP3 zMXnY1?bn*v-vOw~#P%l2Mu@>OhU0B1lU<>oLj?{h-$?v63Vjn1wxDdc!-Tu0h5y}_m-0n5fsxvKIFq-F0iC4ejh?oQbArB&PJu?Ynuq3Gu)O1 z5?PC0a9GI?;zQ}Q7IzA5Gxwb4nqu-&pIzFGpH0i|`v&7@OR1dNS&V}NCHX|}O3ljg3@jki{S_#Zxyn%`>Pn2DKdxA%u7-%OBb^*qk0OyV(W z-^0!Pscgazph3Tk_ccc<_Y24P6Z34*+7v#O%ir2L{cAjCt|muNa&nXQx587T7AihV zus@#)r)o#Ag@ahT$Q~4G`GIlH&+mqlqi5CMsAO!&Uog#zR$1k;YBqQfeXJE7`{1w3 zS#Mpan$wEcaQw{yy0a0e^r3i(x8>NratmYA<`x)O(>S}oJ7H9uF8g^9zw?sCKp;K4 z3``_i$0|^+C*`sNhe|3U{e=INF6Qt=ksO|U)vfk50@QX+IyjbGeHm7yl_pGlI?SUX zraHbLv3KTC8sP71XncF5VDXd@YzZ5Vxl!;|OG9u+-hH*_QR~wKd~};rz@gB*RUTxt zTzL9Mpaole!~Qav1P!`l>_tD)YL1lSs-Tv%!{&i;rt^IVZ93d=?=0RCyVlp(LEsE{ z+);;*1phlj><^m;)Q9)-EGv1GszB}jvoX2wF`#|VKmT)HcAP9tvl$@U#uCr@CS&&) zJ`_qOWo6qO_9s|S=NET%#VbHjPz(45&Ko3TE;dnewTcLChPp_f&bh`q@9`7XdJEKx z2p;W3%xqYC1E+ki1*1=PE>?W)E8XdJWc!{4!8x{@Jws zr1ukZf>>W%g5bNaHHSmVv`yyWW!+hB7+O8p=X=4)rHRd|_KPKz>T~b#K z7)o?A1)HqynV9y@c81E3N%m|8=R9{TUO{aj{g4PeFIXvP>z1}^oDN|9^6PL$aCEuA zx){;TxnYl4L$mwIGy(sV422_{-~9e&2BlABZboA=&;zC{YsfMA`wrL$ghFv)Ropw?UF zr!>@#f@47d)D_s@H7&IfP@Vcpr+-SO@e#v7nIhCTN-bb`1smBZ?V za=PO$B?q{k;Apwc2pW;*3sc|Er{hml@nJg9}N?*qW_t%~I^(@r#iMF95p7U!G-rQc@@uv`U}X^Jx9+X1RWZi~In$?Au8Y-P zLrYWO#n#h6`1F(dp5NFt21k((o;3-8P7p#xf8&z{_ePnE9rugDHG=4mSw3428~LL6 zzq2}K>Y9qoom!D*K8rTUQl*I6@ULV;+S#RXhpK>~7U|i_O5*47y?wFs(K8okJbqB( zxFh0M)Banh4;= z`E(pjd*HX`8aH}^06@-WA4S6NvH^DNaq5l@Z#_MM_&Vy}x$v}IHUsl8p{ngJN%o8k zRBcUGpP)N)7$SoTYeL;^uGc`Gz#zA6e2~k}HBZr6o;ELE% zU}A6%aNvnu=gj}cqKUAjs>*vmPxtx<4(dCqvBb$>qB)vPJ<6{1_5B`!Zp?c<=;-z9 zDC)E^+u@iP1tA4Gv=fS~=HoS|!vv_4?ZM9aZOiStxlPsX2}IKb|6(iqJozwC&1i~) ziV8zuuz63$X6gaN(nRo|dWC@f-O#Ijk2^2`WNlJtnzB**Dz7peD zvu4Lw%q^N~#|7izR6s+?Z?wW$y$~FvU?k2WcH4e&tQ?BGa$gu&JV|_fho>2&O$pjQ zWq5{12ipE_ruP}0*?iT(0)r~f^=+9o)gvZ$*VNoL%e~{b?Za>12sX)o@9DSJx|&cd zVo-jIO@l_7%H1(AN@Pr(c2^&gvk)0d8Z@d@qOaS7Ww9vQVzaVm)&x0@ixI-0OIdD?fNy#?m_b7p5#{76X1*qTT2 z*1GJHCI&G1_gT~yU8V5yIz&#lOGyq$i;(lX#wp;|`t<)czALqFxZ3h(N)kKg=Sk1o z+d((DAH($BF&7|ci{SzI!LS}1Lq^!D%vIoHqGPIr9lnp*3#*jfwak^GdX z0vYl868fFDPcvH*Tc>_f6)#3KTjj&9@#Te)ig?SYNV+YFO4Zn!(kD&=ygLE%5nELy z$N((!nxB$^r&iFZ`)<#g>SZs`B>!Bqsl@@wlOX!L#}=n2ZU6Roh61@nq#n7+-6qAo z!Y8iAZ6*oxH5!j&M&!3T?&n6%UA-fXSwWx;RWVS4o3N*DLPAATpZ~cWD&=t8XcF|HSwaq+O5F~yx5~2*LEh}> z?pyMfbyRzPLk@U{EUWmMd(MqZT$qE{EeiQVbse^#U-yDKEeQSNWoFPThNk4pF265D z%3OP+M!V$|@(;%n-!%DZj9jTv+oje@psU9&RgGxjVhj97IS6zakTfcF25jVaa!AbW zfChWx-24;$xyG1Ty3pCKo3Y8g`cVf6?qerj@YwVciquX)$@9X)8kO=ZseZ1|*??P2 zWQ4CUIh)ww;R=X&w1z{yxE0>%7-a^MA10yf4eow4FKph)i5qTO3xPEAVYKe1rwQ_R5?(Nkhm(|ZC&+qj_e||Q+#K=-i zd%WfvIIHC;j;5#nRbFETs zWLuaCSF6zxIThe-6bJg1wub}%bsVBQO_KXoJd5LJN@R5qW z7Sb`YCd2f%z`UWeTkFaRi=YWM8f()P0GV;%MVQCc^@ouOj;8O!@o{&$`W@gdjpozgoaDXS{nGMYF}pca;t6g-!_jrk&xckBI#dL9sIwfzf0`H^+Ae}z>X4P!F=Cx@L? zyvX!>jL4^3KH2TgqC2KvxF8Ed+?-qENr~W$}HV5yTdF|E|T?E4HOkSKv`c7_T+XYYxDBZYz4T$y>9$96v4*s0?4?DQULJ*(4b5r8kQ zF5z$oYlE2t%Ix<2$iwmxNF3s8IU;@7QX0`{g4%5ju{&=vck1nh71n&~CFLcyG0@mO zHeTDOGi`{({J|pEwO{ZzUx1uca;UnCDacFckr8I#6_iyQxiBsIKOf&q)Aw474F?{X z8AhZ+r3vAiK9%v#Tnq|N6V6XnTQriV)H*(~^ON}i zty-{9inN1(_1wqv&C>Bn^0GaVKeVV>VzRfHb`_F`o+_9890kx7?d6ebbtL3Lh!VS? zv3%|p4K6c%9LW1rii>xbv%@}Yx4S{j51W8#pttG!3*3~BrFssrIM!tiVz zUDk&8od@}$o|CtinIW_1i??)%k-vS-r(WY%H5?d!6BOj1x?n*^ga6U!fOZRP;&9c` zmG0~;a`Qy`FnQ-Fit7_Qck7!BKzm>1Lo@=gHnHWd7jm}P+2Wpfr@3b!xU`9jn}oG+ z&S5pjCjSi9Z;3YahHViPT!8ox^!k74`9oq^Y&e(@^(zoBfdG)R_lH1$m%R%w05|Cq zmhZ9SEHpw^H3EBl)#Lo7YY0_P0Cf@>#)xloW!ycPLP+%X2W z44(oXf-H3J=S$lzGc?r=nJf1;R1%bYPq~mW{niy^6nmU?azV^jNa8v?bT%Uq|3;s+|w}`T5;Z1%_RW?@|&U#+x0Jwt|XE zd!v#-pN4RmTpGG6PrtFbb|*2+aKG#2y$h+PQ@N-X%iTbHopO&`l``9^&%zr(xy!za z!E`_XzVv2f^GS>kLFcMEy-_UiYTl#RgS)NqyQq(G0Z}a~9Pblpd93v6k9TRoi^cxH z1?dqfLL^8jG%^JjPsQy+pZF9QboF>1Ep_c{Qf{KsS1~oQ+37uwmK4e=R{L6SA<5Rx zLvN1m4aM3rT-`rUsL92-G$J{go{UYhLbq~WX=3ypRNk>%-h#aYo&=Lm4f|Wck8;6p zv(`_%A$azNdj`V*`<=8(``uDtO;n}mTz7RCfGx1i`-BeM53t$d0+hB_g#XDMgvN<2 z(r1$dg^d@Y_P?QJ{t4laIGp|z<(H&rtM(pc?!#@EO@_%SWeu1wtSqa76zZwn7?JBe zp3juR6I5@;nrp>2czjsY$gfSsD<}@fixC#FOk;#C^*F!PH1){&HUtNtz3g%kLQ-w3 ziG8EVtE`*eymNH7j#GQ=f~(gT9tB8yAcP4(37rKdSUi}g8F~MS0u%`cL**@c6yr5i1^HSwcEWO z3#p^mx>#5>UxwV0WX#7wewl~umBkLzCU%#riH%L zMSp|7GSp59U;KPZZAs`av6 zGWioWGAfQ3Ff#rTV~V6~FE)`2&_$ni-h#k%3rwCfDE=a`Mce~V5O(63*}I2gxA_sj z=L#yb&-hYIyufJNx@V`a}p`^3cS@}-;{Gne4Bj`@9x zoK`ad=V2B?V`}HpD43nnQ`0es`!_8}rCR#sTEny~A*)9IJm7dYh-kCcEeeAevgA!Rb=@%5)y+!CwZpBt=0DhS8qzixGeJk@K zgfj$#)C7{#u1Ca5!4G_8`Um4`!q?hPSu+K?Q6R$6XU*23_z3jL+z@9pEd{K@9SPoF zxRCXjm1V_~t?h2-?CsZUkhpO_ajW>~Hbm7q>~ zgJ^nVAGKZ5z%p#CYl_bIyGylPP>PEomAV_(_XM0v^mbQM6BCnf7bJ2qYwBJ?(jiT) z&@)h73Hr`cfL@{&@LFe{``-;q;881?B5&UUf$sfekQce{7qrG4->-%7{EB-(zswPg zXKQjj&vDSv3(2fP7o1Pq=n@&Vv`rUT9b3{bv+G>*HTCh#)WMM6-2VDCArXAB#S?d@ zX|dxRA%9geAF{Q!b@2-+tZ`X#>-j?dQRyJbsnv|Ox&JZCvI|FNL3U+HrfCX;sBmgR zNIu;^V8>@Y_VM%%mF+L@2c-PneZTxIH4Nj*2Wf&Trl*RdrNcWy0`~{7AnCEU_h!4g zs{|1+^Pn1<7)ntVxtxBx$5#*_+7Tt>3zP`?F48vy()*he&59q6v5n%-;!gFqr*l}J z37uY8OfKfzz1ysKXP|JehFRh>&=u9+e5O|DOg|;q7|XTAD`t-ITnWD9a4@Nv6alB+x(?D5mIQK4|%Xc(W62sXp!R$V0Nr6!G<6!&rl2INp`2jeX}zFMT_ zIhPHMdMHOn)RHalyI#qIsc4>O;e30Daxm||`mvk_10LbN>u6^&Ef0Ol$*LF`I6;Ih z6iRua#~HWa6W1;jy=30`yYT1PT7}@1WJHwW2Ez9m8$Bl!^r7W&>IHo)7!xQ}uI1hP z6Ssd&Ii?LEZ>$j|8^X0PS@d)|in4(pl{zR_tAQ6j&{sW<`Q*Za2BOFcH}{+h|WEbOl0*Uf9|UX zV4*bp!fKZ7e8d8xVyFCR%7I+Ui4DD*+zTPEU0vyx8rXmUcW{>Vldu08(ZnibaHkwz z>j_A|w8t*_AlL!gjFB|FzX=6DhI0H7Y*stS8g;2hAb#nRQRN0lK`wY=K4Tv=SrTEF zpYPts161UV?V1Q1f1$n)4~w_Te$nlzgqlC;T=1k^NPHFaf9ZNIEfoy$I%bNKBD5h0BLA65f45k^1UTgK{rJjJ+a^aVpcfuFGp{2k^+iVP6 z+lc8?sQ=*b>-=o-*-ZCW-B~)3$A8S!s@S?m8gp1FqTZ;qNh@mW2q>8ndepX-A9{Z} zw3G9#q2$q8behy-FJ0U1m`(vt3Ad$x%v3&guT}Qx9M;14td8z{k>!ctmyVA63ca>U zu_rPGf&9K<_FvLQiH1)k3z_46`4tStx}59(WzY|ewv|xMYey3(dl3HR$A47Vg0u{= zZ518P1~`eoXJnzC7nkBTDp5^v8n4GcvaedjuSV1Vh{ZT|1E5Nrz>sJ@B%;{{(Snc=wRLxC`C0aeWh1(yc__I!-#u5=0tDVP0Pp`2+q^Cy z@x1IVaM-2yxo{U7O*Y&TFQ%vhxZBN~{o27FR!g2oYVi`jZwhdjV~zuNS^&x<0te`W)J-9jPXz6C-oXX*Tg z3?b0lnf!QpOG4v7kVf}gcDrhBUWg~ToFH-_^z?G3+R@|yOF-$F^Kz1)1g0@pJb*!*zmA-@v&wuQcnm*LUehe0EAMbpun*mFMxv zhU?4@<_C{$mzN(qfcS)~n{KGR?uI*>Sry$=5pCYjwg%oOXsf_8;=Sh4$I6Z|dW=T^ z1+-j5dcgH-D1PU;_!F>ZX?LQIIhi3f;cyZ6i^G^#UV@7>&v;@~s_W`t%COeRUzs=f zg^w|JzZXZW!maLO?1}36{^LhX1#pPC$^ndRE-euZ(+%I@i6GkU8hDfir3?&02o&P- z=_CFgQc=er?s(+z*esi&db( zj*Zivu-(NyOV?+pDJog6eKr;*m)JB?hBDGc?CjdFsn|L$u&fobulE%i(#-Lv41s8dY9L)^|5&GjZh4*RoMi>-gkk;d4 zM70e?oer4T#N5No*_a)?uf!8PT51)?48e3LU5o5y2UDidP}IM7M$D002)K@C3T^jMOAfE(;fTnKWxhsFD)${T_(+H7X{LT*4ShA0z+Oe#1jPy19g{23Mw7XX+q}<;{wS|jgmCX4fMgsYHv+sgS#I%6HS7@ zv=26XkU?__`>N-Bc@Qei`i|Z9kJK^^fTrorzES&e{Y-l~RWqsO=+jHdUNmOdHj$}P zmtrSzI(D_3`;*M?%BL2wNKq}fYoDs{)QesBZ(F&;IimJz6NJ*bCB!RMjZJrqSD#K{ z_3mWARW&`s14X){cG0q>Ken<2u@XN)@71f8DUHj&QnSmgBtp5I@#ecd{SoLr;4!^Bi<2$w0lpk&KX zKA#$6#it79u<_my_x;(-EJn8DSvyIU9FDkOn|{xG(rft7c~uD|dQC$44#u&~S0>7X zd$D0qnlPC8oOuMmH*|)|9`JoTRst=1F8cyW&$CF`vJ^HSLxyGc2b1KB$0iq33Kg1M zoHC0;>zr}bHFM$8X*KU8lcKpCM|Nkm{lG3g5yH0*-1=jfgMHs5C)=4~o0jr@(@r*| zF|nD;uwv+_lSEfu-CFu4S<5J^{!>23)OS@_ac@lou8k7a%9SeDil!N=myMRlN%F%r zD*Cu1;14rI12?`Jw91RdQ>mXS65!=?=Bum}u2^tH8cVk$^zUFDpsn9cYJC(dpUZXZ!)#DzasIf~7@+1QFFpyn)PFCI>uxE<7`)_JLu zk-mLV3NsVB!FK6>gi)X}m)6)-luMG@+kU5Z@j6TuW|=Ss?pAImj{5hh=E_AwNvW@c z=rVNNTEe5M#LL0dxo-Z`hG=rhE!);;R*7#VGXQbFXL*0hs3nke(g>A(M~CY%iS}|z zu{y=Ql@}7!m>PFDT^1P25E@hfly!%Q?QIS`bM@~w(DE$EX8;NiX0jl>1hv}8JH#j5 zaLA zjb*Hzw@U^{QoW(^5kNs9zf)X?V6#Kn&iRX7r9KI&+L8-`D|S(O zoHU`F8m)BnF9V85g&RCcnBzs#MiWn z8aeoSb=bpXxIE;Qkjt$t!}l3^nz( zGsK~@p3eE*`;0UkgLJ!KcFHR&rzw>SlQvA{{++lc>++!CxWgyAw@_oe5QJt-{f0&r z?_#9roSichTRiA5@lL>l19s&ZBWl~jP4D`jh7A?kK&*nRY7>wTj$T<9_e@?jtX8Q| z%Sb8$aP|AZFTb4*MRu{a4T$Go`|L{{xxo)4Hi4lkX-+0m6{T{~xDblQE7qmL*&FxXVe~KZ<(ZMS$=$zXrENO{RFD#9nm^ zqlls%%ylyi(`)W#evdi1Bm1h)qXRRul2fWAB#yNwj}XpV9ex)Z7^R50i7pVs^u=pi z^K2C;B6EBvt}C>{SjT!kjlmcpe{HLS_mF5V4Y+j|uScEBUO)%J9c=M)I&ptWKBVB# zsq^%({d#UXBpF?wkJO5Z-zog0RJS65+Z+7b9`(nnbxBXF676wCHc=hBjvyhP7BCdb z(B4|~NqgB?+-uewRu)@vsKO+dW6_Stq64-kB6-oLI;QOy37PJlDUAo}u%bg=%%Iyd zi*P-Sh1Pl@+y)lmgOYX=@(c$ufh|1NGHAx}qwyBGr8rT|+UQ}F`bYlrmw&me%?lP&#t*X$5a{JqVg zRA$@p*8cQJO{+u=7$~RxPswqp;{6KJ@GGldg;tln9hTLv~&PqK$%5n*=AuyQYTBg%jt=wZ>;_cU|t>mzKc7il4!%D8?&tWR43MP1%L!F*v3*E~$l#Di< zWTUY@)fF`00A=T3C}Rb9#>;tMjIWQ2w1yYialH3oQ(RnR$GL@2p9Iv%b(V1Lmzu(pH{CL1QPN4I$K+?)U&bJ4e}WiwJLQoKYX3 z^UFlB(Sr4zjLouLG#$8U{8m5aE#iKGzHILrP@vh9)W`jm>6WC0%^2q6g8Jfw1c7Nz zuZdLrqu+A20{y1Vzt_Ag+B9JJJ9yKjfgJn4O9SikuXErB@^k5k_i~c!pc@YJj|(B~ EKY! Date: Tue, 6 Jul 2021 00:29:26 +0300 Subject: [PATCH 178/290] 2 try --- docs/en/interfaces/http.md | 2 +- docs/ru/interfaces/http.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 87ac4dc47ee..c5bfa7e54d2 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -18,7 +18,7 @@ Ok. Web UI can be accessed here: http://localhost:8123/play. -![Web UI](../images/play.png#) +![Web UI](../images/play.png) In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 934e0399b6b..b9c7edce7bf 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -17,7 +17,7 @@ Ok. Веб-интерфейс доступен по адресу: http://localhost:8123/play. -![Веб-интерфейс](../images/play.png#) +![Веб-интерфейс](../images/play.png) В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). From 1bbfbff00358e77a7934cb3d23cbe0f02b7592ad Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Tue, 6 Jul 2021 01:04:09 +0300 Subject: [PATCH 179/290] Regenerate ya.make --- src/Functions/ya.make | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 039c166c1a5..d4636c6c1e2 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -79,6 +79,8 @@ SRCS( JSONPath/Parsers/ParserJSONPathMemberAccess.cpp JSONPath/Parsers/ParserJSONPathQuery.cpp JSONPath/Parsers/ParserJSONPathRange.cpp + JSONPath/Parsers/ParserJSONPathRoot.cpp + JSONPath/Parsers/ParserJSONPathStar.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp From 0bbcf6879a271564c8d7abc6c7f5773e377c21d8 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 6 Jul 2021 01:08:26 +0300 Subject: [PATCH 180/290] Links --- docs/en/interfaces/http.md | 2 +- docs/ru/interfaces/http.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index c5bfa7e54d2..0f497f9af80 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -16,7 +16,7 @@ $ curl 'http://localhost:8123/' Ok. ``` -Web UI can be accessed here: http://localhost:8123/play. +Web UI can be accessed here: `http://localhost:8123/play`. ![Web UI](../images/play.png) diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index b9c7edce7bf..fcd9b949ad8 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -15,7 +15,7 @@ $ curl 'http://localhost:8123/' Ok. ``` -Веб-интерфейс доступен по адресу: http://localhost:8123/play. +Веб-интерфейс доступен по адресу: `http://localhost:8123/play`. ![Веб-интерфейс](../images/play.png) From 94a024c7c845cabba924e668c93fef2ea3900db2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Jul 2021 01:12:49 +0300 Subject: [PATCH 181/290] Fix warning --- src/Interpreters/AsynchronousMetrics.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index db0fd1f7c43..11ecf547714 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -43,6 +43,9 @@ namespace ErrorCodes extern const int CANNOT_SYSCONF; } + +#if defined(OS_LINUX) + static constexpr size_t small_buffer_size = 4096; static void openFileIfExists(const char * filename, std::optional & out) @@ -62,6 +65,8 @@ static std::unique_ptr openFileIfExists(const std::string & return {}; } +#endif + AsynchronousMetrics::AsynchronousMetrics( ContextPtr global_context_, From 575dfa18e129cc849e2538c61097e0a5f9587c43 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 6 Jul 2021 01:26:11 +0300 Subject: [PATCH 182/290] Update contrib.md --- docs/en/development/contrib.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index f372da8859f..ac39c496c72 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -95,7 +95,7 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li 1. All external third-party code should reside in the dedicated directories under `contrib` directory of ClickHouse repo. Prefer Git submodules, when available. 2. Fork/mirror the official repo in [Clickhouse-extras](https://github.com/ClickHouse-Extras). Prefer official GitHub repos, when available. 3. Branch from the branch you want to integrate, e.g., `master` -> `clickhouse/master`, or `release/vX.Y.Z` -> `clickhouse/release/vX.Y.Z`. -4. All forks in [Clickhouse-extras](https://github.com/ClickHouse-Extras) should be automatically synchronized with upstreams. `clickhouse/...` branches will remain unaffected, since virtually nobody is going to use that naming pattern in their upstream repos. +4. All forks in [Clickhouse-extras](https://github.com/ClickHouse-Extras) can be automatically synchronized with upstreams. `clickhouse/...` branches will remain unaffected, since virtually nobody is going to use that naming pattern in their upstream repos. 5. Add submodules under `contrib` of ClickHouse repo that refer the above forks/mirrors. Set the submodules to track the corresponding `clickhouse/...` branches. 6. Every time the custom changes have to be made in the library code, a dedicated branch should be created, like `clickhouse/my-fix`. Then this branch should be merged into the branch, that is tracked by the submodule, e.g., `clickhouse/master` or `clickhouse/release/vX.Y.Z`. 7. No code should be pushed in any branch of the forks in [Clickhouse-extras](https://github.com/ClickHouse-Extras), whose names do not follow `clickhouse/...` pattern. From 5a810fc061127afa761066679bedbda307ae0a26 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 6 Jul 2021 01:39:40 +0300 Subject: [PATCH 183/290] 3 try --- docs/en/interfaces/http.md | 2 +- docs/ru/interfaces/http.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 0f497f9af80..f4237cc2eae 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -18,7 +18,7 @@ Ok. Web UI can be accessed here: `http://localhost:8123/play`. -![Web UI](../images/play.png) +![Web UI](../images/play.png#) In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index fcd9b949ad8..83a6a30a071 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -17,7 +17,7 @@ Ok. Веб-интерфейс доступен по адресу: `http://localhost:8123/play`. -![Веб-интерфейс](../images/play.png) +![Веб-интерфейс](../images/play.png#) В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). From 999ce1c867271693f711fd180f282e2ac7d46a5b Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 5 Jul 2021 19:18:57 -0400 Subject: [PATCH 184/290] Enabling all TestFlows modules and fixing some tests. --- .../snapshots/common.py.tests.snapshot | 48 ++++ .../tests/array_tuple_map.py | 213 +++++++++--------- .../rbac/tests/views/materialized_view.py | 8 +- tests/testflows/regression.py | 14 +- .../testflows/window_functions/regression.py | 2 + 5 files changed, 165 insertions(+), 120 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot b/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot index c8b57ffdd1c..18b58b0cfdc 100644 --- a/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot +++ b/tests/testflows/extended_precision_data_types/snapshots/common.py.tests.snapshot @@ -1035,10 +1035,12 @@ a mapAdd_with_Int128_on_a_table = r""" a +([1,2],[2,4]) """ mapSubtract_with_Int128_on_a_table = r""" a +([1,2],[0,0]) """ mapPopulateSeries_with_Int128_on_a_table = r""" @@ -1563,10 +1565,12 @@ a mapAdd_with_Int256_on_a_table = r""" a +([1,2],[2,4]) """ mapSubtract_with_Int256_on_a_table = r""" a +([1,2],[0,0]) """ mapPopulateSeries_with_Int256_on_a_table = r""" @@ -2091,10 +2095,12 @@ a mapAdd_with_UInt128_on_a_table = r""" a +([1,2],[2,4]) """ mapSubtract_with_UInt128_on_a_table = r""" a +([1,2],[0,0]) """ mapPopulateSeries_with_UInt128_on_a_table = r""" @@ -2619,10 +2625,12 @@ a mapAdd_with_UInt256_on_a_table = r""" a +([1,2],[2,4]) """ mapSubtract_with_UInt256_on_a_table = r""" a +([1,2],[0,0]) """ mapPopulateSeries_with_UInt256_on_a_table = r""" @@ -6280,3 +6288,43 @@ a \N """ +mapAdd_with_Int128 = r""" +mapAdd(tuple(array(toInt128(\'1\'), toInt128(\'2\')), array(toInt128(\'1\'), toInt128(\'2\'))), tuple(array(toInt128(\'1\'), toInt128(\'2\')), array(toInt128(\'1\'), toInt128(\'2\')))) +([1,2],[2,4]) +""" + +mapSubtract_with_Int128 = r""" +mapSubtract(tuple(array(toInt128(\'1\'), toInt128(\'2\')), array(toInt128(\'1\'), toInt128(\'2\'))), tuple(array(toInt128(\'1\'), toInt128(\'2\')), array(toInt128(\'1\'), toInt128(\'2\')))) +([1,2],[0,0]) +""" + +mapAdd_with_Int256 = r""" +mapAdd(tuple(array(toInt256(\'1\'), toInt256(\'2\')), array(toInt256(\'1\'), toInt256(\'2\'))), tuple(array(toInt256(\'1\'), toInt256(\'2\')), array(toInt256(\'1\'), toInt256(\'2\')))) +([1,2],[2,4]) +""" + +mapSubtract_with_Int256 = r""" +mapSubtract(tuple(array(toInt256(\'1\'), toInt256(\'2\')), array(toInt256(\'1\'), toInt256(\'2\'))), tuple(array(toInt256(\'1\'), toInt256(\'2\')), array(toInt256(\'1\'), toInt256(\'2\')))) +([1,2],[0,0]) +""" + +mapAdd_with_UInt128 = r""" +mapAdd(tuple(array(toUInt128(\'1\'), toUInt128(\'2\')), array(toUInt128(\'1\'), toUInt128(\'2\'))), tuple(array(toUInt128(\'1\'), toUInt128(\'2\')), array(toUInt128(\'1\'), toUInt128(\'2\')))) +([1,2],[2,4]) +""" + +mapSubtract_with_UInt128 = r""" +mapSubtract(tuple(array(toUInt128(\'1\'), toUInt128(\'2\')), array(toUInt128(\'1\'), toUInt128(\'2\'))), tuple(array(toUInt128(\'1\'), toUInt128(\'2\')), array(toUInt128(\'1\'), toUInt128(\'2\')))) +([1,2],[0,0]) +""" + +mapAdd_with_UInt256 = r""" +mapAdd(tuple(array(toUInt256(\'1\'), toUInt256(\'2\')), array(toUInt256(\'1\'), toUInt256(\'2\'))), tuple(array(toUInt256(\'1\'), toUInt256(\'2\')), array(toUInt256(\'1\'), toUInt256(\'2\')))) +([1,2],[2,4]) +""" + +mapSubtract_with_UInt256 = r""" +mapSubtract(tuple(array(toUInt256(\'1\'), toUInt256(\'2\')), array(toUInt256(\'1\'), toUInt256(\'2\'))), tuple(array(toUInt256(\'1\'), toUInt256(\'2\')), array(toUInt256(\'1\'), toUInt256(\'2\')))) +([1,2],[0,0]) +""" + diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index d1a5171d00a..550122c5b86 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -50,18 +50,16 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = f'Array({data_type})') with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['arraySplit((x, y) -> x=y, [0, 0, 0],']: with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f""" - SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) - """) + execute_query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," + f"{to_data_type(data_type,1)}))") with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() @@ -69,18 +67,15 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = f'Array(Array({data_type}))') with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in [f'arrayZip([{to_data_type(data_type,1)}],']: with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f""" - SELECT {func}array({to_data_type(data_type,3)})) - """) + execute_query(f"SELECT {func}array({to_data_type(data_type,3)}))") with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() @@ -90,9 +85,7 @@ def array_func(self, data_type, node=None): with When("I insert the output into the table"): node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,1)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['empty(', 'notEmpty(', @@ -125,20 +118,17 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", + node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", exitcode = 44, message = 'Exception:') - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") else: with Scenario(f"Inline - {data_type} - {func})"): - execute_query(f""" - SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)})) - """) + execute_query(f"SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") with Scenario(f"Table - {data_type} - {func})"): table_name = get_table_name() @@ -146,11 +136,10 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") + node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['arrayDifference(', 'arrayCumSum(', @@ -171,12 +160,11 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", + node.query(f"INSERT INTO {table_name} SELECT {func}array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}))", exitcode = exitcode, message = 'Exception:') - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['arrayElement']: @@ -192,20 +180,18 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") + node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['arrayPushBack', 'arrayPushFront']: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f""" - SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), {to_data_type(data_type,1)}) - """) + execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}," + f"{to_data_type(data_type,1)}), {to_data_type(data_type,1)})") with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() @@ -213,20 +199,18 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = f'Array({data_type})') with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), {to_data_type(data_type,1)})") + node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), {to_data_type(data_type,1)})") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['arrayResize', 'arraySlice']: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f""" - SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1) - """) + execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() @@ -234,20 +218,18 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = f'Array({data_type})') with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") + node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), 1)") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") for func in ['has', 'indexOf', 'countEqual']: with Scenario(f"Inline - {data_type} - {func}"): - execute_query(f""" - SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL) - """) + execute_query(f"SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)") with Scenario(f"Table - {data_type} - {func}"): table_name = get_table_name() @@ -255,11 +237,10 @@ def array_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into the table"): - node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}, {to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)") + node.query(f"INSERT INTO {table_name} SELECT {func}(array({to_data_type(data_type,3)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,1)}), NULL)") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") @TestOutline(Suite) @Requirements( @@ -281,11 +262,10 @@ def tuple_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple({data_type}, {data_type}, {data_type})') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT tuple({to_data_type(data_type,1)}, {to_data_type(data_type,1)}, {to_data_type(data_type,1)})") + node.query(f"INSERT INTO {table_name} SELECT tuple({to_data_type(data_type,1)}," + f"{to_data_type(data_type,1)}, {to_data_type(data_type,1)})") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"tupleElement with {data_type}"): node.query(f"SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)") @@ -298,9 +278,7 @@ def tuple_func(self, data_type, node=None): with When("I insert the output into a table"): node.query(f"INSERT INTO {table_name} SELECT tupleElement(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), 1)") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"untuple with {data_type}"): node.query(f"SELECT untuple(({to_data_type(data_type,1)},))") @@ -313,12 +291,11 @@ def tuple_func(self, data_type, node=None): with When("I insert the output into a table"): node.query(f"INSERT INTO {table_name} SELECT untuple(({to_data_type(data_type,1)},))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"tupleHammingDistance with {data_type}"): - node.query(f"SELECT tupleHammingDistance(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), ({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") + node.query(f"SELECT tupleHammingDistance(({to_data_type(data_type,1)}, {to_data_type(data_type,1)})," + f"({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") with Scenario(f"tupleHammingDistance with {data_type} on a table"): table_name = get_table_name() @@ -326,11 +303,10 @@ def tuple_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT tupleHammingDistance(({to_data_type(data_type,1)}, {to_data_type(data_type,1)}), ({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") + node.query(f"INSERT INTO {table_name} SELECT tupleHammingDistance(({to_data_type(data_type,1)}," + f"{to_data_type(data_type,1)}), ({to_data_type(data_type,2)}, {to_data_type(data_type,2)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") @TestOutline(Suite) @Requirements( @@ -355,13 +331,17 @@ def map_func(self, data_type, node=None): with When("I insert the output into a table"): node.query(f"INSERT INTO {table_name} SELECT map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)})") - execute_query(f""" - SELECT * FROM {table_name} - """) + execute_query(f"SELECT * FROM {table_name}") with Scenario(f"mapAdd with {data_type}"): - node.query(f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))", - exitcode = 44, message='Exception:') + sql = f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," + f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + if data_type.startswith("Decimal"): + node.query(sql, exitcode=43, message="Exception:") + else: + execute_query(sql) with Scenario(f"mapAdd with {data_type} on a table"): table_name = get_table_name() @@ -369,16 +349,27 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))", - exitcode = 44, message='Exception:') + sql = (f"INSERT INTO {table_name} SELECT mapAdd(([{to_data_type(data_type,1)},{to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + exitcode, message = 0, None - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + if data_type.startswith("Decimal"): + exitcode, message = 43, "Exception:" + node.query(sql, exitcode=exitcode, message=message) + + execute_query(f"""SELECT * FROM {table_name} ORDER BY a ASC""") with Scenario(f"mapSubtract with {data_type}"): - node.query(f"SELECT mapSubtract(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))", - exitcode = 44, message='Exception:') + sql = (f"SELECT mapSubtract(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," + f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + + if data_type.startswith("Decimal"): + node.query(sql, exitcode=43, message="Exception:") + else: + execute_query(sql) with Scenario(f"mapSubtract with {data_type} on a table"): table_name = get_table_name() @@ -386,15 +377,21 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapSubtract(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))", - exitcode = 44, message='Exception:') + sql = (f"INSERT INTO {table_name} SELECT mapSubtract(([{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}], [{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") + exitcode, message = 0, None - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + if data_type.startswith("Decimal"): + exitcode, message = 43, "Exception:" + node.query(sql, exitcode=exitcode, message=message) + + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapPopulateSeries with {data_type}"): - node.query(f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", + node.query(f"SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}," + f"{to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", exitcode = 44, message='Exception:') with Scenario(f"mapPopulateSeries with {data_type} on a table"): @@ -403,15 +400,15 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3], [{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", + node.query(f"INSERT INTO {table_name} SELECT mapPopulateSeries([1,2,3]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}, {to_data_type(data_type,3)}], 5)", exitcode = 44, message='Exception:') - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapContains with {data_type}"): - node.query(f"SELECT mapContains( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}), 'key1')") + node.query(f"SELECT mapContains( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}), 'key1')") with Scenario(f"mapContains with {data_type} on a table"): table_name = get_table_name() @@ -419,11 +416,10 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = data_type) with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapContains( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}), 'key1')") + node.query(f"INSERT INTO {table_name} SELECT mapContains( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}), 'key1')") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapKeys with {data_type}"): node.query(f"SELECT mapKeys( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") @@ -434,11 +430,10 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = 'Array(String)') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapKeys( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") + node.query(f"INSERT INTO {table_name} SELECT mapKeys( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") with Scenario(f"mapValues with {data_type}"): node.query(f"SELECT mapValues( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") @@ -449,11 +444,10 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Array({data_type})') with When("I insert the output into a table"): - node.query(f"INSERT INTO {table_name} SELECT mapValues( map('key1', {to_data_type(data_type,1)}, 'key2', {to_data_type(data_type,2)}))") + node.query(f"INSERT INTO {table_name} SELECT mapValues( map('key1', {to_data_type(data_type,1)}," + f"'key2', {to_data_type(data_type,2)}))") - execute_query(f""" - SELECT * FROM {table_name} ORDER BY a ASC - """) + execute_query(f"SELECT * FROM {table_name} ORDER BY a ASC") @TestFeature @Name("array, tuple, map") @@ -465,7 +459,8 @@ def map_func(self, data_type, node=None): ('Decimal256(0)',), ]) def feature(self, node="clickhouse1", stress=None, parallel=None): - """Check that array, tuple, and map functions work with extended precision data types. + """Check that array, tuple, and map functions work with + extended precision data types. """ self.context.node = self.context.cluster.node(node) @@ -481,4 +476,4 @@ def feature(self, node="clickhouse1", stress=None, parallel=None): with Given("I allow experimental map type"): allow_experimental_map_type() - Suite(test=map_func)(data_type=data_type) + Suite(test=map_func)(data_type=data_type) diff --git a/tests/testflows/rbac/tests/views/materialized_view.py b/tests/testflows/rbac/tests/views/materialized_view.py index 26fefb23bf9..d2192e81cf7 100755 --- a/tests/testflows/rbac/tests/views/materialized_view.py +++ b/tests/testflows/rbac/tests/views/materialized_view.py @@ -2103,7 +2103,7 @@ def insert_on_source_table(self, grant_target_name, user_name, node=None): with When("I grant INSERT on the source table"): node.query(f"GRANT INSERT ON {table1_name} TO {grant_target_name}") with Then("I attempt to insert into the source table"): - node.query(f"INSERT INTO {table1_name}(d) VALUES ('01-01-2020')", settings = [("user",f"{user_name}")]) + node.query(f"INSERT INTO {table1_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) finally: with Finally("I drop the view"): @@ -2152,7 +2152,7 @@ def insert_with_insert_privilege(self, grant_target_name, user_name, node=None): with When("I grant INSERT on the view"): node.query(f"GRANT INSERT ON {view_name} TO {grant_target_name}") with Then("I attempt to insert into the view"): - node.query(f"INSERT INTO {view_name}(d) VALUES ('01-01-2020')", + node.query(f"INSERT INTO {view_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) finally: @@ -2201,7 +2201,7 @@ def insert_on_target_table(self, grant_target_name, user_name, node=None): with When("I grant INSERT on the target table"): node.query(f"GRANT INSERT ON {table0_name} TO {grant_target_name}") with Then("I attempt to insert into the target table"): - node.query(f"INSERT INTO {table0_name}(d) VALUES ('01-01-2020')", settings = [("user",f"{user_name}")]) + node.query(f"INSERT INTO {table0_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) finally: with Finally("I drop the view"): @@ -2248,7 +2248,7 @@ def insert_on_target_table(self, grant_target_name, user_name, node=None): with When("I grant INSERT on the target table"): node.query(f"GRANT INSERT ON {implicit_table_name} TO {grant_target_name}") with Then("I attempt to insert into the target table"): - node.query(f"INSERT INTO {implicit_table_name}(d) VALUES ('01-01-2020')", settings = [("user",f"{user_name}")]) + node.query(f"INSERT INTO {implicit_table_name}(d) VALUES ('2020-01-01')", settings = [("user",f"{user_name}")]) finally: with Finally("I drop the view"): diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index eef6dadb4bb..c2e143a4b1c 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -23,14 +23,14 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): with Pool(8) as pool: try: run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) finally: join(tasks) diff --git a/tests/testflows/window_functions/regression.py b/tests/testflows/window_functions/regression.py index eb4ab07edbb..778a829082f 100755 --- a/tests/testflows/window_functions/regression.py +++ b/tests/testflows/window_functions/regression.py @@ -41,6 +41,8 @@ xfails = { [(Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857")], "tests/:/misc/window functions in subquery": [(Fail, "not supported, https://github.com/ClickHouse/ClickHouse/issues/19857")], + "tests/:/misc/in view": + [(Fail, "bug, https://github.com/ClickHouse/ClickHouse/issues/26001")], "tests/:/frame clause/range frame/order by decimal": [(Fail, "Exception: The RANGE OFFSET frame for 'DB::ColumnDecimal >' ORDER BY column is not implemented")], "tests/:/frame clause/range frame/with nulls": From 7b94573506de67be804d767cf731265d6e010029 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Jul 2021 03:31:39 +0300 Subject: [PATCH 185/290] Remove obsolete trash about libraries --- docs/en/development/style.md | 12 ++---------- docs/ja/development/style.md | 14 +++----------- docs/ru/development/style.md | 12 ++---------- docs/zh/development/style.md | 14 +++----------- 4 files changed, 10 insertions(+), 42 deletions(-) diff --git a/docs/en/development/style.md b/docs/en/development/style.md index 78e3c8fc966..c495e3f0417 100644 --- a/docs/en/development/style.md +++ b/docs/en/development/style.md @@ -749,17 +749,9 @@ If your code in the `master` branch is not buildable yet, exclude it from the bu **1.** The C++20 standard library is used (experimental extensions are allowed), as well as `boost` and `Poco` frameworks. -**2.** If necessary, you can use any well-known libraries available in the OS package. +**2.** It is not allowed to use libraries from OS packages. It is also not allowed to use pre-installed libraries. All libraries should be placed in form of source code in `contrib` directory and built with ClickHouse. -If there is a good solution already available, then use it, even if it means you have to install another library. - -(But be prepared to remove bad libraries from code.) - -**3.** You can install a library that isn’t in the packages, if the packages do not have what you need or have an outdated version or the wrong type of compilation. - -**4.** If the library is small and does not have its own complex build system, put the source files in the `contrib` folder. See [Guidelines for adding new third-party libraries](https://clickhouse.tech/docs/en/development/contrib/#adding-third-party-libraries) for details. - -**5.** Preference is always given to libraries that are already in use. +**3.** Preference is always given to libraries that are already in use. ## General Recommendations {#general-recommendations-1} diff --git a/docs/ja/development/style.md b/docs/ja/development/style.md index f4b3f9c77dd..596e29f4414 100644 --- a/docs/ja/development/style.md +++ b/docs/ja/development/style.md @@ -749,19 +749,11 @@ CPU命令セットは、サーバー間でサポートされる最小のセッ ## 図書館 {#libraries} -**1.** C++20標準ライブラリが使用されています（実験的な拡張が許可されています）。 `boost` と `Poco` フレームワーク +**1.** The C++20 standard library is used (experimental extensions are allowed), as well as `boost` and `Poco` frameworks. -**2.** 必要に応じて、OSパッケージで利用可能な既知のライブラリを使用できます。 +**2.** It is not allowed to use libraries from OS packages. It is also not allowed to use pre-installed libraries. All libraries should be placed in form of source code in `contrib` directory and built with ClickHouse. -すでに利用可能な良い解決策がある場合は、別のライブラリをインストールする必要がある場合でも、それを使用してください。 - -(が準備をしておいてくださ去の悪い図書館からのコードです。) - -**3.** パッケージに必要なものがない場合や、古いバージョンや間違った種類のコンパイルがある場合は、パッケージにないライブラリをインストールできます。 - -**4.** ライブラリが小さく、独自の複雑なビルドシステムがない場合は、ソースファイルを `contrib` フォルダ。 - -**5.** すでに使用されているライブラリが優先されます。 +**3.** Preference is always given to libraries that are already in use. ## 一般的な推奨事項 {#general-recommendations-1} diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index de29e629ceb..6e1230b4831 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -824,17 +824,9 @@ The dictionary is configured incorrectly. **1.** Используются стандартная библиотека C++20 (допустимо использовать экспериментальные расширения) а также фреймворки `boost`, `Poco`. -**2.** При необходимости, можно использовать любые известные библиотеки, доступные в ОС из пакетов. +**2.** Библиотеки должны быть расположены в виде исходников в директории `contrib` и собираться вместе с ClickHouse. Не разрешено использовать библиотеки, доступные в пакетах ОС или любые другие способы установки библиотек в систему. -Если есть хорошее готовое решение, то оно используется, даже если для этого придётся установить ещё одну библиотеку. - -(Но будьте готовы к тому, что иногда вам придётся выкидывать плохие библиотеки из кода.) - -**3.** Если в пакетах нет нужной библиотеки, или её версия достаточно старая, или если она собрана не так, как нужно, то можно использовать библиотеку, устанавливаемую не из пакетов. - -**4.** Если библиотека достаточно маленькая и у неё нет своей системы сборки, то следует включить её файлы в проект, в директорию `contrib`. - -**5.** Предпочтение всегда отдаётся уже использующимся библиотекам. +**3.** Предпочтение отдаётся уже использующимся библиотекам. ## Общее {#obshchee-1} diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md index bb9bfde7b9b..dcbfbc79e33 100644 --- a/docs/zh/development/style.md +++ b/docs/zh/development/style.md @@ -742,19 +742,11 @@ CPU指令集是我们服务器中支持的最小集合。 目前，它是SSE 4.2 ## 库 {#ku} -**1.** 使用C++20标准库（允许实验性功能），以及 `boost` 和 `Poco` 框架。 +**1.** The C++20 standard library is used (experimental extensions are allowed), as well as `boost` and `Poco` frameworks. -**2.** 如有必要，您可以使用 OS 包中提供的任何已知库。 +**2.** It is not allowed to use libraries from OS packages. It is also not allowed to use pre-installed libraries. All libraries should be placed in form of source code in `contrib` directory and built with ClickHouse. -如果有一个好的解决方案已经可用，那就使用它，即使这意味着你必须安装另一个库。 - -（但要准备从代码中删除不好的库） - -**3.** 如果软件包没有您需要的软件包或者有过时的版本或错误的编译类型，则可以安装不在软件包中的库。 - -**4.** 如果库很小并且没有自己的复杂构建系统，请将源文件放在 `contrib` 文件夹中。 - -**5.** 始终优先考虑已经使用的库。 +**3.** Preference is always given to libraries that are already in use. ## 一般建议 {#yi-ban-jian-yi-1} From 504d2c0c56492480eda7b6dd16dacefcb3e12712 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Jul 2021 05:31:09 +0300 Subject: [PATCH 186/290] Remove old code --- src/Interpreters/Context_fwd.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Interpreters/Context_fwd.h b/src/Interpreters/Context_fwd.h index 99c7d29f084..2564912a297 100644 --- a/src/Interpreters/Context_fwd.h +++ b/src/Interpreters/Context_fwd.h @@ -21,12 +21,9 @@ using Scalars = std::map; class Context; /// Most used types have shorter names -/// TODO: in the first part of refactoring all the context pointers are non-const. using ContextPtr = std::shared_ptr; -using ContextConstPtr = ContextPtr; /// For compatibility. Use ContextPtr. using ContextMutablePtr = std::shared_ptr; using ContextWeakPtr = std::weak_ptr; -using ContextWeakConstPtr = ContextWeakPtr; /// For compatibility. Use ContextWeakPtr. using ContextWeakMutablePtr = std::weak_ptr; template From 0269e34f189975508e44378d1d5e391d9c42cc63 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 6 Jul 2021 08:45:54 +0300 Subject: [PATCH 187/290] tests/queries/0_stateless$ wc -l *.sql | grep -P '^\s+0' | awk '{ print $2 }' | xargs sed -i 's/$/\n/' --- tests/queries/0_stateless/00292_parser_tuple_element.sql | 2 +- .../queries/0_stateless/00842_array_with_constant_overflow.sql | 2 +- tests/queries/0_stateless/01271_show_privileges.sql | 2 +- tests/queries/0_stateless/01668_test_toMonth_mysql_dialect.sql | 2 +- tests/queries/0_stateless/01669_test_toYear_mysql_dialect.sql | 2 +- tests/queries/0_stateless/01670_test_repeat_mysql_dialect.sql | 2 +- .../queries/0_stateless/01671_test_toQuarter_mysql_dialect.sql | 2 +- tests/queries/0_stateless/01672_test_toSecond_mysql_dialect.sql | 2 +- tests/queries/0_stateless/01673_test_toMinute_mysql_dialect.sql | 2 +- tests/queries/0_stateless/01787_map_remote.sql | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/00292_parser_tuple_element.sql b/tests/queries/0_stateless/00292_parser_tuple_element.sql index bb28b771dce..6d43ac9c738 100644 --- a/tests/queries/0_stateless/00292_parser_tuple_element.sql +++ b/tests/queries/0_stateless/00292_parser_tuple_element.sql @@ -1 +1 @@ -SELECT ('a', 'b').2 \ No newline at end of file +SELECT ('a', 'b').2 diff --git a/tests/queries/0_stateless/00842_array_with_constant_overflow.sql b/tests/queries/0_stateless/00842_array_with_constant_overflow.sql index b31efb89686..ffd5fecde10 100644 --- a/tests/queries/0_stateless/00842_array_with_constant_overflow.sql +++ b/tests/queries/0_stateless/00842_array_with_constant_overflow.sql @@ -1 +1 @@ -SELECT arrayWithConstant(-231.37104, -138); -- { serverError 128 } \ No newline at end of file +SELECT arrayWithConstant(-231.37104, -138); -- { serverError 128 } diff --git a/tests/queries/0_stateless/01271_show_privileges.sql b/tests/queries/0_stateless/01271_show_privileges.sql index efd6ddb200c..e3210a7ae00 100644 --- a/tests/queries/0_stateless/01271_show_privileges.sql +++ b/tests/queries/0_stateless/01271_show_privileges.sql @@ -1 +1 @@ -SHOW PRIVILEGES; \ No newline at end of file +SHOW PRIVILEGES; diff --git a/tests/queries/0_stateless/01668_test_toMonth_mysql_dialect.sql b/tests/queries/0_stateless/01668_test_toMonth_mysql_dialect.sql index fa2e1e41555..bdde3e7b825 100644 --- a/tests/queries/0_stateless/01668_test_toMonth_mysql_dialect.sql +++ b/tests/queries/0_stateless/01668_test_toMonth_mysql_dialect.sql @@ -1 +1 @@ -SELECT MONTH(toDateTime('2016-06-15 23:00:00')); \ No newline at end of file +SELECT MONTH(toDateTime('2016-06-15 23:00:00')); diff --git a/tests/queries/0_stateless/01669_test_toYear_mysql_dialect.sql b/tests/queries/0_stateless/01669_test_toYear_mysql_dialect.sql index f7cd84314e2..afd79e2c1ba 100644 --- a/tests/queries/0_stateless/01669_test_toYear_mysql_dialect.sql +++ b/tests/queries/0_stateless/01669_test_toYear_mysql_dialect.sql @@ -1 +1 @@ -SELECT YEAR(toDateTime('2016-06-15 23:00:00')); \ No newline at end of file +SELECT YEAR(toDateTime('2016-06-15 23:00:00')); diff --git a/tests/queries/0_stateless/01670_test_repeat_mysql_dialect.sql b/tests/queries/0_stateless/01670_test_repeat_mysql_dialect.sql index 29fe81012ec..ae2bdb46412 100644 --- a/tests/queries/0_stateless/01670_test_repeat_mysql_dialect.sql +++ b/tests/queries/0_stateless/01670_test_repeat_mysql_dialect.sql @@ -1 +1 @@ -SELECT REPEAT('Test', 3); \ No newline at end of file +SELECT REPEAT('Test', 3); diff --git a/tests/queries/0_stateless/01671_test_toQuarter_mysql_dialect.sql b/tests/queries/0_stateless/01671_test_toQuarter_mysql_dialect.sql index b6fa41f8b49..369f2b47723 100644 --- a/tests/queries/0_stateless/01671_test_toQuarter_mysql_dialect.sql +++ b/tests/queries/0_stateless/01671_test_toQuarter_mysql_dialect.sql @@ -1 +1 @@ -SELECT QUARTER(toDateTime('2016-06-15 23:00:00')); \ No newline at end of file +SELECT QUARTER(toDateTime('2016-06-15 23:00:00')); diff --git a/tests/queries/0_stateless/01672_test_toSecond_mysql_dialect.sql b/tests/queries/0_stateless/01672_test_toSecond_mysql_dialect.sql index adb72b9843c..0306fde14cd 100644 --- a/tests/queries/0_stateless/01672_test_toSecond_mysql_dialect.sql +++ b/tests/queries/0_stateless/01672_test_toSecond_mysql_dialect.sql @@ -1 +1 @@ -SELECT SECOND(toDateTime('2016-06-15 23:00:00')); \ No newline at end of file +SELECT SECOND(toDateTime('2016-06-15 23:00:00')); diff --git a/tests/queries/0_stateless/01673_test_toMinute_mysql_dialect.sql b/tests/queries/0_stateless/01673_test_toMinute_mysql_dialect.sql index 4ac7106158a..5d188b5b95b 100644 --- a/tests/queries/0_stateless/01673_test_toMinute_mysql_dialect.sql +++ b/tests/queries/0_stateless/01673_test_toMinute_mysql_dialect.sql @@ -1 +1 @@ -SELECT MINUTE(toDateTime('2016-06-15 23:00:00')); \ No newline at end of file +SELECT MINUTE(toDateTime('2016-06-15 23:00:00')); diff --git a/tests/queries/0_stateless/01787_map_remote.sql b/tests/queries/0_stateless/01787_map_remote.sql index 854eafa0a50..748316c8044 100644 --- a/tests/queries/0_stateless/01787_map_remote.sql +++ b/tests/queries/0_stateless/01787_map_remote.sql @@ -1 +1 @@ -SELECT map('a', 1, 'b', 2) FROM remote('127.0.0.{1,2}', system, one); \ No newline at end of file +SELECT map('a', 1, 'b', 2) FROM remote('127.0.0.{1,2}', system, one); From 26009227868d056fd27d250e95337e9e44924283 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 11:36:39 +0300 Subject: [PATCH 188/290] Followup fix --- src/Storages/MergeTree/DropPartsRanges.cpp | 12 +++++++++--- src/Storages/MergeTree/DropPartsRanges.h | 1 + src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/DropPartsRanges.cpp b/src/Storages/MergeTree/DropPartsRanges.cpp index e9cf07fb51f..0dfcc18ea3e 100644 --- a/src/Storages/MergeTree/DropPartsRanges.cpp +++ b/src/Storages/MergeTree/DropPartsRanges.cpp @@ -9,12 +9,13 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -bool DropPartsRanges::isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const + +bool DropPartsRanges::isAffectedByDropRange(const std::string & new_part_name, std::string & postpone_reason) const { - if (entry.new_part_name.empty()) + if (new_part_name.empty()) return false; - MergeTreePartInfo entry_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); + MergeTreePartInfo entry_info = MergeTreePartInfo::fromPartName(new_part_name, format_version); for (const auto & [znode, drop_range] : drop_ranges) { if (!drop_range.isDisjoint(entry_info)) @@ -27,6 +28,11 @@ bool DropPartsRanges::isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & return false; } +bool DropPartsRanges::isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const +{ + return isAffectedByDropRange(entry.new_part_name, postpone_reason); +} + void DropPartsRanges::addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * /*log*/) { if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) diff --git a/src/Storages/MergeTree/DropPartsRanges.h b/src/Storages/MergeTree/DropPartsRanges.h index 23f38b70420..fea6ce3be4e 100644 --- a/src/Storages/MergeTree/DropPartsRanges.h +++ b/src/Storages/MergeTree/DropPartsRanges.h @@ -21,6 +21,7 @@ public: {} bool isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const; + bool isAffectedByDropRange(const std::string & new_part_name, std::string & postpone_reason) const; bool hasDropRange(const MergeTreePartInfo & new_drop_range_info) const; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8fa69bb2c36..f607a559564 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -996,6 +996,9 @@ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & pa { std::lock_guard lock(state_mutex); + if (drop_ranges.isAffectedByDropRange(part_name, reject_reason)) + return false; + if (isNotCoveredByFuturePartsImpl(entry.znode_name, part_name, reject_reason, lock)) { CurrentlyExecuting::setActualPartName(entry, part_name, *this); From 9665d4109267cd62da0dec9f3c6d842ee90833ba Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Jul 2021 12:17:26 +0300 Subject: [PATCH 189/290] Force parentheses for DISTINCT ON --- src/Parsers/ParserSelectQuery.cpp | 39 ++++++++++++++----- .../0_stateless/01917_distinct_on.reference | 5 +++ .../queries/0_stateless/01917_distinct_on.sql | 14 ++++++- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 3f6607be0bc..8272a130422 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -80,12 +80,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr limit_by_length; ASTPtr limit_by_offset; ASTPtr limit_by_expression_list; + ASTPtr distinct_on_expression_list; ASTPtr limit_offset; ASTPtr limit_length; ASTPtr top_length; ASTPtr settings; - /// WITH expr list + /// WITH expr_list { if (s_with.ignore(pos, expected)) { @@ -97,7 +98,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } - /// SELECT [DISTINCT ON expr] [ALL/DISTINCT] [TOP N [WITH TIES]] expr list + /// SELECT [ALL/DISTINCT [ON (expr_list)]] [TOP N [WITH TIES]] expr_list { bool has_all = false; if (!s_select.ignore(pos, expected)) @@ -108,18 +109,25 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_distinct_on.ignore(pos, expected)) { - if (!exp_list.parse(pos, limit_by_expression_list, expected)) + if (open_bracket.ignore(pos, expected)) + { + if (!exp_list.parse(pos, distinct_on_expression_list, expected)) + return false; + if (!close_bracket.ignore(pos, expected)) + return false; + } + else return false; - limit_by_length = std::make_shared(Field{UInt8(1)}); } - - if (s_distinct.ignore(pos, expected)) + else if (s_distinct.ignore(pos, expected)) + { select_query->distinct = true; + } if (!has_all && s_all.ignore(pos, expected)) has_all = true; - if (has_all && select_query->distinct) + if (has_all && (select_query->distinct || distinct_on_expression_list)) return false; if (s_top.ignore(pos, expected)) @@ -266,15 +274,18 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) select_query->limit_with_ties = true; } + if (limit_with_ties_occured && distinct_on_expression_list) + throw Exception("Can not use WITH TIES alongside LIMIT BY/DISTINCT ON", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); + if (s_by.ignore(pos, expected)) { /// WITH TIES was used alongside LIMIT BY /// But there are other kind of queries like LIMIT n BY smth LIMIT m WITH TIES which are allowed. /// So we have to ignore WITH TIES exactly in LIMIT BY state. if (limit_with_ties_occured) - throw Exception("Can not use WITH TIES alongside LIMIT BY", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); + throw Exception("Can not use WITH TIES alongside LIMIT BY/DISTINCT ON", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); - if (limit_by_length) + if (distinct_on_expression_list) throw Exception("Can not use DISTINCT ON alongside LIMIT BY", ErrorCodes::DISTINCT_ON_AND_LIMIT_BY_TOGETHER); limit_by_length = limit_length; @@ -348,6 +359,16 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } + if (distinct_on_expression_list) + { + /// DISTINCT ON and LIMIT BY are mutually exclusive, checked before + assert (limit_by_expression_list == nullptr); + + /// Transform `DISTINCT ON expr` to `LIMIT 1 BY expr` + limit_by_expression_list = distinct_on_expression_list; + limit_by_length = std::make_shared(Field{UInt8(1)}); + } + /// Because TOP n in totally equals LIMIT n if (top_length) limit_length = top_length; diff --git a/tests/queries/0_stateless/01917_distinct_on.reference b/tests/queries/0_stateless/01917_distinct_on.reference index 09e5879c7f6..b5b231e5786 100644 --- a/tests/queries/0_stateless/01917_distinct_on.reference +++ b/tests/queries/0_stateless/01917_distinct_on.reference @@ -1,3 +1,8 @@ 1 1 1 2 2 2 1 2 2 +1 1 1 +2 2 2 +1 2 2 +1 1 1 +2 2 2 diff --git a/tests/queries/0_stateless/01917_distinct_on.sql b/tests/queries/0_stateless/01917_distinct_on.sql index f9b12ca6f05..b7875719c92 100644 --- a/tests/queries/0_stateless/01917_distinct_on.sql +++ b/tests/queries/0_stateless/01917_distinct_on.sql @@ -4,8 +4,20 @@ CREATE TABLE t1 (`a` UInt32, `b` UInt32, `c` UInt32 ) ENGINE = Memory; INSERT INTO t1 VALUES (1, 1, 1), (1, 1, 2), (2, 2, 2), (1, 2, 2); SELECT DISTINCT ON (a, b) a, b, c FROM t1; +SELECT DISTINCT ON (a, b) * FROM t1; +SELECT DISTINCT ON (a) * FROM t1; -SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { serverError 590 } +SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 590 } + +SELECT DISTINCT ON a, b a, b FROM t1; -- { clientError 62 } +SELECT DISTINCT ON a a, b FROM t1; -- { clientError 62 } + +-- "Code: 47. DB::Exception: Missing columns: 'DISTINCT'" - error can be better +SELECT DISTINCT ON (a, b) DISTINCT a, b FROM t1; -- { serverError 47 } +SELECT DISTINCT DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 } + +SELECT ALL DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 } +SELECT DISTINCT ON (a, b) ALL a, b FROM t1; -- { clientError 62 } DROP TABLE IF EXISTS t1; From 3cef256f7905db0c797c5b93faa3806ef3f2ec94 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Jul 2021 12:22:08 +0300 Subject: [PATCH 190/290] Errorcode DISTINCT_ON_AND_LIMIT_BY_TOGETHER -> UNSUPPORTED_METHOD --- src/Common/ErrorCodes.cpp | 1 - src/Parsers/ParserSelectQuery.cpp | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index e165e184cd3..f4ceef2896a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -557,7 +557,6 @@ M(587, CONCURRENT_ACCESS_NOT_SUPPORTED) \ M(588, DISTRIBUTED_BROKEN_BATCH_INFO) \ M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \ - M(590, DISTINCT_ON_AND_LIMIT_BY_TOGETHER) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 8272a130422..2b7f6bcaaf9 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -17,12 +17,12 @@ namespace DB namespace ErrorCodes { - extern const int TOP_AND_LIMIT_TOGETHER; - extern const int WITH_TIES_WITHOUT_ORDER_BY; + extern const int FIRST_AND_NEXT_TOGETHER; extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED; extern const int ROW_AND_ROWS_TOGETHER; - extern const int FIRST_AND_NEXT_TOGETHER; - extern const int DISTINCT_ON_AND_LIMIT_BY_TOGETHER; + extern const int TOP_AND_LIMIT_TOGETHER; + extern const int UNSUPPORTED_METHOD; + extern const int WITH_TIES_WITHOUT_ORDER_BY; } @@ -286,7 +286,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) throw Exception("Can not use WITH TIES alongside LIMIT BY/DISTINCT ON", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); if (distinct_on_expression_list) - throw Exception("Can not use DISTINCT ON alongside LIMIT BY", ErrorCodes::DISTINCT_ON_AND_LIMIT_BY_TOGETHER); + throw Exception("Can not use DISTINCT ON alongside LIMIT BY", ErrorCodes::UNSUPPORTED_METHOD); limit_by_length = limit_length; limit_by_offset = limit_offset; From 481211692955d7c654fb6b6b61b8807d08a9feee Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 6 Jul 2021 13:08:09 +0300 Subject: [PATCH 191/290] FunctionSQLJSON ContextPtr build fix --- src/Functions/FunctionSQLJSON.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 9e469c4ebac..497909b5242 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -144,8 +144,8 @@ template typename Impl> class FunctionSQLJSON : public IFunction, WithConstContext { public: - static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } - FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) { } + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + explicit FunctionSQLJSON(ContextPtr context_) : WithConstContext(context_) { } static constexpr auto name = Name::name; String getName() const override { return Name::name; } From cd89138d3eb82458a63b0452991022b2599e70ef Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 6 Jul 2021 12:36:44 +0300 Subject: [PATCH 192/290] FunctionsLogical const result for non const arguments fix --- src/Functions/FunctionsLogical.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 3806ee7511c..f427c9a9440 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -575,12 +575,12 @@ ColumnPtr FunctionAnyArityLogical::getConstantResultForNonConstArgum if constexpr (std::is_same_v) { if (has_false_constant) - result_type->createColumnConst(0, static_cast(false)); + result_column = result_type->createColumnConst(0, static_cast(false)); } else if constexpr (std::is_same_v) { if (has_true_constant) - result_type->createColumnConst(0, static_cast(true)); + result_column = result_type->createColumnConst(0, static_cast(true)); } return result_column; From d32d8ceff65980d9d73fe93fb211a6a40eb61e73 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 6 Jul 2021 06:58:16 -0400 Subject: [PATCH 193/290] Changing TestFlows output to use classic mode instead of just new fails. --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 9fa028fedca..c20e742fea1 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -73,4 +73,4 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --no-color -o new-fails --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n1 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] +CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n1 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] From 53b23775a9867793a3ea9864a82a9d1b9b32327a Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 13:58:53 +0300 Subject: [PATCH 194/290] Fix drop part --- src/Storages/MergeTree/DropPartsRanges.cpp | 9 +++------ src/Storages/MergeTree/DropPartsRanges.h | 17 +++++++++++++---- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 11 +++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/DropPartsRanges.cpp b/src/Storages/MergeTree/DropPartsRanges.cpp index 0dfcc18ea3e..583f91b1e9d 100644 --- a/src/Storages/MergeTree/DropPartsRanges.cpp +++ b/src/Storages/MergeTree/DropPartsRanges.cpp @@ -1,5 +1,4 @@ #include -#include namespace DB { @@ -20,7 +19,7 @@ bool DropPartsRanges::isAffectedByDropRange(const std::string & new_part_name, s { if (!drop_range.isDisjoint(entry_info)) { - postpone_reason = fmt::format("Has DROP RANGE with entry. Will postpone it's execution.", drop_range.getPartName()); + postpone_reason = fmt::format("Has DROP RANGE affecting entry {} producing part {}. Will postpone it's execution.", drop_range.getPartName(), new_part_name); return true; } } @@ -33,22 +32,20 @@ bool DropPartsRanges::isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & return isAffectedByDropRange(entry.new_part_name, postpone_reason); } -void DropPartsRanges::addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * /*log*/) +void DropPartsRanges::addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry) { if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to add entry of type {} to drop ranges, expected DROP_RANGE", entry->typeToString()); - //LOG_DEBUG(log, "ADD DROP RANGE {}", *entry->getDropRange(format_version)); MergeTreePartInfo entry_info = MergeTreePartInfo::fromPartName(*entry->getDropRange(format_version), format_version); drop_ranges.emplace(entry->znode_name, entry_info); } -void DropPartsRanges::removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * /*log*/) +void DropPartsRanges::removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry) { if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove entry of type {} from drop ranges, expected DROP_RANGE", entry->typeToString()); - //LOG_DEBUG(log, "REMOVE DROP RANGE {}", *entry->getDropRange(format_version)); drop_ranges.erase(entry->znode_name); } diff --git a/src/Storages/MergeTree/DropPartsRanges.h b/src/Storages/MergeTree/DropPartsRanges.h index fea6ce3be4e..4d512263058 100644 --- a/src/Storages/MergeTree/DropPartsRanges.h +++ b/src/Storages/MergeTree/DropPartsRanges.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -8,26 +8,35 @@ namespace DB { +/// All drop ranges in ReplicatedQueue. +/// Used to postpone execution of entries affected by DROP RANGE class DropPartsRanges { private: MergeTreeDataFormatVersion format_version; - std::map drop_ranges; + /// znode_name -> drop_range + std::unordered_map drop_ranges; public: explicit DropPartsRanges(MergeTreeDataFormatVersion format_version_) : format_version(format_version_) {} + /// Entry is affected by DROP_RANGE and must be postponed bool isAffectedByDropRange(const ReplicatedMergeTreeLogEntry & entry, std::string & postpone_reason) const; + + /// Part is affected by DROP_RANGE and must be postponed bool isAffectedByDropRange(const std::string & new_part_name, std::string & postpone_reason) const; + /// Already has equal DROP_RANGE. Don't need to assign new one bool hasDropRange(const MergeTreePartInfo & new_drop_range_info) const; - void addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * log); + /// Add DROP_RANGE to map + void addDropRange(const ReplicatedMergeTreeLogEntryPtr & entry); - void removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry, Poco::Logger * log); + /// Remove DROP_RANGE from map + void removeDropRange(const ReplicatedMergeTreeLogEntryPtr & entry); }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f607a559564..b8d36488335 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -169,9 +169,10 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } else { - drop_ranges.addDropRange(entry, log); + drop_ranges.addDropRange(entry); auto drop_range = *entry->getDropRange(format_version); - /// DROP PARTS removes parts from virtual parts + + /// DROP PARTS (not DROP PARTITIONS) removes parts from virtual parts. MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(drop_range, format_version); if (!drop_range_info.isFakeDropRangePart() && virtual_parts.getContainingPart(drop_range_info) == drop_range) virtual_parts.removePartAndCoveredParts(drop_range); @@ -271,7 +272,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::DROP_RANGE) { - drop_ranges.removeDropRange(entry, log); + drop_ranges.removeDropRange(entry); } if (entry->type == LogEntry::ALTER_METADATA) @@ -284,7 +285,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { if (entry->type == LogEntry::DROP_RANGE) { - drop_ranges.removeDropRange(entry, log); + drop_ranges.removeDropRange(entry); } for (const String & virtual_part_name : entry->getVirtualPartNames(format_version)) @@ -996,6 +997,8 @@ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & pa { std::lock_guard lock(state_mutex); + /// FIXME get rid of actual_part_name. + /// If new covering part jumps over DROP_RANGE we should execute drop range first if (drop_ranges.isAffectedByDropRange(part_name, reject_reason)) return false; From 8ab722a6aff23b84e4ebf78bb55cb4f2b957eab4 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 6 Jul 2021 07:06:22 -0400 Subject: [PATCH 195/290] Small test fixes. --- .../window_functions/tests/common.py | 4 ++ .../testflows/window_functions/tests/misc.py | 42 +++++++++---------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/tests/testflows/window_functions/tests/common.py b/tests/testflows/window_functions/tests/common.py index ef694b19a0e..3a6ac95bd9b 100644 --- a/tests/testflows/window_functions/tests/common.py +++ b/tests/testflows/window_functions/tests/common.py @@ -374,6 +374,10 @@ def create_table(self, name, statement, on_cluster=False): node = current().context.node try: with Given(f"I have a {name} table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") node.query(statement.format(name=name)) yield name finally: diff --git a/tests/testflows/window_functions/tests/misc.py b/tests/testflows/window_functions/tests/misc.py index 8251e751ed9..aca24edfe9c 100644 --- a/tests/testflows/window_functions/tests/misc.py +++ b/tests/testflows/window_functions/tests/misc.py @@ -10,16 +10,16 @@ def subquery_expr_preceding(self): expected = convert_output(""" sum | unique1 -----+--------- - 0 | 0 - 1 | 1 - 3 | 2 - 5 | 3 - 7 | 4 - 9 | 5 - 11 | 6 - 13 | 7 - 15 | 8 - 17 | 9 + 0 | 0 + 1 | 1 + 3 | 2 + 5 | 3 + 7 | 4 + 9 | 5 + 11 | 6 + 13 | 7 + 15 | 8 + 17 | 9 """) execute_query( @@ -272,7 +272,7 @@ def windows_with_same_partitioning_but_different_ordering(self): but different ordering. """ expected = convert_output(""" - first | last + first | last ------+----- 7 | 7 7 | 9 @@ -367,16 +367,16 @@ def in_view(self): expected = convert_output(""" number | sum_rows ---------+---------- - 1 | 3 - 2 | 6 - 3 | 9 - 4 | 12 - 5 | 15 - 6 | 18 - 7 | 21 - 8 | 24 - 9 | 27 - 10 | 19 + 1 | 3 + 2 | 6 + 3 | 9 + 4 | 12 + 5 | 15 + 6 | 18 + 7 | 21 + 8 | 24 + 9 | 27 + 10 | 19 """) execute_query( From e9540f06214cec7dbc037e62db4862cc5885bb54 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 14:13:13 +0300 Subject: [PATCH 196/290] Remove debug logs --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index b8d36488335..856b5f1bf0c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1028,14 +1028,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( } if (entry.type != LogEntry::DROP_RANGE && drop_ranges.isAffectedByDropRange(entry, out_postpone_reason)) - { - //LOG_DEBUG(log, "POSTPONE ENTRY {} ({}) PRODUCING PART {} BECAUSE OF DROP RANGE {}", entry.znode_name, entry.typeToString(), entry.new_part_name); return false; - } - else - { - //LOG_DEBUG(log, "NO DROP RANGE FOUND FOR PART {} OF TYPE {}", entry.new_part_name, entry.typeToString()); - } /// Check that fetches pool is not overloaded if ((entry.type == LogEntry::GET_PART || entry.type == LogEntry::ATTACH_PART) From 869a41ffb9542ad994f658847883ed4589d931d0 Mon Sep 17 00:00:00 2001 From: dankondr Date: Mon, 25 Jan 2021 17:09:29 +0300 Subject: [PATCH 197/290] Add leftPadString() function --- src/Functions/leftPadString.cpp | 194 ++++++++++++++++++++++ src/Functions/registerFunctionsString.cpp | 4 + src/Functions/ya.make | 1 + 3 files changed, 199 insertions(+) create mode 100644 src/Functions/leftPadString.cpp diff --git a/src/Functions/leftPadString.cpp b/src/Functions/leftPadString.cpp new file mode 100644 index 00000000000..cdcfb46eb73 --- /dev/null +++ b/src/Functions/leftPadString.cpp @@ -0,0 +1,194 @@ +#include +#include +#include + +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + struct LeftPadStringImpl + { + static void vector( + const ColumnString::Chars & data, + const ColumnString::Offsets & offsets, + const size_t length, + const String & padstr, + ColumnString::Chars & res_data, + ColumnString::Offsets & res_offsets) + { + size_t size = offsets.size(); + res_data.resize((length + 1 /* zero terminator */) * size); + res_offsets.resize(size); + + const size_t padstr_size = padstr.size(); + + ColumnString::Offset prev_offset = 0; + ColumnString::Offset res_prev_offset = 0; + for (size_t i = 0; i < size; ++i) + { + size_t data_length = offsets[i] - prev_offset - 1 /* zero terminator */; + if (data_length < length) + { + for (size_t j = 0; j < length - data_length; ++j) + res_data[res_prev_offset + j] = padstr[j % padstr_size]; + memcpy(&res_data[res_prev_offset + length - data_length], &data[prev_offset], data_length); + } + else + { + memcpy(&res_data[res_prev_offset], &data[prev_offset], length); + } + res_data[res_prev_offset + length] = 0; + res_prev_offset += length + 1; + res_offsets[i] = res_prev_offset; + } + } + + static void vectorFixed( + const ColumnFixedString::Chars & data, + const size_t n, + const size_t length, + const String & padstr, + ColumnFixedString::Chars & res_data) + { + const size_t padstr_size = padstr.size(); + const size_t size = data.size() / n; + res_data.resize(length * size); + for (size_t i = 0; i < size; ++i) + { + if (length < n) + { + memcpy(&res_data[i * length], &data[i * n], length); + } + else + { + for (size_t j = 0; j < length - n; ++j) + res_data[i * length + j] = padstr[j % padstr_size]; + memcpy(&res_data[i * length + length - n], &data[i * n], n); + } + } + } + }; + + class FunctionLeftPadString : public IFunction + { + public: + static constexpr auto name = "leftPadString"; + static FunctionPtr create(const ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + size_t number_of_arguments = arguments.size(); + + if (number_of_arguments != 2 && number_of_arguments != 3) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3", + getName(), + toString(number_of_arguments)); + + if (!isStringOrFixedString(arguments[0])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); + + if (!isNativeNumber(arguments[1])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of second argument of function {}", + arguments[1]->getName(), + getName()); + + if (number_of_arguments == 3 && !isStringOrFixedString(arguments[2])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of third argument of function {}", + arguments[2]->getName(), + getName()); + + return arguments[0]; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + { + const ColumnPtr str_column = arguments[0].column; + String padstr = " "; + if (arguments.size() == 3) + { + const ColumnConst * pad_column = checkAndGetColumnConst(arguments[2].column.get()); + if (!pad_column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of third ('pad') argument of function {}. Must be constant string.", + arguments[2].column->getName(), + getName()); + + padstr = pad_column->getValue(); + } + + const ColumnConst * len_column = checkAndGetColumnConst(arguments[1].column.get()); + if (!len_column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of second ('len') argument of function {}. Must be a positive integer.", + arguments[1].column->getName(), + getName()); + Int64 len = len_column->getInt(0); + if (len <= 0) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Illegal value {} of second ('len') argument of function {}. Must be a positive integer.", + arguments[1].column->getName(), + getName()); + + if (const ColumnString * strings = checkAndGetColumn(str_column.get())) + { + auto col_res = ColumnString::create(); + LeftPadStringImpl::vector( + strings->getChars(), strings->getOffsets(), len, padstr, col_res->getChars(), col_res->getOffsets()); + return col_res; + } + else if (const ColumnFixedString * strings_fixed = checkAndGetColumn(str_column.get())) + { + auto col_res = ColumnFixedString::create(len); + LeftPadStringImpl::vectorFixed(strings_fixed->getChars(), strings_fixed->getN(), len, padstr, col_res->getChars()); + return col_res; + } + else + { + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first ('str') argument of function {}. Must be a string or fixed string.", + arguments[0].column->getName(), + getName()); + } + } + }; +} + +void registerFunctionLeftPadString(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerAlias("lpad", "leftPadString", FunctionFactory::CaseInsensitive); +} + +} diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index f6f95489f82..1c487981844 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -33,6 +33,9 @@ void registerFunctionRegexpQuoteMeta(FunctionFactory &); void registerFunctionNormalizeQuery(FunctionFactory &); void registerFunctionNormalizedQueryHash(FunctionFactory &); void registerFunctionCountMatches(FunctionFactory &); +void registerFunctionEncodeXMLComponent(FunctionFactory & factory); +void registerFunctionDecodeXMLComponent(FunctionFactory & factory); +void registerFunctionLeftPadString(FunctionFactory & factory); void registerFunctionEncodeXMLComponent(FunctionFactory &); void registerFunctionDecodeXMLComponent(FunctionFactory &); void registerFunctionExtractTextFromHTML(FunctionFactory &); @@ -74,6 +77,7 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionCountMatches(factory); registerFunctionEncodeXMLComponent(factory); registerFunctionDecodeXMLComponent(factory); + registerFunctionLeftPadString(factory); registerFunctionExtractTextFromHTML(factory); #if USE_BASE64 registerFunctionBase64Encode(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index d6da7eadd35..ba14e9a3e02 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -332,6 +332,7 @@ SRCS( jumpConsistentHash.cpp lcm.cpp least.cpp + leftPadString.cpp lengthUTF8.cpp less.cpp lessOrEquals.cpp From 75e26b93d066b424eababe08d29eda9f0e95edec Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 15:05:18 +0300 Subject: [PATCH 198/290] Review bug fixes --- src/DataStreams/TTLAggregationAlgorithm.cpp | 2 +- src/DataStreams/TTLColumnAlgorithm.cpp | 3 ++- src/DataStreams/TTLDeleteAlgorithm.cpp | 3 ++- src/Storages/MergeTree/TTLMergeSelector.cpp | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/DataStreams/TTLAggregationAlgorithm.cpp b/src/DataStreams/TTLAggregationAlgorithm.cpp index 12d28ff4aea..287ecb7dd6e 100644 --- a/src/DataStreams/TTLAggregationAlgorithm.cpp +++ b/src/DataStreams/TTLAggregationAlgorithm.cpp @@ -37,7 +37,7 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( aggregator = std::make_unique(params); - if (isMinTTLExpired()) + if (isMaxTTLExpired()) new_ttl_info.finished = true; } diff --git a/src/DataStreams/TTLColumnAlgorithm.cpp b/src/DataStreams/TTLColumnAlgorithm.cpp index 5c0a5e1ae83..1318ea382db 100644 --- a/src/DataStreams/TTLColumnAlgorithm.cpp +++ b/src/DataStreams/TTLColumnAlgorithm.cpp @@ -21,7 +21,8 @@ TTLColumnAlgorithm::TTLColumnAlgorithm( new_ttl_info = old_ttl_info; is_fully_empty = false; } - else + + if (isMaxTTLExpired()) new_ttl_info.finished = true; } diff --git a/src/DataStreams/TTLDeleteAlgorithm.cpp b/src/DataStreams/TTLDeleteAlgorithm.cpp index f1bbe6d4b7d..ea7a0b235ec 100644 --- a/src/DataStreams/TTLDeleteAlgorithm.cpp +++ b/src/DataStreams/TTLDeleteAlgorithm.cpp @@ -9,7 +9,8 @@ TTLDeleteAlgorithm::TTLDeleteAlgorithm( { if (!isMinTTLExpired()) new_ttl_info = old_ttl_info; - else + + if (isMaxTTLExpired()) new_ttl_info.finished = true; } diff --git a/src/Storages/MergeTree/TTLMergeSelector.cpp b/src/Storages/MergeTree/TTLMergeSelector.cpp index ab686c9952d..6a42ce039ac 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.cpp +++ b/src/Storages/MergeTree/TTLMergeSelector.cpp @@ -111,8 +111,9 @@ bool TTLDeleteMergeSelector::isTTLAlreadySatisfied(const IMergeSelector::Part & if (only_drop_parts) return false; + /// All TTL satisfied if (!part.ttl_infos->hasAnyNonFinishedTTLs()) - return false; + return true; return !part.shall_participate_in_merges; } From c6e13e6e2e52e643ad7791088542259a9e15e0c7 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Jul 2021 15:18:47 +0300 Subject: [PATCH 199/290] Add leading zeros in function bin --- src/Common/hex.h | 15 ------------- src/Functions/FunctionsCoding.h | 22 ++++--------------- .../0_stateless/01926_bin_unbin.reference | 8 +++---- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/src/Common/hex.h b/src/Common/hex.h index 69bc6f4f79f..82eff776244 100644 --- a/src/Common/hex.h +++ b/src/Common/hex.h @@ -1,6 +1,5 @@ #pragma once #include -#include /// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. @@ -47,20 +46,6 @@ inline void writeBinByte(UInt8 byte, void * out) memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); } -inline size_t writeBinByteNoLeadZeros(UInt8 byte, char * out) -{ - if (byte == 0) - return 0; - - int clz = std::countl_zero(byte); - for (Int8 offset = sizeof(UInt8) * 8 - clz - 1; offset >= 0; --offset) - { - *out = ((byte >> offset) & 1) ? '1' : '0'; - ++out; - } - return sizeof(UInt8) * 8 - clz; -} - /// Produces hex representation of an unsigned int with leading zeros (for checksums) template inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table) diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 33b26afc8dc..72f2aa1be1c 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -1260,7 +1260,7 @@ struct HexImpl { UInt8 byte = x >> offset; - /// Leading zeros. + /// Skip leading zeros if (byte == 0 && !was_nonzero && offset) continue; @@ -1349,26 +1349,12 @@ struct BinImpl UInt8 byte = x >> offset; /// Skip leading zeros - if (byte == 0 && !was_nonzero) + if (byte == 0 && !was_nonzero && offset) continue; - /// First non-zero byte without leading zeros - if (was_nonzero) - { - writeBinByte(byte, out); - out += word_size; - } - else - { - size_t written = writeBinByteNoLeadZeros(byte, out); - out += written; - } was_nonzero = true; - } - if (!was_nonzero) - { - *out = '0'; - ++out; + writeBinByte(byte, out); + out += word_size; } *out = '\0'; ++out; diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index ace28af5211..96104d0e86f 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -1,8 +1,8 @@ -0 -1 -1010 -1111111 +00000000 +00000001 +00001010 +01111111 11111111 00110000 0011000100110000 From a50a98c595e978c6e3c15e64609501f7ef030cf8 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Jul 2021 15:24:30 +0300 Subject: [PATCH 200/290] Add copuple cases to test bin_unbin --- tests/queries/0_stateless/01926_bin_unbin.reference | 3 +++ tests/queries/0_stateless/01926_bin_unbin.sql | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index 96104d0e86f..f84a858e449 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -4,6 +4,9 @@ 00001010 01111111 11111111 +0000000100000000 +0000000111111111 +0000001000000000 00110000 0011000100110000 111001101011010110001011111010001010111110010101 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql index 3593448d407..555770d09c6 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.sql +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -4,6 +4,9 @@ select bin(1); select bin(10); select bin(127); select bin(255); +select bin(256); +select bin(511); +select bin(512); select bin('0'); select bin('10'); select bin('测试'); From 96536a9cbebe06522e2bd332003f4a59009d7aca Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 6 Jul 2021 15:32:28 +0300 Subject: [PATCH 201/290] Update tests/queries/0_stateless/01917_distinct_on.sql --- tests/queries/0_stateless/01917_distinct_on.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01917_distinct_on.sql b/tests/queries/0_stateless/01917_distinct_on.sql index b7875719c92..e394f219b62 100644 --- a/tests/queries/0_stateless/01917_distinct_on.sql +++ b/tests/queries/0_stateless/01917_distinct_on.sql @@ -7,7 +7,7 @@ SELECT DISTINCT ON (a, b) a, b, c FROM t1; SELECT DISTINCT ON (a, b) * FROM t1; SELECT DISTINCT ON (a) * FROM t1; -SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 590 } +SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 1 } SELECT DISTINCT ON a, b a, b FROM t1; -- { clientError 62 } SELECT DISTINCT ON a a, b FROM t1; -- { clientError 62 } @@ -20,4 +20,3 @@ SELECT ALL DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 } SELECT DISTINCT ON (a, b) ALL a, b FROM t1; -- { clientError 62 } DROP TABLE IF EXISTS t1; - From 5b0bc8a7fbf846cb45012e8009d58341656bf12c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Jul 2021 16:16:20 +0300 Subject: [PATCH 202/290] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 903c72f044a..838a2da9aff 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -253,3 +253,4 @@ 01923_different_expression_name_alias 01932_null_valid_identifier 00918_json_functions +01889_sql_json_functions From 3a69d06fc9444701cc7b50eff12efe0043b6804b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Jul 2021 16:36:18 +0300 Subject: [PATCH 203/290] try fix flaky tests --- tests/clickhouse-test | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 63624246190..4de3dc7c2bf 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -41,6 +41,7 @@ MESSAGES_TO_RETRY = [ "Operation timed out", "ConnectionPoolWithFailover: Connection failed at try", "DB::Exception: New table appeared in database being dropped or detached. Try again", + "is already started to be removing by another replica right now", DISTRIBUTED_DDL_TIMEOUT_MSG # FIXME ] From 1f53404e668e46b50874236f76b426e42d38b872 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 6 Jul 2021 16:58:12 +0300 Subject: [PATCH 204/290] better retries --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4de3dc7c2bf..73bd19f8174 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -29,7 +29,7 @@ import string import multiprocessing from contextlib import closing -DISTRIBUTED_DDL_TIMEOUT_MSG = "is executing longer than distributed_ddl_task_timeout (=120)" +DISTRIBUTED_DDL_TIMEOUT_MSG = "is executing longer than distributed_ddl_task_timeout" MESSAGES_TO_RETRY = [ "DB::Exception: ZooKeeper session has been expired", From 9b3ceda57c74f34316673dcbb37abd8b608d1302 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Tue, 6 Jul 2021 17:10:29 +0100 Subject: [PATCH 205/290] Increment ZooKeeperWatch metric only if the callback is registered --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index a717052a1ba..a7e3a73ddd0 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -566,7 +566,6 @@ void ZooKeeper::sendThread() if (info.watch) { info.request->has_watch = true; - CurrentMetrics::add(CurrentMetrics::ZooKeeperWatch); } if (expired) @@ -773,6 +772,8 @@ void ZooKeeper::receiveEvent() if (add_watch) { + CurrentMetrics::add(CurrentMetrics::ZooKeeperWatch); + /// The key of wathces should exclude the root_path String req_path = request_info.request->getPath(); removeRootPath(req_path, root_path); From 24f5ad8920104a7ca93f3f85e80704b26aaf8688 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Tue, 6 Jul 2021 17:13:13 +0100 Subject: [PATCH 206/290] Subtract number of watch callbacks as this is what we actually count --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index a7e3a73ddd0..37cc1dddce2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -906,6 +906,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) { std::lock_guard lock(watches_mutex); + Int64 watch_callback_count = 0; for (auto & path_watches : watches) { WatchResponse response; @@ -915,6 +916,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) for (auto & callback : path_watches.second) { + watch_callback_count += 1; if (callback) { try @@ -929,7 +931,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive) } } - CurrentMetrics::sub(CurrentMetrics::ZooKeeperWatch, watches.size()); + CurrentMetrics::sub(CurrentMetrics::ZooKeeperWatch, watch_callback_count); watches.clear(); } From b44bd174cc40ba1fdd8e4e185a5e383621529687 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 6 Jul 2021 19:14:22 +0300 Subject: [PATCH 207/290] Change error code for DISTINCT ON and LIMIT BY, finally --- src/Parsers/ParserSelectQuery.cpp | 4 ++-- tests/queries/0_stateless/01917_distinct_on.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 2b7f6bcaaf9..255595caa0e 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -20,8 +20,8 @@ namespace ErrorCodes extern const int FIRST_AND_NEXT_TOGETHER; extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED; extern const int ROW_AND_ROWS_TOGETHER; + extern const int SYNTAX_ERROR; extern const int TOP_AND_LIMIT_TOGETHER; - extern const int UNSUPPORTED_METHOD; extern const int WITH_TIES_WITHOUT_ORDER_BY; } @@ -286,7 +286,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) throw Exception("Can not use WITH TIES alongside LIMIT BY/DISTINCT ON", ErrorCodes::LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED); if (distinct_on_expression_list) - throw Exception("Can not use DISTINCT ON alongside LIMIT BY", ErrorCodes::UNSUPPORTED_METHOD); + throw Exception("Can not use DISTINCT ON alongside LIMIT BY", ErrorCodes::SYNTAX_ERROR); limit_by_length = limit_length; limit_by_offset = limit_offset; diff --git a/tests/queries/0_stateless/01917_distinct_on.sql b/tests/queries/0_stateless/01917_distinct_on.sql index e394f219b62..75dd8c0b7b8 100644 --- a/tests/queries/0_stateless/01917_distinct_on.sql +++ b/tests/queries/0_stateless/01917_distinct_on.sql @@ -7,7 +7,7 @@ SELECT DISTINCT ON (a, b) a, b, c FROM t1; SELECT DISTINCT ON (a, b) * FROM t1; SELECT DISTINCT ON (a) * FROM t1; -SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 1 } +SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 62 } SELECT DISTINCT ON a, b a, b FROM t1; -- { clientError 62 } SELECT DISTINCT ON a a, b FROM t1; -- { clientError 62 } From a8fdc41193e07e1f17bdf5172bcbf93165aac81a Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 19:51:23 +0300 Subject: [PATCH 208/290] Fix bug and add more trash to test --- src/Storages/MergeTree/DropPartsRanges.cpp | 4 +++- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 10 ++++++++++ .../MergeTree/ReplicatedMergeTreeLogEntry.h | 4 ++++ .../MergeTree/ReplicatedMergeTreeQueue.cpp | 19 +++++++++++++------ .../0_stateless/01154_move_partition_long.sh | 13 +++++++++++++ 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/DropPartsRanges.cpp b/src/Storages/MergeTree/DropPartsRanges.cpp index 583f91b1e9d..ab808f59970 100644 --- a/src/Storages/MergeTree/DropPartsRanges.cpp +++ b/src/Storages/MergeTree/DropPartsRanges.cpp @@ -46,7 +46,9 @@ void DropPartsRanges::removeDropRange(const ReplicatedMergeTreeLogEntryPtr & ent if (entry->type != ReplicatedMergeTreeLogEntry::DROP_RANGE) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove entry of type {} from drop ranges, expected DROP_RANGE", entry->typeToString()); - drop_ranges.erase(entry->znode_name); + auto it = drop_ranges.find(entry->znode_name); + assert(it != drop_ranges.end()); + drop_ranges.erase(it); } bool DropPartsRanges::hasDropRange(const MergeTreePartInfo & new_drop_range_info) const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index d326ad10370..18e90952721 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -431,6 +431,16 @@ std::optional ReplicatedMergeTreeLogEntryData::getDropRange(MergeTreeDat return {}; } +bool ReplicatedMergeTreeLogEntryData::isDropPart(MergeTreeDataFormatVersion format_version) const +{ + if (type == DROP_RANGE) + { + auto drop_range_info = MergeTreePartInfo::fromPartName(new_part_name, format_version); + return !drop_range_info.isFakeDropRangePart(); + } + return false; +} + Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormatVersion format_version) const { /// Doesn't produce any part diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index eb82572b107..3752c9deb8f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -143,6 +143,10 @@ struct ReplicatedMergeTreeLogEntryData /// Returns fake part for drop range (for DROP_RANGE and REPLACE_RANGE) std::optional getDropRange(MergeTreeDataFormatVersion format_version) const; + /// This entry is DROP PART, not DROP PARTITION. They both have same + /// DROP_RANGE entry type, but differs in information about drop range. + bool isDropPart(MergeTreeDataFormatVersion format_version) const; + /// Access under queue_mutex, see ReplicatedMergeTreeQueue. bool currently_executing = false; /// Whether the action is executing now. bool removed_by_other_entry = false; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 856b5f1bf0c..aaa76009d74 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -170,12 +170,11 @@ void ReplicatedMergeTreeQueue::insertUnlocked( else { drop_ranges.addDropRange(entry); - auto drop_range = *entry->getDropRange(format_version); - /// DROP PARTS (not DROP PARTITIONS) removes parts from virtual parts. - MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(drop_range, format_version); - if (!drop_range_info.isFakeDropRangePart() && virtual_parts.getContainingPart(drop_range_info) == drop_range) - virtual_parts.removePartAndCoveredParts(drop_range); + /// DROP PART remove parts, so we remove it from virtual parts to + /// preserve invariant virtual_parts = current_parts + queue + if (entry->isDropPart(format_version)) + virtual_parts.removePartAndCoveredParts(*entry->getDropRange(format_version)); queue.push_front(entry); } @@ -266,7 +265,15 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (auto drop_range_part_name = entry->getDropRange(format_version)) { - current_parts.remove(*drop_range_part_name); + MergeTreePartInfo drop_range_info = MergeTreePartInfo::fromPartName(*drop_range_part_name, format_version); + + /// DROP PART doesn't have virtual parts so remove from current + /// parts all covered parts. + if (entry->isDropPart(format_version)) + current_parts.removePartAndCoveredParts(*drop_range_part_name); + else + current_parts.remove(*drop_range_part_name); + virtual_parts.remove(*drop_range_part_name); } diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index 1ce40770e46..b57f94b66eb 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -72,6 +72,7 @@ function drop_partition_thread() done } + function optimize_thread() { while true; do @@ -85,12 +86,23 @@ function optimize_thread() done } +function drop_part_thread() +{ + while true; do + REPLICA=$(($RANDOM % 16)) + part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1") + $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null + sleep 0.$RANDOM; + done +} + #export -f create_drop_thread; export -f insert_thread; export -f move_partition_src_dst_thread; export -f replace_partition_src_src_thread; export -f drop_partition_thread; export -f optimize_thread; +export -f drop_part_thread; TIMEOUT=60 @@ -102,6 +114,7 @@ timeout $TIMEOUT bash -c move_partition_src_dst_thread & timeout $TIMEOUT bash -c replace_partition_src_src_thread & timeout $TIMEOUT bash -c drop_partition_thread & timeout $TIMEOUT bash -c optimize_thread & +timeout $TIMEOUT bash -c drop_part_thread & wait for ((i=0; i<16; i++)) do From 1c39df068eb114b50be6b9608a3c6fca8ce1fbb7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 19:52:54 +0300 Subject: [PATCH 209/290] Remove accident change --- tests/queries/0_stateless/01154_move_partition_long.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index b57f94b66eb..dd16b2dc63d 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -72,7 +72,6 @@ function drop_partition_thread() done } - function optimize_thread() { while true; do From 3dee74df54ad3c1827510601c159abc650fc97c2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 6 Jul 2021 19:53:54 +0300 Subject: [PATCH 210/290] Comment --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index f97ab74bd28..e49d80fc832 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -102,7 +102,7 @@ private: ActiveDataPartSet virtual_parts; - /// + /// Dropped ranges inserted into queue DropPartsRanges drop_ranges; /// A set of mutations loaded from ZooKeeper. From 717775d8c020a88b1cf0dfeda5de7bf245f2051d Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 6 Jul 2021 21:06:34 +0300 Subject: [PATCH 211/290] Links again --- docs/en/interfaces/http.md | 2 +- docs/ru/interfaces/http.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index f4237cc2eae..0f497f9af80 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -18,7 +18,7 @@ Ok. Web UI can be accessed here: `http://localhost:8123/play`. -![Web UI](../images/play.png#) +![Web UI](../images/play.png) In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 83a6a30a071..fcd9b949ad8 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -17,7 +17,7 @@ Ok. Веб-интерфейс доступен по адресу: `http://localhost:8123/play`. -![Веб-интерфейс](../images/play.png#) +![Веб-интерфейс](../images/play.png) В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). From c0798df656ee5ad919bb7a2ba7f35a0ce66717d4 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Tue, 6 Jul 2021 21:08:11 +0300 Subject: [PATCH 212/290] Apply suggestions from code review Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/interfaces/http.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index fcd9b949ad8..895172ab319 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -19,7 +19,7 @@ Ok. ![Веб-интерфейс](../images/play.png) -В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен всегда возвращается «Ok.» (с переводом строки на конце). +В скриптах проверки доступности вы можете использовать `GET /ping` без параметров. Если сервер доступен, всегда возвращается «Ok.» (с переводом строки на конце). ``` bash $ curl 'http://localhost:8123/ping' @@ -29,8 +29,7 @@ Ok. Запрос отправляется в виде URL параметра с именем `query`. Или как тело запроса при использовании метода POST. Или начало запроса в URL параметре query, а продолжение POST-ом (зачем это нужно, будет объяснено ниже). Размер URL ограничен 16KB, это следует учитывать при отправке больших запросов. -В случае успеха вам вернётся код ответа 200 и результат обработки запроса в теле ответа. -В случае ошибки вам вернётся код ответа 500 и текст с описанием ошибки в теле ответа. +В случае успеха возвращается код ответа 200 и результат обработки запроса в теле ответа, в случае ошибки — код ответа 500 и текст с описанием ошибки в теле ответа. При использовании метода GET выставляется настройка readonly. То есть, для запросов, модифицирующих данные, можно использовать только метод POST. Сам запрос при этом можно отправлять как в теле POST запроса, так и в параметре URL. @@ -81,7 +80,7 @@ ECT 1 По умолчанию данные возвращаются в формате [TabSeparated](formats.md#tabseparated). -Можно попросить любой другой формат с помощью секции FORMAT запроса. +Можно указать любой другой формат с помощью секции FORMAT запроса. Кроме того, вы можете использовать параметр URL-адреса `default_format` или заголовок `X-ClickHouse-Format`, чтобы указать формат по умолчанию, отличный от `TabSeparated`. From e7a938f860bcecc9f2062f007636b1a27334f7ca Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 6 Jul 2021 17:31:20 -0400 Subject: [PATCH 213/290] Updating map_type tests due to JSON changes. Crossing out tests with (U)Int64 keys due to double quoting bug. --- tests/testflows/map_type/regression.py | 7 ++++ tests/testflows/map_type/tests/common.py | 20 ----------- tests/testflows/map_type/tests/feature.py | 41 +++++++++++------------ 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/tests/testflows/map_type/regression.py b/tests/testflows/map_type/regression.py index b239d91ccf9..9f9c2b2b261 100755 --- a/tests/testflows/map_type/regression.py +++ b/tests/testflows/map_type/regression.py @@ -89,6 +89,13 @@ xfails = { [(Fail, "LowCardinality(FixedString) as key not supported")], "tests/table map with value string/LowCardinality(String) for key and value": [(Fail, "LowCardinality(String) as key not supported")], + # JSON related + "tests/table map with duplicated keys/Map(Int64, String))": + [(Fail, "new bug due to JSON changes")], + "tests/table map with key integer/UInt64": + [(Fail, "new bug due to JSON changes")], + "tests/table map with value integer/UInt64": + [(Fail, "new bug due to JSON changes")] } xflags = { diff --git a/tests/testflows/map_type/tests/common.py b/tests/testflows/map_type/tests/common.py index a3a0d0ef0b1..6ce1b6ab8a6 100644 --- a/tests/testflows/map_type/tests/common.py +++ b/tests/testflows/map_type/tests/common.py @@ -12,26 +12,6 @@ def getuid(): testname = f"{basename(current().name).replace(' ', '_').replace(',','')}" return testname + "_" + str(uuid.uuid1()).replace('-', '_') -@TestStep(Given) -def allow_experimental_map_type(self): - """Set allow_experimental_map_type = 1 - """ - setting = ("allow_experimental_map_type", 1) - default_query_settings = None - - try: - with By("adding allow_experimental_map_type to the default query settings"): - default_query_settings = getsattr(current().context, "default_query_settings", []) - default_query_settings.append(setting) - yield - finally: - with Finally("I remove allow_experimental_map_type from the default query settings"): - if default_query_settings: - try: - default_query_settings.pop(default_query_settings.index(setting)) - except ValueError: - pass - @TestStep(Given) def create_table(self, name, statement, on_cluster=False): """Create table. diff --git a/tests/testflows/map_type/tests/feature.py b/tests/testflows/map_type/tests/feature.py index 5fd48844825..5d7c900d591 100755 --- a/tests/testflows/map_type/tests/feature.py +++ b/tests/testflows/map_type/tests/feature.py @@ -254,19 +254,19 @@ def table_map_select_key_with_value_string(self, type, data, output): RQ_SRS_018_ClickHouse_Map_DataType_Value_Integer("1.0") ) @Examples("type data output", [ - ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:127,2:0,3:-128}}', Name("Int8")), - ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{1:0,2:255}}', Name("UInt8")), - ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:32767,2:0,3:-32768}}', Name("Int16")), - ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{1:0,2:65535}}', Name("UInt16")), - ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{1:2147483647,2:0,3:-2147483648}}', Name("Int32")), - ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{1:0,2:4294967295}}', Name("UInt32")), + ("Map(Int8, Int8)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":127,"2":0,"3":-128}}', Name("Int8")), + ("Map(Int8, UInt8)", "('2020-01-01', map(1,0,2,255))", '{"d":"2020-01-01","m":{"1":0,"2":255}}', Name("UInt8")), + ("Map(Int8, Int16)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":32767,"2":0,"3":-32768}}', Name("Int16")), + ("Map(Int8, UInt16)", "('2020-01-01', map(1,0,2,65535))", '{"d":"2020-01-01","m":{"1":0,"2":65535}}', Name("UInt16")), + ("Map(Int8, Int32)", "('2020-01-01', map(1,127,2,0,3,-128))", '{"d":"2020-01-01","m":{"1":2147483647,"2":0,"3":-2147483648}}', Name("Int32")), + ("Map(Int8, UInt32)", "('2020-01-01', map(1,0,2,4294967295))", '{"d":"2020-01-01","m":{"1":0,"2":4294967295}}', Name("UInt32")), ("Map(Int8, Int64)", "('2020-01-01', map(1,9223372036854775807,2,0,3,-9223372036854775808))", '{"d":"2020-01-01","m":{1:"9223372036854775807",2:"0",3:"-9223372036854775808"}}', Name("Int64")), ("Map(Int8, UInt64)", "('2020-01-01', map(1,0,2,18446744073709551615))", '{"d":"2020-01-01","m":{1:"0",2:"18446744073709551615"}}', Name("UInt64")), ("Map(Int8, Int128)", "('2020-01-01', map(1,170141183460469231731687303715884105727,2,0,3,-170141183460469231731687303715884105728))", '{"d":"2020-01-01","m":{1:"170141183460469231731687303715884105727",2:"0",3:"-170141183460469231731687303715884105728"}}', Name("Int128")), ("Map(Int8, Int256)", "('2020-01-01', map(1,57896044618658097711785492504343953926634992332820282019728792003956564819967,2,0,3,-57896044618658097711785492504343953926634992332820282019728792003956564819968))", '{"d":"2020-01-01","m":{1:"57896044618658097711785492504343953926634992332820282019728792003956564819967",2:"0",3:"-57896044618658097711785492504343953926634992332820282019728792003956564819968"}}', Name("Int256")), ("Map(Int8, UInt256)", "('2020-01-01', map(1,0,2,115792089237316195423570985008687907853269984665640564039457584007913129639935))", '{"d":"2020-01-01","m":{1:"0",2:"115792089237316195423570985008687907853269984665640564039457584007913129639935"}}', Name("UInt256")), - ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{1:1}}', Name("toNullable")), - ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{1:null}}', Name("toNullable(NULL)")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(1)))", '{"d":"2020-01-01","m":{"1":1}}', Name("toNullable")), + ("Map(Int8, Nullable(Int8))", "('2020-01-01', map(1,toNullable(NULL)))", '{"d":"2020-01-01","m":{"1":null}}', Name("toNullable(NULL)")), ]) def table_map_with_value_integer(self, type, data, output): """Check what values we can insert into map type column with value integer. @@ -281,8 +281,8 @@ def table_map_with_value_integer(self, type, data, output): ("Map(String, Array(Int8))", "('2020-01-01', map('key',[]))", '{"d":"2020-01-01","m":{"key":[]}}', Name("empty array")), ("Map(String, Array(Int8))", "('2020-01-01', map('key',[1,2,3]))", '{"d":"2020-01-01","m":{"key":[1,2,3]}}', Name("non-empty array of ints")), ("Map(String, Array(String))", "('2020-01-01', map('key',['1','2','3']))", '{"d":"2020-01-01","m":{"key":["1","2","3"]}}', Name("non-empty array of strings")), - ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{1:2},{2:3}]}}', Name("non-empty array of maps")), - ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{1:[{1:[1]}]},{2:[{2:[3]}]}]}}', Name("non-empty array of maps of array of maps")), + ("Map(String, Array(Map(Int8, Int8)))", "('2020-01-01', map('key',[map(1,2),map(2,3)]))", '{"d":"2020-01-01","m":{"key":[{"1":2},{"2":3}]}}', Name("non-empty array of maps")), + ("Map(String, Array(Map(Int8, Array(Map(Int8, Array(Int8))))))", "('2020-01-01', map('key',[map(1,[map(1,[1])]),map(2,[map(2,[3])])]))", '{"d":"2020-01-01","m":{"key":[{"1":[{"1":[1]}]},{"2":[{"2":[3]}]}]}}', Name("non-empty array of maps of array of maps")), ]) def table_map_with_value_array(self, type, data, output): """Check what values we can insert into map type column with value Array. @@ -294,12 +294,12 @@ def table_map_with_value_array(self, type, data, output): RQ_SRS_018_ClickHouse_Map_DataType_Key_Integer("1.0") ) @Examples("type data output", [ - ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{127:1,0:1,-128:1}}', Name("Int8")), - ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{0:1,255:1}}', Name("UInt8")), - ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{32767:1,0:1,-32768:1}}', Name("Int16")), - ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{0:1,65535:1}}', Name("UInt16")), - ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"d":"2020-01-01","m":{2147483647:1,0:1,-2147483648:1}}', Name("Int32")), - ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{0:1,4294967295:1}}', Name("UInt32")), + ("Map(Int8, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{"127":1,"0":1,"-128":1}}', Name("Int8")), + ("Map(UInt8, Int8)", "('2020-01-01', map(0,1,255,1))", '{"d":"2020-01-01","m":{"0":1,"255":1}}', Name("UInt8")), + ("Map(Int16, Int8)", "('2020-01-01', map(127,1,0,1,-128,1))", '{"d":"2020-01-01","m":{"32767":1,"0":1,"-32768":1}}', Name("Int16")), + ("Map(UInt16, Int8)", "('2020-01-01', map(0,1,65535,1))", '{"d":"2020-01-01","m":{"0":1,"65535":1}}', Name("UInt16")), + ("Map(Int32, Int8)", "('2020-01-01', map(2147483647,1,0,1,-2147483648,1))", '{"d":"2020-01-01","m":{"2147483647":1,"0":1,"-2147483648":1}}', Name("Int32")), + ("Map(UInt32, Int8)", "('2020-01-01', map(0,1,4294967295,1))", '{"d":"2020-01-01","m":{"0":1,"4294967295":1}}', Name("UInt32")), ("Map(Int64, Int8)", "('2020-01-01', map(9223372036854775807,1,0,1,-9223372036854775808,1))", '{"d":"2020-01-01","m":{"9223372036854775807":1,"0":1,"-9223372036854775808":1}}', Name("Int64")), ("Map(UInt64, Int8)", "('2020-01-01', map(0,1,18446744073709551615,1))", '{"d":"2020-01-01","m":{"0":1,"18446744073709551615":1}}', Name("UInt64")), ("Map(Int128, Int8)", "('2020-01-01', map(170141183460469231731687303715884105727,1,0,1,-170141183460469231731687303715884105728,1))", '{"d":"2020-01-01","m":{170141183460469231731687303715884105727:1,0:1,"-170141183460469231731687303715884105728":1}}', Name("Int128")), @@ -716,7 +716,7 @@ def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): ) @Examples("tuple type exitcode message check_insert", [ ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", - 0, '{"m":{1:"Ready",2:"Steady",3:"Go"}}', False, Name("int -> int")), + 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> int")), ("(([1, 2, 3], ['Ready', 'Steady', 'Go']))", "Map(String, String)", 0, '{"m":{"1":"Ready","2":"Steady","3":"Go"}}', False, Name("int -> string")), ("((['1', '2', '3'], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", @@ -728,7 +728,7 @@ def cast_tuple_of_two_arrays_to_map(self, tuple, type, exitcode, message): ("(([[1]],['hello']))", "Map(String, String)", 53, 'DB::Exception: Type mismatch in IN or VALUES section', True, Name("array -> string")), ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", - 0, '{"m":{1:2,3:4}}', False, Name("array of two tuples")), + 0, '{"m":{"1":2,"3":4}}', False, Name("array of two tuples")), ("(([1, 2], ['Ready', 'Steady', 'Go']))", "Map(UInt8, String)", 53, "DB::Exception: CAST AS Map can only be performed from tuple of arrays with equal sizes", True, Name("unequal array sizes")), @@ -767,7 +767,7 @@ def cast_array_of_two_tuples_to_map(self, tuple, type, exitcode, message): RQ_SRS_018_ClickHouse_Map_DataType_Conversion_From_ArrayOfTuplesToMap_Invalid("1.0") ) @Examples("tuple type exitcode message check_insert", [ - ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{1:2,3:4}}', False, + ("(([(1,2),(3,4)]))", "Map(UInt8, UInt8)", 0, '{"m":{"1":2,"3":4}}', False, Name("array of two tuples")), ("(([(1,2),(3)]))", "Map(UInt8, UInt8)", 130, "DB::Exception: There is no supertype for types Tuple(UInt8, UInt8), UInt8 because some of them are Tuple and some of them are not", True, @@ -1188,8 +1188,5 @@ def performance(self, len=10, rows=6000000): def feature(self, node="clickhouse1"): self.context.node = self.context.cluster.node(node) - with Given("I allow experimental map type"): - allow_experimental_map_type() - for scenario in loads(current_module(), Scenario): scenario() From ad4c069b4ebae636e777ccd39190add441203cb1 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 6 Jul 2021 17:50:12 -0400 Subject: [PATCH 214/290] Fixing syntax error. --- .../tests/array_tuple_map.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py index 550122c5b86..938beabfff4 100644 --- a/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py +++ b/tests/testflows/extended_precision_data_types/tests/array_tuple_map.py @@ -334,10 +334,10 @@ def map_func(self, data_type, node=None): execute_query(f"SELECT * FROM {table_name}") with Scenario(f"mapAdd with {data_type}"): - sql = f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + sql = (f"SELECT mapAdd(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))" + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") if data_type.startswith("Decimal"): node.query(sql, exitcode=43, message="Exception:") else: @@ -349,8 +349,10 @@ def map_func(self, data_type, node=None): table(name = table_name, data_type = f'Tuple(Array({data_type}), Array({data_type}))') with When("I insert the output into a table"): - sql = (f"INSERT INTO {table_name} SELECT mapAdd(([{to_data_type(data_type,1)},{to_data_type(data_type,2)}]," - f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]), ([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + sql = (f"INSERT INTO {table_name} SELECT mapAdd((" + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," + f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") exitcode, message = 0, None @@ -361,7 +363,8 @@ def map_func(self, data_type, node=None): execute_query(f"""SELECT * FROM {table_name} ORDER BY a ASC""") with Scenario(f"mapSubtract with {data_type}"): - sql = (f"SELECT mapSubtract(([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," + sql = (f"SELECT mapSubtract((" + f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}])," f"([{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]," f"[{to_data_type(data_type,1)}, {to_data_type(data_type,2)}]))") From 42a844546229e56c51a3ec986467ced52e4ed972 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 7 Jul 2021 01:12:56 +0300 Subject: [PATCH 215/290] Fix constness of custom TLDs Before this patch the functions below returns incorrect type for consts, and hence optimize_skip_unused_shards does not work: - cutToFirstSignificantSubdomainCustom() - cutToFirstSignificantSubdomainCustomWithWWW() - firstSignificantSubdomainCustom() --- .../URL/FirstSignificantSubdomainCustomImpl.h | 28 ++++++++++++++++--- .../0_stateless/01601_custom_tld.reference | 6 ++++ .../queries/0_stateless/01601_custom_tld.sql | 8 ++++++ .../01940_custom_tld_sharding_key.reference | 1 + .../01940_custom_tld_sharding_key.sql | 2 ++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/01940_custom_tld_sharding_key.reference create mode 100644 tests/queries/0_stateless/01940_custom_tld_sharding_key.sql diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h index 4670d610725..70cf30a7384 100644 --- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h +++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h @@ -60,14 +60,25 @@ public: return arguments[0].type; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get()); FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue()); - /// FIXME: convertToFullColumnIfConst() is suboptimal - auto column = arguments[0].column->convertToFullColumnIfConst(); - if (const ColumnString * col = checkAndGetColumn(*column)) + auto column = arguments[0].column; + + if (const ColumnConst * const_col = checkAndGetColumnConst(column.get())) + { + const String & data = const_col->getValue(); + const String & res = scalar(tld_lookup, data); + + auto col_res = ColumnString::create(); + col_res->insert(res); + + auto col_const_res = ColumnConst::create(std::move(col_res), input_rows_count); + return col_const_res; + } + else if (const ColumnString * col = checkAndGetColumn(*column)) { auto col_res = ColumnString::create(); vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets()); @@ -107,6 +118,15 @@ public: prev_offset = offsets[i]; } } + + static String scalar(FirstSignificantSubdomainCustomLookup & tld_lookup, const String & data) + { + Pos start; + size_t length; + Extractor::execute(tld_lookup, &data[0], data.size(), start, length); + String output(start, length); + return output; + } }; } diff --git a/tests/queries/0_stateless/01601_custom_tld.reference b/tests/queries/0_stateless/01601_custom_tld.reference index e056505f273..04204ebf02a 100644 --- a/tests/queries/0_stateless/01601_custom_tld.reference +++ b/tests/queries/0_stateless/01601_custom_tld.reference @@ -22,3 +22,9 @@ foobar.com foobar.com foobar.com xx.blogspot.co.at +-- www +www.foo +foo +-- vector +xx.blogspot.co.at + diff --git a/tests/queries/0_stateless/01601_custom_tld.sql b/tests/queries/0_stateless/01601_custom_tld.sql index 688dd419858..ceb00d5ff19 100644 --- a/tests/queries/0_stateless/01601_custom_tld.sql +++ b/tests/queries/0_stateless/01601_custom_tld.sql @@ -29,3 +29,11 @@ select cutToFirstSignificantSubdomainCustom('http://foobar.com', 'public_suffix_ select cutToFirstSignificantSubdomainCustom('http://foobar.com/foo', 'public_suffix_list'); select cutToFirstSignificantSubdomainCustom('http://bar.foobar.com/foo', 'public_suffix_list'); select cutToFirstSignificantSubdomainCustom('http://xx.blogspot.co.at', 'public_suffix_list'); + +select '-- www'; +select cutToFirstSignificantSubdomainCustomWithWWW('http://www.foo', 'public_suffix_list'); +select cutToFirstSignificantSubdomainCustom('http://www.foo', 'public_suffix_list'); + +select '-- vector'; +select cutToFirstSignificantSubdomainCustom('http://xx.blogspot.co.at/' || toString(number), 'public_suffix_list') from numbers(1); +select cutToFirstSignificantSubdomainCustom('there-is-no-such-domain' || toString(number), 'public_suffix_list') from numbers(1); diff --git a/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference b/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference new file mode 100644 index 00000000000..0989a305613 --- /dev/null +++ b/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference @@ -0,0 +1 @@ +foo.com diff --git a/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql b/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql new file mode 100644 index 00000000000..5d38cfb18dc --- /dev/null +++ b/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql @@ -0,0 +1,2 @@ +select * from remote('127.{1,2}', view(select 'foo.com' key), cityHash64(key)) where key = cutToFirstSignificantSubdomainCustom('foo.com', 'public_suffix_list') settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1; +select * from remote('127.{1,2}', view(select 'foo.com' key), cityHash64(key)) where key = cutToFirstSignificantSubdomainCustom('bar.com', 'public_suffix_list') settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1; From b0a3a7180f19dda2e29cf09a1e2ff84ed5d530c5 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 6 Jul 2021 13:48:13 +0300 Subject: [PATCH 216/290] Replace print() with logging.debug() in integration tests --- .../test_replicated_mutations/test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_replicated_mutations/test.py b/tests/integration/test_replicated_mutations/test.py index 40a2b15ffaf..12a49ec22d8 100644 --- a/tests/integration/test_replicated_mutations/test.py +++ b/tests/integration/test_replicated_mutations/test.py @@ -1,3 +1,4 @@ +import logging import random import threading import time @@ -90,7 +91,7 @@ class Runner: i += 1 try: - print('thread {}: insert for {}: {}'.format(thread_num, date_str, ','.join(str(x) for x in xs))) + logging.debug(f"thread {thread_num}: insert for {date_str}: {xs}") random.choice(self.nodes).query("INSERT INTO test_mutations FORMAT TSV", payload) with self.mtx: @@ -100,7 +101,7 @@ class Runner: self.total_inserted_rows += len(xs) except Exception as e: - print('Exception while inserting,', e) + logging.debug(f"Exception while inserting: {e}") self.exceptions.append(e) finally: with self.mtx: @@ -128,7 +129,7 @@ class Runner: continue try: - print('thread {}: delete {} * {}'.format(thread_num, to_delete_count, x)) + logging.debug(f"thread {thread_num}: delete {to_delete_count} * {x}") random.choice(self.nodes).query("ALTER TABLE test_mutations DELETE WHERE x = {}".format(x)) with self.mtx: @@ -138,7 +139,7 @@ class Runner: self.total_deleted_rows += to_delete_count except Exception as e: - print('Exception while deleting,', e) + logging.debug(f"Exception while deleting: {e}") finally: with self.mtx: self.currently_deleting_xs.remove(x) @@ -185,10 +186,9 @@ def test_mutations(started_cluster): assert runner.total_mutations > 0 all_done = wait_for_mutations(nodes, runner.total_mutations) - - print("Total mutations: ", runner.total_mutations) + logging.debug(f"Total mutations: {runner.total_mutations}") for node in nodes: - print(node.query( + logging.debug(node.query( "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) assert all_done @@ -233,9 +233,9 @@ def test_mutations_dont_prevent_merges(started_cluster, nodes): t.join() for node in nodes: - print(node.query( + logging.debug(node.query( "SELECT mutation_id, command, parts_to_do, is_done FROM system.mutations WHERE table = 'test_mutations' FORMAT TSVWithNames")) - print(node.query( + logging.debug(node.query( "SELECT partition, count(name), sum(active), sum(active*rows) FROM system.parts WHERE table ='test_mutations' GROUP BY partition FORMAT TSVWithNames")) assert all_done From 5e120b8d37a17c09232b490e3c438085e7c6ac8d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 7 Jul 2021 09:42:13 +0300 Subject: [PATCH 217/290] Fix stack-buffer-overflow in custom TLDs due to StringHashTable copy 8 bytes at a time ASan reports [1]: ==164==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7f0209dd4abf at pc 0x00000b75b7c5 bp 0x7f0209dd4760 sp 0x7f0209dd4758 READ of size 8 at 0x7f0209dd4abf thread T4 (TCPHandler) 0 0xb75b7c4 in auto StringHashTable > >::dispatch > > const, StringRef const&, StringHashTable > >::FindCallable>(StringHashTable > > const&, StringRef const&, StringHashTable > >::FindCallable&&) obj-x86_64-linux-gnu/../src/Common/HashTable/StringHashTable.h:283:21 1 0xb75b7c4 in StringHashTable > >::has(StringRef const&, unsigned long) const obj-x86_64-linux-gnu/../src/Common/HashTable/StringHashTable.h:365:16 2 0xb75b7c4 in DB::TLDList::has(StringRef const&) const obj-x86_64-linux-gnu/../src/Common/TLDListsHolder.cpp:31:26 3 0x1c4a6046 in void DB::ExtractFirstSignificantSubdomain::executeCustom(DB::FirstSignificantSubdomainCustomLookup const&, char const*, unsigned long, char const*&, unsigned long&, char const**) (/usr/bin/clickhouse+0x1c4a6046) 4 0x1c4a3586 in DB::FunctionCutToFirstSignificantSubdomainCustomImpl, DB::NameCutToFirstSignificantSubdomainCustom>::executeImpl(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long) const (/usr/bin/clickhouse+0x1c4a3586) 5 0x10d96e34 in DB::IFunction::executeImplDryRun(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long) const (/usr/bin/clickhouse+0x10d96e34) 6 0x10d9648b in DB::FunctionToExecutableFunctionAdaptor::executeDryRunImpl(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long) const (/usr/bin/clickhouse+0x10d9648b) 7 0x200ed79b in DB::IExecutableFunction::executeWithoutLowCardinalityColumns(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long, bool) const obj-x86_64-linux-gnu/../src/Functions/IFunction.cpp:212:15 8 0x200ee436 in DB::IExecutableFunction::execute(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long, bool) const obj-x86_64-linux-gnu/../src/Functions/IFunction.cpp:257:22 9 0x20cd6f6f in DB::ActionsDAG::addFunction(std::__1::shared_ptr const&, std::__1::vector >, std::__1::basic_string, std::__1::allocator >) obj-x86_64-linux-gnu/../src/Interpreters/ActionsDAG.cpp:214:37 10 0x2124c8a7 in DB::ScopeStack::addFunction(std::__1::shared_ptr const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&, std::__1::basic_string, std::__1::allocator >) obj-x86_64-linux-gnu/../src/Interpreters/ActionsVisitor.cpp:570:51 11 0x2125c80d in DB::ActionsMatcher::Data::addFunction(std::__1::shared_ptr const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&, std::__1::basic_string, std::__1::allocator >) obj-x86_64-linux-gnu/../src/Interpreters/ActionsVisitor.h:169:27 12 0x2125c80d in DB::ActionsMatcher::visit(DB::ASTFunction const&, std::__1::shared_ptr const&, DB::ActionsMatcher::Data&) obj-x86_64-linux-gnu/../src/Interpreters/ActionsVisitor.cpp:1061:14 13 0x212522fb in DB::ActionsMatcher::visit(DB::ASTFunction const&, std::__1::shared_ptr const&, DB::ActionsMatcher::Data&) obj-x86_64-linux-gnu/../src/Interpreters/ActionsVisitor.cpp:971:17 14 0x2121354e in DB::InDepthNodeVisitor const>::visit(std::__1::shared_ptr const&) obj-x86_64-linux-gnu/../src/Interpreters/InDepthNodeVisitor.h:34:13 15 0x211e17c7 in DB::ExpressionAnalyzer::getRootActions(std::__1::shared_ptr const&, bool, std::__1::shared_ptr&, bool) obj-x86_64-linux-gnu/../src/Interpreters/ExpressionAnalyzer.cpp:421:48 16 0x21204024 in DB::ExpressionAnalyzer::getConstActions(std::__1::vector > const&) obj-x86_64-linux-gnu/../src/Interpreters/ExpressionAnalyzer.cpp:1423:5 17 0x230f7216 in DB::KeyCondition::getBlockWithConstants(std::__1::shared_ptr const&, std::__1::shared_ptr const&, std::__1::shared_ptr) obj-x86_64-linux-gnu/../src/Storages/MergeTree/KeyCondition.cpp:385:103 18 0x22877f9e in DB::(anonymous namespace)::replaceConstantExpressions(std::__1::shared_ptr&, std::__1::shared_ptr, DB::NamesAndTypesList const&, std::__1::shared_ptr, std::__1::shared_ptr const&) obj-x86_64-linux-gnu/../src/Storages/StorageDistributed.cpp:280:34 19 0x22877f9e in DB::StorageDistributed::skipUnusedShards(std::__1::shared_ptr, std::__1::shared_ptr const&, std::__1::shared_ptr const&, std::__1::shared_ptr) const obj-x86_64-linux-gnu/../src/Storages/StorageDistributed.cpp:1091:5 20 0x2285d215 in DB::StorageDistributed::getOptimizedCluster(std::__1::shared_ptr, std::__1::shared_ptr const&, std::__1::shared_ptr const&) const obj-x86_64-linux-gnu/../src/Storages/StorageDistributed.cpp:1015:32 21 0x2285a9c4 in DB::StorageDistributed::getQueryProcessingStage(std::__1::shared_ptr, DB::QueryProcessingStage::Enum, std::__1::shared_ptr const&, DB::SelectQueryInfo&) const obj-x86_64-linux-gnu/../src/Storages/StorageDistributed.cpp:500:40 22 0x2183a4b2 in DB::InterpreterSelectQuery::getSampleBlockImpl() obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:616:31 23 0x21828db4 in DB::InterpreterSelectQuery::InterpreterSelectQuery(std::__1::shared_ptr const&, std::__1::shared_ptr, std::__1::shared_ptr const&, std::__1::optional, std::__1::shared_ptr const&, DB::SelectQueryOptions const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&, std::__1::shared_ptr const&)::$_1::operator()(bool) const obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:506:25 24 0x2181b652 in DB::InterpreterSelectQuery::InterpreterSelectQuery(std::__1::shared_ptr const&, std::__1::shared_ptr, std::__1::shared_ptr const&, std::__1::optional, std::__1::shared_ptr const&, DB::SelectQueryOptions const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&, std::__1::shared_ptr const&) obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:509:5 25 0x21817cbe in DB::InterpreterSelectQuery::InterpreterSelectQuery(std::__1::shared_ptr const&, std::__1::shared_ptr, DB::SelectQueryOptions const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:161:7 26 0x21dd0eb5 in std::__1::__unique_if::__unique_single std::__1::make_unique const&, std::__1::shared_ptr&, DB::SelectQueryOptions&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&>(std::__1::shared_ptr const&, std::__1::shared_ptr&, DB::SelectQueryOptions&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2068:32 27 0x21dd0eb5 in DB::InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(std::__1::shared_ptr const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectWithUnionQuery.cpp:212:16 28 0x21dcd0e7 in DB::InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(std::__1::shared_ptr const&, std::__1::shared_ptr, DB::SelectQueryOptions const&, std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectWithUnionQuery.cpp:134:13 29 0x211afe79 in std::__1::__unique_if::__unique_single std::__1::make_unique&, std::__1::shared_ptr&, DB::SelectQueryOptions const&>(std::__1::shared_ptr&, std::__1::shared_ptr&, DB::SelectQueryOptions const&) obj-x86_64-linux-gnu/../contrib/libcxx/include/memory:2068:32 30 0x211afe79 in DB::InterpreterFactory::get(std::__1::shared_ptr&, std::__1::shared_ptr, DB::SelectQueryOptions const&) obj-x86_64-linux-gnu/../src/Interpreters/InterpreterFactory.cpp:110:16 31 0x22273f97 in DB::executeQueryImpl(char const*, char const*, std::__1::shared_ptr, bool, DB::QueryProcessingStage::Enum, bool, DB::ReadBuffer*) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:524:28 32 0x22270ce2 in DB::executeQuery(std::__1::basic_string, std::__1::allocator > const&, std::__1::shared_ptr, bool, DB::QueryProcessingStage::Enum, bool) obj-x86_64-linux-gnu/../src/Interpreters/executeQuery.cpp:913:30 33 0x23905879 in DB::TCPHandler::runImpl() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:312:24 34 0x2392b81c in DB::TCPHandler::run() obj-x86_64-linux-gnu/../src/Server/TCPHandler.cpp:1622:9 35 0x2ab1fd8e in Poco::Net::TCPServerConnection::start() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerConnection.cpp:43:3 36 0x2ab20952 in Poco::Net::TCPServerDispatcher::run() obj-x86_64-linux-gnu/../contrib/poco/Net/src/TCPServerDispatcher.cpp:115:20 37 0x2adfa3f4 in Poco::PooledThread::run() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:199:14 38 0x2adf4716 in Poco::ThreadImpl::runnableEntry(void*) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:345:27 39 0x7f02e66f2608 in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x9608) 40 0x7f02e6619292 in clone (/lib/x86_64-linux-gnu/libc.so.6+0x122292) Address 0x7f0209dd4abf is located in stack of thread T4 (TCPHandler) at offset 447 in frame 0 0x1c4a2c6f in DB::FunctionCutToFirstSignificantSubdomainCustomImpl, DB::NameCutToFirstSignificantSubdomainCustom>::executeImpl(std::__1::vector > const&, std::__1::shared_ptr const&, unsigned long) const (/usr/bin/clickhouse+0x1c4a2c6f) This frame has 16 object(s): [32, 40) 'ref.tmp.i168' [64, 72) 'tmp_data.i.i' [96, 104) 'tmp_length.i.i' [128, 136) 'domain_end.i.i' [160, 216) 'ref.tmp.i132' [256, 312) 'ref.tmp.i' [352, 360) 'tld_lookup' [384, 408) 'ref.tmp' [448, 472) 'ref.tmp11' <== Memory access at offset 447 partially underflows this variable [512, 536) 'ref.tmp14' [576, 632) 'ref.tmp20' [672, 696) 'ref.tmp65' [736, 760) 'ref.tmp66' [800, 824) 'ref.tmp67' [864, 888) 'ref.tmp68' [928, 952) 'ref.tmp78' HINT: this may be a false positive if your program uses some custom stack unwind mechanism, swapcontext or vfork (longjmp and C++ exceptions *are* supported) Thread T4 (TCPHandler) created by T0 here: 0 0xb51940a in pthread_create (/usr/bin/clickhouse+0xb51940a) 1 0x2adf3a9f in Poco::ThreadImpl::startImpl(Poco::SharedPtr >) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread_POSIX.cpp:202:6 2 0x2adf699a in Poco::Thread::start(Poco::Runnable&) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/Thread.cpp:128:2 3 0x2adfa998 in Poco::PooledThread::start() obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:85:10 4 0x2adfa998 in Poco::ThreadPool::ThreadPool(int, int, int, int) obj-x86_64-linux-gnu/../contrib/poco/Foundation/src/ThreadPool.cpp:252:12 5 0xb582c25 in DB::Server::main(std::__1::vector, std::__1::allocator >, std::__1::allocator, std::__1::allocator > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:915:22 6 0x2ab511a5 in Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 7 0xb56a89c in DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:392:25 8 0x2ab956f7 in Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 9 0xb566519 in mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:171:20 10 0xb56224a in main obj-x86_64-linux-gnu/../programs/main.cpp:366:12 11 0x7f02e651e0b2 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x270b2) SUMMARY: AddressSanitizer: stack-buffer-overflow obj-x86_64-linux-gnu/../src/Common/HashTable/StringHashTable.h:283:21 in auto StringHashTable > >::dispatch > > const, StringRef const&, StringHashTable > >::FindCallable>(StringHashTable > > const&, StringRef const&, StringHashTable > >::FindCallable&&) Shadow bytes around the buggy address: 0x0fe0c13b2900: 00 00 f3 f3 00 00 00 00 00 00 00 00 00 00 00 00 0x0fe0c13b2910: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x0fe0c13b2920: f1 f1 f1 f1 f8 f2 f2 f2 00 f2 f2 f2 00 f2 f2 f2 0x0fe0c13b2930: 00 f2 f2 f2 f8 f8 f8 f8 f8 f8 f8 f2 f2 f2 f2 f2 0x0fe0c13b2940: f8 f8 f8 f8 f8 f8 f8 f2 f2 f2 f2 f2 00 f2 f2 f2 =>0x0fe0c13b2950: f8 f8 f8 f2 f2 f2 f2[f2]00 00 00 f2 f2 f2 f2 f2 0x0fe0c13b2960: 00 00 00 f2 f2 f2 f2 f2 f8 f8 f8 f8 f8 f8 f8 f2 0x0fe0c13b2970: f2 f2 f2 f2 f8 f8 f8 f2 f2 f2 f2 f2 f8 f8 f8 f2 0x0fe0c13b2980: f2 f2 f2 f2 f8 f8 f8 f2 f2 f2 f2 f2 f8 f8 f8 f2 0x0fe0c13b2990: f2 f2 f2 f2 f8 f8 f8 f3 f3 f3 f3 f3 00 00 00 00 0x0fe0c13b29a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Freed heap region: fd Stack left redzone: f1 Stack mid redzone: f2 Stack right redzone: f3 Stack after return: f5 Stack use after scope: f8 Global redzone: f9 Global init order: f6 Poisoned by user: f7 Container overflow: fc Array cookie: ac Intra object redzone: bb ASan internal: fe Left alloca redzone: ca Right alloca redzone: cb Shadow gap: cc ==164==ABORTING [1]: https://clickhouse-test-reports.s3.yandex.net/26041/42a844546229e56c51a3ec986467ced52e4ed972/functional_stateless_tests_flaky_check_(address)/stderr.log v2: Replace String with string_view in custom TLD for scalar v3: use ColumnString::getDataAt() --- .../URL/FirstSignificantSubdomainCustomImpl.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h index 70cf30a7384..3aee0141073 100644 --- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h +++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h @@ -69,11 +69,11 @@ public: if (const ColumnConst * const_col = checkAndGetColumnConst(column.get())) { - const String & data = const_col->getValue(); - const String & res = scalar(tld_lookup, data); + const ColumnString * col_str = checkAndGetColumn(const_col->getDataColumn()); + const std::string_view & sv = scalar(tld_lookup, col_str->getDataAt(0)); auto col_res = ColumnString::create(); - col_res->insert(res); + col_res->insert(sv); auto col_const_res = ColumnConst::create(std::move(col_res), input_rows_count); return col_const_res; @@ -119,13 +119,12 @@ public: } } - static String scalar(FirstSignificantSubdomainCustomLookup & tld_lookup, const String & data) + static std::string_view scalar(FirstSignificantSubdomainCustomLookup & tld_lookup, const StringRef & data) { Pos start; size_t length; - Extractor::execute(tld_lookup, &data[0], data.size(), start, length); - String output(start, length); - return output; + Extractor::execute(tld_lookup, &data.data[0], data.size, start, length); + return {start, length}; } }; From 5bc05337128f0295a7d2e7ce20bc17bb517a053f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 7 Jul 2021 10:42:33 +0300 Subject: [PATCH 218/290] Add a note for padded to 8 bytes keys in StringHashTable --- src/Common/HashTable/StringHashTable.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index b05d119e0e9..d30271d65db 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -237,7 +237,12 @@ public: // 1. Always memcpy 8 times bytes // 2. Use switch case extension to generate fast dispatching table // 3. Funcs are named callables that can be force_inlined + // // NOTE: It relies on Little Endianness + // + // NOTE: It requires padded to 8 bytes keys (IOW you cannot pass + // std::string here, but you can pass i.e. ColumnString::getDataAt()), + // since it copies 8 bytes at a time. template static auto ALWAYS_INLINE dispatch(Self & self, KeyHolder && key_holder, Func && func) { From d87607b160e6757c57ff0f14a9c66a565ea55262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 7 Jul 2021 11:09:38 +0200 Subject: [PATCH 219/290] AsynchronousMetrics: Don't assume temperature is always positive --- src/Interpreters/AsynchronousMetrics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index 11ecf547714..5c49adf6fe7 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -1024,7 +1024,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti ReadBufferFromFile & in = *thermal[i]; in.rewind(); - uint64_t temperature = 0; + Int64 temperature = 0; readText(temperature, in); new_values[fmt::format("Temperature{}", i)] = temperature * 0.001; } @@ -1041,7 +1041,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti for (const auto & [sensor_name, sensor_file] : sensors) { sensor_file->rewind(); - uint64_t temperature = 0; + Int64 temperature = 0; readText(temperature, *sensor_file); if (sensor_name.empty()) From 35ca8b97ac6a4a1897968b48cd86fc1170221981 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Wed, 7 Jul 2021 12:48:39 +0300 Subject: [PATCH 220/290] Set distinct_on_expression_list to null white transforming to limit_by --- src/Parsers/ParserSelectQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index 255595caa0e..b1f7570878f 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -367,6 +367,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// Transform `DISTINCT ON expr` to `LIMIT 1 BY expr` limit_by_expression_list = distinct_on_expression_list; limit_by_length = std::make_shared(Field{UInt8(1)}); + distinct_on_expression_list = nullptr; } /// Because TOP n in totally equals LIMIT n From a47a9ef39a9ed6c8e5e0c26f56cd3210ca901864 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 7 Jul 2021 13:38:56 +0300 Subject: [PATCH 221/290] Update distinct.md --- docs/ru/sql-reference/statements/select/distinct.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/distinct.md b/docs/ru/sql-reference/statements/select/distinct.md index 6616f421486..f57c2a42593 100644 --- a/docs/ru/sql-reference/statements/select/distinct.md +++ b/docs/ru/sql-reference/statements/select/distinct.md @@ -6,7 +6,7 @@ toc_title: DISTINCT Если указан `SELECT DISTINCT`, то в результате запроса останутся только уникальные строки. Таким образом, из всех наборов полностью совпадающих строк в результате останется только одна строка. -## Обработк NULL {#null-processing} +## Обработка NULL {#null-processing} `DISTINCT` работает с [NULL](../../syntax.md#null-literal) как-будто `NULL` — обычное значение и `NULL==NULL`. Другими словами, в результате `DISTINCT`, различные комбинации с `NULL` встретятся только один раз. Это отличается от обработки `NULL` в большинстве других контекстов. From a24686d300b67f2ec34060ba38a222aa959e509e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 7 Jul 2021 15:04:07 +0300 Subject: [PATCH 222/290] fix serialization of type Map to JSON --- .../Serializations/SerializationMap.cpp | 45 ++++++++++--------- .../Serializations/SerializationMap.h | 4 +- .../0_stateless/01939_type_map_json.reference | 10 ++++- .../0_stateless/01939_type_map_json.sql | 17 +++++-- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index fa882bef7ca..ff8bc518dc0 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -80,8 +80,13 @@ void SerializationMap::deserializeBinary(IColumn & column, ReadBuffer & istr) co } -template -void SerializationMap::serializeTextImpl(const IColumn & column, size_t row_num, bool quote_key, WriteBuffer & ostr, Writer && writer) const +template +void SerializationMap::serializeTextImpl( + const IColumn & column, + size_t row_num, + WriteBuffer & ostr, + KeyWriter && key_writer, + ValueWriter && value_writer) const { const auto & column_map = assert_cast(column); @@ -98,17 +103,9 @@ void SerializationMap::serializeTextImpl(const IColumn & column, size_t row_num, if (i != offset) writeChar(',', ostr); - if (quote_key) - { - writeChar('"', ostr); - writer(key, nested_tuple.getColumn(0), i); - writeChar('"', ostr); - } - else - writer(key, nested_tuple.getColumn(0), i); - + key_writer(key, nested_tuple.getColumn(0), i); writeChar(':', ostr); - writer(value, nested_tuple.getColumn(1), i); + value_writer(value, nested_tuple.getColumn(1), i); } writeChar('}', ostr); } @@ -170,11 +167,12 @@ void SerializationMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, void SerializationMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - serializeTextImpl(column, row_num, /*quote_key=*/ false, ostr, - [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) - { - subcolumn_serialization->serializeTextQuoted(subcolumn, pos, ostr, settings); - }); + auto writer = [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) + { + subcolumn_serialization->serializeTextQuoted(subcolumn, pos, ostr, settings); + }; + + serializeTextImpl(column, row_num, ostr, writer, writer); } void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const @@ -188,11 +186,14 @@ void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, cons void SerializationMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - /// We need to double-quote integer keys to produce valid JSON. - const auto & column_key = assert_cast(column).getNestedData().getColumn(0); - bool quote_key = !WhichDataType(column_key.getDataType()).isStringOrFixedString(); - - serializeTextImpl(column, row_num, quote_key, ostr, + serializeTextImpl(column, row_num, ostr, + [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) + { + /// We need to double-quote all keys (including integers) to produce valid JSON. + WriteBufferFromOwnString str_buf; + subcolumn_serialization->serializeText(subcolumn, pos, str_buf, settings); + writeJSONString(str_buf.str(), ostr, settings); + }, [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) { subcolumn_serialization->serializeTextJSON(subcolumn, pos, ostr, settings); diff --git a/src/DataTypes/Serializations/SerializationMap.h b/src/DataTypes/Serializations/SerializationMap.h index bf68689f1e4..6523d5388d0 100644 --- a/src/DataTypes/Serializations/SerializationMap.h +++ b/src/DataTypes/Serializations/SerializationMap.h @@ -60,8 +60,8 @@ public: SubstreamsCache * cache) const override; private: - template - void serializeTextImpl(const IColumn & column, size_t row_num, bool quote_key, WriteBuffer & ostr, Writer && writer) const; + template + void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, KeyWriter && key_writer, ValueWriter && value_writer) const; template void deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && reader) const; diff --git a/tests/queries/0_stateless/01939_type_map_json.reference b/tests/queries/0_stateless/01939_type_map_json.reference index 9b831c29608..4c19bc3c0dc 100644 --- a/tests/queries/0_stateless/01939_type_map_json.reference +++ b/tests/queries/0_stateless/01939_type_map_json.reference @@ -4,5 +4,11 @@ {'key1':1,'key2':2} {"key1":"1","key2":"2"} 1 {"m":{"key1":1,"key2":2}} {'key1':1,'key2':2} {"key1":1,"key2":2} 1 -{"m1":{"k1":"1","k2":"2"},"m2":{"1":2,"2":3}} -{"m1":{"k1":1,"k2":2},"m2":{"1":2,"2":3}} +{"m":{"2020-10-10":"v1","2020-10-11":"v2"}} +{'2020-10-10':'v1','2020-10-11':'v2'} {"2020-10-10":"v1","2020-10-11":"v2"} 1 +{"m":{"11":"v1","22":"v2"}} +{11:'v1',22:'v2'} {"11":"v1","22":"v2"} 1 +{"m":{"11":"v1","22":"v2"}} +{11:'v1',22:'v2'} {"11":"v1","22":"v2"} 1 +{"m1":{"k1":"1","k2":"2"},"m2":{"1":2,"2":3},"m3":{"2020-10-10":"foo"}} +{"m1":{"k1":1,"k2":2},"m2":{"1":2,"2":3},"m3":{"2020-10-10":"foo"}} diff --git a/tests/queries/0_stateless/01939_type_map_json.sql b/tests/queries/0_stateless/01939_type_map_json.sql index 4ad25f3c073..df782334c90 100644 --- a/tests/queries/0_stateless/01939_type_map_json.sql +++ b/tests/queries/0_stateless/01939_type_map_json.sql @@ -11,9 +11,18 @@ SELECT map('key1', number, 'key2', number * 2) AS m FROM numbers(1, 1) SELECT map('key1', number, 'key2', number * 2) AS m, toJSONString(m) AS s, isValidJSON(s) FROM numbers(1, 1) SETTINGS output_format_json_quote_64bit_integers = 0; -CREATE TEMPORARY TABLE map_json (m1 Map(String, UInt64), m2 Map(UInt32, UInt32)); +SELECT map('2020-10-10'::Date, 'v1', '2020-10-11'::Date, 'v2') AS m FORMAT JSONEachRow; +SELECT map('2020-10-10'::Date, 'v1', '2020-10-11'::Date, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s); -INSERT INTO map_json FORMAT JSONEachRow {"m1" : {"k1" : 1, "k2" : 2}, "m2" : {"1" : 2, "2" : 3}}; +SELECT map(11::UInt64, 'v1', 22::UInt64, 'v2') AS m FORMAT JSONEachRow; +SELECT map(11::UInt64, 'v1', 22::UInt64, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s); -SELECT m1, m2 FROM map_json FORMAT JSONEachRow; -SELECT m1, m2 FROM map_json FORMAT JSONEachRow SETTINGS output_format_json_quote_64bit_integers = 0; +SELECT map(11::Int128, 'v1', 22::Int128, 'v2') AS m FORMAT JSONEachRow; +SELECT map(11::Int128, 'v1', 22::Int128, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s); + +CREATE TEMPORARY TABLE map_json (m1 Map(String, UInt64), m2 Map(UInt32, UInt32), m3 Map(Date, String)); + +INSERT INTO map_json FORMAT JSONEachRow {"m1" : {"k1" : 1, "k2" : 2}, "m2" : {"1" : 2, "2" : 3}, "m3" : {"2020-10-10" : "foo"}}; + +SELECT m1, m2, m3 FROM map_json FORMAT JSONEachRow; +SELECT m1, m2, m3 FROM map_json FORMAT JSONEachRow SETTINGS output_format_json_quote_64bit_integers = 0; From e361f3120f306335126a3753ea377d73c12d76fb Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 7 Jul 2021 08:34:46 -0400 Subject: [PATCH 223/290] * Fixing race condition between starting `tail -f` command and the file removal that follows. --- tests/testflows/ldap/authentication/tests/common.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index ec6a66c0257..e1615f3ee97 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -153,7 +153,10 @@ def add_config(config, timeout=300, restart=False, modify=False): with node.cluster.shell(node.name) as bash: bash.expect(bash.prompt) - bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + bash.send("tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + # make sure tail process is launched and started to follow the file + bash.expect("<==") + bash.expect("\n") with When("I add the config", description=config.path): command = f"cat < {config.path}\n{config.content}\nHEREDOC" @@ -170,7 +173,10 @@ def add_config(config, timeout=300, restart=False, modify=False): with Finally(f"I remove {config.name}"): with node.cluster.shell(node.name) as bash: bash.expect(bash.prompt) - bash.send("tail -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + bash.send("tail -v -n 0 -f /var/log/clickhouse-server/clickhouse-server.log") + # make sure tail process is launched and started to follow the file + bash.expect("<==") + bash.expect("\n") with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) From 31e0e5cec7f437e19e10db031910f027c64291b3 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 7 Jul 2021 16:22:15 +0300 Subject: [PATCH 224/290] pass buffers explicitly --- .../Serializations/SerializationMap.cpp | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index ff8bc518dc0..a76784695a9 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -103,9 +103,9 @@ void SerializationMap::serializeTextImpl( if (i != offset) writeChar(',', ostr); - key_writer(key, nested_tuple.getColumn(0), i); + key_writer(ostr, key, nested_tuple.getColumn(0), i); writeChar(':', ostr); - value_writer(value, nested_tuple.getColumn(1), i); + value_writer(ostr, value, nested_tuple.getColumn(1), i); } writeChar('}', ostr); } @@ -145,13 +145,13 @@ void SerializationMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, if (*istr.position() == '}') break; - reader(key, key_column); + reader(istr, key, key_column); skipWhitespaceIfAny(istr); assertChar(':', istr); ++size; skipWhitespaceIfAny(istr); - reader(value, value_column); + reader(istr, value, value_column); skipWhitespaceIfAny(istr); } @@ -167,9 +167,9 @@ void SerializationMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, void SerializationMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - auto writer = [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) + auto writer = [&settings](WriteBuffer & buf, const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) { - subcolumn_serialization->serializeTextQuoted(subcolumn, pos, ostr, settings); + subcolumn_serialization->serializeTextQuoted(subcolumn, pos, buf, settings); }; serializeTextImpl(column, row_num, ostr, writer, writer); @@ -178,34 +178,34 @@ void SerializationMap::serializeText(const IColumn & column, size_t row_num, Wri void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { deserializeTextImpl(column, istr, - [&](const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) + [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) { - subcolumn_serialization->deserializeTextQuoted(subcolumn, istr, settings); + subcolumn_serialization->deserializeTextQuoted(subcolumn, buf, settings); }); } void SerializationMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { serializeTextImpl(column, row_num, ostr, - [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) + [&settings](WriteBuffer & buf, const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) { /// We need to double-quote all keys (including integers) to produce valid JSON. WriteBufferFromOwnString str_buf; subcolumn_serialization->serializeText(subcolumn, pos, str_buf, settings); - writeJSONString(str_buf.str(), ostr, settings); + writeJSONString(str_buf.str(), buf, settings); }, - [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) + [&settings](WriteBuffer & buf, const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos) { - subcolumn_serialization->serializeTextJSON(subcolumn, pos, ostr, settings); + subcolumn_serialization->serializeTextJSON(subcolumn, pos, buf, settings); }); } void SerializationMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { deserializeTextImpl(column, istr, - [&](const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) + [&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn) { - subcolumn_serialization->deserializeTextJSON(subcolumn, istr, settings); + subcolumn_serialization->deserializeTextJSON(subcolumn, buf, settings); }); } From 7f292c600643aa763b3cde8de62ee7b61ffe8853 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 7 Jul 2021 12:02:50 -0400 Subject: [PATCH 225/290] Changing output back to new fails only. --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index c20e742fea1..d39ec12fb82 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -73,4 +73,4 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n1 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] +CMD ["sh", "-c", "python3 regression.py --no-color -o new-fails --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n0 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] From 9b09f215c40721b79bfe0f9afbc0628c5e487bb3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 7 Jul 2021 20:44:30 +0300 Subject: [PATCH 226/290] Fix max parallel stream for joined pipeline --- src/Processors/QueryPipeline.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 14b60d0b14c..2b882ee93ab 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -350,6 +350,7 @@ std::unique_ptr QueryPipeline::joinPipelines( left->pipe.processors.insert(left->pipe.processors.end(), right->pipe.processors.begin(), right->pipe.processors.end()); left->pipe.holder = std::move(right->pipe.holder); left->pipe.header = left->pipe.output_ports.front()->getHeader(); + left->pipe.max_parallel_streams = std::max(left->pipe.max_parallel_streams, right->pipe.max_parallel_streams); return left; } From bff1fa1c58a39b1bd83418f07521e63d7f36e65d Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 7 Jul 2021 20:51:07 +0300 Subject: [PATCH 227/290] Add tests/performance/join_max_streams.xml --- tests/performance/join_max_streams.xml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/performance/join_max_streams.xml diff --git a/tests/performance/join_max_streams.xml b/tests/performance/join_max_streams.xml new file mode 100644 index 00000000000..1505e1d6e6e --- /dev/null +++ b/tests/performance/join_max_streams.xml @@ -0,0 +1,5 @@ + + SELECT * FROM (SELECT 1 AS k FROM numbers_mt(1)) t1 LEFT JOIN (SELECT 1 AS k FROM numbers_mt(10000000000) WHERE number = 1) t2 USING k + SELECT * FROM (SELECT 1 AS k FROM numbers_mt(1)) t1 LEFT JOIN (SELECT 1 AS k FROM numbers_mt(10000000000) GROUP BY k) t2 USING k + SELECT * FROM (SELECT 1 AS k FROM numbers_mt(1)) t1 LEFT JOIN (SELECT 1 AS k FROM numbers_mt(10000000000) WHERE number = 1) t2 ON t1.k = t2.k + From b9357402e0e6c77e4357a87d40336dbbeb139606 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 7 Jul 2021 20:55:03 +0300 Subject: [PATCH 228/290] Test image from the CH blog --- docs/en/interfaces/http.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 0f497f9af80..a4e3d2f71df 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -18,7 +18,7 @@ Ok. Web UI can be accessed here: `http://localhost:8123/play`. -![Web UI](../images/play.png) +![Web UI](https://github.com/ClickHouse/clickhouse-blog-images/blob/master/en/2021/reading-from-external-memory/all-single-read.png) In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. From 02c6b55630ba988721172a127a5e1b6514918c18 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 7 Jul 2021 21:29:33 +0300 Subject: [PATCH 229/290] Back --- docs/en/interfaces/http.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index a4e3d2f71df..0f497f9af80 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -18,7 +18,7 @@ Ok. Web UI can be accessed here: `http://localhost:8123/play`. -![Web UI](https://github.com/ClickHouse/clickhouse-blog-images/blob/master/en/2021/reading-from-external-memory/all-single-read.png) +![Web UI](../images/play.png) In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. From c6f8f2844c025aee5253a896a1868339661f606c Mon Sep 17 00:00:00 2001 From: ImgBotApp Date: Wed, 7 Jul 2021 19:12:16 +0000 Subject: [PATCH 230/290] [ImgBot] Optimize images *Total -- 79.00kb -> 57.71kb (26.95%) /docs/ru/images/play.png -- 36.44kb -> 25.98kb (28.71%) /docs/en/images/play.png -- 36.44kb -> 25.98kb (28.71%) /website/images/index/hackernews.svg -- 1.08kb -> 1.00kb (7.78%) /website/images/yandex.png -- 4.35kb -> 4.08kb (6.28%) /website/images/logo-180x180.png -- 0.69kb -> 0.68kb (0.57%) Signed-off-by: ImgBotApp --- docs/en/images/play.png | Bin 37317 -> 26602 bytes docs/ru/images/play.png | Bin 37317 -> 26602 bytes website/images/index/hackernews.svg | 8 +------- website/images/logo-180x180.png | Bin 702 -> 698 bytes website/images/yandex.png | Bin 4457 -> 4177 bytes 5 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/en/images/play.png b/docs/en/images/play.png index 583bd8a7ad9cccacefe859b0144911438b771946..b75aebe40895e1e6d92aa2189e2c910df3eaff5f 100644 GIT binary patch literal 26602 zcmb@t2UJr__czSF3V2aLrAfC!=%Vy0N>n-tC3F;N(t9sn5x4{aA)z-3ASLwP0wN+( z0@4X3)X+l-z2-gWectu_*ZSV|e(PJ`C+D2u*)@`@6;zxDbBRGostmz#p(?RyG} zBjC~7B?=0U2NV=5FDWRb6DcT|om1-}vcL-}bJgcc6y)=NpD?)y;K^UkFAUr$C}_W( z|NIfhPs;{8q;^+U2U8QMsp+rKg?)+mKtW-JQ&v*c@g5_dI;a`>pcapdVTBUM-(ZXO zZCQTLw)$)|{MQ4n`$iXkm4ZEg)nmhFjLl0ay6ch2njZo2n3`4h=o9u0Ga(8dY8d&A zhwu|wmR{=8nA4_ID#O3&u#V|S^GYbwOJs+$LBG35Pe1PW?4%T*Y3Ty<8SJ-}CnRu* zyq6`NF6Bbu)Z?R65`l#MRZ6v`;}X zz57r~>X49NR$N#J0*AEFF%l?skRc%=gs$7-^Ziw49VwzP)-TN(+*%4K~o%zsbVl(;0{YxXQ-D@~UlwGat0Ny4p-W z-NpUu810$m>6a_`;bH2{8G1hU2DWqa`%3qe}G{O!2)!GjD zPoAAv={Ve3JCYE70}gMWYpD8$eyQF0+I&OIA zyi`rXzZ#an`o675Ja7EfSjk9;S;^&$qGpkL>8ibQJ%;9~p#AkpLH_oYl@)3(|IHR^ zE}bbTAX2OQ$6n5J#ip2o?tANwK9X(=Y5)x*C(Q1!k<;nPqC&It%saz-Q5-VE!^745 zr%H7vBTYFvW)LQ3W?{1?|Ap?Px%O!CJdeU*hi0Jb$-&km>mK7Ke;*f@tv7rC+rF0s zo!UjTk=7eFnvT~Sm?8)x&1bs+J4+-e-onzl2);G87z{?HA>MAJm^ZKy(mf|w0e2;iRb`ya^9UuV5~U&{B7_pqRaI4;%@xouxIvp; zB0k@kB=lhRjh)AlS;qD^`|r8!-`{PK%t~3=*SS(v8c7N^wl22r0tX+0q7DYN$j%RT z*MgFV4!35u4hJ>3NPRQ;OSBu`56kfQvbIIiTlFgn+tgRQuj3~_r;xHoX` z5ueJIjh3LG=9DQF6&1ceyzoAYWYXyw z;>D3!YOlafe}>Vih3QyPh-v3i%{#u0FKXwjWb^oRXdeMckAvVo>G<^W<;&w3w~>(% zudtVvkx*&o98?$s=@O~3O?VOe#B18@5U>` zxzn`WBhy)xD($h-u5eZ+El5urmJ+l>_BiVaa+MF*nH^)xLT7ZV(xV%yO#-{%=VIib zgIpacfteC^MNgFsR;w!1w;P6_@d*iWhlr`Oy_&X+Q-W|%es_+nI+%BfXNB5qBfh){ zjyjQHr)54l95(>tMn?~TrGz5#Fh#VPmoZc#1!4Xe8b&Zwh})o8^4_~v3@sXGQ9YV= zVJ;&7omUIbQIQYE{1GQRaih|(o4aDDPfnC$YS4e;O--V@RRqTErF~N=+@1Pz+qu zm-S1j=i!?;Mn}*GPeyf5NRvt(msr4pF;tUX^l z7w@}TbiQgdgJx95eKs)rDtgFVgR!hDzC(dfS;4GIq{4SZ$rr`265~1&uzD3) zH)%DacgMu$vc3&&ZXURM2g=KeO*n`c9Bnh-wq#Xl+kGN~2Bq{ZZ!K?{i%KhqO+QGI zZZ|u4muAbS;_ckkFqhA-Vp~5ITgQRcwl(1On(`WJ2>YJfIy3!EE4R&Jxz`k5Q3Q&3 zs)q=0_H_Y@ysVu3d}U(yBz-uk0+bTb>WM8v5SH|FrH<{5uF~45sF)XwnD;O+7|_xt zsz+sha#h!UeM(z#6i2Xv@R`Em7~A?~kM+5K}}BN#2j(FWd?;=ua@vV6-MMJ)F#=3i*&F0a9ID;B~EKN~B)_#lHF&OE%q2*x~E{n^-fPBG4$ z^OBY0`pLm#&z>%W%Bj~bKO8%aBV8{G?5)Y_?*G*B*N)d7z*rFr)%*(wEVvAHHdFzv z_s~ejFU=F-b4$P}lvhm(GMv4vb6c-pCM@Mi51j8dNkyK7vr=j5 z!SK}bk+@_oBC(QxTfwTq&Bol;{?;Sv9G{PKuhY1@ru|*63m+!Vf2O1^v^svN@3tr% z7h7H@x%R-|I2R@c9@)It;`EEf(o` zZT2!yk%@?#y=}Vu`aGY3z+zR$WSHu^Bbmd6|d>taH|=u_ZcC>`qV&g^K3W)%M3=vfG1UlP zyL2FnTip_cKfw%f(S$t7e4!Sr-SUT>ry3%8T`WZ=F6Mk)=nICZmVFH-@T;6l1&4tC zPI(<`8f)nyzEN+gx3{TRLBT={N6r0pdwaVj{Pww~>-msGI+qA}+2)&(LZR))MDDJ~ z58{e$Z71K!PGix(N;@jpQ%^{oJf6beSN*O z!rB9w_AE3yU)>-+Zs=*$-&+7le{bnp{msqK?@pO2q?0qa%9Gbl=UO&T{JqUfA64wU z@j#8+>#dr9R=4kTH3$AK-;2@+tDq|ZmP%Lta-e#SGN3dBFSKCa!ON52)M3)t_^$@Y zq3O4iYd1Rurwy^y(K^mw3?*fdG<0L?!|Q>$-~g+HLQuF%Zb!D7MH|4I`1QVS_D=)m z<+Py^{H7x-N;Mr5a!l!FsdSoGqyfX@^-F(gSp_ck)vDx&3Pzt%?=J@`LjNK_qI;vI zv4@ujEDWNey_hF9yB`fs;HK;(jH)hIis$-ailp1)M zM;J4!zBuHceMB(oTvxY@sTk8V78>zn_SJPeDeg3@NZ{|Z&Ir)+dDU|{1Kbw~YgzXv zb}Pp*pS`Q;+Ry{@dS#cG6oOzSrju>(a{j?!FGf*&V(X{93=W-7BcD+IJAKbP$7N@n zYp0#^NYy`|a|P_|Fy_pgEjs8TwNfBIC44F_7sRG>r)k+;?QrcM$fLLA$Rsh-mk+~3 z^@7?$OqIQ|K!XkDu3MC$g;np0`|fs#SYz_+oZAY_mhN|jFaHsSv)|xkU-Qyue>yXC z<8mxuW?+{R2h53Bwk?aPTYxoNNeO28WHr(?FU&r6g|0xGmcZmoVA2qz9d`>RyAWLq zhNeHY_DiKEQSibb=lb-7bgwIAJJ?6UM!Lbaz&R6QTTtU|h*ALYyIo9XD>hh)Bj0sc zpb0A%x2srN;c>HSF|1@5s=V$zD*8M@9+;Ww znT=Bp4`1(2#B9!V_hcuk?Va@6@O3WO`Uf^n701t=gqW@v8;7iR`Z^qA0X7Wj@=ESv24kLXtF^KIp!*bia#wvQcnDNmJrd?sCI9>-m^=%NK=170s` zWxZ-%$-ImA+mMSeK)%~xy?IlsIPaz9NJIf>d|b%PCo`cRDt|B?%v}3(b-$U6>DTo1 z${eZieqy(%Mk5~^NVH%5)Y@5njd1neu6AdC2?x7-t#>BO1&mvGQj*=Ec#YM#j{Bp# z*d_B|quA3TDn;H~K6mid-=(umc~*9)FrXzPe@1@znv>W+Rl^@iB`244m>-^(ba}PBeFEmuV31m``1pr5*a_{p{Y(uC^OMP$Xs?8GcTL-cs+)wwI$ zjAgYG()1tX+m(`d{-!GAuBdZlMQNk&pB5Dam2FP-%uC`c?&3)WEsnoj1} z{-0=%WPwxqh?N2GWOz9Xhg-n_IJroew!)JGhClwpssd;H^C2pWw&JxwaW~YE>RtX& zzG;NVtwQ&{d$$Jv{;DGQ4M5wNJ4AFbUkdp%ooQ~E@&3)LTmB(WRv$9$^GKUS@=UO z)cHn-UFgZoKeUnMyh=+L_G)Wqw>sWLK~<>AoknuMW}RjwZ`lBI-N1Ga2C*=}t95H$ z*&oB=PJe`NX&~yn?i9!9t9Oh`E;(2|$utJLSdF;xCZxS^V1hANzV_U=aTZ~Xex6HJ zI0lb3>d>(S<5Jga0MUh%cU!rwXAHMEhrr8O*(+0s7bxkOLIAsvGW@TctnANzk4)}s zLq4nMBD+1a^L3>FHDHvgTgwdAP~T5>W9*xdlbn8#!VwG}@XL|#{Za0!Uy|u8EBR8) zfZEk)Joti@_M$W3TRqDSL(*b+6AD0wnhE_Qx>}IX!n#&b5PI}0qa&)E5+AE@A&D1s z;r`w%zZ?G~J;ZX+-_ayxA=7}pIz@(>nth==Eh$AGvX8RoKKD%A!t-t=T%qU**06U8FA0|QUstI3;I5nzX@ZkMj^l*5wA1^%JTH92s=Hur zKWB2^@VdFV$;dBl4(!ug>S^E}=(zd~0K3=T4l$?pBZ$W^6JP4*Of1EdHePl_ zJK5Vaq%she>7?1Un0%GTLQJ38y@*GwQp6w=o~AhGMuAbHp|)jpw} z{(tINl``=x2us{}HvLL!S?orVraO9d!P3Csvvz5JK^_XEO5)!|sNsG@P3z_QZb!Y- zDg9~ZGFqH=r=t*-v@Wq2T92k*DV3CYKubkkG-hOa5DaKuSL$v)k%jIxSgcXasS zt1z5Yed3FIvn`4Zb$+tRFF?C5c>e&~1Tb$J z!S6WnC5S4l5889_7J$(7lU9~0cN$vmGYZ)-F!FV-xHq4N7&I-dwOwBa5>1iVgW7>q zqnh1ReidV05ITpZ(-!wGHi^zH*=^s{`-}Zbk8Bq`y+(!A{X~+ape;Y(nv-n3f4mZx z2~ASPsRDNNySX(xvo$2xY&`i1uPL~?vNBtgJ!ezLG`O^#Uq!m%>M0)MC;SR9^)NHe z`f~SQws<0)B~N-}HgxK=h?FoSpXOxkTzvVQw#2%MLZ6nJ7`VrgQ?Rofie0lHi!ryOuhwK?V zBG`B=!q=bh58| z_}lYIDl|)SrE_v=(Jm!RU4eR?xflWgHg46$Pvv#qjZ^7tuJ^yEaaXuWrjF`_)_2nG z*@(&(PTtQ*23C2IUc~22d!Z<40Uv&g8^C-aFhTZ!SqRvL>pJ!CQ!i<%g)nFJEtz{4 zjwBbo?km%%=opieX8dJFYO3+_bm#eICJ40i)C+Rk?U%a#MSmfgfe^1jpCEk3wSm!Gr)Wa5;7lQ|84y>#qPL}-Fd9=m1U$o>b)kI0Wf0cx%4Q-%L&) z(TdVy=ju`rfdjQuI+;q=lERtVsl&L-?fRhFlPfnGPPTyv$#D_4>i)%X=-v$v7nqaN zM#k9=PmadPvS!c^=EtQ5+J}orP;si9)ck;g2tD; zax!eVN@rzO)4cGI9trY}rVT*~rUgMR;y6n{ofBpC>RNIWGw~5F{APvyP+vd4Ayj#7 z?dbTp5h0%t$8rURfaG>{hv`T!l~&~se*!`ygKR02A_nn)7B5H#yAHmBSU(WA_-r7` ztY3`+F^}rI|C~u|sM8asS^4|Jqiq4`mn6BzWo0-HRZ40iQ&o5#b(f$d!B1Sk9jT3SZo!4L)*IeB@e<5K7mf|rdRqS44LBT}RoWX9816aC zuN83GUR%gt8q}&;IHDR_{zn~aT^O~#Ws?s_v7=!=x74g6hVc2tHDhXObon)7E6%iP zmNX{Lr31E*&Q>dT^IL9V(p0%HLL-p}h_4Zoj*))Wf%cE3NNx)7h2Klj#2OR%)FOEd*{NHL-sT=(4Q8+P#kU*+CXi~&x6iKY-_MqHEsIMxKep9X z1>$)0XY$z|F;>mKaos$EM30%-ciqRvC47w2eucZq2sYrY2(D;mW;aKd^NpD6-rPx- zba`^zbP`U_14PZ!95_S{DV-8Da|Z&On@NVk4ywFf^Tn>Y@%zW8K#a0sRCBPSrsLrFThf9g=z#}hMVuM6WB0He~6Pl=+px4*wXI%3v#f{6ROK}>x zb6W=h(iTt8I}Y4DfA(y4%lpWK`O-x`*%`onz&yV77>xTH01rn^pLQsSeCy#?;p2?_ zp4C$3Xx9Ng_Vz(5-(ih>zL-nJ5#J+He}9t|073xAf(~%#X;gap`=Kz{2!Q8k&0w2G z8c@@O0uTVi9__9IFsc{73W#A0_smBNk_QqF{`S5iFLSLCmHqAl6QaYO$e#F*qw8Xm z3hH$l5F|F=;9RhQQsjYtA*8V#^P8K_cVb410%Kp?3k?fHbljtRz!(AtzET`G#LOGrVkzv zuPrfIS$3L`YO%Cfja&dZ-Q0Yz+OTJM9tVEGjp&q;zOOthED2z}EXuJ+L;w#$SscK6 zP1Vl33Pf-L9s-zhXZTEw08giUJb=DbC^IFY*_GPF*In1vogUFY~Bn0-bUOLjWXFP(Z-AIq(!f3e5bs z+J62j%x3;Bgw8mid;9|epK|Q&?LlDhGT%AY{;9E+UwCJSp0BeD3VF*1^Us0uR-u8E ziIGs!jou?6Ey$)yHG?jYfzfzGV>-pM`4;|eB)b@>0As$uCOZQEI>QT~{uGU}?`8Vw z#ey{Ch8Om}*NnX%;ky3h((U(+UnUI4C$`8pIE_i>+FC+11!2{3m+l`chgiAVIUuf` z*SYqjKrN&gTAp)>@-*8*inliE=ZXaI^!!!1W#>>_EciYQ0zX!Ek?J$WuC9LFdz|)c7`#(noBo&*P-JKhLUm?rs7birFrmqTk~(Ajvr#!?C)utacfIdkU`h6%*-djnV0E=e0rjtOv40p zbpSqOK2hKW@i20dQo2@u^`Rkd_EqT7M)&g@Y!dWrk8nX2&>hRRgfBlss1sys+G{@5 zGcd=w3Ig3%*{4^R;@$t!D^T={F>8)A&sZtRKTbACM?3vv-K3)r3&B2SAIWr~LbA2< zKJrbJIl$nM_PhX3nN;==z=VHMeQEIe&3Rc~EonYGS}pk%-6O96vU9Y8rl#X`1|Y;) zM&0t9c|N%3*}a$-KY+Or9zzti8%zZr6M@xpywsP4{Z*Wr4!M~kMBHau+R&hw7YyC! zA4!LxiZ^ESfv-%zLJFnd>pm-+^50gMPP1a@3XXu)qw)m7X)mtBbG58s_tF8em?p4% zWr{W*^1=-)yNj|~E`0^6RiO%Bw9)fE*f#!m+fGrf0?SK$O2; zsys;gO{yAV-{FP3uYA;HcJ>wE9dn5dxPyI;Kj;n^l>Q_W$X$4w;Lm_=>goZ-123P0 z`!urWVMs=0DY9~(wjxOsUT?M0wL`CbrWah?!ok0l|FKFBw)hW-JZIN;87^(TmW02h znU!Z8GrAOCDUpi>6-ZD6%EzC5AxZm;IlGi7{6FnF*I3ix@dnbfW5(%u|TZz94e2@Ph$U|FpYrUa2X7*uNQX$efI8z zsCAUJJmR`(Z63O{ z#KzvFCBQ_IV%vREeRR%d8yokdCuREgrr+c1Q@phnfeVhJOII1&#tX*@WvW}F z(=1Ujd-&jpkzy2}1sLEGC}aspNYe0-W%HG?P${53>41uvBMMC8-VaI^ve)5BTe7Ne%n@_ zW4l_+YVp#Fk#U_-!j4P|vcn8wHg2}BA>^W4==+dUD?gSO@ykow^iV}pu zBeZ^D!eLW9+{erAm6uvQo06hEN$0?A)nV9j{KYo|9(ZUyci6}zs<#Am9i|=N=E!UE z2@A5~j7{wATj)Mr6R|eg9asbe$@!9zYsr&3NFf6tH_O1(vayX#=sAF6o~*|M_#5nC zmd)l7O~?m_q?Q-m)((Y35%07fI77a_kl$?^fbcLyDXDz8+8G_=?Tq$z-m2GQsLHw2 zrVWxv7*W+@0I)mPqt~$v+toL>JPYtV5mY@B z&$6*`r8JL4Uhy&ffbz|3V#2(%+4_FZK;vk9MJb>HR)9iXl71S1=0PH3|F8}g+7FIv zzUm5aXSkYgH8(cU4%5_=iD!d;gYd8#p0;^;*TG|V#M#_;^@5=B1dAV&7C*&mX~;}R zm-$s%+I*AU&7W>9A^;q-sAqJ=I#Q5gx-QB*?)DmxuLGoG!9#^^+ZE+Gxm%jOx{8t> z)H02+2an|B4bn&KoWnQb=R^`~+7i*XWePxy-(Seq8Y$Fz z13@V)VRtqh>7c`a_1p&3!SSj<-FBeVb`Yg!tFJ3JHNEq{`IQI6iojlu>?p&>`Ox zl>u!f>o;jYJh_VgsW{lkSP+5(5YpEiAWx_7qnz_w{qb4BrVIRf$i&=^TTFY`oDdq< zE-J!cE3-h#olDkecZ$!7oiX~h5fDtFf=tEwy%yCwNM3QcaT&j){hVSfp?mGct6avM ziJRfn3GgRioF2HZkcaPJcN_xlqg_W7huYHJWO5md!-ZUTlUTWeZR&4cX#+xadYi@J z4Y##9XwC`0Ss3gL`QG_jvIH31ndt z3fbhgb3_%Uz}?~caQ=33o(7}*9nQwb1hhP3=F=|u}!~&a4 z7if42HPW=%p1Sg+1d6UB@hLCE?}Qi&ULCvsIADPY+Tqaf?I_7a;Jl#D%dnjP#*4saqbbj06SoSJARW4FU+ zd$Wr%K`*CbwEG@TdC!fuB)6LMHAb`l#J}gsH{&SEKg1s*Ndp|-WW)T3q|@A|V#iTu zvxcH^rwT-{Md0kI*i_eXxOFr~q0~vl;TrjDff#@j1_iN|D{P!pZJV^?<$awqO36#S zyLZyNy$Es?M#(@cK4xYl+UC9Wv*f$EG#lgJnA>SL#=Q?=Lu| zC+nJCEx`nR;+9!@k=Dyb$@n3r^TSoX4}eOj@ps-Tz@5TTXI6SAyV`u6PSr5T`%|Z& z7`{pUbXlKRmG0i29v&6O$k@9xhsCYOv`s-pYa2c;VC7V4!-@^8F-NTnz|oeV=Ps3#Z=) znVmM6ACglJfH|%d-qyU)*ErH0%XptWi^`1}ytHzRTAV7(_t}<6PEh(AkZi>`jHMrA z&QgA9o6JR^-3;f=Q{e^r@THTrNCS2H+O4xcX12u?yu(Rc7 zD;5#GU+z8c#INb_ZM8KV+eNu! za|cs6fke5z5q4=$!5GA$+AJ`0%(VU+%x2xsvijH-qdD+3Mamo$6zRL5z3*kgUw<{1 zud@^J`Z?WNmmZQQUl=d1)}1wYr6t%0$g`YlcWb8`VV4!p zk;jO?lXK!6gWbLK1lx2WY^u32GjL&RqRxwM114WV{yDyQhVmEn2E=dXwH(=dXSyjl zIr%gUFDsIBm=0tUYhr>PU!y&J{a`;PvTK8L)T?q)I66X|Lq-R&UsbTRPsdOs!ANV> z?Ebo7k=RPTH*`ttc#q3>`+#6c#L9c_{5)e-%)@zY{%FVz#u-U?9TA@%+wBn*oHv)s zVwx}dMXi?rZ%@0TxuP5EN)D2vQV&%3!sO%=FiSBt`$PU4?t6cBO@-FQ04!TC$7UvascX#ridh1fDFl1c<%U(ykzXiJ8+L0`}Yz`7lJwV6GO3fTC079a+c zV_Ht18lL03f*JCL{;m7XeurCgg!I|oyc++z-bbfd)cYfs-jzC$GdDQclKievd~rMy zSLJACy7H*JpEA1$Cx_+Ht=}vCR+E_GQ zh+|%nu!Ea|xtuO)`fmW-3^=A^Kf)_Sku6ua=r+(euLG`JP9R6y`;b(2mwJyVX3C1;4y5`?R5=+B@~<2R1D z<;5VAyPVNzMzGwHvtvVJ{m)e^^Ve2Ba-7i##<%oOkN^+7J)uAct6pAeC@;Seu7v6X z?VnGPMw7a&I(+i)%E5JM5*(@#1KIl=R3CZ@QE zSgD2+xcj3rmq%qHn99A}gI0w$E-KD@7wBTO`w00zM5&_66NYXes>aNBm8l5I@WskU?G9BN(q$qWSLoIB+=ARdn~M9 z38@$BwepN8paL)rP<&e$!_Mo?XdsSa@J0T7NF)Wyp^LM0|k=b7yk6q`@@RPgz1asPQQT!5PA z=tF-(bqf?Z-;%^LuKNps&Rb7T$9p!OLU zQG&q|BjDBm4j~9G{nP}}nlY%ykUKJFGZ_9Zuf)MMWKjGTrLo{HRT>D6YN{^7vS~ME z83>E}((pY6+?3%4bH<=cywf(KD_}@&i9=<*p%^2Wh8K4465tB`TnT@(68=o*=YLgvUKN0?ihDFZmItfknyxhoZo$a|a`9h*?=hq`69if$gFC-Q_p^5miO_f9`d4ysdx;4;wx$ z7idrQND6xMY-Rmf-_O~SgGUEOS5aj+HiV_ra?S867%-rp3>!kGmf$bS1wvAYI4~;T3IRj z8-Tl@-6Mu*r%}}Bb0I2G2ScnOo>0i0DCxe@g0x3JNL5QA69svRXw$)(10Wp`QA`CH zL8b`)Rmt+GE7u?EYl48q6W&(g0f#}^iU$%K7x?}$TCt*Bl_b7=5liMq#4d2(G!%1a zA1UO={8ZgmkIhwPNNKbMUl-Mu23*lS>sU+Pq~p&YtHejdfB6#gyh1mXy@K;bDoYY8 zglax?j~5Pz)!va+1bPU>luf}gukrB!jK5;XmOP%C{upWm5v6?WD)G;eRR&b~iVxy` zY=fc>lQ%fk!#wTo9qO?URl6Rc+|l=ZShwxK;qo(YZh*1H`yYoA?F2+Js`7jCMpgm$ zLWAL=!+j9o&jeb#VfsG#BQzMkaYq|MNRE*;%H57NF?Z6dzmHAqU#iP^uxy7OxC}^d zQ~uD*UVFB&7(Gos>^>#?RVv6J%PN~up%7PBBf!tc;QVsb4jpY?9ven;Ymn(7C&5=WKEG950}(!dUZXt7z#fFoOg3N$!q=ApljOt%zQ#606aW-aXLRBf=^qs}0%z)U@Sw ztq%bN2Z)sF%wzZHBtw7I1j|cel5V*>zb+<)5(Y% zKx`tE$?TUSU6kLG2#cHt1n+@p!_zVg0VbOF4^MB+=FxjZrgtwKf0oYDDD?X7$_R#y ztVDE_vjb5Ha6p76be-xvRSj5r;0eLVDyo)G{?Fns6j7N3Ec`SBR)$_~S2c6Ti2b#z z>gylxbwPg&K+)RGyt8=&L)BKfL4QV3P3pA+C%AyXV=laW_d2GN(_M18LVTFj9R*@w zm#pyz@=kcCZHs>;J1^U%u-8rQU4L%k*!lu+*e@Arpg_8)+kQ5$b4MU08U^1Pi*|j} zG^`&C*VI0}u8B}&7`kBF;reX=n#&AiM|&KNEv%^&WKWH~{24Bd$t><*`RC9u5(UJ; zz{ox5_=>0a5Fa&%V>gGn)U0FiP=qw5W%TDlWQg%qUmzT_%d23&u-Wnsv=Vu>$PwF; z@N_-iH_J797Mjr)(HRrHnqu;YXV$s2z+-{Lf@eq|59fSnFZcO76T+|l()Sd_y516|L>_d4^W(s`2T^y z6#uu`_#Z>~@5(w1|7Q~a*CC(x=09`t-<1Ib?w95ApX+~<_)q=wZ~rp||3`uP@5=vk z!v8~}1PcC(Tl+Vm-U`%31n)0mZ(-Re|D@61^XM(lNa*_ZeVCE3K$8B&y)PBTfY|?! zq5me;2;>F!3JL-zZqEDH=uJN91yDV%`A)!$6Z{1oQcUA^>2pxPkrpGnlqcS&*MB#- zz03m~zZ4%21FrY^RsQ#LHzE%ZZ{p)qsm}RLK{hL}yt1-ThoLG1`3q?Hx9_Noi3YD; z`!jOD>27Q_Dvu5FkTJ*m&c8HSb`c>Vlm_wi;YwQ!KBIgpPc zvEBWw|AJnCImL+lk6ApnYeIQC-<#iF2ZvVY%ri4Oz<6PXzc8>%xBVWpsVfRzA9IBk zXUy}dgv3LQb*y07v;2XM;G79v`rpu->B~Vr=(Q>(n=a*$bMi(#ITvBs#+}Be5i0BL z+`o~wMjZ%`EU8PmE|9c1_n(ZnYSg^BrH@gp;5l@>z+BKCAX zYDcN|u3TR=!!CL#oWTJC9EDQFKdr2Z{|)Ss{><`(yArZL0eu6`354%ORDyG+rA-$M z^=6uG{Y&$AF}FP^h~rcb!Ha zRyvHHS5i?loh~o7h12v>{X0k<#{3C2n$8ZibOf|KRYxH~={J*n(s_&T6FNIMmOG3M z19$gOIv}1Q&3nI`lxt5+U#N)F=cIys+MDaxV1f{3-0|p^m`83_TYTkAXGek6{qZgE3_^zujckZf@nljk4W^TD zR5g-@o>z7SX#EFXJ?~A?wo_BObkoExY}F))EydPg`GhO}N&lX(3Twl({h?y37cC2U zsAX(}{f&RuEN5evoZ63dU1Xymh`%3g2+_+1S}WYW12nhioZZ&3P!hD?yDv<3TI~r+ z;yjT^!IO7m_Q_RJ2PN7)0Ucc#X9O=nRRsNz>6nRG*L94Rn&?@(=93CtlOI|98DkFM zvB`?eO4)>Xp?x$tQ0fMY$7p@eB>HGIfw^(7MznRYwghI=*iYMyzu-6wReN)valbEA(4dcAKtRT^ zFg_8p);zybKE9SKyU&SH0i=WGexe!%RRKzR?XN2m^EKlQFc=FuOS#6IM?1m^57vH5(pe~WFf}60-8=3?~4X}o*eer z>&Y^XC>@uUZ#FwxokQ?ARMRJLS}2r@_P?&!RkO1D9_(gPy>|xS>c`mgQD)So^2$bk z7j_?GU-ZW%A+$CpTvFzCik|y5q9&qB<^%i>pG8l4o}5e&VQ7?1t`-m{uYS~3hOl2G z4Z%@qFEUZ*v*0IJX2vZ|wmn5WJ)eX_I`UqhHF=>gj{FIU;DMcuV0&qUj>9%OgHF7r z{7JCYH1cx+y=9b@aP>)5X;t#w9)&e?u#Y!+_D3Cp%ZsySe2kdJGm(`b<`kwGh__t5 zxl0)kAeTlIH}Y%GA2>U%n^``Jb(-%Er)|Jm*?GCAFUfg{z#a{soXw<1bNs5NJhC|R zwmMy|aYJruv2=W#!=z^$foNIb@4P#e>uHp*_Z%VAK&*}q(9UU+m&93Dq_KuC$UmG@ zu&mhinIi$Kv|&-^RmJ{<{s1yZl|#J2&XD)-%hodX?Y&`ccQqKeC8mVVY)wL)%-H!25pW0^Z>g(qXlYu95j~3tU3p-}D%WwMISM2s) zR1Msl`Osyxx+QGPDHVWfb*#u}suk$X$?52r7|60#GEw9?oY>o&01Ku9^0{WK>hDfy zY#o-7Li99TOqrZHIH>gVwS3NK35NCrm7faT?xbyI5ATQ-0FHpQ&A){?I(%$fl#W29 z3WH^0pryn7+K{IDzBx%Bp^e)Ke-1PaobqtcHVAHaZy&B(;>nlrbt1DFDLCbrfa$~0 z^~QSSX_^7cK3#)veH5^M8z#40%%&C}A-Wp2+a;WLkysJgChP`{M!JwGr4fOrL9KIiIa)%|jtT5Af!=hsEtAyr z?dR*qPFU{=6JpdveSSx8gT46iR&-dve)jk(^ri2?`^mumW@L(+^G;fWhy+XBMl)lR z_nT2ao^3vVf`0-wuzX`>&D-MD$@d%tZOYsLZmrg>W8AJcaBhMS(-ale89#Yuv%aT2^A= zB!+S-=Mim6mi0rK^uSu@z1M&O+o|g)RVaTTAJFP1e;jF~$Ivi$(EgNpLx3^?_NU96 z2>ybD$6=0Fy++qKGBix<+@`p8_5&se&N?QLc&D9F2adaz+~u4-wf5DX&tbibGx-OG z59)6i2Z?$t>ogQqEN&-l$y}a8WbtrR@0m73yRA+F#M^(y?HY&K*Lc-GLRjthdg7$z zgFdG9t+<;cb3{*MG(K8PX>|7_wQLuFk_>3m*5Xt*YvdeMaXkVm9*p;Qk9(z~8>;TR zws|$7C#^P?xHythF*i~%Yx`;IMEjbIx)UvcW7-KCW9+SG96HFnsW%GC5Im!Py3F|N z8k~%#>844eh@YkJ!feh=x==%rwd+C>4Eelfge$HDWRiP&h<@oLcS+AJ$19Yh8jD6^NL1VPS2_f6n{aucXb2!;pd!pX#_94Oc$t9eTvXF*&iTmmPa@SNG`CyW#`8aB) zgKWZj=+!CEITsTYw{n;v9<)XfscUr`W z%GL={jtoM`$bQH~I2dcTk-{YVZmdO8<4_o6XOP{s+I5ASNk zjaT$aK$(!{)$?&!7mT^D@T+Tzina?Eskpcu`*Qf@1h+_40|dP3G;Ot&u5GxD+AnQt z4d)d=?CP~;9U}VJ+rz6cz`j33-C8Ial@XblP-Q$B$_Z6XUBOfVv$!DVs|qGXBHDjZ zJ_EVE7~gwwP1b>Ny-+nCG2&tS7|DPKYzpNO8z0z0)T%Dha(onvKX0u7bL?q&Z%m{4 zim-kj87&j@^)c&aww2O8I~tqIB*la?&-sT^^RS;cw9AnJw+%&VxuDwu+F#zW4DR$X zG$PI5V8Ja~XB&nEF6MU@>0?&Wl`VS({JO^@E0%AaXuL+p6}~UrsUdmg8D$==D`0Ab zd6NaQ5rGR2Si#eO>1*=IK#}$ASnilhMsoC;+gm=`2d>N8VJpM4))RSM^;yY@Av%Z* z?oCx02Wqr`oJ}(?Dv|j(SbY?vAJb1y$|})8KH4t6QG3sNB1K^cv8arTE5~MXdn;6w zdnHRt{Xtb7y7}-`dDt$DS=8$e7nBtKtvn15HC+zp`hEDC;&O>}8M8_ZF6ZlSc!g1< z=o?X=sm-bc^wmv8#R10!87?!SZAm7i88or40n7D`t1I^0B7F@=*E)L*YnhoOXlT@H zFLLoRJk8G@@#RPMx;n48`%+O^6b4ts_iab{H>eFDmuS3>Zjk)x@(!*=D7oIgbjtYL zjA6s|cGu7Gn*|a;@XM)Fe^1m@-j(MnKOfMzTWM0u+P22yVFD8ouG0j8QPJmt4OA6; z?KoI|6D4|oM5lU@@YOvYw&KEFpLAErUrWZVt$h9)Ffu{gG z`WK4SnOX9r1V61mIS}%h=_2Rju3C$J27!jW@=wXJi9(zffrdEE()EJ)fCsqkTHgzz z1Au5mvimsfY&{~OSFAo&$(f_Bpxw#aHDp2cMa5@c4x;MLTQsj`W>%`1m+5#<)?e;R z_Hm0FAtx6X${U$k3!3aa+U zN-snB%H^*FBmEyw5%oO}ORF6psfybfP+*bQa8#~Jj~d781~s_^1k2xjwOn#TDMDHa z(`;4=*}w!iF5u6brMk&-W;@9zgqe|xX_l~NCD+_-RzpupM8GuD9;&LkAKQ}BiGzTk z_1nr;Clc<;nsr4FE*EZUfRT~bEmOYUT@h@I0dytbkdLjP`q!Tb(^}bE`AcsaymB+r zB-s>f?526mp)IdP>&2NZ&WLCbdn$IlTXIXKmb$LvI#h-n*^MwH4tZ)(X(R>0-IIO&6H)DUy!;(4UF{JHyHVu+!6Wl z)t$YLx3k~{CO@~$ect^ab8qb?V@FB9rQYny8 zz~WT*4{Qu$WnA1&^hd4Q4j0QMPn*cr>UOYX5Qa84^T*1k(ZEbcI62ksWMvPFOd1uN zD>yw$X)vEKF9ZK9K_=2(pk`{jZn>)COG0eP&p-EagM4}GFPH?9jWKnhRV%CperZF< zZ3VMk#QtUT z#G0(%9rMlC*FfNTpN+5k+JbvYx|ZjLgmuP%jswgEN;s8aZ%{9Gb;!Y&&3XY{;*&k; zDC;5+E1P5FoCBV$42F5`d~818_C?$y0(9(q^E{PedMTPW89F*2)^2LR(o!wNR31o2 zb8!d$nE3INam^)ptFGNf(H|vUsoygmS=q5m^(k`IG?yQk@Mvo&5V*ox*`BZ1G&5^n zQK#rEG=b=-rE6lq3l=>V_Q)npFt{MhiG@Sy#Hw}fI|+MT9O>2WbKF}nV5p%#WQ~3+ zty5-JZcX`{n)Z={cVEGc1^0AcV6AO0TzfC;rn8!K&CtW;eF@_ZrBEZk{n_#v3+L`HZ6Cn@%Xo8ioO8fIUCEwSP|wrg zi7VY{77=HmUvvKx(NF~ER3dGjd(n7h&P+T1wcF$w8oRQ2}yS{x!S?*|vo2BL-3 zNqC4YN`x$H>6S!MB&HAzg*V4MRrlv3f*wTgzeQEYmmw9yba`el9?C$=-2q_;Dsbww ze_(sTE0B$}kb60Capn`*Q6Cdz92tbZYVJA8MQkZ2prh9R4BJWuaY)Vmz=7^6nP6h7 zR3N#bJ2w|1G;!G!Rc%^iALljMT@+;g<{$Nhlb8|iogP+eU7RK?w=S=|KRFf^`o2(r zq|G(o;4erIRhoOeRoi7}XCHK*d2g$3mZ?sTHWytVmiKIS8y2;F_sx59;d>t+E;meU zI1QxKh&iW|0t@cx?A7vih8bu|ViY{4xkzW1>s(+{>{%4qpIevY~2F}-D0NORIT3sb;6uH73+GByOx)5ykK4GSly3 z>KV9l;84dn2u^#k?Nr)`ZmVu9tRW{>x;T6Cmnl}mf&6YNI86bQl=oD1Z6B3t_bE{N*6gb>?o{f!hb`4`><@*MamQ6N8#n;e z9Sm6BiMh>uwHaUP!pgn5Wm~%TNJ!l(KLlY7p73lDLDO+vHI(X879?d{UAXm+wrJ?~ z{QZe@L2>6;WDWoP*Uz z449C$`m`Q&E)EKHPrX1CpLzV)XXMln$`b5WUKQ4Y2JY4_HJI&9LVfnuq$c*)D;u`G zUg0=52e*^mmo4~3_uWu&O1I?c>_9pujXRR&l-FmoF&9` zX?xwWht_`VaqD!m>+I93VkfE3G|#7Wdipk^;DgOL#g5 zyj-@XzXO_e>gzh5z4KPB%qv-;vluMVNpq*MEg!5i&;T};V22X8p?gM5wbU18g(k#EqsbKSgnD3 zZ|BuDAtGn>NrEV1;eTKW{9y62=9Vj7ry1~$lKbT*aL%>;b0mZKbILESrN1rjY}oW6 zZ@3Ow6ngDC*ymYi{S3TE@38GHSZX<`xRbVLL4tOL1^VawP+UKGyla82uT?*G=~(^&jVj87ccT1di}E?5P_J^AwsA+>%gINxA?`xeF8eEG(m z*wb8t1;u$KSh*V4374I|V(J{il*`@Qloggz(;w5&>~5RcLahU<{w9dK*z!#PgSRo$ z7UbZpT{|_rpIQr|l>EehzH4aFUHNnW?DY2!L%n7u^|G6l?a^PV%jDMAR>zZ>#IoLx zx8A$Ob`NE#c%GHiMC#et)SAsYJ;o|(0Sb$#;O=s zC@vR!ShNR*^7@kRcmW$S!N>?a6zbn1niIY^vN%KYTw%tkvWvy*tVtKue96u8-(#jI zP5C2wQjR$6XW=fYG=*yx`SvowZ9!-{bzPc5O*X_}^`uTo2sKvZK-wkbJx5-oq;~!l&~^qWf0oj*uZN1UlocXr0DbkM zRqCH$+h)@`n1>%j5p3D zx|Gb;b;ZesHlv*~wT;!XLE^g_?Ycc!%xmhjGC6MXU%NMn|G9f3dvA`fD~mL`ZZk%h zSLfvSRY=9V+P2KX1Y(KA1{mzFVnPC^t6+=qP!f9*?xb zraj|SS|xlx2PQFdOm1fX zA=o8#vNE$O3A4bjf7GoM7WbI1HHR0DE?0cCP)K`H7ePhqE~#rkqrbW;D;|D)Mp{3P0JC-y zUd_`nQirk~UH`sJWg}ut>5*3d?CItLZ=Bqi&19BT72#JPmJBi!v%Up zR;#T>m()Nc`T$3l74OIn_v^NiO`8{kX4K-SjMbP|FlD46zv2 zzDdd)vu|@-;6GT(&tA_a{mM1TzEL}Ws8Tz2%w7*98+TREI?h*FtZ6E=Od6b!_84%D z_eHFc%mP84^f2?uL)xx5_$ZlVaF`CdaG39{5F()UEw5$~$`QY0WXIk|2&w;2Ayf?d zoB;{eMl19IDRIA+mCx-ToZvOxiXzAe?JfW(dy<_C8F`7x&W8v^x$ZF^zP1PohWZJB zmxIcW&_A3B*h6@6n8_^`0vZbwoUov}k|IdDYQg^XWN5OC^qPmRhTDuT)qlBFiKa>V zs7Pj*t)E(gwXZXA0EO6#tzf4*FX+!)=UGdo8#}FgeII?`H~ku%zJBlxUBf=|&Ty0Y z8P3!!6S&%ZB^V%eyNyy43yFv?6Q`cw;WTnFE=i3z=t=80#lUj=iqg4f!J&*?ax1~> z0jtlISWYPUO-I)fviYxvswkaXRSj63U}9W2p>&RM&;4q$R`#;zZh-a=o(2@_dn=Lze%lrFO`ttgChyVN)-&@IAle;r&Z*dU5+1J)4@sZ~n zd_DR6*Q0rXOm2(YO#iJGrux!HU*@qd07RjomcXeDU5r0z)~6^Bp$po$Ga38=XNKxt zH-{(RfeR8MJ8KDk=8&i=5V=Sq6;i8hkGhf&>u8 zgX}q2R8()+nNg-NFu=*;B>XL?XOC+ph*Rpn`zQ0zeT_^_!M{cZjYEzVpB4n480q8o zsw&2gnIW>lNLcww5ewT$qu%*d!m#ftrPd`-ofEzku<{&2_n~pUSv>Ryy#-}K*4Og- zXE^@YeJ6d|CI%o6-Y)^F^C3D-b~P9jO*kETHk66`!bznYbwB#*=bf9-QL@;f^yR%d zJh%UTG2+~b@yN~_Ubg`B;R$LSgwrsOG;F5PHOi!XCg7Hk-qYFaOA1vR6IYUXUOeYE z2NRgtH-GwJvf?dQ9eyy)ecWF^fgubD{(m^5e^2J5VeN5X*nn%UkQ+r`1!(ZGzXtw0 zf%1RxKmNY1Go2=U~|f4-ctWeK@j zW8*~C-HmuF6ATDNpsKX2o|%?Y&*$5Ch=m-kPYr(}A!!LtiL6;Yg)?^O^Aio(?sjbH%r3_xn!PVPw?TJVrbhR{7XR6MY-X;}Rt z$bs1J>?4KOqhS|Z23eP(Z9?oIS_4EK{qt+tqUT;VJrJ`9dwO7Jb>Mk#V78JoU0Cl+ z19%AG>J2*{4F83`%Z61zZwfjMFICnQPzLo8<&REaMGo>5MbFrY%}}vcgB~Qomn`j~ z5RxET5x1cRbgDcTO?E~uW!iG@vE5#y&G!rCy?~2gOf-77jk7Gj@ypp~$_hp_h_sKh zryb13*>Eij$c3;Ib%UpZ-^gYfg`lI{rN}34m-1h`+x8?sH$C^A;W-X7V*UA&LHZgW zq#B@%zuDQ{m;)|o)7A6V70pB*dJl&^Y&-{8s@(&XC_mcTbMZZ1M&p&IB=aCNq*3ys z2bPB%9&VV5>wrA&Y=VT037@@#clC)Lxzq6yE@;4F98qOTUIpg=c@3MF8oG+(ES6^w zS+5J$Znz9hS~_wqdvd&>{W1hVw?gms`J6`n{A!ykF$c^o3;>5sw+~o7ME<|0{r(?t zmghkuPaG`SXaL&#lz7+|5gL2<>-IbZ-hbhP|Fc2huU&8(Tu8*r(fHb9tR!oo_07JU zO)C|D>=>xuV2F8A(vq3q12U6vz6-!Xj2U_(2WkD`rZaU>9uBQwHu09zn$xLU7(S44>wfJP~s;qBnLEPG^R1Hc^Ec) zDy5-M-sM6-3Z9pTea2ait?!v<0JW!IcM+}5g**JqI2PdDw*4~ZC;>1o+*dUHOZ(_!=fW!SC$)gOUnFq1$KQ%{D0r4wD^M)++o;vO^N>i@;kY{ zXdv~UTMH`Y(Qy&SK{w>~(HAt|rg*_3<-)PUbA}vGSkB})D6dpr1nOhRb~U)mZj^16 z!af>{k`w*OUe_ci1wfN7{b%^teNF;JX_1&LYlhkD;VE@O3Fl)8`Wl=y33Y<}Wag{%X*UupIF$Z{{Q1KJv! zhO?`vse5|w$8}Yps&$uqp^TWNGqyYFU#gu8!S-Atm{s*g94|c8u}U*pPiFPu^_upS z02uIJ>PZNU)%)>5ePs6;@XY%MtmU`zpUlq6SlhI z^1#+lkk7&%9Ei{h`$!!jt?`Q#Yz|QhXUKm0o6db;Z9yrj($n8`(V^LJmesS2xXI;p zl)MOTYf;QALu+$T?fQl9CLF&LmopF6JZQGn^|PHrf#e+KEMM~$8)&ewUw3n=fl?=Mz$`vR^EKnnG+>1b_kMschG_m z6uGpWrVmMxpnbo{G2dJo+6eLpgqmt zf=(*L8d~spc*omUg1>3eqk$>h(aF#B7nfLEiY_6ss2hMC`XdhV{wQl!KMSG45M>P@ zCY8f+yk`_tuNJ^0>$5m$+y!Sn_8w}x-PUP{>d3PG0^mbFQ*@07Lb^E))JEU~JX?9Z zu?g>CfKV4JaB(L;>#un_b&KwzWkI5LUaE3iKE&EoJpkV08RME2a)bk3%JiPW#U6K% z26&w3jgY8Z2&;*@3+&^-bYRYTATy42WQ`MTaL*rfQbq)6$YC>WTo*ixSSVf(sN09kq>*4tlI(rbSqb zb}#}16)@6QA7nj>(Nffvlpw}ho@3f8AUetZ5rBVtdFjx zA2@vLHQpE|C?JgvKtK@t|6ZRG zX%TUO522hTWksR(p&-x^KyrSgnu353fk=u7s<>yIZTP9osb@WYP|8hnVpx@k#1klw z)KCAWgFPDtsi2Ec-V0_t4CyvR?>or? z#$SxFQbjkLq=bi!sP?<8Ehb4fY*w!NnoY5~0A)fuo_gzxkP+yoUdEbppNu~L_S?x9 zpLm@(Fa67<4o;DzeP(dMl-cE!M@(dJ!Oy@8LtR2*SP4_`b7wu8lSjG(n@W=C?Q;>DPkGG)UU=m3b>&5bIe<%VXHa2HLIC|&^Y6T);aL#`N1r4B}ps4O{6=RaVS+th^Fpf$nS^(1>K5MW<*N)H`C^!F7pFn4eV&dTSHh`jze=ywn} zE84)e{nd;(p~IgO)y_W^cDZV0fs?)VJee95YVW6A?-A_Qi&w*>KR;t&a9jq!^$OwQ z;u_o7M6Na2IX&H&e%ovxIbUl^Zq@iVLGMyOAd+T_Bl+oKh4ItPeo#E&gx(Ai6O?v~o3C2L}g|X5w|$&UybR>0}0GCZ?}iS~z8D6Z_g0iMMUEvlz%QG7(0Le^m`?hDvWbEU8Hv?{cqW;%6fVP&s=%8UJp1m z8*Q+O4%R;>Ep`wrYj7;!Co|~o{N0MSt+$H+2zC%)hJnQj?Wemr`4;En*#La?&j)>| z>(y)L7Hnzc3{n4E>3}Pjq8ST&F1+t4x3ZrDkeR9-_D9hLo{vbCl$9|}MdPQZ6*}L} z?QVg!zg{uVrhmMKFk~1U9DJP1e(&e(ypC$~++;PMTwl0_`S<7#jp(#$_AmFg$n&2u z*3X$A^J#-V99#X~k6^d&)pOEhVQskZ5TBo)G3ZOSYaI8bfQgQekFUi@Aq-1H84>=c zso>jsWUHmULM{A|V(QSvRo*>4tW8wBPU)<|##mjTJyDYWQ9lKqLqiCC-33FkWlR@- zsq(#E^?BdTh)4J`2txNiL z?*9C8puP3HSRS0S-s}*Vtmg^y9aGQUa1jC;0bw7wgYj3*!otGpd|yQlCo@UO$N=pG zM}r+0M@lKi<@|Q_vh|Ubl|>-%w1*0^v$G>1A@RFrh+wz#CNQ?Iqi&-5uNQK&|E)53-1!RRd0s!E#{r~zund6U}t9sRCeOH#?sOf4FiKgg1fb~l?e-e zvr8G!n7!tJu7NXii2RzGSU@e9`1tVV-xU=V`Q9IQ&z5V9d&W>o?4GBEkzx}P`rW!- zbfFPKBtJ#gRP~4RZf11>UI>`24wk_^0Ozkh=Y|lZIT6oWH9Tbd5=TZy8!=&F@HfxQ z%?Z1>uyfjN-mPtJZOH>BNGX?*)7+d=P(Wr!>74rn$YYuFvYGYOR+lHT^6{ zgK5G?Z4>4bV~Ob{6a&938IYPLs@>2?Noejl`FU_*KUYWiisBvPs<58%jy}1i9WCu$ znJDe)Qg85&85R}y1tnxm29tYJ2Ybp*TZI`O-XjFj>oS92vAfI~&cvh^X~)!Zg$NX! zmq$dc#cRYg6H=@>zT%Tkq#vH<%x9;;#?Qc1|@99E;aO6)sWNe*Hlxjl(rj z*&nZmf)8PYo=3TwUFw^9ALiLUgR2eJfs>0VSo7}pE9T}dM*6wAx#hZTNwn&f)vo8O z+=Ng>%*?Tyo13CY-!cSz(zf33n?T6O$f`}pFyi9kngMgdWYGD&HxiE(CH8)t{T^nX zo=k&z3MH;h_14sFO5`osQjmG^!E0ttwUJtxueT1&|lW}sZ zpb#872YRiOl1CUc)!R5m-6mol^5cZxhMe#5%#GtDwyNh%k--gf@wgpL)Lt$(+T403 zHx?EK8myNzIy~8bUyO&BM-3d}c3+)~Y_)sD14=mqD7=o&dO&7XLj(Hvn;S{0Fmz(h14QP6y+ufRg-OdCo2G_&_M6THiO%XafFGJKtMe5CXqf2jACS zLZ{9DrlWg92P)v!JdnKFF8nnV-wGrV$T-H){I{*dwsnkI8;d zn3EaYyA>*M)CvlHw*+{6?h4?X7XiR7fiUQ`4?_`pf3>orsi~>y9~=a|ZPWUFdN7eb zzqBL|Yvms7%m1+M^qsdG*xgThK7Uj>Y*)#G3yJ3R(q#R00vpJ2_Sw96#3$!g=cTeO zu{3i<@Z$&X*V*b(Jp(6m>*m!tNzL(dxTt&0Up!W|){fW_jNK$fEO&>;Ck*px<`3Ty zpJmi+7$8PVu;Q8yV=5yINDjX;f-&H8Gsj7Un;?y#nWf+Gck#1xxSyFFRKx_@KCM;CT} zJzxSGfufSjI({P@@NP_8ob(;t3Gxv*`vH}8}1?bbG(i_7m!H9BpVdNX^(;?3}cp83^aE56?oiKlnvq>n9% zqSkH8wLX_Z?zJ|_sqxxAUPuVLP&7F~PAx4u=Y1}KW(v`{?llDa|+tUg;^g|Y) zzz*lD?3;kw0_psCy9B$~X142k;{cTF{v8_5?pNe@KxgCtuawGaj=@&Ft-0a!_WbAd z%>2tOWS#HRn7-T-mOKdAaxN|F^_{FwAP%K`*4^f1I{wYs)_bJ>YnA?u-^UL6JKw^F zIy9WW%NCaL6N$OaYu}gGzC41hwKX>`yr3Tw>v&p`Ignp9G?oMCUs{)YqT(s#9d8e0 z?TQ9f{%06~kdb1$0U1litvQg*fwUye4sQQ*xr~(q;oaFhF5&a@^En5nSL00%C-c;o zn59f$#i@%Y`FEwY%PpMIp^B)sva<7wA$48$`E7H%SUcY0v=zE<@NM0Gp8JZ=+UdvY z{qYT~5%$T6m5XX8TQLfW--N+}XiRiQqfGYgNCS{3v%eMniK56uJTna38mf%<@BF0G zinnuNwe^)m30K@q2IChrW>PdN8ylW*{tLCG{zAF;7>HSNbVhm9Q4$@>?`}2&Vx8n4 zumG+FxLeYWy zJwLWI7SrsUNEvs={0?1xE(oGYTSUo3n+FRus19QA_~DU7#X;yk)aKAc7oU@NMFE!M zy-D65+q@ug*_Y3zI}G{Fdt2NlnIqvwzCnR@nUfL{5 z>$T`pQ&Zzd$?4iDv)ls!9hqHQYqK~^VMwV-`KzSHKlu+3QrdyiCjS1~B&1xwD{BHK z{jub{0|Fn?iSf!es#S*}iBVtRd=fFlSTO(xWGtC+Xy0+7-8cq=^=~2r#$;-mGVz$Z zlE$TY)SnX^LfL!`q}D3noSl0DkG})I8}sw0;-K1Yv?i{uu6jUlm>A<1lJC#E=zmQj zATP|x_%#Z8x9P!DUay{{q_z^U9T!9O+eMK!zNzK7W7R>ukhiRVi1kC2P6%@j^N!8T&AHj!!1Po+`dEXYVp`+%1jM} z9L3o5&Azb;Fb#Vs4l&Pk3mj-lp9i_|KeEejSzp0%A6_G)HrHw_bivw}?XhpEEJ(f& zlQr5stXBzRn&o~BgP7sI`y&j2bbv89w>SiS66PPWJYh4$ZX|9)3lcvL9UCV$-x@qx-kW^Icx9FXj6KkPuJ%Ud!K6rR7m6SD#yK;uV9$k&~5&E2DNZ zi9%8*i`=-ZYmMT`^NePfJ6SjIW$e?AZQMiZiVSUBv3p%i&@rJn@l5VWF4-Gqc?||C z6>8;34YRM%ck1t^=hu6_uLRRkU$$d_FkmNJe-FmY&j0ikc?xON4cYoQ=Q#>-oC7N z4RuD=cuQ`e#tGQ>EqNV3$G~+Sh*r!H;=!FD4h!*Xf9Ewd0F%2nODPT-xSw z&POeN@4`_63q$?)_=q9>0~cwY7r(UV(U+?&`x51`&075CzG~QBcCC6E5c*aP1cqDU zRY*Fncv8v*!$SqiDniS?R6QAoR!Cbj#A_dMOIvo-CShr+{JVNrrt%4+pNE8$Mqn+x zyWDr4V6xXQD}^V;r7J-!E{vL5AFL8M2&>n`f?B$CIO60XZIjF~xMPC~Aei@?oR9n5 zyd22^_AcXxh5Om$$%Pf_M1quFNGSbtBJht1Y-n^UeC z`GA3JTR3Y`1Xt2usyaU21(kV(ySpY|8VLtMR!k9#O%d)Q#0T_mcu6cZXE#y!V5S}- z&3U%Rr^HPRS5R_M4#jq{9?K<-mf-VBC2X6muFCXZPJ3)&9u9Euly8er+b)ZnfMP#g zVJ#p~u(f!t)Lob1K%isv4n$y&^4{wj}@&Y9Sa_B2PI&_R=sY0Q$ z1tX4hAkDw4$1FaiS&oeQu?Q4O4i4A#FOP`l?mJ017bmT!ZQ%&`c4X zpEe>?9&>#&#`ppuTLh=_xOoWkjc&oFN1wcNMO0C7PG1#{0?vxUtd84YBb5tnpzk)9 z-+Yk=Rw*lwcb`a=-lL=PJ`xg~GZ4S|@gqtNBkulX6QJYd;PHn6d zKbEx=^~B`h;)bj!C|eSSf+c-UQ6XZ-?BMU5Jyf($?>pqJzit#j<5+MWNg0yIc%eY0 z5MrH~nGA@P^NG=*%;FIw)6dS2#Ea0ClOXM5cLm#nT-(5U*-JoWLQn7-rk7TJ^3y~k zp#J)4oP!xJE&eUCZF6}7EX4%Br7c?o{lws?4^iv%C)3%g_%&Mmk6`P$;gxu~An^Pm z#I&?i%S8f%B}n{Ms?<&Yv1-HiU2AqjSJBwBHDRNKTYR78rNY>9h+s4KT47y0J1im+ z+UyQyWG0><(plw3Vjk-W$9Q^W2WlSkaXrA#m7{3B4=Hw317t*H+6Q% zKt}j=B(eLFbm$F{Q6jx1tc6*Jv>IM*hWG@!Tr|%nqJ%-w;!uQ$Ke`2V8uhHoyh5oN zC}d&!@E7LO;#b-z2@1h~_uuHzH|R3#)VkKn;s!hQ3u@#n4Ks)9s^!efG%L@>WYl5H zV!{8ZpIyI#j4Mj8ZRM(GV>NJ%<0Cl2VMk!1J04a7BQ@vN+8B@d2FL67Svtr%6p5Fw z%ART!3qQQ`lT7|`$-1R%3RG~cbE_d5nJi(~ z7aK5cO=dhR9zpFJy;<=zOLpp!_2jc;-p?56W0U=T?VI*!$+H2HsUw@?AcrYm!(+Pl zd-p-7CSg_iG_{R3QdMmy*g9q+Z@T0AgCP|OmOdv#1&58VPQ2Ik(DMgg?2HpwQ! zLQMjQu2fYWct7_@PCmh?s+YZi-bgDet7g}89FT^FhW5vu2&Yv6OrZ3gpH~4O8Ta#s zrN{KHCtMIa1=8j68mOj z)8)u)2ze|(9nn6Q(k`?S?y=q$7fca(d9%WxC(M&6EPM-Ur!IohMV56{%<@W1hy>re z7t6A;8Pd`UT3rOb32}%1HlNUplomS^I`LUt)B|omidv(BC64DuWF+`}cYg!{ZGpv| zXU7yCF7C3bny-0Cb=2O)S#jewbO_|mjHKCf$j&gM9PGEVH*h&=s3G37{yh^p3E8GA zy?sMDq}{BN+Uaxq%$#j?q)w%#`>>`PN>z&%b7o zd3|AY^G8Tp4w=kl`>F-Ug0aYpU#5)3LNKtH)2nG+j@q$N@-x2Z$@VGWd48qRR(8j0*TRd#7NCd=6;?+&aCa%Lmw7IPb>Wb zL!#~$G>0>WE#dJ2xk=5$f{Qib)yHG!SPswUsZEcDDOaY?S)-klLzJ!ad(|83@_$0! zA`1#zc=KQ(q#9roFJC?v^>`U7LvvAZm=~Cz=ETxx%h=eGUJx;^HD^TYd5rr5IV_Id`m4M;?h95vb1_#9+EXO9amn& ztE@FstHM@&3L5ilpk@)g#?RVbBgj0zH}F^;&597t@|Xb(`Vq(p_W8xFtbT1}SY{1P zCQ-&HDl6|IXvnaOtZ1T{l_0o(PnLjA z#Uu;?@UnDrCUez@@MYJBufWHv(?!?EJGGndMayBkCN9vwIlO$oxWp(I8h8Ho6UaiK zd}NZxe)Jd^I6Eg?U0rF?oATmMh3pqfC!1w9FzaXKj3D_v3U{4%X6neR$>t2f+#bT% z00{Em0>2I@t1eX`kfv@1;E(DvZwn{))_MCd=Fw#@@9CYVXc+Sjdcx_(RAhw zo?41e&O>q!>Pcde8pA#2KqQg2B3q-EfXNu-ON~3SV5eQJSO#feo^H7_F)2U1#93pG z6BpD(PW1R9l07Y1x+^tL$nHejLNt4 zh=3Hp|5_>tZ>tf2ME%DZXvNE~Dh}$bl1c%AdK>XF2Ll`|v02*(JwvxFyP{lu(Tn%< zo+;j3X;Rjr?4lE4joo-j*|l-V3HQM;Q|Pu+mWe6>@w% zDiR`}?hWB;3q(0h_Xhk2c;NerxnI{UKiU#VW%#wfdr8hM9w0u`bDS|ips+9$$_Gz@ z;$Y(lseQvkDqqGTaLkpueHy~$FpN#?w=Bj@-$fK#NM}Ez+Y}(y;1Dc9k!=p}N5-B; zX>!1?!1qbXsOfKoJYuB|9r+2QKS_qT7Dhz^dK!3PtrR%VkoIJ@WaB*FR|+X6?0i3y zr@LXchG4Xqh|wJ$TiNampH^2b$L|x(EjgnllVt65;DWU?#*!VH%{VRNH?O1@B0PRM zX8L1PORGR-bC=B?TbjvdH@IJVYtMPNIssIo5(H+kziVT zgq2uqB-kKv6#vLG-og|#@o%}8c(^E1cDWfEJkc{YGck2`YeCvUX8xZDvGg{=2VQjBEJS&W-l%Whpqe?03w+(VJ4T!v+vVxXAgD-XZ77H zRej3%(8$Q1l#b43_A_c=4^aS)^bvq}k`fbt3he2Ax$K&1v|Yn^#gj&!p0;Y(^gYfM z4rz8d#Q<6+n{XQ)9UbjAqqGunA`TABeSLiqp4TokZnifHKZJUKCS~ykHyvFhfUzg( zzwTh_v^s|YEh5XF0Nf_KExqYPX;27ujFU&84f$8F46Yvik3ag1!^}Szk*4-CHa0f? z#(4l&y%bUS-3dCqv2niP)(Ie{BI@eu(}Z7vPKeQ5k*t$|el>CE_ek`;eN`bvbQe|X z5@sdMD|NQXozJu70Vp}+mAw}9xs|^H!e_e*etBcMY`E;na+eN)A85SDm>pYxH6LhlI;B7kr9idfUPX_TNjcPG`Bv01 z{I`x2G#=N%@p-|N11Fo3K+b4aK_iAXR|^>P&`D$QV*mn>B%%HlQO9rAXdcC4pq4YH ze|VnKf+pph?()bujB+5bo51gdKJo)9l=;u>%8cOKHsZYI1vg4JL^zh?!~SFO6fIdV zssBv3IDCZ41XF6hQ7h<14MRkS{NU4oJX*MYZ}IAuWl=+8*5)w%*?JvvBKSe077hjc zQ0jxX!X%Q%?mi@>2_{219*tdx(|VCQwOw53mhNCY>pi+Sx8E!D9}0Tj2sh+;G$;W( zGZP(@lSPha8M%5#Mhfca*FjQl(A{m!_qP?Wf$5}r8>hXP)!dGFt08Rg%!DnXGdmcP#1<93ZubWb3h{2 zNBT+#D-(uKd@<0MVO$W$wFqCEV)#h)P%M7`P8m3wZPsR)ZL*69ax?`I38J6RKAcn}yDO>L z!aBn22o2&fZkTnXt_sDm;CS7gdYw)lem#UC+eUw>+*D% zRm{=dJ;m>B$&Z|aW6>@v;`!MV=$yU{DoqXn+4J$)Sq9r(KPz)0W8F$*k3q^%a&25t zka*9uP?J3E;^L+d=Zp!NM2@^**AB}0M!fnt@`5YyB3inFCV^i>U-T1+KUl?d~B};Rq#FpmMw4ie(peDRRo1X#L(G1 zrDdj6FnMQ|`6oyOu(aQSIH;n7vE}i{m*<5|OCs(%H;-4POl&7qOh^W42ewENlB9Yp z4vka&-A{&j9D~er?Ntmb5LC^#iRE9hyW%_AP;)P-?3^{sylvm2ZVZ)C-@+&OQqd!< zgc(|{rGN1PnOm%W+-LK@OwOZ4*HwA23o$bs* zgmOVNv+#byM49YXcAUutZRRhuv+9~62esJ%&9=?P=_=npv|Su3zP|4TIl<=cqkn6- z4*0U-FZ;(!?n!=sHX0RZ1!3!&fG%rqZ|~z?ygU|8nk7jMW5{0s`(ev38^FW=Hh?Wg z-(1g@KQ&Y0`T)Sth1&H6j@_KmL>jyI^Kr4DF8V#tIcIVDOAg`#P-rLL_8}uE_U^aW zY|j&F-v`J3 z_-33^lyljSRva*Fau~9Nq`LeQ>hPPqQo36i0#A6$p>(w4Pd8MI9trcce6V|t;YxY2 zHi5&>e6@c(^Gfr6wMQS3@Jp3r*)|pNEk#PAkMfO50Wdh-PmS7pa&t=0}=}T9TTwY@#zPfgJo0Yn{LGvsD#IoS92>ZR^?p zd`>DNUpLJyZoo`u$yz=>4E}?|vm{06aJF93ZB}Cd_SL^Y1sNMM|B&}MG0}pQw3|Di zuqmq2ko=l^*8@Hu>@E%M=*A_U=e2U1V3nsYY)RyIt@Et+WHb+}EgzT=i4bUdX-EXA zX8*4|1;^+*TSAo70wJS>To_>g@-psS!u1MSsppZmcs-hf80uKZ%=QD zXLCoZ*n-pOqJ_W9X?Z1(iMC=nl!R{OkdUnVZ!qMBEudh-AlBixIpXuk?qw1~?njVp;Fn`aJCMi;kv1gWr9%j2*UCfBak7}O^b<&shri4iU#30=bzck*Ga zU?4AjxqvU5E-*z|(v}yl*zUn&|k{h}rnV~-@M`c*UyByCt9(!QSX8ll0I-H9TeLKPSKr8VoxcJwvaO~}& zxII!$OoSbQGVJ4o)7WBizlB)xYz_|kteH)=Yk-oPr(4v~NRH2wAR$CSS)d1%k<_?l z{$8(+>mO$dmZi8T&&>Op8!3cqS$yh_*>^&B@1ex6a(pK77DH+~bR0=bEW0n_>Tkqn z3`WG_z^i#cfHrcyIgTv}3!zD2&q)`sjQ``To-p@4D!=3&Bi`FBjW>WP|Mk@y;JZnx zt79^@Ux5NS=U>{zw+$W;2K~;n4o9R5n}{D4Z5{zs09T`+v2oGswZ5LM^P%JN>70}f zK|Z)|B%X4m!|S0vdS4PTWdaQXVB&zLxP6}Jo8$C(+E3S!!kb$HmzP z*&k2Y9nzs5K|g-{&C?l=mf7W7cgm_drHqV7fNtI9Zu$g(#jhTnpJzH!Tv(bWE4GxS z&(^YN!D>+{VJa*k0PV8`lPLuIdwfCv*fUKXV4DE(1!ualjFXcw9zTdTg*653Ziu6fbh6N#-Pf zZt2{&!m5fuvvj`q^_}63MDESEg%~}Z$oW@9{0VVx`nDLLBUI}=2hI_I9iB(x;l>=k zXvU-UbJo-?H>OvyGD7gw<5#TYYYtXgIW+Y%vwL^C2NFeB*m z$5MwXrngV=G+~obTJ?nBMjXe&wyaqGK12PGzlwr8qTio_XY-h}lPqeuAAc_;dT#NK zry~dzJzIDj5C*tLR{C0dRW@3siEiYi zT}dbfeP&7}q|AxT&Kt#yN-peWe$_T~2M8>TX#%9}$u|~OPVfvQp~MXNZy&@bKbvYD zKB7}UJ~GT!wKkjtO$MIlQ$i8;OisAw=VipT=+;0d2T&v(zwwwOHC<1(kV(y&aV_Z{ zcF)`OLS*oIhB|GYpAK_ZLk)P8y>vcXWxuQ!^c&ru@*GvuNNyw_B#Y*ctjihcv)fh> zqI8KLPUgUczX$sgfT^n0+5)`MS(C6}B(D&T59ZoDSfDB>5K6 z#$AnwnvoI7{iELl+2^TJg+)KB^LUK0Yy7W4#wRBqy3M>jbbauD09ka8vOIqS6>TqF zeTxhM4#`;UgCZ~pKs(iR83u?#+xTFdONA)_KMNzmR1g&RaI&Rq2a~mA`xMC2Gx$8> z0CHnjA`_7K|7}`f(Bs?#h7OfE=-RZT^v;5fi+lU-eLaBbebocScfV{b_~jj7qy3MZ zky>wTY}|nw4tYAB$|{(B3y_F10ICL(z+-n+r|oKeScP`eq&0%8VQHjeftu~1J7Pwj(cHij^C^k2N&1l_bC;+pf;YXt`~E5`Gk=BB0+?e1 z0|Wg70~>{LJ}u*%)is~ZP?tY<28c~5y;;$2w;|QiE`B(<34-B96Vq`R6l5P1$@Mxt z)K~|moXMxaQIaMZ>I|pw%1_%!<~~_iYG(74h`+6c0-Amx0&W zNl>>k?li^ExeGy!y!`os{K3`4vjt(LJ-b8dk25qu1jEVTV_x4NwKC8`hS*mjfJ_W7 z2=dU5Ggoo&mkj886Q8|4fCAII6rw)}*=&By4=5@S>dLhmtQ6$|T6Ol^%F17QMxz03 z9(1owXypG(EN0;d@*3&Ls;RNr39hbY+S%PbJ~`RBKVOS*H45ny1mY$Du8@=v(@|eT z5dWhPuaAj%seUfvq^|x1up6}qJob=V+u9s>Fny!kLi&1IoDSXhV#VEm`$J*8J_dZI z+hPohiTMn$qwTJWzq}n+CJCBY|3^md{I=0L0Fciq8&n|#q3H|^zzl$<>)#~%uciX@ zV(%Wg$G`G#ZcYXw31Ho6bp=!}^Xa2)Z*LpfI{ZgCmei!fuv2S;wt(&(FVaPF0fqqZ zBy)30q%W^bJel7n%L@O0lywqi;pRhtM*KGqh!i6oOQJUhHb)|^etrPuAGy4f!bI}` zNcREOI?yr!U}&Kj3Rwc2(7x^8`9B4AfIF`%*>BhYfeV4Bd)t|(#c>}Nu>6maG?lLAi;QJgvbdps) zUldLO4r?!izMkSrHLB7zWGA!gHyRM$9rADHD|Sv@aXa9c#^>xVY`cD_g^@ek-+tGk z-8Co5naYozpIziNq*xb7yS$3AlUQ`^jYo5-I=jFi*v!)0j9btDZ*mt@3sAnT^Ufw= z=)Cyjy7P!@x&3~QXO%K_Xnn@K3m9q3U(cAKZa-mMtcre%b*n0Z0#7os`584F_TLaf z_7tvjm5Z$E!>5WX1o+GTS*1G<07XZC9g^&{(B`c(tjNtV4lPWsi_ z5c+`6r$Ag^C!4RoWuk_Mhehd|{@Vy|T?p3LuFI`#pL=Yg)uZ0muvSX%)X2z48SYrB zZk80CSkW|q2L3;x>7q)5eIqzptCs;ixNt!|%|E;}Mdf@vb8wTQ_)jK%;T~rUFxd+# z=v(rWtgAvc*-RaZJcs|*t-8MGR3sc2Aq5zr(+1SkRt6`W#rJ1P_kaIjpDf#SlXGMO z8zDK*zpcl~!NFm=7`&x`rl<+=PaU^6i2*T{kX_`PlV!`Z$3V}Rg~^jF$Kw?+8t9ae zFCkMeW-Ss{Tr`T;ha^M;;u1y$FO%+3*a5r0$WHiWNznGVH^@gOuUWu%vIoP9H8w3x z{fMUjPwgI2pyv?@(`%v#zE=dpQ>V>daH9!vC@Cih)hprH+0js2WGw&A$i`l)>(y;C zaFPFv;on36SrQQsh6^{WmMre9_)`B;k6Aj>ww7%a$eyrBMO{kum4AR?v6Y3!20|r4 z#H-0vMHAbJ>xk!(iuOWEu$W?>orD*UYm?q&0VNsYdk+oEohjCV1GKdlIRy0#6*-Kg zQhbcA)ULtrR>Sh~uW>9THL~sEX4N$bHXS(BWZ;sel=%ga`Im-!f;iH3lA&rOKP0H~ zQHRSc^k=^&{%_}cJ4~v7g~o|dm9QGrXjH@Fl#VB1+~M^!nkN@@5kdDRkJb)Ewqvg$ z$b2B16s#Gf8)pA7%P%a?=s9y7MJZYqy(r3S&rG;+WTEi&l}3Ski4{PK3nFunPY~y* zpQ0`4F+hh{L0^W&!A>wW(dIQB6@#ty2_$C$xb9X`tix1D-p7I$45YhNCpiU(fdc`# zHE5>0NFxlkgjMD&8>ZN3Y*GpH&a4*9vwEq$1|A%u>?HO7i-Nr!hNV@}xCmkr)P;Q7 zKeZ}cKj_5rod)mElhzkf> z`V&{X?hv(dKBEk6`PfH1WWnBE?DF1$31fRw2aYC5-P~Q2ot^}N{(}D` zunJ!&5)u@nV3^Ul91J_El}BAREX+N?5Xf#iuOK8kqw<8O9FM@BVDe~!zn3_}lG|lt zOHIm1j8ljfP!LH0>_%6y$!XEA?)YcQT)E-=JRc=1p_u*BJR-sKNtx+G#T9z_X1}7F zQLIy`;c41fG8jp0S*ch*4{cbo-uDx%Y<>4s5qX~6+R%}}M4Ne2=^zORHo5k(Fq{n6qnzF_ae+v4M4HMG52w-QTTJ#TGPq}@z5bu!Ibm4j4L->&Q{h@uNhy*_qFDKG zn4;q5ezz&lCj(XHjVhC7E;cq!}75(6zYla_XYSrg%q`NPXGrBnE zetFuEG>D?1)CV%nG@7hm;jTob2Q6}mM%&I~|2{ut^ zrpeQp3O{-%NVT=8a)aB?VEMMd9Nn<7JzOLTqp)ld%Qq5_)})6StnMIvR>y80CEY+( z#c+*{)o|)65-<~HS=JQ((37eriou>lE{nC6hJHKrOHlC~Ly_^oF3&9~Ne`kSt)(MH zT8!n?5R+WU1W|I$M3}4U>I?t#a*X3wHDtqXGF4Pg4Y60+R{HDAwF{Ah)P?;#Mp0zY zy)ca7SS$(ZxRREfm*>hbFXF_a>Kfn3#Mpis*6PnA2Rn8ldV)J*ID)Y*x9Xt94XoK) zG~7R@+**^fdL%{zCuO0iZcsm>(MB6uS(f*bG{w+}Ayq5QM4>Wcn;2Vho*?DoYugnB zOKOY=_0ebzLF@XO-h_XN#j2D`_(jH;+G9V^gi{MBb&RViAcQSH?cYjmArLLU2UJ@G zlz9c%yFD%p!2EHTbh25WO;*y33($&FFi%!0NM^=?^0~j-OOU*veN*}l&N{_0Qa9EA zveCsNHN?u1;2Ay|?moXNwA+s)=fi#e_QsZUR7_X5t|#Z=2D`tHtz6WGT{c=0!H~~q z_scuFbOoo(BH@;*YM4R`Gyx_d0TbD_Ja-TTID4TwU?9e=Dhg9lcuf~R2t0CL$IwU# z8;em1nKu0=DW&k6={5Lqf!wbo`Rzup4bnL!Wa;RG-!8pTS(7oVv1>?j)qZvnjG~RM zRyZf0yiEoMO}%g}l$DWH4Swy)b#$B-PL)+P=2kMLi6tfu5*bu98850V3I%Wq*rbM< zpjk0?@_mSS7CtT5grFc16m=t_!ph9i1c2D`^Piu^gKwO+P*GM;b1v3ZPdenx&ZsDi z<7xRG9v*q^bBCsy6#IMaK9EEVZ`CVqF!X6oQwxFc=k-IJAplsV;GG79Ckf)Vs)mDv1lk zi7HI{(5Tk)hHB3smvhtear3==wfM7as!e7O4#R(Z-dRMcg; z16&V`^pX|gRqFDHbegAB!2RlNgbYb%?^`v z?Z$Y0@k^-n!7rpClDTn5(Y4(<=CVZE;~a_cIx~!#U~Ul%_uO=4PgQWFtwY6x3aoRJ zcCqqsrs34XdCpDaOCII!)0Tt1_HjFl$iG4*fO&%tO1S~Q*jAbcwCN2Gb=7ywI6HSu z1KD7>!7qJuqX=%^EYWhSD~hnO;A$<&H~da&{r1$<;3~;F$tTcq&BchSUr76^l@j_E z4LB@)X$+oO!~77=KQ(-AODk43#au218Fv3u3~ZPKGD-{*OwXB9L2*&SSrbAznh9`W z^2Y6IX!-siuwM6d6F~I-*Gzhdac-Yv}INl#6?Xbyd0&qInTwy&StN3 zAc=4h;5W03a}QKEuWv?JMvu^Lq-aVknP=bN*v}JP+zz&h9uqZ9%8>JCaC9nS4wbxY z>XXf4)G=ifb2GyuVeFdMg<`cx*6bh>u5;OHna2m#wwKUa(N&Yg4nBg43#2{ow1E&zeR�}ivy$=#|&X-VE%FzVuB($rXus7$lMEZ16CTU>~C2Rny&IW$DP~nYsJ;0?Iz%c0 zR?6XV5M&5yjPh%!{1IRZ2);J|OK#;lE44T8KKi-uxs|CNHN=VQsPxG?W}^vd6gl>hvXmuKD|V?! z1!P$dldt0BS5-{}HS*x=Q$><9cy|7;O}y`4Jv=z6r@l<-m#2`R%xep3=BK?e zo*OKn#pYGcS|nBI{5@nW>9tSNH1&?K1vCW{X_IsPO1C<;e&N4fY%EC}F$h-a1S@5W zN8J#8+}n9+e(}vf-+E%d?9}{TK?3$?)U-r-l&R=7`b&G`gwF-~n)`bL;aAgeFX~wo zFkU^c_`^CAs~Z%mw>~W6ON<eV^s0 zcX~?$inv%vYLq|_W-$nG7!fiGwFu*5EqaRkRaGV4hr$+7lZ=U&H@?L#ebDR5{ zYjx!39mRL`9g3VQ2wrn!^G}65oQMuqRS7BOq33xi;NWy3uS^6ApNQ!^AjzCRrjg9V zSf6_?!?vn`W1-`$Q+!ho9tVQs53E~+V`ACj=^wPDq?nJJlZRYhPcH9gOH$PIvaLTj zZyTD#v9PJPx2HbTJ7>{~U7Y9>6BF6kE-7YTZ95-|FVh`vdF?jaJ^sS1O=~XcClT)r zYcG!v`Y*URMr$Rs2&k6Jf+OYv(tYb(*Virqf4twl^o^8la9y^&Z~72y&Mp5R_TDNg zt|xdGB@hViZi8EJcSvw2K!QUE?(Q-Hf?M$5Zo%DM1HlRI?m9Tk+zI)e`*!bIXRY&a z9{z7Ld-rasuKK#V`|IlUtF_07Uu*GPK8=9(#Pc-Z1%XYtZ2=uA%)hDMj_yd=at)5 z(>31<2U_ld-rb{JsXyw9_i=#RCl={;t<>}7ssfUsFYH@mnPJKK!#S9C&Zu+_q znOmCkw>`%+%6*~UIio(kosaY13as=D7o6f&imWg$1b4`GR!q&2jMLY^Q^Xf~ldb6) z7#PS`v9CnihowTg#^2aI+tJ9y<+n;fpU<-0^BL<%GLk^QwVK{R+?vzA^g5VZNBr)` zgAk{wye8*GsagE8OcH6FoEy8g=lhKC7nw18o-4o)ouO!W}Z3r!7q19O!%~ueA@PaCS zvr@q{8yB=By9LiOX6vk8&gx)OdnSPOti?SUtc(|OL_%4xa zhGcT>Z#WCxsOdbRnW;%L25uBw)2jHTCyO;OLl)iLDqxLcWH^`27e768fW~&}i-Z{M znZL^>Wz{i@v5D>Y`8>Bh`CJZGls`Q^&CC*edd>~Z58>J||LTN|zY+3t#HV3-D{Baq z$6tvHM!qrX#knn#-Yxt_{3~a%kC0?bUzx5as=9GJOAOKg2>6>!4!a8)$zUi@E@!}_ zG~1_XR>yerg6wU@%eyI)tqp{HH}7E;ByKYfosxeKPm)BF6gtXzUjC`E4$W8@t7+oC zl$zhFEw7ny5v1Hn{qo=Gol`J}C-wsB+ff6d(OAqhx9gQIUb_OjhPmM4l}YENTT*LV z3XR&tilq+oe_u>%5J~wpurjzQ{VMZS_*STYA*7G`@@kc%aG3RSdJp(VqvhcEuvbgs zZ$tBi%!LPKiG;a5r4ugp7W(JgU2%9~F>_Qy9qOV-g#=S(F7hkmPn3ARNqByvs<&BP z$iPPKYjvAi#-t@h#zK}Vwp|=Pou7|5(GC5UYB7>P;Z8|A4r;xlqK1#-_EcAhjH3k8D`Z=rh6(mQuCrSs>aRUSv|qp6l!P+fgh3Eg`J zwis1@dHpNa7J4#?^wSL8;)rjEK|M1zi^}GgzmAUbU7g7$9YWjMFk=_#8XAi-Ryy1o z`2Juc#n@4+E%nW=l*r(ntx8=LrdGR7jh*$CSeh!V%$s9It4>e!RFLP9{1wX73bjDd zkb-xnV`S8Ujsxh#%IvBFaY^VgbZ`rFY)98&jItycKPNxY+>WQ?Ha&ey2aPnan5sL^ zV^Do+Z9GzFCTOjDa2SCTb5fX+!EL*!%qOly_Q7laG^GvV;m#)#OKe5OCG;voH*J1V ze{d|5-?!SWp(b8`B`lw(Wudw`rQw{naS4536^;63yMmjGCk5g$|7bL-E>mjm5}{jl($8_ zaG0?Mom?=nLo3pQL6M&nmO`&Ks3FtNZKj}tV4*Fl9PL9-4x+1WYDr$iShbF;q3&DT z3%Ul|HA(K*->fZH8-grumEGK4LB-S~;&Kh+Qr(^tytvEz@b$F9E_C(-SKgFcbt#Lq z(iJbS6im-dy-fjSLYp$0u8^_I*&?S-k}8@~nsu1FJ6#ZrI0LsM06B;*(?Dq5%x%zX zenm}z5K$^^ei=qTgHf0A9Sh46pE?dYRyc2UdqGmiif!cHdo-bz=85`1rPv7z&qnSK z({ns-tT3B>UwkoidcSGiOk3z^H#&Zrtuh__WHmU>2E*}% z15v8y+pG6j#BMHFOYK2KX3{VCvGQRK&qk8Z-dMqc9wI^W{-lt_#@05^T1_m0VsbHm zpOTd7HBLFUS4%6Uj)IaCzpA>>tUNWkWJ$!p!NIA2h5qqz1m?rd63K$G>QK}tLys5aO)9KNyG zhUMpx<3-LC*71f(=1t5=*c_7%7@ydzsmZvMF>kckK5;kuBN_49+EOlpv2tT|h)LtZorf zy+BF2qkcC+tWup@zejC*;%+eJO8MxF)Y1H^D_3dX2P>-{kjw2vy)|FX=#pj0Nq%{I z+9`Q)NrtvJnrsZ7baMVSyGXgsYF$)j@TV59hyA0YyrTNd_553<@=9BjG#$Z`g;iyh zwS6}?-ch%=E5{a#iq<|Ttu0?b*VmJC%~7`nwsgx?8Dp7z2g?gRg|Zl}I>e!~4c~=v zWgK?Q7x-q+_7_>kDhe-n-p@1~7t+s4Ej7^P`nE1Klmw@a+CmTh{nhH7Z)M#BGl3xk zVS^Hqhp85;i}Jdf1m&*!bp}UU?By0pXdssY^swIHu1q`4%%{U#IdPd1znVp$%DKn|eV>y3g3z+m z8al^aThBPWZ`N{fn%R~pSr(F+4?{IO4Y$D%%RVtD=j|S<#4UvB)Yom-!`$5&6Q1gO>$w&ij}oQte~uqbM6Ct7r?@>WA3eHj5x zBTG2LBfAG&at2OWKIEq9(ld=-WU>l+zlS!0i&dkkB&AT5WU1kl4nM@BES^l)9lO-p z=t|2t(gLc16}*d^Xf_o=vb}UvC#@DSbL>|LB=y|3$XC;|Cw13-VYk;44fTm*+pO%d z43mBP8Wk|rE4mAlL{x-l)I4txanR}aS68In%VM<3(kf2pB_qDea>I(8xb(w3>@Rmj zHFzNPeX(SrSX`(s_jS>>;ah@Ql-;5tnK;9H8kgA9ib*1qO_5fMg$1P_qF!{!(^3jb z-;|aH&2z(Wj;xC4$xWZA5&7z?1g6FsF0{2bSL%59RMqF+KS~0#$+#sCEi3_I!J;if ztp^b^?97Z($3aJ=S$eMMjDcIHj zW}Q0!u{E*0{v9DeBCjEcEGukk7MGT^+{js%Wy#I9h;eyfHI^ zRTr)ko+ZnrRrbYU>bfx7@=iFR!%)fG8D96vnU)A!jKPHk4F!cq3xS44Tl&R@6blAP z0|rG~X9CyqRtq8wpZ92Wr>nmT=Xno%Jhy{L<`<{Mk50+v5R{{MYisMHD|Q#FE$#BS z>!s!=8v^SbKmYX@48Nf3@_@on!~Es)KyaI zOh-2`G(H}ZAI4c>f3EuSF2D6_cks&~GOUe2@`9L*=0rCCIwjq-it;6uQ_}w=`X3Se zj~2xJUnzpw{Lj!S+nI90%S3|4Qa*(fMfZQ1a3aTi)t`SB-jZfA!c@~R&%XTq#p2xq zOE{5RbN|0=z2(T~Ds63Zx%q0k!u&rVIAsFfx*Hlz=1L_bCMvhu!Bn92aKQ4L6u)HH zq7FSG+0j$^G+e2w=?>|uX=`f>Bl#8M?_)0ZGN#M92VSTeLnAx*oH_#s*4Nh)j`ntT z7VxBAfM>5sM$V)+P&o~b zV+BshzkDj!C|9Uze6cC@^vCcS5Arm=M^~PeX}BH3rl>pe{hwS`)E3@HvH4wyUHnoJ z?QLDtNBCY{FFnvLVO#$XK2EewtQd&Cr1E1lO*t_cW@WOBflGmn zx)_8GKch{7F76#715t9Uos=Ojm1~soA`{m2;P|r@ssCXBE^51iCWSmkY#gr^i3OGa zpW@V;31;tWR2H~M{lc}N5mT}6=J@!|NvgszdQOk|{Lxw}JDgYaEmHn^qiL8Jca`avKuFAL0(oG(&cXGvxZG_6HBY z>y$;}ON`>-r*tjzHBaMpZSvVvIyzTj*O2DYU`=)DURqZ%9(7ZJNK=`xI{c;3*k(JN z;YDFzD|LcZ- z+WB8;SHJ7VGN$lopB)=0pq9=?`@2a{smf7E$3t-mLg%_uwxVW+WB1RFkKDNq=ZRW# z^Vf>L!A$=Sjyra9(@wGaIqQOBZE|1`=a*%>pmBFJ|T9j$(4;2373$N{;8zc*n{Irz$3>d^PaI2xR{(R^a}ZN{ySC=NmcE z8=JlL46nZ{a0$KGAxL`RaH{7IGKigi9xnFKps?c892oDu5={M0RKchlcJ+G8o*i^l)Z}5mkKm5EH-jQ{e>k@^|2R$l zutgb8{o&K|fSKRX!Dv~;azftse7Q?>4AiLnMm0hrBL7&dPeE(0ArbD6WXIklmzP5% zt*_$A>DOYo&Y=c9wL(9ZK9}Y>w_X`f2nHeoMCG#E0-kc4w>kQT!~qiR=5l3sDyaBR z)ZINQPL3yU%sbPxgn&0oY4=Nj%wJDl4zw{~R}vuD1+DhzShSC1ZJ|3R2Ho6;2Y9e0 zd=vZeH?8tGA2@Cfc_3>qo3C@+83>I0sp1p9czNUj=|VzxcX8OhBifXp49Bnh;_55% zvB3XeZA9t|Tj$Wxn`lSISI+XuSSu4lixzT=UJ@X{VRq>iMf$f$}JhJ8N6#b>%y%lhlKX$$xhg19Ck+AkO;;dASLE3fmv-z@*%jfe&y}aYozPlx46>GPm*xc6_NmB(HW&#?gr`Ny8 z(&zIl_^r1$XIq179{wHo^<|zW0y+NFE@z(pH+~-bVXnAqJ?anXBsFayDt4W_?&cz9pBvWqklP`T`k}(b0(xh3CB50uv+-RPMh@ zjT4N{wYPnQiyC5N$&y2RHT1;7EBnrcB`1Iygwd&dPC!7Ub4eL&u7Pp5k=EI6dZ=pL zBh>=*kt3gp*WXDZgZR@0$wT0=JER2EdFkuo17{pSd_>o1?i-8lKAEWT@6% z8Vfgc&@k@nO1Vfeo{JsKk75X#(f~Zlh}uU%{__W{F@0HaBgUP#M_OBDKzQ~EcdxGzlByXD*qx-G zjdYb0?6!tnSd%I+?}33=lhqE^p?sMTYro}Ae;MA&qR^QW0~SPV<_Ze6TLyq$$|}5- zSG?F*_1CS!x_Jsj;3T@zhfqn1*L}tlEAT?@;zzd|pCY?_0$F&!%9ZmD7>*D+f!rs- ze>9QAE>PVfBN=tW zXl?nt#Y(x_z=j=)HrHG>dSXeI82U5~5L0vjfA{^>vtG{E&lj-6x%D&4jlnf3C!iKk z=Ot|i#0nnpFq5}D^BHVRD(Vt@e(0*PJhKIiz*z%RJw}KP z7Y_V1Jk-#LqgmP6h3K-1-o+$jc|`bpzMFZ4U4*b9yj`g+DT)*v4UQZkpgfbO44U+O z&4hpX38cV<%mafiyfh#-UZa(Kd%77?ZRThF$#6BeDK6Amz*PHdz-Nlb+wvSg)(`S^ z6-l1`?_|^I8_rA(9tU9VJ2PfQj2*-vpi8Dz$WUjY9oV844Cp-P@$2K_u6PR<%)fNO z?2WP3d-)Jk-y3%W-rT3a3`ur-|etH$cPXf^8 z`E(#OxM2^B$}-UEx*Uk9F%i`htO^+;U~{59vG~Gvye<>M)C^GvcI0$Hq{|2XA}TK%kb{O^HL02@A=DtvXf89yAIzvuEaS|Z?nw5Qc+!<$>~?u#d)?3 ze?F7aA`0+#mZ4)?`d?fC5ZZeqPwm6W*TW1M81&;=#$Ro&Go4+4drPDn?K5^ZG-4R|JEkxkmlLlf>fVoKKI?gsyKh1PC;3M~m5!GCM8va6ie=JU!!5W^~t(9BmH*hbx8a+ zC7%CRY;x9CABCQ#snt67-eZn~<3T=$DyP8Qy}s6p?8>hEdx>65(%iG1%1Q7cjOh{a z!h{y=9>(RB2;2D=`+x!ZL&P`l6Ql?n>aPg*f@iB3U_j(~?p0r(0;c=CLW*@rVa6~c zXc@{XauJ(+K%Rb}DNFe??;mJ?`hOECvpawZKy;g3qlmnA!D09~l*J{17E{(ah%~Ht#Dhhc{lmkC?0aB7t`mzAg_T`V!(d z@V0f#>3Yp4(gJf+q_PGGq%Gh==PiU5()XYcsu2@J3^(Xeexge%1ju#S@z$=jdzBMF zx!F$7?|Jvw=E5{aouvibdTghZK*f0X?sLmX^$*(XC}z2{1WnaA9{;hQ5Q-IVWt^zh zq`&pQY(muvcR-9fdgSmY$;)UxlS;O{jnu|3>3_62XZu@^mCqFFkeP+S z>RxRGlnnXY5V*k<{VI+-E~E{?;XfU57KK*`!_v75bHGOa*f9xW&WCL$ie*`O7!D3*Ox7 zid-x1+pjgTzXMQ}iS134jSz!n49DA2Cc8pEhYB22zLEHC6#6!dlAUo?~nZ#q% zzK5InQ`v+cK!bi6?`w`!?iY^nC+69rwJCfmm%p`h`qy~OTuqLkabY~+_=|k}nZ_BZLSf_~^NDPW-~yxjU}G*O~&pq zd?=Jm%F4Dm>`$A~^{hHcdln9l2`*TPvRfjFy#%AMJ)O2QsA$hl2&sB&(%^tM&^%|@d`DLKY{IhBM zN$)4-1hKxj1i^P-YYv$L9+feVLc*sS_eH$$hxxGEwOvC#)5azMYRq-;e@xc%yQHof zFqG(K3N~5YGcoO*?F^M6lkC|H&Ux-wyn@<5`XLc`Ua(To)-7$-I32+H<=5ei;OKIJ zbuprwbHg69hGzGZX#)N!845=@zxxwvj;zQ?gvzut=rtU3_WBG5E1rfL7d|Q;t3g+~ z@8kV16seiArsSF+lphH}qdhEk)v@`NBsZtkz z<#fkiN)B*6!O?P?5i}%GW~iBz^Mxe4*b81N@_KG@4kNwDcu)&j-v@xz{()bZ{1s2t747~a;8tKTo;@D@aZS@J-@MQ42~inJZlmFogjpY{>CQ@?u{}RJMI^QYXs3BvwXH5Hu6RB ze`j^f)HM~EJGCOsd=_nxrAiUC;a|yyw6ja&4pjj|Ez+}JKm zaYw|j$g@v3T*h)os1MR+&o+k)h2Nw~3C^d-d_l9CSj}+8)U;lWhA2G9r+XM;H4(s# z^XWL6_P}q=HE#3-0f3y%K8l3jWdrQkMH695Rh9RCp6>Mz9MpGIV~LZ&L~}HodX!!1>-#+d-I(`!(9!GH zQPgQ;w!<+o3PK8WXeSg|&Btp_hY3(8+k>6;+m_pPbDOH&6Nsh>{>4`IdGcYPn$Z*o z6%~fSVDp}g&C~;krHSA_^$G#|yP;S69(P~>$l9dPG-adyN2s$Y;d`kABYmuRd?m)Q zX3dVVm|HZ}jtj=asep!(-)M!idLcMS!AP7%?6&>lSUD7V<-Rbmc#`<|4o@>kn-a8r z%J2-24z&H)SGGsz*%huBo|gmV3u<+lSx25p0tG-qUZbbv2<_ z#Gw2Zn+Aqv1u{Dq4 zt##QYO$=c2@3W{ax=P{Yb%>m9my#Ti79r<%jZ?s__38g@d{=7UaJA*nlq7b}&y$|F zw}WnQKZfbMV=h3@7Q+MZgJC^3hK#UPnXACZM8{MKJA5Cr7gi~|Yo8JQq`*HhPLi=! zbmaqE{Nn;##M`$c=>x*}5*HAF3C{@phHN&dNJQ;P$VCqeXgk1bA5+WzhF36`;+G@y`FK=dXqvY^rIBMY}$V z>*z%5QVsrt1J^grbkiGX?-;aRUqRr{i7#@V#_Rsm zaB5nIbz$FUOgW~zuAnQuoXGR}nt&;_aDHh>5I+sX!VAou9E|cqQ*@TLxH!@?$f4zy zw3p_2NXR9+1BJb)#pX~@?Sj*amo*Qe$p`$p`k4&s1UKHv8F67$phkm#0!sGycv)34 zS_mmHDKrpOB|l(Rpeu4&?4qSuYMXhmAV~aXBt#ifTU3g4rIp7Z;3UpHORT)g6WL?% z-M8c|>!|kph8*w?Syu5i_naG-xG)E?TNLt#>N;#ezwQNfS`hlj%gmrx3{Acw0js>oxQaZDhf$D z_b-$wiDu2UfjTz4BJwm6aQ)E!)Dq|LYUSj9-P@~2E~}qOp5N<<{`_osiIJt!Uhlj2 z`4CvxEA`hkb1_144!l)uXoyXf4Vd@G$y-AG#=wO9UFEBQRNh`&vBhy)^lg-lPdM6< zCYH9Al!+@f;NjeRV}FG7RB7SE&3Iu`oF_aSs9j*mkll=JvNMgXy(vXZcmmDks^H9a zw>1#vofTi*#oix2PMcC1xnb=qzraSnX9c~;F?zpU+sj=x`lhmgCrN-rk$02N19NX! z^XsJc1zkffZw4URqT0~nTA5cja{O= zXT;0gP=6DKW~SwP_nJPYBe9M``FMh@Nli;mD)?(rc5!5QebfwdaNoUiACc{V;SYW# zXo&j$0#nPzxG=MuD|}6C#dL}rGb_(=G7(?0Ee4p^#l;m=oe*}BqByHzXpnXR9z`RU z`-H?k)w*D{F{e#&C%zxf>B0`g!^t>Al?n-L^lHM2Xa0exG|7)$MhGZBi*BI}8EWKD-r#Nu%Qq>St=@jhOYgucen3)g>rn$y+-%bK3e_T4Xfgn^aS z$nLB4QGd(+!DxcX3Mv=N2x?`&IPK03Jah^wYli>>M;~hX^igO9)Aup6z;|4r=US!O z$hI&Ou2!QXaw@>z`ns4SIp%E7hDuf&w1_dE)1Ua8M6^3rGX{&TJ^6BdpF@gDB7J2} z&Za8#(jrL=-`^WiX0F!k1eTCr2I2~GY~_jC=Bl?VUP1R=Qcrem>=`3c@)FTj;Ug7$ zEu>>)O@`@jfq6q`x7L*t7C{qiG}fjo05apii!hI?>klIn98KSc8tnU4ObH1Q zW^C#rNHac@p!UhoFMKg5pINu%;zO?GJEeCm|%rQ<#d1x_XWKSjsxjW@7DVf^*kWbYWo+0@+0eJ{|c)(8pdS!PYye) zc#-M%7?Dr6e6rh}$;~@EzwZ+->vr#Z^-e@&pBy6FZp&#M&lXe8sAswWz>VAa$Gf%5 z0n|@fx9jNOWB0sT!N^G}gZ$h5jTvNscva-krXpUGH-n+htBed%{`zk`5oa6nXMQaP z!b6FQh6pII=}T9(1`0Nx)VlOAZQ;{zX`m3|s zugq6Ypzx`-ZW4<5fuTxJN=?jSG^U#F0rB*T$)sGqOVc2bcvlylI`rQZeB=CL*?FkWk%$Fa$7A$_=Myt-QblEnLmF6y=_G!<<` z;G&)2e)3A9Oj~~-QN8l=0sKn-)H|IPM+yV8xia%IkL`T8u~Wt8+37vhdRC{=A^=}p zUBclG)&?^Pl-ce3k%#3YkT}HGazy&Dr8J_^1hv~5Vt3wT?$p~0E3Em}OUg@ZW1z8n zY`nHlXW9^n`GZBSYro)cz5qF?{c0)`Fe$#8To2BEEJykCGISQaF+RG!=>PX0g5G8g& zWBJ@I8eC@jIFR?zLNwd7NL~6*3;3X*cNI5T+xs2U+sR_wllnpGYXv7HZx|sCC?ubh z$@G^yIC!UA8{e-KnXS>X=^L(Dj3=pDl<6vL-i(HG3iq)SF%1XZVt)DI4}N9Wrj};3 zy`hNRx!zX-gBSBrIe}h7aB=TKP3!1H3!cSAjH zOPbw@Cr}T;hx!Y~9yvz6r>@(DCH3w{fOaxBg~ltXcsf3e52Pi;Bwx*u;SZ+$;0HwW zb5P60)kf>=w>>-yA8e43*ALIj63OS9_1r$zv=K=5mrWyXU8KC`X!= zfWFd!#U6S}-pTV$3=4dK1e=J5+xAw{0E|fu_l%k$(7&Vr4rpAv<#a}&mwGeJv^qb` zmlb553D!kdzEC)71#Yq3P0gj1jCIJng&9^sggSq;`GB9Fq?2ClN6Y~eTDs4st}{9} zhE%_Qw#1u!7}RdqJ0evu6lhAv@M#K&XA>EZfjTQy0aDj07vQjAJFJaA z8f^9=k5{oR^dyQ^9M0s$ar?+<|hFMAhW0B+JJ zEZ<|tS!jf;ZYC=}F{LJFj1WWL%)DnLMEnojopD2lw+2VmSY?}VX<1ZHPH$ucv$p2f z+3$l25~R-ZtlFx=ULmli@3oZnb1{!7R`xefhFE%(1DrL4+pM^Joa+iG- zgXw?(eCf@|=93s7g3eWSdZSq2)x1Zs2X|ZJcTpeX0-{<}INm4H@>uEBAMetF7mNLY z3(_M}gh-H5Xk-d5o{HOtKJh6q=<4x2TI$-@q})WMuVQLqv(tMVEh&^$toF6uLXxeW zhu$3B8;Z4MxVnFyP?L*uX+&}~JsF#1g>L1%(!}UHsJvsjyajs)JP9VB8uqt>ALWAI zX04xiL-6bk_Y8&s_B&~n_PeFPny5+8i2#ph4 zq|YV^3L7s(?SDhd{1d_-aX9@c$}dULR_#5?+=tsRn+%gv${H|VSXovDDb!QDF(TJ{ zJfA6rC#c?xHP?!5@c6K(kzbpNS5O>|7b7fUnZ^iP>T!OnY3h;lZ3qrPd)eh8grwS5 z6Z=M!S6Mf`dFSYE9jEr#1y`>xJPMHZKnN3n5;_Y^uy`;}GxGiu1t=0!aAX@)c3^Bu zzb-8TM;m^EFlPR4s3b}>Eu8!EHTO`OT#@a3aa`G3nTc3zG=3V$oB}_0{&^&onJt(> z_mKf@ns*zyYCje~(}%{^(GxBW8qK#{Ut?t`*aq;sX&*`?6c${G_OIsPy}Z1Xeltkg zn&siwUK8!;h^5mYjv@Y9$6e2QVq%5M3t;ti_tEHJcv4%%?hss*?DRC95%HPvYPWkm z$bV+X-*l#|^y@si!Gb%hL=Sgg*JN)xN$J(z8#s1umyAI8D7(treI%{6@C2gJ;kggB z+(S@z>FOaZW$Lz(-+RAro6fiP)BFC-FBLcB)b%J!0$wWDYv;4}VF57geShkfO$&Xg zi~azk{WoBD6W8Xyig01+EvIvD$syg^&;R0be@tkCV&F~blNf*J{UE4w54bVq+o`23yaj>j7MMI|Q2a$=i?|1#Ane35vv&{0Zu294 z(b=8RYSZ_Qw$Sq_Dcn&rvHnC{0w_VgGpkmEoew(GGmy3jhKiSK**jGsq1x-h_D2+r z>YzFKBk@j1OLkpYOl9=ygRgvC{xXhU_kLL-(b!IbWe{Au);8Ukp zR53opAq$WlOz0W2*6I>{t`xh-2;e;OiP8jS0E^J)$I70I_KAtt_&ciH(#?;QGQ7}8Dr>0{N_itK|O11RMxz=w#Wil!CR(|gs-)ovStc&qd@ocX06F1#j@DlIM-8&EOujg5xOQ=h81KQS+6%&=S=DnXs} z2GR7!K5Dz9fo0fO*A$)acb96ppcEHFDs?xm?+G}U=WCx&8HPLL&HLizn_* z(_+UtLjI~`K4fcc>*5zuSmUzf*7Jq@qtZc=Q>z(mbN^$OWfzXlg6ztYOw$wwQQ_2t zkbJs-z>d#)?BnSjD%)S)4@mjD`+oUbY8b|q57GoxOivX@ONV!a1nv)DLDFMy@6C2~ zR|z6u=0PZqe9`j(J($C5p0Idt-46kOHCRzDemPA49KC>4#rz{e6>i; zb1oYi^-zwEs3lw8cfFDaQ_(!n!uj?PmS|0k8lT|S?aDoV1 zD3tO-k27w;C$3#6dda-=cj3>owF<#2$%rV$4TSGCHhNAd=tIll)C>ApFeXr{T+6%n zCvN|ma!eaS-dH0_HiT)!4$@n)Gma<4r=;7&Qb z))SC^X^&mX4rqsk4Af?V*#e8xU#vLwPT zKi|EN2dKy!+cgn3{z82p9u{wv{i54b2{nJxx!_5;koYR-|I+nbS}GXgb<7keMQB%6 zRu7F@P8nWuY{yX|;LL9mlxj>^_(oEj2&LNHJ6`(APsd*D$dUZQM_5*(?AD+^uw(ky z)9w)^g;q*@)zUg$9Ptj+zFDdIlUCH$5l}KE^r&quKlJ`| zXeZ}eL&>AH=rpOvUb?p1F`WXQ5^hWXn5lf~UaRcWIjn{8SsmT^BFhuOFC88C6?$!z zVoziW0{MNz?7yUq5)Ge77Ba{C@+%mObvf7n%b*_`Z7ZRi*N!Go_8|PrkN>E!1!);# z+bTMo4R8{F&&WbOFD}JxRHB;TG+vK?WM8$4UyY{!5sPu`20)cKfg#a+NJQ0d!)fF? zpU|HL0H{HTEO+__{9saGl_Lv^{QCrE-|s47-omac^BDJ6MF5kX0POcU*hv+>kz!0~ z90xEkSln`w?=(D47X=49&na>yYbKx#FnryuF7Mvgkjjad1I})P=cN9Tyhd-nCyL*E4fv$e1dSu!7PIxV4tap}ezoa+pBF`P|I7ydx`jf%eG7zG&eHh} z8A719Gx_oImW0NEAdT*~>~`ty{rEm*Fr3er_ zf}9m8zae}f#c*&yrGP_8_h%3FKgWdI?PmPZ#EwHSSQ|0Xvgu(Z*~)QZX33ZHbzSkJ z0_B@E8`x0IISBqE6?MT z4cD0+%nu&hE-ycJ0PzV|H{DQs-3@m%vnsl$BHFy4Z4JCn&{lzG#Cy%7kCh!`^car- z3TU~A^nmNvQ2fqw@h4!-((XhZb239}!r>zB7l$#gyaX3%p7F$}RM*wPlwqxrzcO#| z3m;?delLz#g->tTv{R+rW?M)6G61yHSj14N*Nf05Gcgu z(?|S0q@q&)^?atiRh{Gyf$pYmoBIjV-0Ji9ar;VJZ|6rc=5*v!{o1OnxgQ>N7pp*n z9UG@TVY`camaflGQ&h5C`)n*sF0pB(3}vK?*x9vTQ?YeiU|B1yZ5aANa9CNoqcqvB z4x=o+k&${~w2U{V-nKmt(*}^SB#n)mjX}N83oh~)Ihq>`BJ{^+3h&eEjW8(0Ag#y8 zh-w>(Ivp^viMfZFvoSk(Ux_DpwA3n&8G`9hx)#~Z4yH_@p{RfFjF=-yQ-4v$3a-s; zhwd|{lWNYc@8-nOSQ?s-m!0KQyFK$5bht&8CIh=<6!x@X?Ajs3bAFJ83R=d;FoULn zkv)R-D?X3YDU_$XM8MT%fr1{m8>8vrq3D=t18VYu0s7s+f1pTeIlW+Qe-&|-WzGmr zXk7D3-Y77m6TraSiCf7q5OURqi@xY()i;eV$f!H&;)>QwUi zB{ye{fv?J)qBU?|8Uov66ZLp{Dy}d8;frzlYv@FAqdM~6sd2M|>hgI)-f&!Sy^My< zc?hKC*Jv7<&L8L%xkN0ZPVPa8i81#(?gZ#IM0{7@n(6Y4Q~ojLQwM&@i1fOnJt7>|N2a0q@?V@E%e{5w5VkLfp-m6zFQyQ0lrDm5~NrZAaEM@YA%eloLb2Y`J>f06Kj1@I;xE2{Zb7&!)LM+H$ct78%V7i zQm7hZTsJIA+4x)G;Qw@Fay0vkvABKJfSO=C%U4;`>Gaelz_k~uZ2XAwz$>f;z3_U2 zkky2|;OdIxtOQ#2T=oT$o@bG=Whrbvh78N>4<^YMk4-M76e={i zIAs=x);Z&{2m8KBPPQ|}HZA4*rk!j^ zV`4LxVa3oh9ljMhM zRP=F2z#nFa25x*cXq6X@r&2#vB*4q(%vV_{T(RJYG?s2h=-t@BbR zBYpd#6lNxLgYDA&2%|t{F0HYvD3>I)xBX7-;&qrR%rapN+^yVB9QE&0&6SIWl2Ts> z(PikkwS-4iiI;<^bKU%>4bkM1TehvytPGcZg5A z;hrD=$;wwA1bIrn@DMf=Chzt5z0&Hw&<1%3>#Tu7Q}N!vlbX9ThlfnSmPb77u-9Rd z0XX!^T?5yJiwwRJ~elu?F_M*eg0-ap7C4{Ju}-Lwlacq%sk?@3|`JW1fEAU#1USsKKi zP?rit>#7t%)M9hk0KLGC3W6h(04NUCT{E2WF`BANz6+4Vj8ohzsiR*bRU4^q}VMmr;B7lNIey6w&!DffDo%0vFN_`SkwIvq>SL~wn zIB7yTHCpNDUj`JB3O9I?Fv%|u1`j-!K?v_k1@GU%kmlhqt(B%ly3GwSx$RnKg5LNhpYP$AtCi^ffheF6c=4=j|Q&(JVvtk<+ z!wO?W%0)9nla%G-%ao9H5HiO;mU74`pP0inQX({2VUt6VecK$94my|)-`afd|G(e+ zKF|AH_w(NOdp-A$#ncZ{R7{+*7h5F#FsO~qNIut1Y4ot2tT?tzXp!b;G`+KNh_7iC zHFEIv>adk-Rj~z74v(oFF@q~(PqnC=1*(p?EpeQy2Ish-tbQv@29iY(jNJD)tw*<+ zV?z{7&5w{rIqE9q)NmSGyOtz(3Bs*yh-+IMtGOU|WQL5NRL0Q@ET64FMA3QCZCE`y zMUtvxPmL$C!&Smlv@Fex0bjRWzj&hYr<72Pf?MbP^2u8HrVMazitnIy4KFx;8(5iM z=?{1uSfM5wRT6(jPIIEBw{jjU4T^@ko|miL6kd6w-K<@&KGkLR!Tm3Ql2-(88EWcn zXNW^*J)QHr_Zew82I+Re?37nlPE#ruCT*C?{X20@*5yINafeTKZ=uF`AqdTw`VEaL z-o;4KIXh=2ws_EA;+=p82kgo-M%1>4o8I+54I3)7fmj7s)g~Yx9KEtI?wP!5Sglf_ zmXTBh;Oh5*Uw%6sitJ)-8xYUG_Su&@a)Td8Yyv}7(wt1BDoW*|aUm3qU4Sw}e-!n+ivZziehqGknoRLPiM{F; zMiE6jnCoU3rq|rf{2p_1NA^{pM+atRC8tzLNE~ZV9wD5!I{Yp+FiH_~6I~#N>5JF4 z=GiJxMCSNTTvup?v5xh88iO%H{@PXt?;+7#8gT0_UXMDLy?_pcJJ{mqbmIP$d`Q8e zQ|IYn`}N#(NHV%UAE^}+zf<^0scuCAw>S8=J?f8D>yn;UCEDYPY@#}L9YI1oEnp~= zp}n=}llHQ)xYw*VtSq+TP=!e@$D$pPMF(t8MDn6fbxhka5;EO8QyLG{VMT|$m_fH^ z7U6mt3$67+xD71A2PN$$GWOL3x_wb8>U^^g4LF9GW=Gz2U^ zlUf=<54}4*q&2>sM6Y%{kMw@H2Qq%b|5>6hk6eR58wE>oBZG+^*C1OWa- z0$E6m&ai(`ip#Y{sHl1)Tkip7bLkuw=GTi9eTC}-;r#+=OmfaZ_|HOwj;?MLT6iOA z51Yax4iD+uNi(vh7#tEJN(q`k4yRx3b-EZbPCaiTB$?+1?Xb@dMiX~0a2pD0)?kOJ zG?}R2#Ia1zB==lD2uA@$zCU)d&g$BUsCe9 zf^T|-o71z^eG=e44oPc>us(Booqgb$Z$jv5R6Q}gS)%{_e))pk%isA=BCApEP@xQR z;n1TEO)Hb2aW1ln309ERe7sS@oCXShWrGcAvvr#sdBtN$*F#n)CR_UPuGt};_mRAHVICQcTEkM3n$to&r6XIpf@GM0MyLe&9CnE3PnwnLKY{= zLj?{ZIS#Tp8hc!?{OD0HcAlNkxh}gQixM<@f90z}-QF*%h{#q*L_JYZ!`~XS!(sOQ zjb08`3L;7JD`%(JVI`M97PAg@c~m8Xyz#sM-3|3fCz+7oQ^s(7jPJ=zcgfmNar_c; z4K~TgE64;!38hLv2&pTks86-@9jhdMpU7P^;hC>d=y z$wp&+sw-%~0m{z7P{s=IjF#3<9P4GrntDsj&lp4#;e~?z0mT@4kJpF zv-n}}SnhrQhg9w0lFL0aW__V!2FzQRq^&%Og2qlj8$zft-SGi>c8;>$77^aqIHNv5 z=a-3MqXp|b8JlIhXgYAy_^p1-Tg3eWec9eMpg^-HsgL_B(=AC0n=#DC1@*-V2?Eoc zUK6SKN5AE41^P{!f3JC0v}wTbckrf513C78mj>47U+2INn-tC3F;N(t9sn5x4{aA)z-3ASLwP0wN+( z0@4X3)X+l-z2-gWectu_*ZSV|e(PJ`C+D2u*)@`@6;zxDbBRGostmz#p(?RyG} zBjC~7B?=0U2NV=5FDWRb6DcT|om1-}vcL-}bJgcc6y)=NpD?)y;K^UkFAUr$C}_W( z|NIfhPs;{8q;^+U2U8QMsp+r$<(MT@M?qnQQ&v*c@g5_dI;a`>pcapdVTBUM-(ZXO zZCQTLw)$)|{MQ4n`$iXkm4ZEg)nmhFjLl0ay6ch2njZo2n3`4h=o9u0Ga(8dY8d&A zhwu|wmR{=8nA4_ID#O3&u#V|S^GYbwOJs+$LBG35Pe1PW?4%T*Y3Ty<8SJ-}CnRu* zyq6`NF6Bbu)Z?R65`l#MRZ6v`;}X zz57r~>X49NR$N#J0*AEFF%l?skRc%=gs$7-^Ziw49VwzP)-TN(+*%4K~o%zsbVl(;0{YxXQ-D@~UlwGat0Ny4p-W z-NpUu810$m>6a_`;bH2{8G1hU2DWqa`%3qe}G{O!2)!GjD zPoAAv={Ve3JCYE70}gMWYpD8$eyQF0+I&OIA zyi`rXzZ#an`o675Ja7EfSjk9;S;^&$qGpkL>8ibQJ%;9~p#AkpLH_oYl@)3(|IHR^ zE}bbTAX2OQ$6n5J#ip2o?tANwK9X(=Y5)x*C(Q1!k<;nPqC&It%saz-Q5-VE!^745 zr%H7vBTYFvW)LQ3W?{1?|Ap?Px%O!CJdeU*hi0Jb$-&km>mK7Ke;*f@tv7rC+rF0s zo!UjTk=7eFnvT~Sm?8)x&1bs+J4+-e-onzl2);G87z{?HA>MAJm^ZKy(mf|w0e2;iRb`ya^9UuV5~U&{B7_pqRaI4;%@xouxIvp; zB0k@kB=lhRjh)AlS;qD^`|r8!-`{PK%t~3=*SS(v8c7N^wl22r0tX+0q7DYN$j%RT z*MgFV4!35u4hJ>3NPRQ;OSBu`56kfQvbIIiTlFgn+tgRQuj3~_r;xHoX` z5ueJIjh3LG=9DQF6&1ceyzoAYWYXyw z;>D3!YOlafe}>Vih3QyPh-v3i%{#u0FKXwjWb^oRXdeMckAvVo>G<^W<;&w3w~>(% zudtVvkx*&o98?$s=@O~3O?VOe#B18@5U>` zxzn`WBhy)xD($h-u5eZ+El5urmJ+l>_BiVaa+MF*nH^)xLT7ZV(xV%yO#-{%=VIib zgIpacfteC^MNgFsR;w!1w;P6_@d*iWhlr`Oy_&X+Q-W|%es_+nI+%BfXNB5qBfh){ zjyjQHr)54l95(>tMn?~TrGz5#Fh#VPmoZc#1!4Xe8b&Zwh})o8^4_~v3@sXGQ9YV= zVJ;&7omUIbQIQYE{1GQRaih|(o4aDDPfnC$YS4e;O--V@RRqTErF~N=+@1Pz+qu zm-S1j=i!?;Mn}*GPeyf5NRvt(msr4pF;tUX^l z7w@}TbiQgdgJx95eKs)rDtgFVgR!hDzC(dfS;4GIq{4SZ$rr`265~1&uzD3) zH)%DacgMu$vc3&&ZXURM2g=KeO*n`c9Bnh-wq#Xl+kGN~2Bq{ZZ!K?{i%KhqO+QGI zZZ|u4muAbS;_ckkFqhA-Vp~5ITgQRcwl(1On(`WJ2>YJfIy3!EE4R&Jxz`k5Q3Q&3 zs)q=0_H_Y@ysVu3d}U(yBz-uk0+bTb>WM8v5SH|FrH<{5uF~45sF)XwnD;O+7|_xt zsz+sha#h!UeM(z#6i2Xv@R`Em7~A?~kM+5K}}BN#2j(FWd?;=ua@vV6-MMJ)F#=3i*&F0a9ID;B~EKN~B)_#lHF&OE%q2*x~E{n^-fPBG4$ z^OBY0`pLm#&z>%W%Bj~bKO8%aBV8{G?5)Y_?*G*B*N)d7z*rFr)%*(wEVvAHHdFzv z_s~ejFU=F-b4$P}lvhm(GMv4vb6c-pCM@Mi51j8dNkyK7vr=j5 z!SK}bk+@_oBC(QxTfwTq&Bol;{?;Sv9G{PKuhY1@ru|*63m+!Vf2O1^v^svN@3tr% z7h7H@x%R-|I2R@c9@)It;`EEf(o` zZT2!yk%@?#y=}Vu`aGY3z+zR$WSHu^Bbmd6|d>taH|=u_ZcC>`qV&g^K3W)%M3=vfG1UlP zyL2FnTip_cKfw%f(S$t7e4!Sr-SUT>ry3%8T`WZ=F6Mk)=nICZmVFH-@T;6l1&4tC zPI(<`8f)nyzEN+gx3{TRLBT={N6r0pdwaVj{Pww~>-msGI+qA}+2)&(LZR))MDDJ~ z58{e$Z71K!PGix(N;@jpQ%^{oJf6beSN*O z!rB9w_AE3yU)>-+Zs=*$-&+7le{bnp{msqK?@pO2q?0qa%9Gbl=UO&T{JqUfA64wU z@j#8+>#dr9R=4kTH3$AK-;2@+tDq|ZmP%Lta-e#SGN3dBFSKCa!ON52)M3)t_^$@Y zq3O4iYd1Rurwy^y(K^mw3?*fdG<0L?!|Q>$-~g+HLQuF%Zb!D7MH|4I`1QVS_D=)m z<+Py^{H7x-N;Mr5a!l!FsdSoGqyfX@^-F(gSp_ck)vDx&3Pzt%?=J@`LjNK_qI;vI zv4@ujEDWNey_hF9yB`fs;HK;(jH)hIis$-ailp1)M zM;J4!zBuHceMB(oTvxY@sTk8V78>zn_SJPeDeg3@NZ{|Z&Ir)+dDU|{1Kbw~YgzXv zb}Pp*pS`Q;+Ry{@dS#cG6oOzSrju>(a{j?!FGf*&V(X{93=W-7BcD+IJAKbP$7N@n zYp0#^NYy`|a|P_|Fy_pgEjs8TwNfBIC44F_7sRG>r)k+;?QrcM$fLLA$Rsh-mk+~3 z^@7?$OqIQ|K!XkDu3MC$g;np0`|fs#SYz_+oZAY_mhN|jFaHsSv)|xkU-Qyue>yXC z<8mxuW?+{R2h53Bwk?aPTYxoNNeO28WHr(?FU&r6g|0xGmcZmoVA2qz9d`>RyAWLq zhNeHY_DiKEQSibb=lb-7bgwIAJJ?6UM!Lbaz&R6QTTtU|h*ALYyIo9XD>hh)Bj0sc zpb0A%x2srN;c>HSF|1@5s=V$zD*8M@9+;Ww znT=Bp4`1(2#B9!V_hcuk?Va@6@O3WO`Uf^n701t=gqW@v8;7iR`Z^qA0X7Wj@=ESv24kLXtF^KIp!*bia#wvQcnDNmJrd?sCI9>-m^=%NK=170s` zWxZ-%$-ImA+mMSeK)%~xy?IlsIPaz9NJIf>d|b%PCo`cRDt|B?%v}3(b-$U6>DTo1 z${eZieqy(%Mk5~^NVH%5)Y@5njd1neu6AdC2?x7-t#>BO1&mvGQj*=Ec#YM#j{Bp# z*d_B|quA3TDn;H~K6mid-=(umc~*9)FrXzPe@1@znv>W+Rl^@iB`244m>-^(ba}PBeFEmuV31m``1pr5*a_{p{Y(uC^OMP$Xs?8GcTL-cs+)wwI$ zjAgYG()1tX+m(`d{-!GAuBdZlMQNk&pB5Dam2FP-%uC`c?&3)WEsnoj1} z{-0=%WPwxqh?N2GWOz9Xhg-n_IJroew!)JGhClwpssd;H^C2pWw&JxwaW~YE>RtX& zzG;NVtwQ&{d$$Jv{;DGQ4M5wNJ4AFbUkdp%ooQ~E@&3)LTmB(WRv$9$^GKUS@=UO z)cHn-UFgZoKeUnMyh=+L_G)Wqw>sWLK~<>AoknuMW}RjwZ`lBI-N1Ga2C*=}t95H$ z*&oB=PJe`NX&~yn?i9!9t9Oh`E;(2|$utJLSdF;xCZxS^V1hANzV_U=aTZ~Xex6HJ zI0lb3>d>(S<5Jga0MUh%cU!rwXAHMEhrr8O*(+0s7bxkOLIAsvGW@TctnANzk4)}s zLq4nMBD+1a^L3>FHDHvgTgwdAP~T5>W9*xdlbn8#!VwG}@XL|#{Za0!Uy|u8EBR8) zfZEk)Joti@_M$W3TRqDSL(*b+6AD0wnhE_Qx>}IX!n#&b5PI}0qa&)E5+AE@A&D1s z;r`w%zZ?G~J;ZX+-_ayxA=7}pIz@(>nth==Eh$AGvX8RoKKD%A!t-t=T%qU**06U8FA0|QUstI3;I5nzX@ZkMj^l*5wA1^%JTH92s=Hur zKWB2^@VdFV$;dBl4(!ug>S^E}=(zd~0K3=T4l$?pBZ$W^6JP4*Of1EdHePl_ zJK5Vaq%she>7?1Un0%GTLQJ38y@*GwQp6w=o~AhGMuAbHp|)jpw} z{(tINl``=x2us{}HvLL!S?orVraO9d!P3Csvvz5JK^_XEO5)!|sNsG@P3z_QZb!Y- zDg9~ZGFqH=r=t*-v@Wq2T92k*DV3CYKubkkG-hOa5DaKuSL$v)k%jIxSgcXasS zt1z5Yed3FIvn`4Zb$+tRFF?C5c>e&~1Tb$J z!S6WnC5S4l5889_7J$(7lU9~0cN$vmGYZ)-F!FV-xHq4N7&I-dwOwBa5>1iVgW7>q zqnh1ReidV05ITpZ(-!wGHi^zH*=^s{`-}Zbk8Bq`y+(!A{X~+ape;Y(nv-n3f4mZx z2~ASPsRDNNySX(xvo$2xY&`i1uPL~?vNBtgJ!ezLG`O^#Uq!m%>M0)MC;SR9^)NHe z`f~SQws<0)B~N-}HgxK=h?FoSpXOxkTzvVQw#2%MLZ6nJ7`VrgQ?Rofie0lHi!ryOuhwK?V zBG`B=!q=bh58| z_}lYIDl|)SrE_v=(Jm!RU4eR?xflWgHg46$Pvv#qjZ^7tuJ^yEaaXuWrjF`_)_2nG z*@(&(PTtQ*23C2IUc~22d!Z<40Uv&g8^C-aFhTZ!SqRvL>pJ!CQ!i<%g)nFJEtz{4 zjwBbo?km%%=opieX8dJFYO3+_bm#eICJ40i)C+Rk?U%a#MSmfgfe^1jpCEk3wSm!Gr)Wa5;7lQ|84y>#qPL}-Fd9=m1U$o>b)kI0Wf0cx%4Q-%L&) z(TdVy=ju`rfdjQuI+;q=lERtVsl&L-?fRhFlPfnGPPTyv$#D_4>i)%X=-v$v7nqaN zM#k9=PmadPvS!c^=EtQ5+J}orP;si9)ck;g2tD; zax!eVN@rzO)4cGI9trY}rVT*~rUgMR;y6n{ofBpC>RNIWGw~5F{APvyP+vd4Ayj#7 z?dbTp5h0%t$8rURfaG>{hv`T!l~&~se*!`ygKR02A_nn)7B5H#yAHmBSU(WA_-r7` ztY3`+F^}rI|C~u|sM8asS^4|Jqiq4`mn6BzWo0-HRZ40iQ&o5#b(f$d!B1Sk9jT3SZo!4L)*IeB@e<5K7mf|rdRqS44LBT}RoWX9816aC zuN83GUR%gt8q}&;IHDR_{zn~aT^O~#Ws?s_v7=!=x74g6hVc2tHDhXObon)7E6%iP zmNX{Lr31E*&Q>dT^IL9V(p0%HLL-p}h_4Zoj*))Wf%cE3NNx)7h2Klj#2OR%)FOEd*{NHL-sT=(4Q8+P#kU*+CXi~&x6iKY-_MqHEsIMxKep9X z1>$)0XY$z|F;>mKaos$EM30%-ciqRvC47w2eucZq2sYrY2(D;mW;aKd^NpD6-rPx- zba`^zbP`U_14PZ!95_S{DV-8Da|Z&On@NVk4ywFf^Tn>Y@%zW8K#a0sRCBPSrsLrFThf9g=z#}hMVuM6WB0He~6Pl=+px4*wXI%3v#f{6ROK}>x zb6W=h(iTt8I}Y4DfA(y4%lpWK`O-x`*%`onz&yV77>xTH01rn^pLQsSeCy#?;p2?_ zp4C$3Xx9Ng_Vz(5-(ih>zL-nJ5#J+He}9t|073xAf(~%#X;gap`=Kz{2!Q8k&0w2G z8c@@O0uTVi9__9IFsc{73W#A0_smBNk_QqF{`S5iFLSLCmHqAl6QaYO$e#F*qw8Xm z3hH$l5F|F=;9RhQQsjYtA*8V#^P8K_cVb410%Kp?3k?fHbljtRz!(AtzET`G#LOGrVkzv zuPrfIS$3L`YO%Cfja&dZ-Q0Yz+OTJM9tVEGjp&q;zOOthED2z}EXuJ+L;w#$SscK6 zP1Vl33Pf-L9s-zhXZTEw08giUJb=DbC^IFY*_GPF*In1vogUFY~Bn0-bUOLjWXFP(Z-AIq(!f3e5bs z+J62j%x3;Bgw8mid;9|epK|Q&?LlDhGT%AY{;9E+UwCJSp0BeD3VF*1^Us0uR-u8E ziIGs!jou?6Ey$)yHG?jYfzfzGV>-pM`4;|eB)b@>0As$uCOZQEI>QT~{uGU}?`8Vw z#ey{Ch8Om}*NnX%;ky3h((U(+UnUI4C$`8pIE_i>+FC+11!2{3m+l`chgiAVIUuf` z*SYqjKrN&gTAp)>@-*8*inliE=ZXaI^!!!1W#>>_EciYQ0zX!Ek?J$WuC9LFdz|)c7`#(noBo&*P-JKhLUm?rs7birFrmqTk~(Ajvr#!?C)utacfIdkU`h6%*-djnV0E=e0rjtOv40p zbpSqOK2hKW@i20dQo2@u^`Rkd_EqT7M)&g@Y!dWrk8nX2&>hRRgfBlss1sys+G{@5 zGcd=w3Ig3%*{4^R;@$t!D^T={F>8)A&sZtRKTbACM?3vv-K3)r3&B2SAIWr~LbA2< zKJrbJIl$nM_PhX3nN;==z=VHMeQEIe&3Rc~EonYGS}pk%-6O96vU9Y8rl#X`1|Y;) zM&0t9c|N%3*}a$-KY+Or9zzti8%zZr6M@xpywsP4{Z*Wr4!M~kMBHau+R&hw7YyC! zA4!LxiZ^ESfv-%zLJFnd>pm-+^50gMPP1a@3XXu)qw)m7X)mtBbG58s_tF8em?p4% zWr{W*^1=-)yNj|~E`0^6RiO%Bw9)fE*f#!m+fGrf0?SK$O2; zsys;gO{yAV-{FP3uYA;HcJ>wE9dn5dxPyI;Kj;n^l>Q_W$X$4w;Lm_=>goZ-123P0 z`!urWVMs=0DY9~(wjxOsUT?M0wL`CbrWah?!ok0l|FKFBw)hW-JZIN;87^(TmW02h znU!Z8GrAOCDUpi>6-ZD6%EzC5AxZm;IlGi7{6FnF*I3ix@dnbfW5(%u|TZz94e2@Ph$U|FpYrUa2X7*uNQX$efI8z zsCAUJJmR`(Z63O{ z#KzvFCBQ_IV%vREeRR%d8yokdCuREgrr+c1Q@phnfeVhJOII1&#tX*@WvW}F z(=1Ujd-&jpkzy2}1sLEGC}aspNYe0-W%HG?P${53>41uvBMMC8-VaI^ve)5BTe7Ne%n@_ zW4l_+YVp#Fk#U_-!j4P|vcn8wHg2}BA>^W4==+dUD?gSO@ykow^iV}pu zBeZ^D!eLW9+{erAm6uvQo06hEN$0?A)nV9j{KYo|9(ZUyci6}zs<#Am9i|=N=E!UE z2@A5~j7{wATj)Mr6R|eg9asbe$@!9zYsr&3NFf6tH_O1(vayX#=sAF6o~*|M_#5nC zmd)l7O~?m_q?Q-m)((Y35%07fI77a_kl$?^fbcLyDXDz8+8G_=?Tq$z-m2GQsLHw2 zrVWxv7*W+@0I)mPqt~$v+toL>JPYtV5mY@B z&$6*`r8JL4Uhy&ffbz|3V#2(%+4_FZK;vk9MJb>HR)9iXl71S1=0PH3|F8}g+7FIv zzUm5aXSkYgH8(cU4%5_=iD!d;gYd8#p0;^;*TG|V#M#_;^@5=B1dAV&7C*&mX~;}R zm-$s%+I*AU&7W>9A^;q-sAqJ=I#Q5gx-QB*?)DmxuLGoG!9#^^+ZE+Gxm%jOx{8t> z)H02+2an|B4bn&KoWnQb=R^`~+7i*XWePxy-(Seq8Y$Fz z13@V)VRtqh>7c`a_1p&3!SSj<-FBeVb`Yg!tFJ3JHNEq{`IQI6iojlu>?p&>`Ox zl>u!f>o;jYJh_VgsW{lkSP+5(5YpEiAWx_7qnz_w{qb4BrVIRf$i&=^TTFY`oDdq< zE-J!cE3-h#olDkecZ$!7oiX~h5fDtFf=tEwy%yCwNM3QcaT&j){hVSfp?mGct6avM ziJRfn3GgRioF2HZkcaPJcN_xlqg_W7huYHJWO5md!-ZUTlUTWeZR&4cX#+xadYi@J z4Y##9XwC`0Ss3gL`QG_jvIH31ndt z3fbhgb3_%Uz}?~caQ=33o(7}*9nQwb1hhP3=F=|u}!~&a4 z7if42HPW=%p1Sg+1d6UB@hLCE?}Qi&ULCvsIADPY+Tqaf?I_7a;Jl#D%dnjP#*4saqbbj06SoSJARW4FU+ zd$Wr%K`*CbwEG@TdC!fuB)6LMHAb`l#J}gsH{&SEKg1s*Ndp|-WW)T3q|@A|V#iTu zvxcH^rwT-{Md0kI*i_eXxOFr~q0~vl;TrjDff#@j1_iN|D{P!pZJV^?<$awqO36#S zyLZyNy$Es?M#(@cK4xYl+UC9Wv*f$EG#lgJnA>SL#=Q?=Lu| zC+nJCEx`nR;+9!@k=Dyb$@n3r^TSoX4}eOj@ps-Tz@5TTXI6SAyV`u6PSr5T`%|Z& z7`{pUbXlKRmG0i29v&6O$k@9xhsCYOv`s-pYa2c;VC7V4!-@^8F-NTnz|oeV=Ps3#Z=) znVmM6ACglJfH|%d-qyU)*ErH0%XptWi^`1}ytHzRTAV7(_t}<6PEh(AkZi>`jHMrA z&QgA9o6JR^-3;f=Q{e^r@THTrNCS2H+O4xcX12u?yu(Rc7 zD;5#GU+z8c#INb_ZM8KV+eNu! za|cs6fke5z5q4=$!5GA$+AJ`0%(VU+%x2xsvijH-qdD+3Mamo$6zRL5z3*kgUw<{1 zud@^J`Z?WNmmZQQUl=d1)}1wYr6t%0$g`YlcWb8`VV4!p zk;jO?lXK!6gWbLK1lx2WY^u32GjL&RqRxwM114WV{yDyQhVmEn2E=dXwH(=dXSyjl zIr%gUFDsIBm=0tUYhr>PU!y&J{a`;PvTK8L)T?q)I66X|Lq-R&UsbTRPsdOs!ANV> z?Ebo7k=RPTH*`ttc#q3>`+#6c#L9c_{5)e-%)@zY{%FVz#u-U?9TA@%+wBn*oHv)s zVwx}dMXi?rZ%@0TxuP5EN)D2vQV&%3!sO%=FiSBt`$PU4?t6cBO@-FQ04!TC$7UvascX#ridh1fDFl1c<%U(ykzXiJ8+L0`}Yz`7lJwV6GO3fTC079a+c zV_Ht18lL03f*JCL{;m7XeurCgg!I|oyc++z-bbfd)cYfs-jzC$GdDQclKievd~rMy zSLJACy7H*JpEA1$Cx_+Ht=}vCR+E_GQ zh+|%nu!Ea|xtuO)`fmW-3^=A^Kf)_Sku6ua=r+(euLG`JP9R6y`;b(2mwJyVX3C1;4y5`?R5=+B@~<2R1D z<;5VAyPVNzMzGwHvtvVJ{m)e^^Ve2Ba-7i##<%oOkN^+7J)uAct6pAeC@;Seu7v6X z?VnGPMw7a&I(+i)%E5JM5*(@#1KIl=R3CZ@QE zSgD2+xcj3rmq%qHn99A}gI0w$E-KD@7wBTO`w00zM5&_66NYXes>aNBm8l5I@WskU?G9BN(q$qWSLoIB+=ARdn~M9 z38@$BwepN8paL)rP<&e$!_Mo?XdsSa@J0T7NF)Wyp^LM0|k=b7yk6q`@@RPgz1asPQQT!5PA z=tF-(bqf?Z-;%^LuKNps&Rb7T$9p!OLU zQG&q|BjDBm4j~9G{nP}}nlY%ykUKJFGZ_9Zuf)MMWKjGTrLo{HRT>D6YN{^7vS~ME z83>E}((pY6+?3%4bH<=cywf(KD_}@&i9=<*p%^2Wh8K4465tB`TnT@(68=o*=YLgvUKN0?ihDFZmItfknyxhoZo$a|a`9h*?=hq`69if$gFC-Q_p^5miO_f9`d4ysdx;4;wx$ z7idrQND6xMY-Rmf-_O~SgGUEOS5aj+HiV_ra?S867%-rp3>!kGmf$bS1wvAYI4~;T3IRj z8-Tl@-6Mu*r%}}Bb0I2G2ScnOo>0i0DCxe@g0x3JNL5QA69svRXw$)(10Wp`QA`CH zL8b`)Rmt+GE7u?EYl48q6W&(g0f#}^iU$%K7x?}$TCt*Bl_b7=5liMq#4d2(G!%1a zA1UO={8ZgmkIhwPNNKbMUl-Mu23*lS>sU+Pq~p&YtHejdfB6#gyh1mXy@K;bDoYY8 zglax?j~5Pz)!va+1bPU>luf}gukrB!jK5;XmOP%C{upWm5v6?WD)G;eRR&b~iVxy` zY=fc>lQ%fk!#wTo9qO?URl6Rc+|l=ZShwxK;qo(YZh*1H`yYoA?F2+Js`7jCMpgm$ zLWAL=!+j9o&jeb#VfsG#BQzMkaYq|MNRE*;%H57NF?Z6dzmHAqU#iP^uxy7OxC}^d zQ~uD*UVFB&7(Gos>^>#?RVv6J%PN~up%7PBBf!tc;QVsb4jpY?9ven;Ymn(7C&5=WKEG950}(!dUZXt7z#fFoOg3N$!q=ApljOt%zQ#606aW-aXLRBf=^qs}0%z)U@Sw ztq%bN2Z)sF%wzZHBtw7I1j|cel5V*>zb+<)5(Y% zKx`tE$?TUSU6kLG2#cHt1n+@p!_zVg0VbOF4^MB+=FxjZrgtwKf0oYDDD?X7$_R#y ztVDE_vjb5Ha6p76be-xvRSj5r;0eLVDyo)G{?Fns6j7N3Ec`SBR)$_~S2c6Ti2b#z z>gylxbwPg&K+)RGyt8=&L)BKfL4QV3P3pA+C%AyXV=laW_d2GN(_M18LVTFj9R*@w zm#pyz@=kcCZHs>;J1^U%u-8rQU4L%k*!lu+*e@Arpg_8)+kQ5$b4MU08U^1Pi*|j} zG^`&C*VI0}u8B}&7`kBF;reX=n#&AiM|&KNEv%^&WKWH~{24Bd$t><*`RC9u5(UJ; zz{ox5_=>0a5Fa&%V>gGn)U0FiP=qw5W%TDlWQg%qUmzT_%d23&u-Wnsv=Vu>$PwF; z@N_-iH_J797Mjr)(HRrHnqu;YXV$s2z+-{Lf@eq|59fSnFZcO76T+|l()Sd_y516|L>_d4^W(s`2T^y z6#uu`_#Z>~@5(w1|7Q~a*CC(x=09`t-<1Ib?w95ApX+~<_)q=wZ~rp||3`uP@5=vk z!v8~}1PcC(Tl+Vm-U`%31n)0mZ(-Re|D@61^XM(lNa*_ZeVCE3K$8B&y)PBTfY|?! zq5me;2;>F!3JL-zZqEDH=uJN91yDV%`A)!$6Z{1oQcUA^>2pxPkrpGnlqcS&*MB#- zz03m~zZ4%21FrY^RsQ#LHzE%ZZ{p)qsm}RLK{hL}yt1-ThoLG1`3q?Hx9_Noi3YD; z`!jOD>27Q_Dvu5FkTJ*m&c8HSb`c>Vlm_wi;YwQ!KBIgpPc zvEBWw|AJnCImL+lk6ApnYeIQC-<#iF2ZvVY%ri4Oz<6PXzc8>%xBVWpsVfRzA9IBk zXUy}dgv3LQb*y07v;2XM;G79v`rpu->B~Vr=(Q>(n=a*$bMi(#ITvBs#+}Be5i0BL z+`o~wMjZ%`EU8PmE|9c1_n(ZnYSg^BrH@gp;5l@>z+BKCAX zYDcN|u3TR=!!CL#oWTJC9EDQFKdr2Z{|)Ss{><`(yArZL0eu6`354%ORDyG+rA-$M z^=6uG{Y&$AF}FP^h~rcb!Ha zRyvHHS5i?loh~o7h12v>{X0k<#{3C2n$8ZibOf|KRYxH~={J*n(s_&T6FNIMmOG3M z19$gOIv}1Q&3nI`lxt5+U#N)F=cIys+MDaxV1f{3-0|p^m`83_TYTkAXGek6{qZgE3_^zujckZf@nljk4W^TD zR5g-@o>z7SX#EFXJ?~A?wo_BObkoExY}F))EydPg`GhO}N&lX(3Twl({h?y37cC2U zsAX(}{f&RuEN5evoZ63dU1Xymh`%3g2+_+1S}WYW12nhioZZ&3P!hD?yDv<3TI~r+ z;yjT^!IO7m_Q_RJ2PN7)0Ucc#X9O=nRRsNz>6nRG*L94Rn&?@(=93CtlOI|98DkFM zvB`?eO4)>Xp?x$tQ0fMY$7p@eB>HGIfw^(7MznRYwghI=*iYMyzu-6wReN)valbEA(4dcAKtRT^ zFg_8p);zybKE9SKyU&SH0i=WGexe!%RRKzR?XN2m^EKlQFc=FuOS#6IM?1m^57vH5(pe~WFf}60-8=3?~4X}o*eer z>&Y^XC>@uUZ#FwxokQ?ARMRJLS}2r@_P?&!RkO1D9_(gPy>|xS>c`mgQD)So^2$bk z7j_?GU-ZW%A+$CpTvFzCik|y5q9&qB<^%i>pG8l4o}5e&VQ7?1t`-m{uYS~3hOl2G z4Z%@qFEUZ*v*0IJX2vZ|wmn5WJ)eX_I`UqhHF=>gj{FIU;DMcuV0&qUj>9%OgHF7r z{7JCYH1cx+y=9b@aP>)5X;t#w9)&e?u#Y!+_D3Cp%ZsySe2kdJGm(`b<`kwGh__t5 zxl0)kAeTlIH}Y%GA2>U%n^``Jb(-%Er)|Jm*?GCAFUfg{z#a{soXw<1bNs5NJhC|R zwmMy|aYJruv2=W#!=z^$foNIb@4P#e>uHp*_Z%VAK&*}q(9UU+m&93Dq_KuC$UmG@ zu&mhinIi$Kv|&-^RmJ{<{s1yZl|#J2&XD)-%hodX?Y&`ccQqKeC8mVVY)wL)%-H!25pW0^Z>g(qXlYu95j~3tU3p-}D%WwMISM2s) zR1Msl`Osyxx+QGPDHVWfb*#u}suk$X$?52r7|60#GEw9?oY>o&01Ku9^0{WK>hDfy zY#o-7Li99TOqrZHIH>gVwS3NK35NCrm7faT?xbyI5ATQ-0FHpQ&A){?I(%$fl#W29 z3WH^0pryn7+K{IDzBx%Bp^e)Ke-1PaobqtcHVAHaZy&B(;>nlrbt1DFDLCbrfa$~0 z^~QSSX_^7cK3#)veH5^M8z#40%%&C}A-Wp2+a;WLkysJgChP`{M!JwGr4fOrL9KIiIa)%|jtT5Af!=hsEtAyr z?dR*qPFU{=6JpdveSSx8gT46iR&-dve)jk(^ri2?`^mumW@L(+^G;fWhy+XBMl)lR z_nT2ao^3vVf`0-wuzX`>&D-MD$@d%tZOYsLZmrg>W8AJcaBhMS(-ale89#Yuv%aT2^A= zB!+S-=Mim6mi0rK^uSu@z1M&O+o|g)RVaTTAJFP1e;jF~$Ivi$(EgNpLx3^?_NU96 z2>ybD$6=0Fy++qKGBix<+@`p8_5&se&N?QLc&D9F2adaz+~u4-wf5DX&tbibGx-OG z59)6i2Z?$t>ogQqEN&-l$y}a8WbtrR@0m73yRA+F#M^(y?HY&K*Lc-GLRjthdg7$z zgFdG9t+<;cb3{*MG(K8PX>|7_wQLuFk_>3m*5Xt*YvdeMaXkVm9*p;Qk9(z~8>;TR zws|$7C#^P?xHythF*i~%Yx`;IMEjbIx)UvcW7-KCW9+SG96HFnsW%GC5Im!Py3F|N z8k~%#>844eh@YkJ!feh=x==%rwd+C>4Eelfge$HDWRiP&h<@oLcS+AJ$19Yh8jD6^NL1VPS2_f6n{aucXb2!;pd!pX#_94Oc$t9eTvXF*&iTmmPa@SNG`CyW#`8aB) zgKWZj=+!CEITsTYw{n;v9<)XfscUr`W z%GL={jtoM`$bQH~I2dcTk-{YVZmdO8<4_o6XOP{s+I5ASNk zjaT$aK$(!{)$?&!7mT^D@T+Tzina?Eskpcu`*Qf@1h+_40|dP3G;Ot&u5GxD+AnQt z4d)d=?CP~;9U}VJ+rz6cz`j33-C8Ial@XblP-Q$B$_Z6XUBOfVv$!DVs|qGXBHDjZ zJ_EVE7~gwwP1b>Ny-+nCG2&tS7|DPKYzpNO8z0z0)T%Dha(onvKX0u7bL?q&Z%m{4 zim-kj87&j@^)c&aww2O8I~tqIB*la?&-sT^^RS;cw9AnJw+%&VxuDwu+F#zW4DR$X zG$PI5V8Ja~XB&nEF6MU@>0?&Wl`VS({JO^@E0%AaXuL+p6}~UrsUdmg8D$==D`0Ab zd6NaQ5rGR2Si#eO>1*=IK#}$ASnilhMsoC;+gm=`2d>N8VJpM4))RSM^;yY@Av%Z* z?oCx02Wqr`oJ}(?Dv|j(SbY?vAJb1y$|})8KH4t6QG3sNB1K^cv8arTE5~MXdn;6w zdnHRt{Xtb7y7}-`dDt$DS=8$e7nBtKtvn15HC+zp`hEDC;&O>}8M8_ZF6ZlSc!g1< z=o?X=sm-bc^wmv8#R10!87?!SZAm7i88or40n7D`t1I^0B7F@=*E)L*YnhoOXlT@H zFLLoRJk8G@@#RPMx;n48`%+O^6b4ts_iab{H>eFDmuS3>Zjk)x@(!*=D7oIgbjtYL zjA6s|cGu7Gn*|a;@XM)Fe^1m@-j(MnKOfMzTWM0u+P22yVFD8ouG0j8QPJmt4OA6; z?KoI|6D4|oM5lU@@YOvYw&KEFpLAErUrWZVt$h9)Ffu{gG z`WK4SnOX9r1V61mIS}%h=_2Rju3C$J27!jW@=wXJi9(zffrdEE()EJ)fCsqkTHgzz z1Au5mvimsfY&{~OSFAo&$(f_Bpxw#aHDp2cMa5@c4x;MLTQsj`W>%`1m+5#<)?e;R z_Hm0FAtx6X${U$k3!3aa+U zN-snB%H^*FBmEyw5%oO}ORF6psfybfP+*bQa8#~Jj~d781~s_^1k2xjwOn#TDMDHa z(`;4=*}w!iF5u6brMk&-W;@9zgqe|xX_l~NCD+_-RzpupM8GuD9;&LkAKQ}BiGzTk z_1nr;Clc<;nsr4FE*EZUfRT~bEmOYUT@h@I0dytbkdLjP`q!Tb(^}bE`AcsaymB+r zB-s>f?526mp)IdP>&2NZ&WLCbdn$IlTXIXKmb$LvI#h-n*^MwH4tZ)(X(R>0-IIO&6H)DUy!;(4UF{JHyHVu+!6Wl z)t$YLx3k~{CO@~$ect^ab8qb?V@FB9rQYny8 zz~WT*4{Qu$WnA1&^hd4Q4j0QMPn*cr>UOYX5Qa84^T*1k(ZEbcI62ksWMvPFOd1uN zD>yw$X)vEKF9ZK9K_=2(pk`{jZn>)COG0eP&p-EagM4}GFPH?9jWKnhRV%CperZF< zZ3VMk#QtUT z#G0(%9rMlC*FfNTpN+5k+JbvYx|ZjLgmuP%jswgEN;s8aZ%{9Gb;!Y&&3XY{;*&k; zDC;5+E1P5FoCBV$42F5`d~818_C?$y0(9(q^E{PedMTPW89F*2)^2LR(o!wNR31o2 zb8!d$nE3INam^)ptFGNf(H|vUsoygmS=q5m^(k`IG?yQk@Mvo&5V*ox*`BZ1G&5^n zQK#rEG=b=-rE6lq3l=>V_Q)npFt{MhiG@Sy#Hw}fI|+MT9O>2WbKF}nV5p%#WQ~3+ zty5-JZcX`{n)Z={cVEGc1^0AcV6AO0TzfC;rn8!K&CtW;eF@_ZrBEZk{n_#v3+L`HZ6Cn@%Xo8ioO8fIUCEwSP|wrg zi7VY{77=HmUvvKx(NF~ER3dGjd(n7h&P+T1wcF$w8oRQ2}yS{x!S?*|vo2BL-3 zNqC4YN`x$H>6S!MB&HAzg*V4MRrlv3f*wTgzeQEYmmw9yba`el9?C$=-2q_;Dsbww ze_(sTE0B$}kb60Capn`*Q6Cdz92tbZYVJA8MQkZ2prh9R4BJWuaY)Vmz=7^6nP6h7 zR3N#bJ2w|1G;!G!Rc%^iALljMT@+;g<{$Nhlb8|iogP+eU7RK?w=S=|KRFf^`o2(r zq|G(o;4erIRhoOeRoi7}XCHK*d2g$3mZ?sTHWytVmiKIS8y2;F_sx59;d>t+E;meU zI1QxKh&iW|0t@cx?A7vih8bu|ViY{4xkzW1>s(+{>{%4qpIevY~2F}-D0NORIT3sb;6uH73+GByOx)5ykK4GSly3 z>KV9l;84dn2u^#k?Nr)`ZmVu9tRW{>x;T6Cmnl}mf&6YNI86bQl=oD1Z6B3t_bE{N*6gb>?o{f!hb`4`><@*MamQ6N8#n;e z9Sm6BiMh>uwHaUP!pgn5Wm~%TNJ!l(KLlY7p73lDLDO+vHI(X879?d{UAXm+wrJ?~ z{QZe@L2>6;WDWoP*Uz z449C$`m`Q&E)EKHPrX1CpLzV)XXMln$`b5WUKQ4Y2JY4_HJI&9LVfnuq$c*)D;u`G zUg0=52e*^mmo4~3_uWu&O1I?c>_9pujXRR&l-FmoF&9` zX?xwWht_`VaqD!m>+I93VkfE3G|#7Wdipk^;DgOL#g5 zyj-@XzXO_e>gzh5z4KPB%qv-;vluMVNpq*MEg!5i&;T};V22X8p?gM5wbU18g(k#EqsbKSgnD3 zZ|BuDAtGn>NrEV1;eTKW{9y62=9Vj7ry1~$lKbT*aL%>;b0mZKbILESrN1rjY}oW6 zZ@3Ow6ngDC*ymYi{S3TE@38GHSZX<`xRbVLL4tOL1^VawP+UKGyla82uT?*G=~(^&jVj87ccT1di}E?5P_J^AwsA+>%gINxA?`xeF8eEG(m z*wb8t1;u$KSh*V4374I|V(J{il*`@Qloggz(;w5&>~5RcLahU<{w9dK*z!#PgSRo$ z7UbZpT{|_rpIQr|l>EehzH4aFUHNnW?DY2!L%n7u^|G6l?a^PV%jDMAR>zZ>#IoLx zx8A$Ob`NE#c%GHiMC#et)SAsYJ;o|(0Sb$#;O=s zC@vR!ShNR*^7@kRcmW$S!N>?a6zbn1niIY^vN%KYTw%tkvWvy*tVtKue96u8-(#jI zP5C2wQjR$6XW=fYG=*yx`SvowZ9!-{bzPc5O*X_}^`uTo2sKvZK-wkbJx5-oq;~!l&~^qWf0oj*uZN1UlocXr0DbkM zRqCH$+h)@`n1>%j5p3D zx|Gb;b;ZesHlv*~wT;!XLE^g_?Ycc!%xmhjGC6MXU%NMn|G9f3dvA`fD~mL`ZZk%h zSLfvSRY=9V+P2KX1Y(KA1{mzFVnPC^t6+=qP!f9*?xb zraj|SS|xlx2PQFdOm1fX zA=o8#vNE$O3A4bjf7GoM7WbI1HHR0DE?0cCP)K`H7ePhqE~#rkqrbW;D;|D)Mp{3P0JC-y zUd_`nQirk~UH`sJWg}ut>5*3d?CItLZ=Bqi&19BT72#JPmJBi!v%Up zR;#T>m()Nc`T$3l74OIn_v^NiO`8{kX4K-SjMbP|FlD46zv2 zzDdd)vu|@-;6GT(&tA_a{mM1TzEL}Ws8Tz2%w7*98+TREI?h*FtZ6E=Od6b!_84%D z_eHFc%mP84^f2?uL)xx5_$ZlVaF`CdaG39{5F()UEw5$~$`QY0WXIk|2&w;2Ayf?d zoB;{eMl19IDRIA+mCx-ToZvOxiXzAe?JfW(dy<_C8F`7x&W8v^x$ZF^zP1PohWZJB zmxIcW&_A3B*h6@6n8_^`0vZbwoUov}k|IdDYQg^XWN5OC^qPmRhTDuT)qlBFiKa>V zs7Pj*t)E(gwXZXA0EO6#tzf4*FX+!)=UGdo8#}FgeII?`H~ku%zJBlxUBf=|&Ty0Y z8P3!!6S&%ZB^V%eyNyy43yFv?6Q`cw;WTnFE=i3z=t=80#lUj=iqg4f!J&*?ax1~> z0jtlISWYPUO-I)fviYxvswkaXRSj63U}9W2p>&RM&;4q$R`#;zZh-a=o(2@_dn=Lze%lrFO`ttgChyVN)-&@IAle;r&Z*dU5+1J)4@sZ~n zd_DR6*Q0rXOm2(YO#iJGrux!HU*@qd07RjomcXeDU5r0z)~6^Bp$po$Ga38=XNKxt zH-{(RfeR8MJ8KDk=8&i=5V=Sq6;i8hkGhf&>u8 zgX}q2R8()+nNg-NFu=*;B>XL?XOC+ph*Rpn`zQ0zeT_^_!M{cZjYEzVpB4n480q8o zsw&2gnIW>lNLcww5ewT$qu%*d!m#ftrPd`-ofEzku<{&2_n~pUSv>Ryy#-}K*4Og- zXE^@YeJ6d|CI%o6-Y)^F^C3D-b~P9jO*kETHk66`!bznYbwB#*=bf9-QL@;f^yR%d zJh%UTG2+~b@yN~_Ubg`B;R$LSgwrsOG;F5PHOi!XCg7Hk-qYFaOA1vR6IYUXUOeYE z2NRgtH-GwJvf?dQ9eyy)ecWF^fgubD{(m^5e^2J5VeN5X*nn%UkQ+r`1!(ZGzXtw0 zf%1RxKmNY1Go2=U~|f4-ctWeK@j zW8*~C-HmuF6ATDNpsKX2o|%?Y&*$5Ch=m-kPYr(}A!!LtiL6;Yg)?^O^Aio(?sjbH%r3_xn!PVPw?TJVrbhR{7XR6MY-X;}Rt z$bs1J>?4KOqhS|Z23eP(Z9?oIS_4EK{qt+tqUT;VJrJ`9dwO7Jb>Mk#V78JoU0Cl+ z19%AG>J2*{4F83`%Z61zZwfjMFICnQPzLo8<&REaMGo>5MbFrY%}}vcgB~Qomn`j~ z5RxET5x1cRbgDcTO?E~uW!iG@vE5#y&G!rCy?~2gOf-77jk7Gj@ypp~$_hp_h_sKh zryb13*>Eij$c3;Ib%UpZ-^gYfg`lI{rN}34m-1h`+x8?sH$C^A;W-X7V*UA&LHZgW zq#B@%zuDQ{m;)|o)7A6V70pB*dJl&^Y&-{8s@(&XC_mcTbMZZ1M&p&IB=aCNq*3ys z2bPB%9&VV5>wrA&Y=VT037@@#clC)Lxzq6yE@;4F98qOTUIpg=c@3MF8oG+(ES6^w zS+5J$Znz9hS~_wqdvd&>{W1hVw?gms`J6`n{A!ykF$c^o3;>5sw+~o7ME<|0{r(?t zmghkuPaG`SXaL&#lz7+|5gL2<>-IbZ-hbhP|Fc2huU&8(Tu8*r(fHb9tR!oo_07JU zO)C|D>=>xuV2F8A(vq3q12U6vz6-!Xj2U_(2WkD`rZaU>9uBQwHu09zn$xLU7(S44>wfJP~s;qBnLEPG^R1Hc^Ec) zDy5-M-sM6-3Z9pTea2ait?!v<0JW!IcM+}5g**JqI2PdDw*4~ZC;>1o+*dUHOZ(_!=fW!SC$)gOUnFq1$KQ%{D0r4wD^M)++o;vO^N>i@;kY{ zXdv~UTMH`Y(Qy&SK{w>~(HAt|rg*_3<-)PUbA}vGSkB})D6dpr1nOhRb~U)mZj^16 z!af>{k`w*OUe_ci1wfN7{b%^teNF;JX_1&LYlhkD;VE@O3Fl)8`Wl=y33Y<}Wag{%X*UupIF$Z{{Q1KJv! zhO?`vse5|w$8}Yps&$uqp^TWNGqyYFU#gu8!S-Atm{s*g94|c8u}U*pPiFPu^_upS z02uIJ>PZNU)%)>5ePs6;@XY%MtmU`zpUlq6SlhI z^1#+lkk7&%9Ei{h`$!!jt?`Q#Yz|QhXUKm0o6db;Z9yrj($n8`(V^LJmesS2xXI;p zl)MOTYf;QALu+$T?fQl9CLF&LmopF6JZQGn^|PHrf#e+KEMM~$8)&ewUw3n=fl?=Mz$`vR^EKnnG+>1b_kMschG_m z6uGpWrVmMxpnbo{G2dJo+6eLpgqmt zf=(*L8d~spc*omUg1>3eqk$>h(aF#B7nfLEiY_6ss2hMC`XdhV{wQl!KMSG45M>P@ zCY8f+yk`_tuNJ^0>$5m$+y!Sn_8w}x-PUP{>d3PG0^mbFQ*@07Lb^E))JEU~JX?9Z zu?g>CfKV4JaB(L;>#un_b&KwzWkI5LUaE3iKE&EoJpkV08RME2a)bk3%JiPW#U6K% z26&w3jgY8Z2&;*@3+&^-bYRYTATy42WQ`MTaL*rfQbq)6$YC>WTo*ixSSVf(sN09kq>*4tlI(rbSqb zb}#}16)@6QA7nj>(Nffvlpw}ho@3f8AUetZ5rBVtdFjx zA2@vLHQpE|C?JgvKtK@t|6ZRG zX%TUO522hTWksR(p&-x^u=^w03PC`KKqN&3RopYqHvClP)UzHxDCMR(F|0~N;t7;T z>ZgCx!JZ9+RM15z?*+0BuilFO+M4Qycj{zu>7>neX{>#_<28M#S6-cRhIAXE_nl+` z<1fZosiK=rQo_SVRQp}l7L%kKHY-6$yloSN1ZNJ-413CW1 z^uZ13^B*wEZEC`7(3;@vdXl{q2(YiZ@y?ZbrH2k6`umC)m^-)wXXW!pL|*?K^g9Tg z6>VVK{%S^?(BaRCYUiH{yIi%hz{y^Fo=lAjwfEDm_Xu|D#j9b`pPw->I4%R=dWCRt zagA+kBG;PioStq>ziqaUoUb({w`%;Gpm(Vs5J|Jek^FSA!uaWCKPV_D^mnkmeDe}3 zhZ($$PK)D@&Q1XW-$z4do{fxXVlNoe({%%Fv*4sq@1Um>Y!@y#N_S4;*e2eq(YyiIc=Yu}f z_3E{A3$`?JhN%CobikEM(Ts&X7vA@jTiMS6$V}A^`=jUr&qt(6%F39gqVdzy3Y~A~ zcDKOVU$2;F(?4EA7%~hF4nEFhzxQ)?UPrZgZnBzBt}ooe{Co6=Ms!*=`*vgm`LsbFj;((0N3h%X>N#n$ur^$Hh|kZ@81yCEHIDmIz(mK#$Jb({5Qe3pj0peJ zRPb#*venXFp%#8fF?HzTD({{i)+Q=mr*u|fW2`RFo+wHGsGkDQp&^96?t&rNGNuc^ zRQX=7`n>OE#3Ote1flyMB80px|5K+2lo>U2jM@D!Kc>C@z-4iAJ-t+K#y>11fEvjJQ4DaCFck@uoNWha?{K~^kUrz_txH|V#WwyO` zcYl64(B67pEDz3EZ*~Yw*7Jnwy54ox2k^d~YMynSrLEy^R+sAU|0y^mq{aJ*#jFk+q(bdfmxhCa0(cF)toNU;eC{cc?^ zy3hzAlAj`Ls`|ruH?z6`F9gh12g~3dfb&Lv_C?K<=bk2Q(WarB~k{5Uq0q4X}^zq>@mdk{7 z-}(WE*=)ZZ@Nf^9#mLl@e$1__i;JSZ{#21{Iu#w=UFR0CILgIx*JpK8wbo0Ontqm} z!8GBcwh8lzvBdNeih*C23`k8A)oy5{BsBM&{5-g@pQ|H$Me&YtRann>N1xo%j+XYW zOqBL?sWYSt3YVyBzy2VU#^IW% z?2p$&!G|zH&!b$;F7-{l5A$rF!PN%qz{$lFta#YS;5s zZbB#`X6D$<%}r6HZy5qUX<+eh)Lx z@-XX}LW&Q>fkGkx|Gl5AkI@q)x(8!bP*haw`NxXD=S~sWIH>OZvE&x}ZP2Ak-Tbn$ z2w=bHB@F)lW;_(_@VnbzQj;e~)b9!RiB8}O|funpsi9qYuM&tgVl}a)-uLs_-K0;Uf$vC-H zPzVm41HINs$s-J!>TMjOZWA#N`EkN;L(cbj=EiXnTh;TX$l!*#c-)RAYA=@?ZEii2 z8w(2q4c1E<9iHsJFUG^mqXrIfyRXhgw%R@70i~P)6kbPXJs`8Hp#lB-&5ffSQO|D3 zmsjGhmyJsb{)5><>4fBDr-SiSKuP|tJm;2od>|B3t?!#>Gy(spo$oC!2!Y?LgYWAu zq0{j!x$kwCujJ^*6T&+&aqPzfDW$P~WGo@!=wOLy`O9DjD~2h9a8*11dzpGw-VFGX z3JltOirt}R{*3*flLZeb4!|+xK>;qYSx{FOALPsP%+KXo(})Ma8@2re>=D@P$7H`J z%*hPy-3k>rY6XS9TLL^jcLi|HivVDkKp6DehoK0)zgk(*)YR1U4-SIfwrTx7J(x(J zUs{rfwQ>*k<$qXr`p(-8?CvK$pFgS`wyR{pg+z0DX|n!0feqw1`)po3;*)c$^HSNC zSem&a`0<1H>umL?o`I9Ob@S?+q~`cJT-3ehFCME}Ye(z|#%_`#mb=5_6NdRT^M~(< z&oXK@3=pFwSaD5UG!9pjppT-Y|-oA*ihc59o?#pQRV8lARFy_r2?@n(2J&-`kz72ofP#M8TS(#IA> zQR}wlTAxcH_gb6e)Oc+lFC>IrD4LugrQDeSp2V;|6K|GI#VN`)maZmg}h0H=tx(sU`f?P-M^`XLKY zV2ATn_D#TTfpmVnU4mU~Guw5&aRAD7{|*gj_bc){pfhrSS4w3y$6%}8*4%Ju&Qh9slNR>pfEcwMzfS?_&r3oo``7 z9U9KxWedyriNxIIweQPoUmn5M+L{{|UeJ$;bv&)e9LTR48p{FnFRjZxQSp@Wj<*N0 zc143K|1%6g$Vjo>fQ%*M)*Q&@Kw6S!2e<#ZT*k_Q@a}9Lm+<-d`J98(tMMj>lX>b( z%u*(>;?%{H{JYZHay3%0;!4tr&&GZ^B?fG$uNuQ6~F#qyfm2+24x(L{a1+o*4#i4OPbbcYe}o z#oM{C+WJbOgez_)gYgR*GbtLCjSWvY|ApF8f1%ua48$xsI-@-5D2Wc`cQ=~>u}<<2 zScCcTZhJK5PoR478a&)HhQjGdV@XsHjqe;<+FLn-9^d(KFa4*3twV|1M7({8M`C|F zMj9Mn7F3j`8Qv>r@Cgqn*_yo^$VF#Z@AlpCyJyGkMXbeuu6;7X;CyEuv(i&4Yy+R0lD5{P4)4;vjS%YIA6!i_gisq5#YB z-X!mjZC;SL?8|4<9ftc89RY2;lr@qDU5lZupdX6TIqj6R zwWR^?0s{l%`;al?;l{EpA z{#bI}0f7(c#CYW!)vCjg#HcTDK8YA&tQdd;GM3CZwC_04ZXAQb`ZtjQV=^^OnRv`y zN#jyH>dy%dp=`beQfn1(&dxo7$KQe9jrsXgaZqhHS`$}SS3Mv&OpNgh$@k}7^uHz% zkQe4;{2B$l+w@>6uUAh}QdqxwRmhZWu^u~ zj$&;3X5Ux^n1($RhnQ!&1r9W&&x73fAKB%%tgqm>53i9?n`<=|x?t_g_Sm;n79`(? z$r|k*)~kdu&2m46LCkR9{Sk&hI=~p5TO5Ku3G)wGp0F8WHxjp@1&JSrj*SzW@6M;9 zk@DLqtnqbMp9vuo{9s@X$y_rE!#XZFS>PaXB{Z1|z82@5aylFEJZk|QuhcCf<8;B$ z%*@hQ48DDb_ujTKk(($jQpXl~KEy zL?Nk@MQ&WywMOyec}BC#ova)9GWKc5Htr#HMTR!6*u5?$=$KHPcqVrwm+TF*yaof6 z3bk^ihS^u>JN0+d^Xom|SAyxNFWWId7_gJAzXxMx=YRT&JcTsshHQPD^BjdZ&Vijw zg6AT`!|Uo<>n4*?QZg9l`yEBT-@G9HmoUBs?AS8%K!8!Dzo(veCBgTriGhUuNTnAe zez46Of^F_>aK^b@N4mDLm4;6SXv*fSmoA6~4K1v!vu=qol^cWdeg{r*%g}US;zyjC zhP#;o&B45-6&AQt{WB=psnT+BuuCBg?d!jR;K#hV7n6(V>vTl7+VRCW1Ox9tE^YHT z=c5+Cci||3g`xg?e8iCcfr~WHi(gvw=*!iXeTnkeW-WemUo~tmyH-672z{#t0>drw zDkL3OJSpXZ;h_R$6`^Hcs-BEPE2OO%;Z{$0H*Q~89^&qG2=Bd`|U zUG6(iFxl&umBN$a(v=_<7e-C34^{~rgw<aNRhAkZ;-2O=;_D=Hvzs;j5dH+a_0o&H8dct|kFKz}eOME9Sj9O+LB zRlWlg>6}lS51pJ<-Ggg(UFWthAwPcPE?9M7rt^h|hquRfd4UoEIrNnt9Xdv`RH0DW zf)PhLkmldjV-_FMEJsHDSOf|s2Z!tWmq)~N_nk1xlFG{W6G)Cq{jSMhs%1s^Q#ZYE zjPAVi+1@uW(4aRay`00o$Ou0iz>Xr_qH zPa6>`kGZ}XV|;;-ErL^d+&qN&Mz>(oqfg$sBC4o3r>_b}0cS;FR>y6yk;;WO(07~5 zZ@$O_tCW?;yHBJ_?@>|_m5?kT8jKDf1yK^qoM0NSOC7JSC@dO|;xQe}vifH4B1N4T zWm=MGU|W^5=ZB_{#v-A+sp9c&#Ue#ZHA`EA{fmm8Hiu|c!p)M6$e3!3s7)y^r#4oK zAIn;bdSY^LaYI%Vlr0HE!ID0ws1UJZcJO!39xB?W_Z{-qUpES%aV$8GqzuVpyilN0 z2(ix0Oa{ct`NU{YX7LD;>1XFh;zj7nNs#ujyMpaOu5Dnw>?NQwp(l6^(@QHq`DvmN zP=Eb2&cTeA7XKF6wz)h3mSTe6(v~fPeqwOchp2V>lj&?#{2DF(N3iwW@JhT~5O{tO zVp`g%5+r^rRqCeyShZpMt~I-%t7z=mny^vAExyn4QekX4M6j8At*|bh9TpJ@ zZFUDUG80b_>8x@iF^_ebr7>$un`=(5%+R&jW{G-houLWbn~z!n{S(@^P>>KfQ-wZ3 zx^{K11O|0@&8M{)DzTb7*XfhCF$pB`vUIGdF;n$uNIG+m;$td1k+Uw;gsssW zAS3)blGuGoI`oFfD3RV0*21hqS`Du@Lwo{VE}CZ(QNo~TaVSE>AKijFje1sPUZK@ev&1up=`Av|(a|^4TQr^i>s% z>2hQ?ggh3Yj%c4tX&2fE_gHU>3#N#?yjfw;6XwYj7QO|wQx`$$BFnleW_cwhM1t?# zi)C5a3~6Zvtu6xJgt$Y0n@?y)N{gKdo%k#+>H)VOMXgc6636o+G7@~gyFY?}w!q@f zvttSm7k61z&DT7nI%@CYthjL-Is|fOM$+s#WM`OB4))vG8@QY_)DZ7k|DK7QglyB5 z-oBw6(r#8s?Q}fF<+eSNaM+4uq&%MtawR#1nmAd`+(3?V$LOhTW=edcd}}Y<=U=nP zyuL8H`6DDPhfL1AsAT9>D9C@NA1`s`59mIWcw8Gyu#-K(}`vU z0`OrnJ>f^CI5e`({hr2F(mNsN1Ie*EjBSvcyObSd-_aue~Q zuHWJhBZ3k$1A|c8W9I%+-TFVN_TNIu(V)QI%+Q!ikA!+=SNtWV_MqCDT6q5)DU7|{ zaYQA5p+Ll!y+h)Xkb?zks84VfPW1wBw}x_rp3!px`!GL8OqYUKNc8tIS3PCYGS4=) zztEA9P$!pf3bPSb4bdPG*)mYLD2d5q)o{1;6MANLj!7v!6y>Ed{MyDdJ#r+h$V`rP z=Jg3Cp^jO!H8C!SoX(BFGWu=L_@)-R0I5mv1GJE5Q8&B6>g>&j4Ox6g?BRA+(##X_ z!LYU$a>|SP;pTbkrAVxb^zbW;Wo&FoFNhe|nlqyHJjVTj92UoJ{nciT8r>@1eIXR| zLCLH;OI?U?1sIi)YaFX#N$AD}2zB^yipgvq;@pfSv1K}BacQ7jSz5g=56K#tjw>(X zRo0rRRbi_>1&w(&P_qbL<7aKJ5oDg<8+fdaW<>~RdCUL?{Rrd)`~2coR=>6~EVG6t zlPF^pm6dl9G-TLCRya!(RaD3*P&*?2LOUlXCjcS~pv%UJEGvHZtZqE}N)X(?Crd!5 zViJY`cv(6*leua{__FK6SK#B->7wi7o!ZU!qUEq%6Bp>;9A3U(Tw;_9jXQt)31lHq zJ~GK;KY9!doShS{uC6rcO?mOBLiP)#lg%<4nDw)AMv(j-g}cr>Gj-(EWOIgKZVzE> z00jAOfnNudRhOy|NK-chaLCJ7^Gyan+0%U}>kOH6z6Y@=c{9Ayu+~cE#<~W|Xgc!- zPc6kK=OMWV^(3)Kjo}`1Ad*O1k*(27z+?>arN$juu+y$qEQ2&KPq*Ben3Nx0;;b>p zi3@5XCwhDl$)1)_w4^SCz!sUZI@W-49fPiAiLQ4?$Okj4MzF8+gVwz{qnL8m;_>Kc z1zS20x}jvh&zE3aAA(%_TvTUUud3d+i89OY0P?r$fHiU*lwsZcf~K2+AzFn!M&(<2 zL_iAQe=QY+x77$hqWef*sSe?o}pWoT~V&S=*9ba z&lGR2G%0IQcF~Ej#%{c%?Akcwg!^EaDRkQ@%S4rcc->au{PNk??g#3~=JmI;crbWb z6$z0~_l9t_1)`j$djtLhJn((R+^_4FA8iSwGW^=#y(H%r4-lW}InEd$P*@lW<%6d{ zaj%l`mrvIOa;-J`Le=7{(^{TNdM{?;?sVq_dyVZ3+-;a0r&5$TkP~BV$ja zG&$f`;QORx)bzJP9g8D}@vjcD|p< z)7>yzLoixQ#OMx>t!#IOPphkz7gJO7EcFkIqCBp=VqO=OpZNHDEE z!b&VQ5^Rt-ihtx8Z($0W__y3kJX{nhyW9*7p6D5ynV34ewV>}&s7|NMt!K<I(eQ%Ly=RQ9Qpvz!Tgob%EIiGs-fXuph{JmK+W@iXM71TT*I(37VQh82RzA>c>c zvcSG8NyAe-uSVZy6~)*}b1Ft%kzWHXvlo|x!&ZI`0Fg|YFq6yV+4pI;vj@9^v-<9p zsy=0WXk=thN=Iii`x!N`hbRC?`Ut=~Nr{O+1@?5mTy{-0+OA=|;z=VpelhWsm723L>%#~*#hVdfujV%}5p{L-X~M5SC&Xy3NY+U}znVDodnEeazN(NSx{E4x z3A2*sl{(wx&ga?k0F<2Z%3cfl+{#}8;j>)@zq~PBHeB{(xl4z@4>+P-S|qlb#x{P- z%aOL!rS(lr+zp`1rO4P>$pQG~f^Bt6%#N+Unh!KNol+o&QlMIKucAlFq#Ww8d@Jf0 z{#!>18jtJX_`G1sfs;*1AZN6zpb>92P|F$9 zKRi!qL6dS$cX?zSMmZ4JP2l%JANc_l%KT?`Wk&FA8*yIqf*Yk9A{@){VgIpsik7UG z)PJU196my2f+;oMs1cOMec1e2j0kH)UUX}w6D+Ac11OLs7y^&VZE+wT?n4+TAMgd6fa8kB&Y znTZa{$s)(Ij9k4VBL#Kz>@JVv|8W5}m~+=XY>U0J4Et@O1r@iXKaRxeDNyfAqWT{b z1^o+@a&ayt%Gk8(VQCqcvJ35~Qk`KGsHO)8m{*sE!iwC(vm-Mzs0+glh0)-xIUo`1 zBYh==l?lTqz8L7sFfNGWT7<7nF?^(YC>Fner;I187 zPj!c13R(*f8%j{HWdFokkded?-Gq=owlUQIq?UQ07M&d*Hnw%l+1_uCL}xo>iMtqA z#|icHh^+O~?X$9@xgMHklAux!NH0fG{IZs;HfyuYHrYi4Ihulq1kulDA5JQg-IdgA zVI5(1ga+{#H_SRxR|f8Hy-T7tHgsK&1B?e9{2JO#-}dKAm4VJ-jmxQ;9qzA^l40It zF<;+KH@~`-GX(c>a|;Ooqsz?9)M;~PAZq4|ucy(Fay4#iy-^*DY-`%wE=I_BM zT+RdC&utr>M7PM@JCN+%{M%9*8d!iS0xc5H(YH;nOaAw3!Via@N!&=F6LnCTb$L3> zD(2|!p5phmBB1c}ZYX{|gBVPR_7GVR+&Qw85QA8|&z()ucW)$6t4lYNY z*l>V+wGGOL1IN;?tN&LJe7gtUC^+mW5XSw)ga2S9MyY)jiX;2BVlxM6Ek>_q1Dm+D zj!2W4X1^5bbu(_jwSEiM%z>$d#Z-cv0#{s&cK48~aD#6gg$hw5=DB zYM2~dUry$XgH98L5w-WSQcNNs8`W`1cL?>&j4a$f<(^u-Y+?gm(V$VX-)3bHvH~(uDo~92DJZTt3?3@o zG^lj)Y#YZ<7Y!DIA>?c;79on%O`kYDAFX^%Y5e`|WhI&Lr&2Dd`K9mu<(70bV(4t1 z(lS#jn7lK~{1YSsSlaJE98}T4*z)+}%k#pfB@uU>o5!nCCbknQCM1Kj16!mBNm4x) zhsLS??kB@MjzQ+R_9})I2&(4W#PYA$UGW`lsJWL^cFr1R-nMU1H-^fnZ{ZVsspt__ z!VE3f(@XRHPcWwHB`OvwIBd91%khPStFDo1w`Y^CZU{3j?Fi`%hm$lE1ahnuLsP^< z`*C%|SAOwxvxC^SHuD$SS#?d3gW7C>X4_`tbd~QP+AaK$o?*xA$=`ULFf4&61>sG2}0R{jlYi4dCH_8^9K$ zZ?0#{pPDIgeE?wSLhbqj$8OGOB8}bq`M6k67yTaSoU=InB?s{VD72Gr`;ZY7d-vOG zw&w}8@5|cO&Dz#`(&nZfFvvE3k|WSZ1Y*&)uAGI11#}`!isk#^CBQmpp*?(R9h;gu zTu|39tgD;jO>ZP&(T2ryD9pkA(SJKG;3SaHYIh zo50~`zS=*Yd8K*3+M|z1_@&CRY@3SsmLet5NBKsj02rL^CrN2Q-t~y^SX4Sy#fX^D zpn`g!V&-=}YD3m~)?u2vi&V@R^CL|~ElEvTHqjZDKn{M0wa(x8*{X^p&depcw)O0P zJ|`8Cubbu;H(;i-WGx>b2LHj~S&|}jI9o62HmflJ`|4kyf{YEBf5>~Bm}o&t+RYtM z*c4T1NPf+|>j9q+c9({BbmJ1w^IEx0u*%aHwj}bq)_K-@GMb0gmJdvbLO2LjuCR(Dn>vMg3) zG<@zfIV!K}=dVh+Fv-VX-UtqbIZ>#4Vm92c!o!=RthCnBERURH%V7eMKW?SAx2Lzn zv$>;HY{6-C(Zb*5w7e3?L|d^ONhY5<#E^vlWSLN4C<4Ja!IL*#0Zy=gs$Ox27nq_fY0C>&?D@Wt81(Z6jB!>Gsfat~#r5G2RgWkZ@N-{#P5#)h8Ki5o z=f;Ci^*laf$qikR%+Mc{qcW`FU5;lRk3BGEvwo;09nM9FzMWuupp|$PT>R@-IQI5X z+#abWCc=(D8TN6)X>2jM-$E>THU|fN*32f`H9$$t(=F;~B**7TkPxDvEYO3>NNU_N zf3H`^^^Y?J%TipFXXbs)jTFMQEIxI|>^q^m_fXL6(;q}lSy)TKFGJysGFmXUr+&)kA&2joX?I&})UlC35-NIK?ROmF@fBG&? z=#xON-B=A&dU_uF=vkg8rR18_x0SL4&)r~1(`#!HbH`+C&IfjnUwfjC)%a7R|IvKl zSfIr0>=*%rc_sjdAS=~*%llT=)5{C2G7Jyk3<3nWT~P!pnFsm3r($G`EVZ)z%fc9B}Nf%$|`e}N-Iz}WE37Hbj+EZX_!t5`UoA(PRjTb72L~Ikx z1my=_TJn8z6T;^VZCbTzsW&QWPnX|L6B=A$3E{3<3FHY$hu6uwV;xCNDoRO2pVT7h zRTCC{x)ZgVlauKy@@u;@eUWulL`}Vr_sKt^v=(nH1j$#{*5T?rU-EuA!@BWWSlJfd zSK}v>WKh}iJ^kQDrTEAQ2dlHhI_qP@ySQyL*S+{86-OFt%6iDlV}aYat0Y0Sz@y}L z7=z8*i6iRAXeJ8(?L6FP#FpE-k6mjNZ-IN%6GJT9XvJvP{k2UpK+iWj-kBy*BK zw{-4XVO2$-SvueQ`p)o1BKPLoLX4hHevw2L~NftHSkH41^J-2wr z(-DMRLKt{%)Sq@d|l?63fmW6XEj4yP6zNnl6;G3 z$2l)R3+QG_&mOdshZ-a&^0UKa% z*9s&kZWnO!SB5}d25^?*l9Cz?J_Sc$GCohVC8edE0IYz&1U(Nv5JWsKrrnr*fDKt~ zIY*I1uPr7a5&9s0^>Ca0foCr-$GxyM$8Xk&gNy6&@%BJTPr;P9P^QkfDn%Z#!ubX~ zIi^X_fYCLajZT+urqkghP(Ur_)N)`10A3U*scbgexl2?P!JFRSeSejenZH750nD+1 zfr0*kfsMjApO$gX>YC4HsLP)_1H`73-mGZ1+mLE$7eAcb1i^5liRm~D3bGH1{V^B#)@8Ib6p1Fq0r0e+00`!)o?ig-e&#Jk-$5epl~c+|Vlj*@Cdrp4}n!#~GR+g5hNFF|TisS{Y~|L+q;%Kqdwk z1bJx3nX5SXO9u44iO*giK!NF93eg{gY&JjU2NV?ub>&(OR*LcftvY*dW#unDqtSpi z54zVTH1hu?7PD{!d5v^r)zsMR1Xoux?d8P(E zi2qTD*T+P>R6iGSQdfTh*o|5Q9(%~GZEX%bn7&bNA$>h9PKWM$vEuH({h=^k9|Jzq zZ83($#C!(W(RNqGU*3)@lLSqy|05%Je%ojr0LbT*4XO}=&~yd{U32>S5pCc zv3HN$<6rqVHzxy;1hDS3x&o?~`Sj7Yx3`UK9sVO6OKQ?#*r~NaTR``Y7wICo07C$H zlDRo0(wA2zp3HBPWrhDg$~uX%aPuKRBmSEQM2eA)CD9uLn^E$Hz=gomz3t4?;E$=I8GmEE8o<eaF1%>bOmnfn=MR9FH@$vm2C(Qr-6C#bK?N|C z>9h~zYB2>74XHOy<3k|Pm#>vSF&{na0fua%m5r3 zki)P>L!-@=9(dwvjak)Z^QX$_otNDf`TwTEh^MsQZUYV1KoD(K_bvp*C6FtQpAvY9 z1h{59_FVQYI)Fjpe+SSy!XH9{02lzAcrZL)ZGh9W5Y&KIhF}vh5dJrmhE1nG!jv7L zgBR%fBi+G7-l}0a!TFM(_s4xB*e~Mc)P7n=N-{m`@g09 zyLX0-6p-via2D?GYU60_DVA%G!4SSQ|5)JxSAUo}I3VW=G@(2`cNr5M@O=&-I?1Y@ zFAApshqaeMUr%wR8dd2UvXfc$8x08W4*56p6+5S{xE*jz<8yWwwp~Be!pNQNZ@+8N z?wS+jOyx(<&o1&BQmhN4U0%i5Ni4ed#-q7Zon2rMY-VY0#;xc7H@OR{1t{Ovd1sR_ zbYA>%-Fd{d+6qx1TUBRz<(Xx>XfHfhQT+{EQk7`)>## zdkWXbB3zt{wUO7}!|>)}W9}N|b2w#kP?Xo$Z0o}jkGyAoL`jPc5{i=cNB+FhNC;e(| z2z|ikQy{Ldlg-!PGEu|B!=m&}|80b~E(B|A*X35W&pkHL>QV1&SSzJ>YGh=j40kM5 zH%p36tY{iQ1OK1UbWx?jz7ZU))yse$T)3c~<{w^~qH;c-Ik-tt{3nyXaE~(vnCt}= z^ey>G)>R>!Y^DxHp2L6ZR$X6oDiRKikOGX*X#;9%D}xix;`_6t`@es%PnK=E$vHBC zjgXw@-`3;g;NUP_4Bk>eQ`Cg`r;gj3#DJJe$S(5D$+G3yW1wfu!sJPomr3_1?10^0WGDQxBxrlw8{{LC*DT;W*@NN58k?4; zeniv%r*@Ah(DR6d=`~RV-z$RQsncdJxY2|-l#~;M>XmTp>}aShGM0a5WMi+@_3Abm zxX6FT@NXi3EQyE*!-X4GOBQ!le5rq_$1ELbTgx^IWKUS6qAsQS%0Ixc*vdj<1EG>2 z;?-oTqKWOqb;R>XMSCG7SWL0cPQr`FwMlQXfRYUHy@!V7&J^pw0ovM&9D;g=iX28# zDLzJ5YS-X*t6};0*Ep7v8rgPnv+9}zn+_amGH}UK%KQSz{7b_~kQG^*ioO2?Bh?(ljV&65kdh@g9uM{5Tn+p*UW zWIm8h3f2tL4YPlkSc9Q!4MZw+Owy4 zpIQ~JA9P~*PJ{PnN_Uu@n@Oz~FugWUvIIOYur`G?bbXmw6s3~XmL}6)`kaL`#03N` z{fVnxcZgdtwWn8l2Apa!pHYUkeC#70vS4p7c6sl>gt0xT14omjZtgD1PEUeBf5Cqe zScNYX2?>f(FwE#&4u&1o%A+nD7Umvc2xK>%R}d1NQF+2sjz?fmFnKh=-%Fff$?dYS zr6y%0#wkP#D2Sv0cB8A<O@hc+x(@B0Z>w!V9+h&)ejZRki~qRqUibdUrDn_TTaYe@bKrgg#O%%ai%2A|}Xsc@~Sq!dXdQLKD8 zOi^)jzuT1OlL0G_-F$9NHHV6V((70DmZ5#gFPc(pi(%qNH8C{%n zzdUV78k$Hnc0?mwW`$h4LrM?RanlkajZ*5eUNr^(DQHx`MvT$1C=>MrW>uk~1e+){ z)8y$)g&#c>q}tk4xxwvcuzXu!j&9i49xf7vQCPN!cV~=qbM@y zUKqx3ES3ayTuDpL%X4Lz7ja@yb&YRiVr)MRYxU=mgB`mNJ;5C@9Kl$ZTXoRl2G;B? z8t$J{Zmr2#JrbjVld@1$H>e-cXrm3SEX#XInqp|gkgAntqEMN!O^mHLPmpr)we1Rm zB{fEb`e?Lyx$5W<$9_HU)O5QvuF1F9_o z%De*X-5wVPVE#BvI@zqxCM#*i1!%=7m?tY0Bs1ec`P^UaB}iV-zA1eNXPx30shjG5 z+2~@C8e-*0@C+Xfcb{Js+U-Y@^Wi>!dt*yFDyFMj*OT*bgWcc9RxWD8E*mY0V94jQ z`{f;7x`I3blv4Q3^cwuQK<-zP{C1<)2I-s1tb1RwB#1az+i3}>5j2BfFg#tJQY*Ir_ z(5x6c`94HE3!fHjLQs$hin<7d;e-CgDwc~sr0jdo`mReh6v95|x3c<@;7>o)^9NNXORF}9oi;k=#>RseOmBa<& zL=~oeXjE%_Q*;hk8mSZ>K9&Z-eB?aTD|&`!@I35(E-Em!NHOdNz00MMq`MWgJ#zP^ zkwsUUaHUW+T1A>fD&^6qZq8MLa`YK8kuB=TL+O!Qh6x=#qTGaVKHPl=t32f!D(bS_ z0j>u|ddUj$Ds_28I?Yq6;mYr|40O@VgJRF6(w4ggsfeYXQklR7HAuo+zTC-)GuwwNFHa?CTR27Tx5kw5FywFBJTqUD-l%V4n_YfHY10#s7`1D&qz=!zW`{|- zc4NH0_$Ac(;1|*m$=tZ3=-Tcab6FzoagM}zof$?=Ft-SXdv3b2rz$wo)}dlT1=hJq zyI6TR({SqHJm;qIC699VY0JT0`?#G&j+1~$wA86^e@rsvG5ptvaEtO=nU%>=lx zda*oHTLy;f{H+A=E&;-aPzUXIe*oabU;XS3Hi zkVLo$@S9o2xd*D7*Eb_9qeo~rQZyx&%(HKB?B|ItZU@^$kBOQlWytw6I64(Ehe}>H z^~q*2>X@>LxtZaSFm}!BLa|yTYjzL`*ST!9&6EF3DOshSfL+a7Ja}yITNj6kb3ohq z;TNO+;02y^j2Lv`ro@mNsi8oe?+c+lCukWbOsI0V@qv_O~nuO;`DxQN%?wo zy&fJZ>qx2^=A(x~r^8p4+9sPiW+#sxt@tP{eW84Fi|{i&N9ozqH_)$fN-%-v_0HNg zE9LMw2r>jUM)@^V{s=II@oF+N=H~f?<+iK2!jV4lw(DVAfCgKeSf7-vu=1_ns=>&DF28k?(MuZzxZaLZ#}VJc4~gFAOZU`YFeT^%2f0k{iVHe!sh~g&HcTB@T+OK7xgR( z7_XjJ{9&Dm)eQ>OTOXG3B}R_U0mIm6#!`!KTjB;%vmN7@J3_?NDe@8f%_d@AyEUBt z?gS{UDYX=tC3YN7I?esbj7CpuNs;UxiM7&FcQ=|6&H_v=8d9S{#xy}8} zwL0?ij^exe4n@ut1g|-=`KLl2PDBT*s)UsC(DS?$aBw=2S0)05PsDT{kYvsu(@17w ztj|4{VOv$evCwhWDZZ%(j|0K+2i7gZF|lm%^bcB6Qq0HA$wMx$CztoLB`Iop+14MN zw+&6=SlHCt+fyIvowI1gE>84`iHU4%mlU(Fww({fm+20-ymp)I9)DrhrZt!JlZf|* zwU@^S{TEyuqqP!R1XRmq!4Y!->ArQY>uZ;QKi=Aj_2EN6qY&`K^U7_j zabaj=B(QFos`7DaC-x!&1YN&k>=vg*+8xiza-v-Ov;v9#c(i!m9tc4*ca5Wwn_`RK zKAYW&neN;#0^W-O!NNg7L5ypY1q+vp>a4PME{+>L@X9#pgm*vMgfFkI=x33(cXpuR zDE^e?^t%`hu%HP^)1?wRv~uCe#nBvmO8x1S@^okR)Wb-7wK%;(r#>E=Vul4>H+@}* z%q`9N+n!?@<-SnwoKc_N&d2$01y*{73r=w>MOGLWf;(h8E2id1#_4O|DdG#g$=37? z3=HI}*jJ+M!%`t#<8SPq?P%oU@>`{#&u7{0`HXcW8A+huT21dDZp~?5dL7KIBYyYe zL5R~-UX$~pavy|6|68wQ#RkLC!Lw<@TTSN`W&`h*B}F_7MLl=Vlr=cx?twdglC3GW zkG*Jr)9w!6kO}D~e^snhnCrDxagVoQ2^vVIeM9)GoJU$(dT1PwvCs3J)GFVe{kcck zAoF*GF0>t~88&Mr<_{yyF70hME_@0YJe>OOzsd#F5HLn(t&D$~$sdf+LbD%TcF~6& zIOBSbV_iXn*8GHyr;WndIx4JNqPBi<$W83emSWo-G-duTC~`vg(+{*u-}He4g8$d@cto%AcN|W@d>!J?94Ihj8tfe|5sf-w63R;?uCal{JLQ zILvxEy$AfG(Q7Bcn()n}IDvu-f(Ns$8LUq?szuFhnW4xw#rn6Zm=4UI(^D;;hP ze1EW!V(h5ZmilH_N@VcPR;8{AQ>$I4#?JamEKL^<<`Y*U`{1>In$iaGaOV?=CAOmC5_*-Pn>N3w zKRA}j?_2HGP!q4e5|+=?vQXWe(s0h(@*qukaFidEVY5(kq@$${?as!|!Lx+do_7ia z9*-xDo{uM3g1WhLyg1gTBfA-WY3?{AL^+k#LX!I5{ehvvi_=v=yw~0o2qSi@g2w4b4F-IGz&eH%X(%*+Dm2=^nz;dkyTiqq>$P zYz}L7seC-s`b+ZW%*w*E$_f^9^;an31WUa31w|!MnI7pLWf8NBZaS&%O2~dM%G;t| zILugsPA-_(p%v-DpvX@OOQBaA)R1ZCHd9bRu+SD&j`pD^2hmkGwInZMtXjv_Q1`9v z1zm&fnk4t@Z`PKp4M7&S%5HA2pknF~ak++Zscz2+Ufkt<_57+E3Z`eK-ll*up-mZ0SIF4qY>`tZNfk{g%{t88oh}GQoPpaBfE+}ZX&|(2<~Hax zzoMoqX zfhg7U?bUlMVmBA8rS>2qGwB!nSotuAXCujHZ>(TJ50RjGe^SU|V{4mdttOU0F}aw( zPf1Gk8mAoFtEH7vM?uMnUsYXbR-PJNvLs^Q;NaB1LjU+U0&}rF;^uC@M%ET>BL$at zRe~8|QdaGw2yOu{wxTx6?1H?r1X`@@4+nllCFeI`cQ&mipvn5uASIp;R2%FY4&PX8 z!}9aU@gipm>v+Q?^CspbY>r6>j8AOV)MQ-Bm^WH%pSYX-k&Jk4Z7CPQSh=w}#Jr7F z#+vli`70oo>+k$mjZcSKKNFLaWmbDss`(}*RL1Xg(CTbT^IDsO9hBgdMwb>AR<{VL zUZ5o1QNJ4@R;kXd-=nrYaW@!qrF?Wo>S%t|m8-PxgOybe$mMpT-kL9Gbjh;hB)_~p z?UcN@BtzR9O*RHkIyrxvU8LM*wJs_%_*0A5!~W4xUQvDKdj73ad8I8%nvP(}!m6^$ z+P<3`@2K0`m1B!VMQb0F)|M}z>+8w6=BV2OTe{_{jIm6}UHdfCE?6vl_cfmBCH>pakPU9~fx&7y{k zggV*@6dV8GWuKUn^L7taVi}!@!lPSVky3du+jQK_ zsmA9LTHHUj%PX;`FE@VH&aKbeIRmFGA9B-l>6u0^GFb(^-$NV0#j4R%l2WKjvea-&hacim7Eh+@j$LYP zbfsk+X#v&13f{#{G@FVb*4=i_Ln=N z8a$BtzF4wQEG|@+`?_e`@GU_t%5G7SOq}68jZ5rl#Uzo*rbw&B!h+HdQ7<~=X(g=WZaU67M1|9V9^$# z)`N%{_UYxn4oO`85~c1ZM>72n?`;;VBk*-B!-Elbf)M9yX=m#q^mWZRwoRJCGFo*@ zPHYP4(q)*{%M$PDRq6hOZnH__5c;6v?l8GVAeYJYl!Sj?{uWX-zN6zuAM zvre7=*qT^g|BetKk=GDJmK8QNi%UydZse@XvgBr4#JD`Lo60h0Mx2wBMAj)|a*O|o zC0(Vi(4S?gQpuTQr6$nURo^BoI82hSnz}8qVU|TwNVj;UE}mRkSN64a!-U%OaG`9d zsteZ%&ywZRD*NIvbzPWkc_*CEVW?#846pm-OiP3<#^Az&hJwPQg+N22E&XCciUotD z0fVBgGl6S)s|68;&wI4G)74*v^Sp;Wp4&kr^NZ8sN2g?S2+C2swYBxp6}yYomUemE z^-}Yb4S{uzpZ|J{axHIn{`z{H9FU_SM5wQqK^SA?2*!iD^uVGKh8J~P&0*DR>MAL9 zrlT7e8XphI596$`KUaNum*4ueJNRV~8P-N1c|lA@b0V96osw=^MfsA-Dd~R_{f`L# zM+@TquN1*-{%7cv?Myl0Wgd!w5Z%H#5VXA4EXJ7vQV)5>Q zC7j5ux&L3b-g4w~mA1CI++9z;$Up|#k7yn7BWufDK)Lu#PAXT_f z+Nt=W+}<}LHnzH3thR0zW=Kewog0>&lauoz_+l`2H+0e#QP*YRm;^F;WL-_(9#8}RwMo#~RmiQN`oGGRZfD{32)TY{nvqVIva0euc z7<`|Z^9g~|%*ZxP|Jqlhq!GtBPr-IYLmwXy!{L7sv_&b|-~d84{--(njt&H8FKwo`-6wy zb;=^~B}Vb^Q@WPkGiQqq^V3;9sW{iY_lCs za?;54hx6r~i}P=~;~#{=+|8x^8qkvQiJRG1r3-6{#u4CBbjwT*Yrg!G`BYTvB$1*z z0tOi{opF6=MxQoWe|))VVU_KloHI%e*Zpn$`#rj153c39rO{uLEy({^wW1c7|8>JZ z?fkE_tKW5F8B=()&yEcgP)lc{{oN#}ROKk7y3STTwH^vHNGoNA6sQ^F*z= z`D;brV5WZu#~r)5X{T8IoOQvmHaRee^UJbbP`su8SA3NCWjv!q6r+ygvJzbQSm8%0 zr>@u%p>Vj67qfD4M=?2agDUoWCCBq z)>rZ5^lLF(=TL*5TA?3HpG)(cTd#~K1OpKPqH@`70Z+Nj+Z=sE;sA+ubGfoR6;%8u z>h2yDC&!aF=ACI;Lcp7)wEHDM=C3C&2ih30D+v(nf>wKUEZRr1w$L3DgKqA_13cIg zzKQ+#n^t+84;(j#Jdm}Q&DS~Z3k46uH zk)y*B1CP9%ofS3*1s!WBo{aH36`k4dV{$?l`21Gb|5}rKBPMlVzxD zd6A&qhtfHBcRFH;UVQi&Ug+hVUd!f} z2q619+#;i`i|+flbnm&O*?if@<@5QXUf%I(-`x_jinUu&Z0_req^SZ8GXag$)9YVk z>GSy&{MOr>v#r535C4w)`Z7-wfgFEomorcQ8$XZzFjw5Q9`y@l9tV)r4FC;G+Y?ZK zq`)AXzT4N~`!vm`%vE)7Ih!<2vp%dJ-;>vOWMCeSr+g=;%a;!gF42fe8`^D)(Qd z#tBB}+S@+DMGY~sWXYku8hT>km3`;Jk`q7;!st{!Cm*T6X3Nb77jJybRB zk!k@qg+Z>scHr%9ZE%2|>U+;?<_CJ@IE^c$7l27mnvld6soa_mC4b>eiBO`6EZ27 zNmE?94)MVbbmfXi!bz&HlwUHu_e}+Ef!jtu17JoN*ofEf&s-d`&Cy&Z4Nv87GF0m> zjfERJXc+f(rCg*KPaya3(d|wFx572YI_Dq-i_P$s5@YM-Z}>Rwz2IqG+rQW%ur-Cz zvxA})LXv9Tnfh;$s!Royw!ZbJ z1NYdw4L9&{2!Z*B;w_&m6}QV2g4E{BRs7mGkH9>(vx2=DOV(=)c10|yh-CFEaq7u@ z!^&}2pUh-*mM!oA1-%eZy38F0`K?x+AN84G;>|M>&fn7*vI5#!F=qmq@XfK73$1@z84>=l`D2U^v>wu0LW|HL`( zY>7VftaYmk;@pUMH?_9#{Y?VTf;~jv>j*@1PxrQ>v@4A&v+IET0#jyf{cdoy-IwPh zMpgtb4D2Je>@qunIpVLci?!MVdY&Ko9hsC}OcD7y+GW_2q6p%%#>|+qE>tr)WQ)qrklhIT0IA2zI|SgzYL#oWB5Le zmN9d+;Twhe=aIGDT>5!ELO-dXdgu!N#gfR$z`&Y9s)zegO$W6-5KhnkxLf**nAp?P zm0me8eE&ZAK-4Q$AxwT1@+(GNA9rv0h^+>=R&U)>_8@8nAUykoyVutUN!1Jn>`v0p zM!L!gc3VR(tVtD^_rSob$!Z7dP`*rvwcm25zYK3>QRvKx0Sh8Fa|MOkEdxL=Wfk7a zD_(4@`s-F<-8=;%a1veVL#QOh>ptU&6?h?c@uS;~Pmx_dfh@dV<;r;n3`Yo^K<<;^ zKblBl7pV1&1ZL@Ai44+t;J4a#0y&z`%0@^V{Uh299nn88Cu2nXtN3kcHuJOoWVjmK6c_3&V5j(L| ziX_kece3g94QHkXj{`9Gof)$t#tvc-&?VC;y4s1~i26UeD`1NsdSGt>(Fc>POlIVfD|&@*FAOpD^WhdZUFo&a(tAx1eSQ^fKfMa!Cjsd4 zd^!*s+^`2mWf^F7T@J+5n272LR)vfausPA5SbSkSUY7}BYKEu-@|dipo=AQ7LX7-O zb>R6Wo9YmiGYWvAWq9}Cc_{|1_x$BR*~us5U59TSSK=I~w^`<2si>~btTir4EpgbY1dFrbS?MtqdDBfKFY>!zwKBNqw-r%<_q_{t8Zy9DS_quhFpG`efalk$yY$IwXFZ z63_oDHaTmnk3vt=)M}l3?=i>0@gSc=l~Z8uUSDfPc4b%oy+ki2Y3|uhy0&2~q?O^;d*@!LwBiFd*_g_o}Z?0n>e6A;mhRFk_ez zv3dKJ)rbiqh8y%KKhY%>0_3{vcx%_%y~+uo z+-#@k_q=;-b72~z&e8&IJ+@Oypklmx_qk=H`UmZG6tmn}f~IO5kN?m+2P`Vj87rw(s=B!MKro6S!ho`;xWL0NlmRJd#J#_*vI=nB5?Z_oCi zO--Qe=N$x?%Qbzu?x0##EOHWaYT0*E@b{XP*dFd1`+~RISc_L!^4R5|%4Z67$jriE zb+0x8N``!H2;5+beig?ZmmlapOG!shXH9Xdg*h=h_u~w2g}hExXm~%Z-u?Kv)Aqiu zM!8Z&HbHg%h0)J}j&XuYpC8xNIBg(#`f8pSY$Vi)xux;+!;lhvU+7AqluLiTQd@I! zMK2P*jZZ3C>0m>9Rrpj7f9sOLuEQDAY*Md-3jInorUJQl+~VQnq1i%@{N)?n1#fP3 zMXnY1?bn*v-vOw~#P%l2Mu@>OhU0B1lU<>oLj?{h-$?v63Vjn1wxDdc!-Tu0h5y}_m-0n5fsxvKIFq-F0iC4ejh?oQbArB&PJu?Ynuq3Gu)O1 z5?PC0a9GI?;zQ}Q7IzA5Gxwb4nqu-&pIzFGpH0i|`v&7@OR1dNS&V}NCHX|}O3ljg3@jki{S_#Zxyn%`>Pn2DKdxA%u7-%OBb^*qk0OyV(W z-^0!Pscgazph3Tk_ccc<_Y24P6Z34*+7v#O%ir2L{cAjCt|muNa&nXQx587T7AihV zus@#)r)o#Ag@ahT$Q~4G`GIlH&+mqlqi5CMsAO!&Uog#zR$1k;YBqQfeXJE7`{1w3 zS#Mpan$wEcaQw{yy0a0e^r3i(x8>NratmYA<`x)O(>S}oJ7H9uF8g^9zw?sCKp;K4 z3``_i$0|^+C*`sNhe|3U{e=INF6Qt=ksO|U)vfk50@QX+IyjbGeHm7yl_pGlI?SUX zraHbLv3KTC8sP71XncF5VDXd@YzZ5Vxl!;|OG9u+-hH*_QR~wKd~};rz@gB*RUTxt zTzL9Mpaole!~Qav1P!`l>_tD)YL1lSs-Tv%!{&i;rt^IVZ93d=?=0RCyVlp(LEsE{ z+);;*1phlj><^m;)Q9)-EGv1GszB}jvoX2wF`#|VKmT)HcAP9tvl$@U#uCr@CS&&) zJ`_qOWo6qO_9s|S=NET%#VbHjPz(45&Ko3TE;dnewTcLChPp_f&bh`q@9`7XdJEKx z2p;W3%xqYC1E+ki1*1=PE>?W)E8XdJWc!{4!8x{@Jws zr1ukZf>>W%g5bNaHHSmVv`yyWW!+hB7+O8p=X=4)rHRd|_KPKz>T~b#K z7)o?A1)HqynV9y@c81E3N%m|8=R9{TUO{aj{g4PeFIXvP>z1}^oDN|9^6PL$aCEuA zx){;TxnYl4L$mwIGy(sV422_{-~9e&2BlABZboA=&;zC{YsfMA`wrL$ghFv)Ropw?UF zr!>@#f@47d)D_s@H7&IfP@Vcpr+-SO@e#v7nIhCTN-bb`1smBZ?V za=PO$B?q{k;Apwc2pW;*3sc|Er{hml@nJg9}N?*qW_t%~I^(@r#iMF95p7U!G-rQc@@uv`U}X^Jx9+X1RWZi~In$?Au8Y-P zLrYWO#n#h6`1F(dp5NFt21k((o;3-8P7p#xf8&z{_ePnE9rugDHG=4mSw3428~LL6 zzq2}K>Y9qoom!D*K8rTUQl*I6@ULV;+S#RXhpK>~7U|i_O5*47y?wFs(K8okJbqB( zxFh0M)Banh4;= z`E(pjd*HX`8aH}^06@-WA4S6NvH^DNaq5l@Z#_MM_&Vy}x$v}IHUsl8p{ngJN%o8k zRBcUGpP)N)7$SoTYeL;^uGc`Gz#zA6e2~k}HBZr6o;ELE% zU}A6%aNvnu=gj}cqKUAjs>*vmPxtx<4(dCqvBb$>qB)vPJ<6{1_5B`!Zp?c<=;-z9 zDC)E^+u@iP1tA4Gv=fS~=HoS|!vv_4?ZM9aZOiStxlPsX2}IKb|6(iqJozwC&1i~) ziV8zuuz63$X6gaN(nRo|dWC@f-O#Ijk2^2`WNlJtnzB**Dz7peD zvu4Lw%q^N~#|7izR6s+?Z?wW$y$~FvU?k2WcH4e&tQ?BGa$gu&JV|_fho>2&O$pjQ zWq5{12ipE_ruP}0*?iT(0)r~f^=+9o)gvZ$*VNoL%e~{b?Za>12sX)o@9DSJx|&cd zVo-jIO@l_7%H1(AN@Pr(c2^&gvk)0d8Z@d@qOaS7Ww9vQVzaVm)&x0@ixI-0OIdD?fNy#?m_b7p5#{76X1*qTT2 z*1GJHCI&G1_gT~yU8V5yIz&#lOGyq$i;(lX#wp;|`t<)czALqFxZ3h(N)kKg=Sk1o z+d((DAH($BF&7|ci{SzI!LS}1Lq^!D%vIoHqGPIr9lnp*3#*jfwak^GdX z0vYl868fFDPcvH*Tc>_f6)#3KTjj&9@#Te)ig?SYNV+YFO4Zn!(kD&=ygLE%5nELy z$N((!nxB$^r&iFZ`)<#g>SZs`B>!Bqsl@@wlOX!L#}=n2ZU6Roh61@nq#n7+-6qAo z!Y8iAZ6*oxH5!j&M&!3T?&n6%UA-fXSwWx;RWVS4o3N*DLPAATpZ~cWD&=t8XcF|HSwaq+O5F~yx5~2*LEh}> z?pyMfbyRzPLk@U{EUWmMd(MqZT$qE{EeiQVbse^#U-yDKEeQSNWoFPThNk4pF265D z%3OP+M!V$|@(;%n-!%DZj9jTv+oje@psU9&RgGxjVhj97IS6zakTfcF25jVaa!AbW zfChWx-24;$xyG1Ty3pCKo3Y8g`cVf6?qerj@YwVciquX)$@9X)8kO=ZseZ1|*??P2 zWQ4CUIh)ww;R=X&w1z{yxE0>%7-a^MA10yf4eow4FKph)i5qTO3xPEAVYKe1rwQ_R5?(Nkhm(|ZC&+qj_e||Q+#K=-i zd%WfvIIHC;j;5#nRbFETs zWLuaCSF6zxIThe-6bJg1wub}%bsVBQO_KXoJd5LJN@R5qW z7Sb`YCd2f%z`UWeTkFaRi=YWM8f()P0GV;%MVQCc^@ouOj;8O!@o{&$`W@gdjpozgoaDXS{nGMYF}pca;t6g-!_jrk&xckBI#dL9sIwfzf0`H^+Ae}z>X4P!F=Cx@L? zyvX!>jL4^3KH2TgqC2KvxF8Ed+?-qENr~W$}HV5yTdF|E|T?E4HOkSKv`c7_T+XYYxDBZYz4T$y>9$96v4*s0?4?DQULJ*(4b5r8kQ zF5z$oYlE2t%Ix<2$iwmxNF3s8IU;@7QX0`{g4%5ju{&=vck1nh71n&~CFLcyG0@mO zHeTDOGi`{({J|pEwO{ZzUx1uca;UnCDacFckr8I#6_iyQxiBsIKOf&q)Aw474F?{X z8AhZ+r3vAiK9%v#Tnq|N6V6XnTQriV)H*(~^ON}i zty-{9inN1(_1wqv&C>Bn^0GaVKeVV>VzRfHb`_F`o+_9890kx7?d6ebbtL3Lh!VS? zv3%|p4K6c%9LW1rii>xbv%@}Yx4S{j51W8#pttG!3*3~BrFssrIM!tiVz zUDk&8od@}$o|CtinIW_1i??)%k-vS-r(WY%H5?d!6BOj1x?n*^ga6U!fOZRP;&9c` zmG0~;a`Qy`FnQ-Fit7_Qck7!BKzm>1Lo@=gHnHWd7jm}P+2Wpfr@3b!xU`9jn}oG+ z&S5pjCjSi9Z;3YahHViPT!8ox^!k74`9oq^Y&e(@^(zoBfdG)R_lH1$m%R%w05|Cq zmhZ9SEHpw^H3EBl)#Lo7YY0_P0Cf@>#)xloW!ycPLP+%X2W z44(oXf-H3J=S$lzGc?r=nJf1;R1%bYPq~mW{niy^6nmU?azV^jNa8v?bT%Uq|3;s+|w}`T5;Z1%_RW?@|&U#+x0Jwt|XE zd!v#-pN4RmTpGG6PrtFbb|*2+aKG#2y$h+PQ@N-X%iTbHopO&`l``9^&%zr(xy!za z!E`_XzVv2f^GS>kLFcMEy-_UiYTl#RgS)NqyQq(G0Z}a~9Pblpd93v6k9TRoi^cxH z1?dqfLL^8jG%^JjPsQy+pZF9QboF>1Ep_c{Qf{KsS1~oQ+37uwmK4e=R{L6SA<5Rx zLvN1m4aM3rT-`rUsL92-G$J{go{UYhLbq~WX=3ypRNk>%-h#aYo&=Lm4f|Wck8;6p zv(`_%A$azNdj`V*`<=8(``uDtO;n}mTz7RCfGx1i`-BeM53t$d0+hB_g#XDMgvN<2 z(r1$dg^d@Y_P?QJ{t4laIGp|z<(H&rtM(pc?!#@EO@_%SWeu1wtSqa76zZwn7?JBe zp3juR6I5@;nrp>2czjsY$gfSsD<}@fixC#FOk;#C^*F!PH1){&HUtNtz3g%kLQ-w3 ziG8EVtE`*eymNH7j#GQ=f~(gT9tB8yAcP4(37rKdSUi}g8F~MS0u%`cL**@c6yr5i1^HSwcEWO z3#p^mx>#5>UxwV0WX#7wewl~umBkLzCU%#riH%L zMSp|7GSp59U;KPZZAs`av6 zGWioWGAfQ3Ff#rTV~V6~FE)`2&_$ni-h#k%3rwCfDE=a`Mce~V5O(63*}I2gxA_sj z=L#yb&-hYIyufJNx@V`a}p`^3cS@}-;{Gne4Bj`@9x zoK`ad=V2B?V`}HpD43nnQ`0es`!_8}rCR#sTEny~A*)9IJm7dYh-kCcEeeAevgA!Rb=@%5)y+!CwZpBt=0DhS8qzixGeJk@K zgfj$#)C7{#u1Ca5!4G_8`Um4`!q?hPSu+K?Q6R$6XU*23_z3jL+z@9pEd{K@9SPoF zxRCXjm1V_~t?h2-?CsZUkhpO_ajW>~Hbm7q>~ zgJ^nVAGKZ5z%p#CYl_bIyGylPP>PEomAV_(_XM0v^mbQM6BCnf7bJ2qYwBJ?(jiT) z&@)h73Hr`cfL@{&@LFe{``-;q;881?B5&UUf$sfekQce{7qrG4->-%7{EB-(zswPg zXKQjj&vDSv3(2fP7o1Pq=n@&Vv`rUT9b3{bv+G>*HTCh#)WMM6-2VDCArXAB#S?d@ zX|dxRA%9geAF{Q!b@2-+tZ`X#>-j?dQRyJbsnv|Ox&JZCvI|FNL3U+HrfCX;sBmgR zNIu;^V8>@Y_VM%%mF+L@2c-PneZTxIH4Nj*2Wf&Trl*RdrNcWy0`~{7AnCEU_h!4g zs{|1+^Pn1<7)ntVxtxBx$5#*_+7Tt>3zP`?F48vy()*he&59q6v5n%-;!gFqr*l}J z37uY8OfKfzz1ysKXP|JehFRh>&=u9+e5O|DOg|;q7|XTAD`t-ITnWD9a4@Nv6alB+x(?D5mIQK4|%Xc(W62sXp!R$V0Nr6!G<6!&rl2INp`2jeX}zFMT_ zIhPHMdMHOn)RHalyI#qIsc4>O;e30Daxm||`mvk_10LbN>u6^&Ef0Ol$*LF`I6;Ih z6iRua#~HWa6W1;jy=30`yYT1PT7}@1WJHwW2Ez9m8$Bl!^r7W&>IHo)7!xQ}uI1hP z6Ssd&Ii?LEZ>$j|8^X0PS@d)|in4(pl{zR_tAQ6j&{sW<`Q*Za2BOFcH}{+h|WEbOl0*Uf9|UX zV4*bp!fKZ7e8d8xVyFCR%7I+Ui4DD*+zTPEU0vyx8rXmUcW{>Vldu08(ZnibaHkwz z>j_A|w8t*_AlL!gjFB|FzX=6DhI0H7Y*stS8g;2hAb#nRQRN0lK`wY=K4Tv=SrTEF zpYPts161UV?V1Q1f1$n)4~w_Te$nlzgqlC;T=1k^NPHFaf9ZNIEfoy$I%bNKBD5h0BLA65f45k^1UTgK{rJjJ+a^aVpcfuFGp{2k^+iVP6 z+lc8?sQ=*b>-=o-*-ZCW-B~)3$A8S!s@S?m8gp1FqTZ;qNh@mW2q>8ndepX-A9{Z} zw3G9#q2$q8behy-FJ0U1m`(vt3Ad$x%v3&guT}Qx9M;14td8z{k>!ctmyVA63ca>U zu_rPGf&9K<_FvLQiH1)k3z_46`4tStx}59(WzY|ewv|xMYey3(dl3HR$A47Vg0u{= zZ518P1~`eoXJnzC7nkBTDp5^v8n4GcvaedjuSV1Vh{ZT|1E5Nrz>sJ@B%;{{(Snc=wRLxC`C0aeWh1(yc__I!-#u5=0tDVP0Pp`2+q^Cy z@x1IVaM-2yxo{U7O*Y&TFQ%vhxZBN~{o27FR!g2oYVi`jZwhdjV~zuNS^&x<0te`W)J-9jPXz6C-oXX*Tg z3?b0lnf!QpOG4v7kVf}gcDrhBUWg~ToFH-_^z?G3+R@|yOF-$F^Kz1)1g0@pJb*!*zmA-@v&wuQcnm*LUehe0EAMbpun*mFMxv zhU?4@<_C{$mzN(qfcS)~n{KGR?uI*>Sry$=5pCYjwg%oOXsf_8;=Sh4$I6Z|dW=T^ z1+-j5dcgH-D1PU;_!F>ZX?LQIIhi3f;cyZ6i^G^#UV@7>&v;@~s_W`t%COeRUzs=f zg^w|JzZXZW!maLO?1}36{^LhX1#pPC$^ndRE-euZ(+%I@i6GkU8hDfir3?&02o&P- z=_CFgQc=er?s(+z*esi&db( zj*Zivu-(NyOV?+pDJog6eKr;*m)JB?hBDGc?CjdFsn|L$u&fobulE%i(#-Lv41s8dY9L)^|5&GjZh4*RoMi>-gkk;d4 zM70e?oer4T#N5No*_a)?uf!8PT51)?48e3LU5o5y2UDidP}IM7M$D002)K@C3T^jMOAfE(;fTnKWxhsFD)${T_(+H7X{LT*4ShA0z+Oe#1jPy19g{23Mw7XX+q}<;{wS|jgmCX4fMgsYHv+sgS#I%6HS7@ zv=26XkU?__`>N-Bc@Qei`i|Z9kJK^^fTrorzES&e{Y-l~RWqsO=+jHdUNmOdHj$}P zmtrSzI(D_3`;*M?%BL2wNKq}fYoDs{)QesBZ(F&;IimJz6NJ*bCB!RMjZJrqSD#K{ z_3mWARW&`s14X){cG0q>Ken<2u@XN)@71f8DUHj&QnSmgBtp5I@#ecd{SoLr;4!^Bi<2$w0lpk&KX zKA#$6#it79u<_my_x;(-EJn8DSvyIU9FDkOn|{xG(rft7c~uD|dQC$44#u&~S0>7X zd$D0qnlPC8oOuMmH*|)|9`JoTRst=1F8cyW&$CF`vJ^HSLxyGc2b1KB$0iq33Kg1M zoHC0;>zr}bHFM$8X*KU8lcKpCM|Nkm{lG3g5yH0*-1=jfgMHs5C)=4~o0jr@(@r*| zF|nD;uwv+_lSEfu-CFu4S<5J^{!>23)OS@_ac@lou8k7a%9SeDil!N=myMRlN%F%r zD*Cu1;14rI12?`Jw91RdQ>mXS65!=?=Bum}u2^tH8cVk$^zUFDpsn9cYJC(dpUZXZ!)#DzasIf~7@+1QFFpyn)PFCI>uxE<7`)_JLu zk-mLV3NsVB!FK6>gi)X}m)6)-luMG@+kU5Z@j6TuW|=Ss?pAImj{5hh=E_AwNvW@c z=rVNNTEe5M#LL0dxo-Z`hG=rhE!);;R*7#VGXQbFXL*0hs3nke(g>A(M~CY%iS}|z zu{y=Ql@}7!m>PFDT^1P25E@hfly!%Q?QIS`bM@~w(DE$EX8;NiX0jl>1hv}8JH#j5 zaLA zjb*Hzw@U^{QoW(^5kNs9zf)X?V6#Kn&iRX7r9KI&+L8-`D|S(O zoHU`F8m)BnF9V85g&RCcnBzs#MiWn z8aeoSb=bpXxIE;Qkjt$t!}l3^nz( zGsK~@p3eE*`;0UkgLJ!KcFHR&rzw>SlQvA{{++lc>++!CxWgyAw@_oe5QJt-{f0&r z?_#9roSichTRiA5@lL>l19s&ZBWl~jP4D`jh7A?kK&*nRY7>wTj$T<9_e@?jtX8Q| z%Sb8$aP|AZFTb4*MRu{a4T$Go`|L{{xxo)4Hi4lkX-+0m6{T{~xDblQE7qmL*&FxXVe~KZ<(ZMS$=$zXrENO{RFD#9nm^ zqlls%%ylyi(`)W#evdi1Bm1h)qXRRul2fWAB#yNwj}XpV9ex)Z7^R50i7pVs^u=pi z^K2C;B6EBvt}C>{SjT!kjlmcpe{HLS_mF5V4Y+j|uScEBUO)%J9c=M)I&ptWKBVB# zsq^%({d#UXBpF?wkJO5Z-zog0RJS65+Z+7b9`(nnbxBXF676wCHc=hBjvyhP7BCdb z(B4|~NqgB?+-uewRu)@vsKO+dW6_Stq64-kB6-oLI;QOy37PJlDUAo}u%bg=%%Iyd zi*P-Sh1Pl@+y)lmgOYX=@(c$ufh|1NGHAx}qwyBGr8rT|+UQ}F`bYlrmw&me%?lP&#t*X$5a{JqVg zRA$@p*8cQJO{+u=7$~RxPswqp;{6KJ@GGldg;tln9hTLv~&PqK$%5n*=AuyQYTBg%jt=wZ>;_cU|t>mzKc7il4!%D8?&tWR43MP1%L!F*v3*E~$l#Di< zWTUY@)fF`00A=T3C}Rb9#>;tMjIWQ2w1yYialH3oQ(RnR$GL@2p9Iv%b(V1Lmzu(pH{CL1QPN4I$K+?)U&bJ4e}WiwJLQoKYX3 z^UFlB(Sr4zjLouLG#$8U{8m5aE#iKGzHILrP@vh9)W`jm>6WC0%^2q6g8Jfw1c7Nz zuZdLrqu+A20{y1Vzt_Ag+B9JJJ9yKjfgJn4O9SikuXErB@^k5k_i~c!pc@YJj|(B~ EKY! - - - - - - + \ No newline at end of file diff --git a/website/images/logo-180x180.png b/website/images/logo-180x180.png index ee9bae1f61ef701514b88f892655ce6b6862a5c4..d89c259e27c8aa861b82f3f0902ef9c14ff7f93a 100644 GIT binary patch delta 365 zcmdnTx{Gy#G-Lf!c6MPN76G9k76t}}Z=NoWAr-gY-rOnFD4#ug-l2RvOvWp zJ70X+edglqw|dj8tLiPx7T&q|w*RhjbqVw2hm0ciu|HOBKe4!K;`?qPdDnj(7x070 zi{@{a#{PJ{h45_&F_NFD@AqN51g9q7o;%+vjye8Wsw3O~-HU+XpjzS@QIe8al4_M) zlnSI6j0}tnbPddP4a`CejjRlftqcvc4GgRd4ETz-*rI62%}>cptAuJW(={{*F|@EU bHn%b~hiJIelAjCIAPKS|I6rOj6ecwQW_ON< diff --git a/website/images/yandex.png b/website/images/yandex.png index 7b35a630afb7645b7d9ea5924143fe1ccaeb9fe9..ad40dcddfc7ea8f942e3d00a44ea97e51b81af2e 100644 GIT binary patch delta 4160 zcmZ`+c{Egi+@5S@%TCDF3?ti&EsQN^gphqoloHKYvge0EB|9V8vqZLx zkgQo}kZduC-uwRZ{`a1H&%Nh-?|IJse7~RPJkRr?UKtg~LrfX&=$Y$*Ks7H}|9Ucl zKp=5%l#MwE6e$G)J&Xr|j)A3z6c8vB4g#%vfIx^W5J(^}@1wOA@PWz8)W`tz?|-lV zTFcT15&}4mo5luuw&6d1t8b%BZuzB za~pkx8hg=XVsVibd1%PHLTeq>*zhPr_7D6#11Q1keTI<;^@|BSwndbho?as*dX1>5 zspls0rV4kQYVyb)TQ8Z+1c6XINhBujehOA8!zA?gTudGDgnNXVp z9pZiq`x{p<$=Xw%?fA&I0&?ivJ_zoR3Oq&GjmOs*&5 zXk`T{_LIJlwAfvS56$%V?`zw(pZdmU<7Qmb_ZzU@#HTAVvZ{QBKHlE(*|W`~U7DJo zQ5>dZH@C>VtY@S`Jv@6ho19b81KM0G(sIw6cpeta%^QDUTH}aMCfl?Y%gGn?go=T+ z!-*y2vexVfc{j{=*v9YkMky3RSRm+dE14kLOT5aG?Cj$5*rk!}l9f#7Q?YfNNJFIR zrE#TQV{F61J~{;SGKv25G@9wlz$jtia2}}P9+{1FI6nz)^@SX_*gs#2d(OGb`OPCnRM9ny|Ei2l3t zIZ)YuuH|Ub9p{h5Zmc~x1^tIU{*oaJwa8OHDu6&B`v(U~^Y6yjkB^V(&tIg-xwyD| z?(Y6JK8{uqDi$&+92v+!Znf`1q*ld%L?rLPB?Qq#W-#r|@BE!SN4hCTDY_er#W!S0pxNn$0Z80YynAO$QfPes#lGMlb_N-*` z*MOLxbV}Thf|mAqzvtLoA#nC4CM5JM_op=-i&-51YMi$)Hdeg*ib{}_lw^vkOH1Rq zNd9j-t@6zq94Bp>IgUc1bja6}NC*Vt(-+fB1-n`w{*NC&a@Lhu7OcK5I|t(9qkKZZ ztwvx2BRCmL4-k>=vy;1E0L>FN1#nl5%}YYQhTD!TRa=MEcDOgEMd+eTYV`l#4D zHpZjI$HmooJl%e@Dr`{K)+XxE9QjA4XkxG6rCFdt`s8~(SPl#~Dr=8Cb+TU(h7ljlVkEOy5o0AePDv=qo@_xiqkS^xFRN3PsfrdM@r_Qqf? z&SJ|I6Bc$YU}}20xuhhaNIJu0d~$MfZmtbGsX(h#MIqGHF9|xQh-$O^i}vLm?cb-4 z{X1F{o3XfYgPtsT@v{2Ea`S8@MiRfU@W4it^6t`0=>b~!Z1QvD^(NERgUzWMobj2N zllhtt9Qkr`a|wmd#1jFE|0J#r3&$INU_RJfcR9|x}7 zEVE$V98*{k$bQ;!-W!Rg8KRU3s2clx7~`>QfK$kfii+ymLXh_n#wI5EY$OtC5u>fGoy0F6Hy-!+ap?RtRaFgu zmm&yR*#WC%1E`OWG!@S&o_5a2sFh>nHKU3)ew$qkZzejt0WNB^f z{P85cRkt`bOE5mf)ReU-YfrG3 z5WI)IeH&8li$+s5f49(2w*)OQDF-4A4Lb{-CL}N?KKaCoC8lndWIk;8Go~vq_b`bS zt~4PXUyrM++greIt*`&6_M{I!5Q3QPn-O=K{zy6PvRi?9?fG!g%*|90R)nt0-f&dK|d5}yU)+Hk&XG}5sxy6O7L z{k36<9EfpgNr|A=3@193K~ywSRDGt9nJ!TM4H%bDZ!2SH@oQ;mY<5;VkeoQYR)8D! z=5ey)Vd%`Dvy$db^!4?XN>$O1R`evE0~s6Vi6F!m1phnO-*;_~)z;7edK208gne#q z?(FRBhM~~N`OWFZm#d-h1bl$MzhQ`$wswSH?8&djL>&jV%LM|yLpcd@QcjcgbSPz3 zEN;Rfj1<^mQc_DI@!tyHHpn`ktWwkkX=8d_&^t@*?l#CUDe>TWUKB`XpXoU;Yk6Vn z0y!}{aA^B;CjjQYC`nrtP9r#@$APateEH)RYgM(%UcmC3Llzj)Y zhNXb#bsIYdJ>DSV!)(shf&wvYvy&6fDI+6;xzfOcb)yD9*I>GUWUq1j`vY4gm?0vh zLjLeH@71eofn)&%S8*n(h=>TS$v`Sua-ta6)zuU_3H^mWRzE*0GVZU$vX#`>e*>)4 zCs=^c1k^H>N?lq~j^EqAAnE4fvazu6P^`gcfZzLF()T#WyA7~gxgGXw>9~S|mzKSN z%M1(-zJL3cy1Dtyj+dQ}`t_^TO+1pWrdO(kIj(PLXr<{1nM{7h16?#4D$p&!#tE1S zT~)|c{=D!i@9ZpA{j|u`wU%FqjgM-nbJWdCJlvX1q6`8aHvY~o;FAG_duMP=CWFP& zp$pA1(eRkrSeX&J{i`J!cNAPnoUkH<1qCTGqP5mu<>auuP7@zzPh?-kXxt3H zeRbsbs9!i0y1B>A5VXC}*$H-YcYo=MfBN(@91^p=pm)>2VjHI;(pSgVd#Ne*6yY~9 zIhk2X#p5}Nml>;GDe_Sl7rmCtV*i~UcHu*UgO!3Y6UTDi#!KV>t`I0qc%+w<0PGe6 zC_OV%crYRNdC09>udQQz-k=0{dBw{vva%M3i)BIMRnneb=Xaq0?CRzQB$#q#^{4~C zQiNrpmAa-T7-#C#4LiX>lRx9#+}x(a3y24PO>$>eB+f$g@nE)cV(yUKn|fz?CH1NA z-)GulqVA%WdxX*nf+9#Wy@R|A2=y1+SQVUx0m$)p%y0geF61 z(?lX9{6@}O)Y4SPc#CDvB^h#XQ}kG=dEZoaYW}%YoG&JrAadiqlhh+R-KzAU<^`uG zMbZQmL(~GG{t0|$h98@n1Ub&{@;C0}H#@-qzOEqoX6UQE}zb zv!k_wYC>hD0u&0R9hg{H_~yG~Fc>^G>De}8S{Z$H> zQTJ_Sxc6@jhQb5{Jn0glGt7*@8N53f8XRnCX-U64t|OGpG=I_sd1YoQ1i}OM!V6mr zrhEkqwwATps ze-0nEyLC$^blujm;O&_J(z%4Lt?h0oVS%x0w)p`V40ghe8;QlF4R=VA| zmbpj_g1ro_69UtgxY@1%u?onI+1LDqHpY<#2Jj$-1pS)W)5s57V;v(SiL!ZZcQtliz+6v|z9_xWUD zHBpUVVoOUN4q`kD|FhiBqXBu}c_$UJa{Q_8uC+`?Vy(^fXT|pACDQL|Yn2of6x7wf z&(G^hOH*h&OQE5mh|H`~G|fc1^Ir{{p|#Ch6?u7cIsOxX21iH!KtWfpzOAmVZf?#U z&j$#yZAszUz6}412Q2E}xgqW_JAGBq4`3jd#Do|+hTQfH@j-Y8`v5PH5?o1995>u;{OLIL)9z*0La+T!k|*$Bkq3y$(rxE delta 4445 zcmV-j5u)zVAn78IB$09ye-Rr=L_t(|+U;F?R8-X(|IQ4EJXDm#5Cf6-Aev~htW=;& zLN0|6L|v?FzW9S~J`i72H^mZ3Yem`ZF36Rosa;{Ulu8S<1Q9{RFb3s{pnxc%BEv8* zm^t^4qr*OXpL6Dn2uX?QehImZE7Io-7g*e__Okhp!C=7%~L@ z^)ImE&xXwtF0Z&4{_`J}YTxf0Or;h2iHn?zsB}q>1o5`(s zdYHa*2Tq@6NoLIgCnswwTgEzl_bxnm02I|HlS;wE10-g40=*u}%b~NgPvYeT0Rc~6 zlWDcurluyzqxPX`f7->xW#q__mR@hVriKP!{kgc%4Gpxp&}!+qb6Mp?M$#%3{nX*o zrAt^F?%TJ|@~bU@2@_z>99G5e-GjP1v%5@73;g&aOEP~x*xNsa3m!4s+1lFv71rsY zLt*7gj?I!rV_r%5`F*dMwICtEx;k1C(-Z{(0T39-5@%+De@bOuNgFpJO9BmHVK8Ql zwS6tj_V)LOpdj1?!>njwkjk4kTN~H%n6|ZrxHvEvc*m$o6c*x=sZ*_uYmxius3?}0 zqM*JW8XCmfVigrE$v^*T-9}p~(<4X1TW_%zS^(I;UsM^bt*kL^V*@j0z|f)A#@q59lH0)vKl_@ZMfHbciMK z^#xyFYs*?N({^_F4!E%q&Yy?gUZDc&>fro&)=ZF;WNlW{qaZ8YK00o ze7J92XAm1}ZC1->8UO+UAT$(by1t(G+Zqk*+zC=?f1l*N_pELDS76%C4#L84l`p^K z{r1U|004S@pJc%TYn%QRm;4Yt>*srsZ*@CvuA^chqX;D6GR-DiwmTs z!S3BG;gKWo_rK$sEiHhZ-^Is+qa)|f==FN7Rx6jw3knKOojO&DY|K(pQo_Q*X3d)A z;^Jase2%G_&3Sown;$u1y}Z1ZFJHcB(IPJ| zFDcr0O~GI=sMYGq%F6uw{Leo7j1kV8H!nImdfvQwJv}{-*Vf(L-Ip$1I(YElkt0Xi z+S&lX&(Ckkk|oQQEgLgtjGdhw=eKk^T~$>TfBK_#c6O5|Pv+BCC=@j{HTp*rEltzo z#*Opx@&b;NO!CA!F_A;<+&Pwxt1Dev%6T$cQ&W?ZlS3i)IrAEo06=DDW^;41zzM5f zuYddPx3LGib?X*Q)0LH#X=!O#`9@@CXLojXiaf_vsZ{y-`9VQJXldw4<(qH5iPhoT ze{a7vS!_i`#X=;c4y)nn)vFtLVgl4^^`=dmuwSgMuIBrp--uAY%F0Tbrh)TW>A(Gr z#bR(UEtj)&dV1)OKkieYD0=2h`u=^aq|VOHZQHg52M3Gn_2b8nudS^WV0!D;t*E|- zhevyRdwzcYbI&~|T=LqrYsJvq+uNI%f0$^po)}LinLc^)T4EY96NbdiwBTd{`en%FwUF(Q!wKv`Lt$=1h`ElMrUQy1H8Af3pL? zojZ5P5?WhZUwrXJRQc6cUuF2iaG&sL?%%(k`3U%GM|*p-i2TPtSfN72#Se$F@n5d_ zq_MGa)~s0%ErLgD@Z`ypQ&UsduU{V#5y3e`a&vPzJ1ogG1|J`vb?erpq@*x!u~3kh zm>6N2*RNj>NJxqCMSOgGT3T8He}7jjaV-_u1L>9GS!v@Z0fB*gW zG0pn=dQ$E9@#C1C-NznYPe(bUwW zR4UbKwOXxKsZ_PKwdB(-Sg_zeu@DF_&5VY^!a{{Y!IZ4gXm;-0iS{Ape?@zHJ0B%Q zMMY%E%a$#>dGn@9rP63L-QC@7ZEfl4>1eMt$@H2vYp|*1lTSWDAD@{;jPUpOzjEaY zS2q}yq)~t|Hiq@$uV}+zM;(g@rTU4`|i5{kd%~k z`SN8>g_f2UY-$348#iuiW;!=EftQt)VZEWLsYwLWt*tnn z6DBf3k5M5i=hW z62hs;P=XFef6k4WL?Q_a3K}XW8Y(Iv zBLkg!z}+2>AbuiFS{s|ep}C`@qo$??6TbA)OH}{*E;HaT3D)%3!=inQ**K7xI(2G5 zKmh95Eh#C{>2x}s?mhM?;@r7&O`UMCdl3W$ftMHSf0(kd5nZw%Apy*BDO(!4ySvYy zKaWaYdF2&>6+t|X965qX1l(lN(a|EkE2~zmLJftjTetS~^!)VGPnc$Sc(^%C1Awrw`|#Blx*C%5nX_!QfVj_P9lnF zAVR@Wm~-EOMx$wLY}~bL7iy7G6!qjUS}_OYf4mfU zDzKMZxNsrLG-hXHWn~TJ3c(h^w3{0wB>{Pm;Oq?3r;B{@va+(QtgNF)j|y*u1|LmL zO_=1FXP)VQvCYlRm?Skdb=R(4oX<;96sdpv_HE3yiFy%Mu3Y&gS1gdLt7}9=1ZVE2 zf4FIL%N7o%Q&K=&HtDu)+p@E>J32ZnaGloabYe!Bxgf`GXJ=<8dX0bx1A^}(dwcts zm>5o`BO)RKcy1>U$Ml2=;Nydb4u*w^Nz};7%G$bhtI>I4oUVg|gWcWTot>Q>92}%l zX>M*VSRoqT-rkcYO(JvlU0hu3d2-gpe=#kQKzux)o4nv)ARK$-%zWm|na!IwQxs)* zl-@_v^pYh@l9G}Q<*)hAAzK)Z8vboAP_YlTlc1{p&KCR31f z^{1z&qeu1q^!B{-&O3ac78DdjMn;kjcFDjsB6DWP(9CqP^z?L#q8R{;Au&%D)22-m zZ(*4W?D!IMz2)TO+`fIAqw@`1eF z2dw<>zWZ+Qm^PlK8IfR-7u=8pa1ID_L2lK^ID*H+FTebPzFo0mMMy}<+O=zq7JyVL z-Lz?winyj`pqRdM=ME;pPSGs{Zfzf7Jy+1J;X*&1%~(Rp;Bm?m93OpL0f!O_ui(xge4un~>6cw8(X5{yeqO8V2?8yXsp z9Xp0ve5jEtlgR=D15pDrGcyzQ>JJpt%tD23UGnquMS6$^6T^oOe<#z%4j(=&P*ssn zLAoRh3k%6DiFjmZXQM{XrcIkr7q`8=eL_M4TDPlLuU@=(alo0z)<0&5fBp5>oXTYS zmUIv!mY#q9d0=OIOto5VX=y2`)6&wickf=* z#KYo50bt&|d6*ff*CVrkjWXPrKMA*OyQjW z#pGJvzI~go0@#LxqNu#QJWfr9Y!t!4!HtcLWCfTc1d+Lxm6erbl%PP?XGuv(Y-}vU zH}+8m8mI{n6%~a>?HiGtoQ(b4AAkH|Du<)rOmogu@$vC>e|2?ArBbO>Ha9na`st^n z4%3c<$uwz0P!u(3(xj@YDuqJP)zzg`D(ma(`4Vmg^0JmMUk*fim`oAQFD#>B+%l`NCVMvop%%2%ygg=GgiA-8k- z`ufVTboNK+?(RlCR1%3KGBT3MI~>4WmQ*TDNl6I_2_c)Nwzd|v9eR3tKKkgR?c29w z2E|~SX35FPu{`%{p##O%*4AJ!R6F#Z^f7%`%xq-4&VIf0_{8Qk67j~_oS=Bfn-OO`CTe*L;}u2v`% zXw*JUf78>aPv<*ql}e={AtAo(J04G-JgL*^#LeCs4ZUIoo?pq6B}<~Nu5L5Wb>Gvc zPcJGeG8ha7g8}mwlK;7dUau#uyw=uMzTYK%zb3D#=;-J;aNq!$C&pmIh7C-zWLH-g zI&}8#-D|QDIy*a$9Xl2!5NHdqapT5Qr%qwHf3jpEs=$K9r2La-rmd~5;-X$?8dNI4 zOn(Ol5FMwj)9KpV+kgJ~=bJZg-nenY#>U3q-#;QE!pFzQ!NGy)zdb!YsKeRT)|L#3 z>hA8wb|8+9j(l4(jYfk71P&Q8#AN7!!C=s8we|J&XU?3dtE*F~R6~aj_4D(Kii+~^ zQ}AF?$e0E-)`YXFoKUOPZEbDk<>gndTw!AOM~xa45D?(!=NA+dWN&XzMi3~KN_4iP jY1(8sI{6hxN5}sGw1xzT^6*D300000NkvXXu0mjf6Eo64 From 9d0c2f83f898172ae06ac2003b036e0c4ed85f48 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 7 Jul 2021 22:14:30 +0300 Subject: [PATCH 231/290] ClickHouse dictionary source secure setting added documentation --- .../external-dictionaries/external-dicts-dict-sources.md | 5 ++++- src/Dictionaries/ClickHouseDictionarySource.cpp | 6 +++--- src/Dictionaries/ClickHouseDictionarySource.h | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index a841bf3bc80..8022221843b 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -129,7 +129,7 @@ That dictionary source can be configured only via XML configuration. Creating di ## Executable Pool {#dicts-external_dicts_dict_sources-executable_pool} -Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary [is stored](external-dicts-dict-layout.md#ways-to-store-dictionaries-in-memory) using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. +Executable pool allows loading data from pool of processes. This source does not work with dictionary layouts that need to load all data from source. Executable pool works if the dictionary [is stored](external-dicts-dict-layout.md#ways-to-store-dictionaries-in-memory) using `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache`, `direct`, `complex_key_direct` layouts. Executable pool will spawn pool of processes with specified command and keep them running until they exit. The program should read data from STDIN while it is available and output result to STDOUT, and it can wait for next block of data on STDIN. ClickHouse will not close STDIN after processing a block of data but will pipe another chunk of data when needed. The executable script should be ready for this way of data processing — it should poll STDIN and flush data to STDOUT early. @@ -581,6 +581,7 @@ Example of settings: default ids
id=10 + 1 ``` @@ -596,6 +597,7 @@ SOURCE(CLICKHOUSE( db 'default' table 'ids' where 'id=10' + secure 1 )) ``` @@ -609,6 +611,7 @@ Setting fields: - `table` – Name of the table. - `where` – The selection criteria. May be omitted. - `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). +- `secure` - Use ssl for connection. ### Mongodb {#dicts-external_dicts_dict_sources-mongodb} diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index fb8660b27ed..42ec73ee520 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -224,9 +224,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) ClickHouseDictionarySource::Configuration configuration { - .secure = config.getBool(settings_config_prefix + ".secure", false), .host = host, - .port = port, .user = config.getString(settings_config_prefix + ".user", "default"), .password = config.getString(settings_config_prefix + ".password", ""), .db = config.getString(settings_config_prefix + ".db", default_database), @@ -235,7 +233,9 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) .invalidate_query = config.getString(settings_config_prefix + ".invalidate_query", ""), .update_field = config.getString(settings_config_prefix + ".update_field", ""), .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1), - .is_local = isLocalAddress({host, port}, default_port) + .port = port, + .is_local = isLocalAddress({host, port}, default_port), + .secure = config.getBool(settings_config_prefix + ".secure", false) }; /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index e7c6d4aa8d2..fe37610b9c4 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -20,9 +20,7 @@ class ClickHouseDictionarySource final : public IDictionarySource public: struct Configuration { - const bool secure; const std::string host; - const UInt16 port; const std::string user; const std::string password; const std::string db; @@ -31,7 +29,9 @@ public: const std::string invalidate_query; const std::string update_field; const UInt64 update_lag; + const UInt16 port; const bool is_local; + const bool secure; }; ClickHouseDictionarySource( From 20cbca87de1e4068113205c3641ceb90178009db Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 8 Jul 2021 00:48:15 +0300 Subject: [PATCH 232/290] Fix 01791_dist_INSERT_block_structure_mismatch flakiness Add SYSTEM STOP DISTRIBUTED SENDS to force messages from SYSTEM FLUSH DISTRIBUTED query context. --- .../01791_dist_INSERT_block_structure_mismatch.reference | 2 ++ .../0_stateless/01791_dist_INSERT_block_structure_mismatch.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference index 3bba1ac23c0..9f376fb3e4f 100644 --- a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference +++ b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.reference @@ -1,5 +1,7 @@ DistributedBlockOutputStream: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done. DistributedBlockOutputStream: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 1)), implicit conversion will be done. + default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done + default.dist_01683.DirectoryMonitor: Structure does not match (remote: n Int8 Int8(size = 0), local: n UInt64 UInt64(size = 0)), implicit conversion will be done 1 1 2 diff --git a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.sh b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.sh index e989696da03..1a96aad3f13 100755 --- a/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.sh +++ b/tests/queries/0_stateless/01791_dist_INSERT_block_structure_mismatch.sh @@ -18,6 +18,8 @@ $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 -nm -q " INSERT INTO dist_01683 VALUES (1),(2); SET insert_distributed_sync=0; + -- force log messages from the 'SYSTEM FLUSH DISTRIBUTED' context + SYSTEM STOP DISTRIBUTED SENDS dist_01683; INSERT INTO dist_01683 VALUES (1),(2); SYSTEM FLUSH DISTRIBUTED dist_01683; From 201bdc4ff59a76de4ddf9c154ba17233d6e6da86 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 7 Jul 2021 21:40:01 -0400 Subject: [PATCH 233/290] Disabling TestFlows LDAP module due to test fails. --- tests/testflows/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index c2e143a4b1c..8932e6bcf8f 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -23,7 +23,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): with Pool(8) as pool: try: run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) From 40a2fc8a18da3ffed18052952f819bb095f745d5 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 7 Jul 2021 21:45:29 -0400 Subject: [PATCH 234/290] Trying to fix collection of clickhouser server logs for TestFlows check. --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index d39ec12fb82..9fa028fedca 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -73,4 +73,4 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --no-color -o new-fails --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n0 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] +CMD ["sh", "-c", "python3 regression.py --no-color -o new-fails --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv; find * -type f | grep _instances | grep clickhouse-server | xargs -n1 tar -rvf clickhouse_logs.tar; gzip -9 clickhouse_logs.tar"] From aa84d6a91379f058e754bc6e4761783b6f6a163b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 8 Jul 2021 05:20:24 +0300 Subject: [PATCH 235/290] Render pipelines in Play UI --- programs/server/play.html | 78 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index 066cd09d16a..35254a29a8b 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -283,6 +283,23 @@ color: var(--link-color); text-decoration: none; } + + text + { + font-size: 14px; + fill: var(--text-color); + } + + .node rect + { + fill: var(--element-background-color); + filter: drop-shadow(.2rem .2rem .2rem var(--shadow-color)); + } + + .edgePath path + { + stroke: var(--text-color); + } @@ -305,6 +322,7 @@ 

     
+    
     

     

 
@@ -447,6 +465,11 @@
             table.removeChild(table.lastChild);
         }
 
+        let graph = document.getElementById('graph');
+        while (graph.firstChild) {
+            graph.removeChild(graph.lastChild);
+        }
+
         document.getElementById('data-unparsed').innerText = '';
         document.getElementById('data-unparsed').style.display = 'none';
 
@@ -461,12 +484,21 @@
 
     function renderResult(response)
     {
-        //console.log(response);
         clear();
 
         let stats = document.getElementById('stats');
         stats.innerText = 'Elapsed: ' + response.statistics.elapsed.toFixed(3) + " sec, read " + response.statistics.rows_read + " rows.";
 
+        /// We can also render graphs if user performed EXPLAIN PIPELINE graph=1.
+        if (response.data.length > 3 && response.data[0][0] === "digraph" && document.getElementById('query').value.match(/^\s*EXPLAIN/i)) {
+            renderGraph(response);
+        } else {
+            renderTable(response);
+        }
+    }
+
+    function renderTable(response)
+    {
         let thead = document.createElement('thead');
         for (let idx in response.meta) {
             let th = document.createElement('th');
@@ -559,6 +591,50 @@
         document.getElementById('error').style.display = 'block';
     }
 
+    /// Huge JS libraries should be loaded only if needed.
+    function loadJS(src) {
+        return new Promise((resolve, reject) => {
+            const script = document.createElement('script');
+            script.src = src;
+            script.addEventListener('load', function() { resolve(true); });
+            document.head.appendChild(script);
+        });
+    }
+
+    let load_dagre_promise;
+    function loadDagre() {
+        if (load_dagre_promise) { return load_dagre_promise; }
+
+        load_dagre_promise = Promise.all([
+            loadJS('https://dagrejs.github.io/project/dagre/latest/dagre.min.js'),
+            loadJS('https://dagrejs.github.io/project/graphlib-dot/latest/graphlib-dot.min.js'),
+            loadJS('https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js'),
+            loadJS('https://cdn.jsdelivr.net/npm/d3@7'),
+        ]);
+
+        return load_dagre_promise;
+    }
+
+    async function renderGraph(response)
+    {
+        await loadDagre();
+
+        /// https://github.com/dagrejs/dagre-d3/issues/131
+        const dot = response.data.reduce((acc, row) => acc + '\n' + row[0].replace(/shape\s*=\s*box/g, 'shape=rect'));
+
+        let graph = graphlibDot.read(dot);
+        graph.graph().rankdir = 'TB';
+
+        let render = new dagreD3.render();
+
+        render(d3.select("#graph"), graph);
+
+        let svg = document.getElementById('graph');
+
+        svg.style.width = graph.graph().width;
+        svg.style.height = graph.graph().height;
+    }
+
     function setColorTheme(theme)
     {
         window.localStorage.setItem('theme', theme);

From e841fae85267228a9f60bbfbd4030883d81c9948 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Thu, 8 Jul 2021 05:27:40 +0300
Subject: [PATCH 236/290] Improvement

---
 programs/server/play.html | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index 35254a29a8b..98770be27eb 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -469,6 +469,7 @@
         while (graph.firstChild) {
             graph.removeChild(graph.lastChild);
         }
+        graph.style.display = 'none';
 
         document.getElementById('data-unparsed').innerText = '';
         document.getElementById('data-unparsed').style.display = 'none';
@@ -627,9 +628,10 @@
 
         let render = new dagreD3.render();
 
-        render(d3.select("#graph"), graph);
-
         let svg = document.getElementById('graph');
+        svg.style.display = 'block';
+
+        render(d3.select("#graph"), graph);
 
         svg.style.width = graph.graph().width;
         svg.style.height = graph.graph().height;

From 15b75be59b0a279cea63097cd841ec8758d84692 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Thu, 8 Jul 2021 05:29:21 +0300
Subject: [PATCH 237/290] Add comment

---
 programs/server/play.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/server/play.html b/programs/server/play.html
index 98770be27eb..da763ec2a0e 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -284,6 +284,7 @@
             text-decoration: none;
         }
 
+        /* This is for graph in svg */
         text
         {
             font-size: 14px;

From 0e4257721ce2eba399898a8b453b2ecc60e88a1a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Thu, 8 Jul 2021 05:50:18 +0300
Subject: [PATCH 238/290] Maybe better dependencies tracking in CMake

---
 cmake/embed_binary.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/embed_binary.cmake b/cmake/embed_binary.cmake
index d15962c05d4..e5428c24939 100644
--- a/cmake/embed_binary.cmake
+++ b/cmake/embed_binary.cmake
@@ -53,5 +53,6 @@ macro(clickhouse_embed_binaries)
         set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}" APPEND PROPERTY INCLUDE_DIRECTORIES "${EMBED_RESOURCE_DIR}")
 
         target_sources("${EMBED_TARGET}" PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}")
+        set_target_properties("${EMBED_TARGET}" PROPERTIES OBJECT_DEPENDS "${RESOURCE_FILE}")
     endforeach()
 endmacro()

From 15dbb5a07a04da12a2f2f67fdfcdf2f197b98a0f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Thu, 8 Jul 2021 06:04:35 +0300
Subject: [PATCH 239/290] Final touch

---
 programs/server/play.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/programs/server/play.html b/programs/server/play.html
index da763ec2a0e..c3e8708f20b 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -301,6 +301,11 @@
         {
             stroke: var(--text-color);
         }
+
+        marker
+        {
+            fill: var(--text-color);
+        }
     
 
 

From ee1b3696a251b3bf24086daedf7decad5391dc8e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Thu, 8 Jul 2021 06:15:30 +0300
Subject: [PATCH 240/290] Fix error in AsynchronousMetrics

---
 src/Interpreters/AsynchronousMetrics.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp
index 5c49adf6fe7..aca92b8866d 100644
--- a/src/Interpreters/AsynchronousMetrics.cpp
+++ b/src/Interpreters/AsynchronousMetrics.cpp
@@ -716,9 +716,9 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti
             {
                 ProcStatValuesOther delta_values = current_other_values - proc_stat_values_other;
 
-                new_values["OSInterrupts"] = delta_values.interrupts * multiplier;
-                new_values["OSContextSwitches"] = delta_values.context_switches * multiplier;
-                new_values["OSProcessesCreated"] = delta_values.processes_created * multiplier;
+                new_values["OSInterrupts"] = delta_values.interrupts;
+                new_values["OSContextSwitches"] = delta_values.context_switches;
+                new_values["OSProcessesCreated"] = delta_values.processes_created;
 
                 /// Also write values normalized to 0..1 by diving to the number of CPUs.
                 /// These values are good to be averaged across the cluster of non-uniform servers.

From 39de7f8a2a18b685e89d0a276a82ede735410c91 Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Thu, 8 Jul 2021 11:16:57 +0300
Subject: [PATCH 241/290] Fix logical error with signed and unsinged offset in
 WindowFrame::checkValid

---
 src/Interpreters/WindowDescription.cpp                     | 4 ++--
 tests/queries/0_stateless/01571_window_functions.reference | 2 ++
 tests/queries/0_stateless/01571_window_functions.sql       | 3 +++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/WindowDescription.cpp b/src/Interpreters/WindowDescription.cpp
index 46e1eb12dc5..4de15af820f 100644
--- a/src/Interpreters/WindowDescription.cpp
+++ b/src/Interpreters/WindowDescription.cpp
@@ -160,7 +160,7 @@ void WindowFrame::checkValid() const
         bool begin_less_equal_end;
         if (begin_preceding && end_preceding)
         {
-            begin_less_equal_end = begin_offset >= end_offset;
+            begin_less_equal_end = begin_offset.get() >= end_offset.get();
         }
         else if (begin_preceding && !end_preceding)
         {
@@ -172,7 +172,7 @@ void WindowFrame::checkValid() const
         }
         else /* if (!begin_preceding && !end_preceding) */
         {
-            begin_less_equal_end = begin_offset <= end_offset;
+            begin_less_equal_end = begin_offset.get() <= end_offset.get();
         }
 
         if (!begin_less_equal_end)
diff --git a/tests/queries/0_stateless/01571_window_functions.reference b/tests/queries/0_stateless/01571_window_functions.reference
index 47a7c062b0b..bbac8e5ac6d 100644
--- a/tests/queries/0_stateless/01571_window_functions.reference
+++ b/tests/queries/0_stateless/01571_window_functions.reference
@@ -13,3 +13,5 @@ select count() over (rows between 1 + 1 preceding and 1 + 1 following) from numb
 5
 4
 3
+-- signed and unsigned in offset do not cause logical error
+select count() over (rows between 2 following and 1 + -1 following) FROM numbers(10); -- { serverError 36 }
diff --git a/tests/queries/0_stateless/01571_window_functions.sql b/tests/queries/0_stateless/01571_window_functions.sql
index 614b98670b2..c6479044b59 100644
--- a/tests/queries/0_stateless/01571_window_functions.sql
+++ b/tests/queries/0_stateless/01571_window_functions.sql
@@ -4,3 +4,6 @@ set allow_experimental_window_functions = 1;
 
 -- expressions in window frame
 select count() over (rows between 1 + 1 preceding and 1 + 1 following) from numbers(10);
+
+-- signed and unsigned in offset do not cause logical error
+select count() over (rows between 2 following and 1 + -1 following) FROM numbers(10); -- { serverError 36 }

From 71d0682a2b32621d5c317072c2b1c96230900fec Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Thu, 8 Jul 2021 12:24:08 +0300
Subject: [PATCH 242/290] Use FieldVisitor to compare offsets in
 WindowFrame::checkValid

---
 src/Common/FieldVisitorsAccurateComparison.h | 12 ++++++++++++
 src/Interpreters/WindowDescription.cpp       |  6 ++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h
index 0f605b7da23..ba3fabd1535 100644
--- a/src/Common/FieldVisitorsAccurateComparison.h
+++ b/src/Common/FieldVisitorsAccurateComparison.h
@@ -117,4 +117,16 @@ public:
     }
 };
 
+
+class FieldVisitorAccurateLessOrEqual : public StaticVisitor
+{
+public:
+    template 
+    bool operator()(const T & l, const U & r) const
+    {
+        auto less_cmp = FieldVisitorAccurateLess();
+        return !less_cmp(r, l);
+    }
+};
+
 }
diff --git a/src/Interpreters/WindowDescription.cpp b/src/Interpreters/WindowDescription.cpp
index 4de15af820f..923e10ed31b 100644
--- a/src/Interpreters/WindowDescription.cpp
+++ b/src/Interpreters/WindowDescription.cpp
@@ -1,6 +1,7 @@
 #include 
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -160,7 +161,8 @@ void WindowFrame::checkValid() const
         bool begin_less_equal_end;
         if (begin_preceding && end_preceding)
         {
-            begin_less_equal_end = begin_offset.get() >= end_offset.get();
+            /// we can't compare Fields using operator<= if fields have different types
+            begin_less_equal_end = applyVisitor(FieldVisitorAccurateLessOrEqual(), end_offset, begin_offset);
         }
         else if (begin_preceding && !end_preceding)
         {
@@ -172,7 +174,7 @@ void WindowFrame::checkValid() const
         }
         else /* if (!begin_preceding && !end_preceding) */
         {
-            begin_less_equal_end = begin_offset.get() <= end_offset.get();
+            begin_less_equal_end = applyVisitor(FieldVisitorAccurateLessOrEqual(), begin_offset, end_offset);
         }
 
         if (!begin_less_equal_end)

From 2304f6a31b77ea3123217f50f5cc6138f61054fd Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Thu, 8 Jul 2021 12:24:37 +0300
Subject: [PATCH 243/290] Remove dots from exception message in
 WindowDescription.cpp

---
 src/Interpreters/WindowDescription.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/WindowDescription.cpp b/src/Interpreters/WindowDescription.cpp
index 923e10ed31b..32129072972 100644
--- a/src/Interpreters/WindowDescription.cpp
+++ b/src/Interpreters/WindowDescription.cpp
@@ -100,7 +100,7 @@ void WindowFrame::checkValid() const
                 && begin_offset.get() < INT_MAX))
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Frame start offset for '{}' frame must be a nonnegative 32-bit integer, '{}' of type '{}' given.",
+                "Frame start offset for '{}' frame must be a nonnegative 32-bit integer, '{}' of type '{}' given",
                 toString(type),
                 applyVisitor(FieldVisitorToString(), begin_offset),
                 Field::Types::toString(begin_offset.getType()));
@@ -113,7 +113,7 @@ void WindowFrame::checkValid() const
                 && end_offset.get() < INT_MAX))
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Frame end offset for '{}' frame must be a nonnegative 32-bit integer, '{}' of type '{}' given.",
+                "Frame end offset for '{}' frame must be a nonnegative 32-bit integer, '{}' of type '{}' given",
                 toString(type),
                 applyVisitor(FieldVisitorToString(), end_offset),
                 Field::Types::toString(end_offset.getType()));

From d4256a8583bd7c1f08a35c868c4455311340c500 Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Thu, 8 Jul 2021 13:49:13 +0300
Subject: [PATCH 244/290] Minor style changes in JoinedTables

---
 src/Interpreters/IdentifierSemantic.cpp     |  9 +++++++--
 src/Interpreters/InterpreterSelectQuery.cpp |  6 +++---
 src/Interpreters/JoinedTables.cpp           | 18 ++++++++++++++----
 src/Interpreters/JoinedTables.h             | 10 ++++------
 src/Interpreters/getTableExpressions.cpp    |  5 ++---
 src/Interpreters/getTableExpressions.h      |  4 +---
 6 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp
index 0198a92f78b..098bf033399 100644
--- a/src/Interpreters/IdentifierSemantic.cpp
+++ b/src/Interpreters/IdentifierSemantic.cpp
@@ -1,6 +1,8 @@
+#include 
+
 #include 
 
-#include 
+#include 
 #include 
 
 #include 
@@ -280,7 +282,10 @@ IdentifierMembershipCollector::IdentifierMembershipCollector(const ASTSelectQuer
         QueryAliasesNoSubqueriesVisitor(aliases).visit(with);
     QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select());
 
-    tables = getDatabaseAndTablesWithColumns(getTableExpressions(select), context);
+    const auto & settings = context->getSettingsRef();
+    tables = getDatabaseAndTablesWithColumns(getTableExpressions(select), context,
+                                             settings.asterisk_include_alias_columns,
+                                             settings.asterisk_include_materialized_columns);
 }
 
 std::optional IdentifierMembershipCollector::getIdentsMembership(ASTPtr ast) const
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index f3857b138da..d820cbbae45 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -311,7 +311,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
         ApplyWithSubqueryVisitor().visit(query_ptr);
     }
 
-    JoinedTables joined_tables(getSubqueryContext(context), getSelectQuery());
+    JoinedTables joined_tables(getSubqueryContext(context), getSelectQuery(), options.with_all_cols);
 
     bool got_storage_from_query = false;
     if (!has_input && !storage)
@@ -328,7 +328,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
             metadata_snapshot = storage->getInMemoryMetadataPtr();
     }
 
-    if (has_input || !joined_tables.resolveTables(options.with_all_cols))
+    if (has_input || !joined_tables.resolveTables())
         joined_tables.makeFakeTable(storage, metadata_snapshot, source_header);
 
     /// Rewrite JOINs
@@ -337,7 +337,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
         rewriteMultipleJoins(query_ptr, joined_tables.tablesWithColumns(), context->getCurrentDatabase(), context->getSettingsRef());
 
         joined_tables.reset(getSelectQuery());
-        joined_tables.resolveTables(options.with_all_cols);
+        joined_tables.resolveTables();
 
         if (storage && joined_tables.isLeftTableSubquery())
         {
diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp
index 5e9a285c5db..099fb5c2f44 100644
--- a/src/Interpreters/JoinedTables.cpp
+++ b/src/Interpreters/JoinedTables.cpp
@@ -161,9 +161,10 @@ using RenameQualifiedIdentifiersVisitor = InDepthNodeVisitorgetSettingsRef();
+    bool include_alias_cols = include_all_columns || settings.asterisk_include_alias_columns;
+    bool include_materialized_cols = include_all_columns || settings.asterisk_include_materialized_columns;
+    tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, include_alias_cols, include_materialized_cols);
     if (tables_with_columns.size() != table_expressions.size())
         throw Exception("Unexpected tables count", ErrorCodes::LOGICAL_ERROR);
 
-    const auto & settings = context->getSettingsRef();
     if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1)
     {
         for (size_t i = 0; i < tables_with_columns.size(); ++i)
@@ -312,4 +315,11 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se
     return table_join;
 }
 
+void JoinedTables::reset(const ASTSelectQuery & select_query)
+{
+    table_expressions = getTableExpressions(select_query);
+    left_table_expression = extractTableExpression(select_query, 0);
+    left_db_and_table = getDatabaseAndTable(select_query, 0);
+}
+
 }
diff --git a/src/Interpreters/JoinedTables.h b/src/Interpreters/JoinedTables.h
index 52581c19999..9d01c081e9f 100644
--- a/src/Interpreters/JoinedTables.h
+++ b/src/Interpreters/JoinedTables.h
@@ -22,15 +22,12 @@ using StorageMetadataPtr = std::shared_ptr;
 class JoinedTables
 {
 public:
-    JoinedTables(ContextPtr context, const ASTSelectQuery & select_query);
+    JoinedTables(ContextPtr context, const ASTSelectQuery & select_query, bool include_all_columns_ = false);
 
-    void reset(const ASTSelectQuery & select_query)
-    {
-        *this = JoinedTables(Context::createCopy(context), select_query);
-    }
+    void reset(const ASTSelectQuery & select_query);
 
     StoragePtr getLeftTableStorage();
-    bool resolveTables(bool include_all_columns);
+    bool resolveTables();
 
     /// Make fake tables_with_columns[0] in case we have predefined input in InterpreterSelectQuery
     void makeFakeTable(StoragePtr storage, const StorageMetadataPtr & metadata_snapshot, const Block & source_header);
@@ -50,6 +47,7 @@ private:
     ContextPtr context;
     std::vector table_expressions;
     TablesWithColumns tables_with_columns;
+    const bool include_all_columns;
 
     /// Legacy (duplicated left table values)
     ASTPtr left_table_expression;
diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp
index 2d9391f4673..d82c7fc1332 100644
--- a/src/Interpreters/getTableExpressions.cpp
+++ b/src/Interpreters/getTableExpressions.cpp
@@ -116,13 +116,12 @@ static NamesAndTypesList getColumnsFromTableExpression(
 TablesWithColumns getDatabaseAndTablesWithColumns(
         const ASTTableExprConstPtrs & table_expressions,
         ContextPtr context,
-        bool include_all)
+        bool include_alias_cols,
+        bool include_materialized_cols)
 {
     TablesWithColumns tables_with_columns;
 
     String current_database = context->getCurrentDatabase();
-    bool include_alias_cols = include_all || context->getSettingsRef().asterisk_include_alias_columns;
-    bool include_materialized_cols = include_all || context->getSettingsRef().asterisk_include_materialized_columns;
 
     for (const ASTTableExpression * table_expression : table_expressions)
     {
diff --git a/src/Interpreters/getTableExpressions.h b/src/Interpreters/getTableExpressions.h
index 6a999729a2f..c4ca01ee3c3 100644
--- a/src/Interpreters/getTableExpressions.h
+++ b/src/Interpreters/getTableExpressions.h
@@ -21,8 +21,6 @@ const ASTTableExpression * getTableExpression(const ASTSelectQuery & select, siz
 ASTPtr extractTableExpression(const ASTSelectQuery & select, size_t table_number);
 
 TablesWithColumns getDatabaseAndTablesWithColumns(
-        const ASTTableExprConstPtrs & table_expressions,
-        ContextPtr context,
-        bool include_all = false);
+    const ASTTableExprConstPtrs & table_expressions, ContextPtr context, bool include_alias_cols, bool include_materialized_cols);
 
 }

From 9f1ffa777f50e8bf31fd99d4ed12a049776c80f0 Mon Sep 17 00:00:00 2001
From: Anton Popov 
Date: Thu, 8 Jul 2021 15:06:33 +0300
Subject: [PATCH 245/290] remove unused code

---
 src/Storages/MergeTree/MergeTreeData.cpp      | 27 ++-----
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  8 --
 .../StorageFromBasePartsOfProjection.h        | 75 -------------------
 src/Storages/SelectQueryInfo.h                |  5 --
 4 files changed, 6 insertions(+), 109 deletions(-)
 delete mode 100644 src/Storages/MergeTree/StorageFromBasePartsOfProjection.h

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index ae3d2220936..f311d58b7af 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3977,13 +3977,11 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
             candidate.where_column_name = analysis_result.where_column_name;
             candidate.remove_where_filter = analysis_result.remove_where_filter;
             candidate.before_where = analysis_result.before_where->clone();
-            // std::cerr << fmt::format("before_where_actions = \n{}", candidate.before_where->dumpDAG()) << std::endl;
+
             required_columns = candidate.before_where->foldActionsByProjection(
                 required_columns,
                 projection.sample_block_for_keys,
                 candidate.where_column_name);
-            // std::cerr << fmt::format("before_where_actions = \n{}", candidate.before_where->dumpDAG()) << std::endl;
-            // std::cerr << fmt::format("where_required_columns = \n{}", fmt::join(required_columns, ", ")) << std::endl;
 
             if (required_columns.empty())
                 return false;
@@ -3999,12 +3997,11 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
             // required_columns should not contain columns generated by prewhere
             for (const auto & column : prewhere_actions->getResultColumns())
                 required_columns.erase(column.name);
-            // std::cerr << fmt::format("prewhere_actions = \n{}", prewhere_actions->dumpDAG()) << std::endl;
+
             // Prewhere_action should not add missing keys.
             prewhere_required_columns = prewhere_actions->foldActionsByProjection(
                 prewhere_required_columns, projection.sample_block_for_keys, candidate.prewhere_info->prewhere_column_name, false);
-            // std::cerr << fmt::format("prewhere_actions = \n{}", prewhere_actions->dumpDAG()) << std::endl;
-            // std::cerr << fmt::format("prewhere_required_columns = \n{}", fmt::join(prewhere_required_columns, ", ")) << std::endl;
+
             if (prewhere_required_columns.empty())
                 return false;
             candidate.prewhere_info->prewhere_actions = prewhere_actions;
@@ -4014,7 +4011,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
                 auto row_level_filter_actions = candidate.prewhere_info->row_level_filter->clone();
                 prewhere_required_columns = row_level_filter_actions->foldActionsByProjection(
                     prewhere_required_columns, projection.sample_block_for_keys, candidate.prewhere_info->row_level_column_name, false);
-                // std::cerr << fmt::format("row_level_filter_required_columns = \n{}", fmt::join(prewhere_required_columns, ", ")) << std::endl;
+
                 if (prewhere_required_columns.empty())
                     return false;
                 candidate.prewhere_info->row_level_filter = row_level_filter_actions;
@@ -4023,11 +4020,9 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
             if (candidate.prewhere_info->alias_actions)
             {
                 auto alias_actions = candidate.prewhere_info->alias_actions->clone();
-                // std::cerr << fmt::format("alias_actions = \n{}", alias_actions->dumpDAG()) << std::endl;
                 prewhere_required_columns
                     = alias_actions->foldActionsByProjection(prewhere_required_columns, projection.sample_block_for_keys, {}, false);
-                // std::cerr << fmt::format("alias_actions = \n{}", alias_actions->dumpDAG()) << std::endl;
-                // std::cerr << fmt::format("alias_required_columns = \n{}", fmt::join(prewhere_required_columns, ", ")) << std::endl;
+
                 if (prewhere_required_columns.empty())
                     return false;
                 candidate.prewhere_info->alias_actions = alias_actions;
@@ -4055,7 +4050,6 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
 
         if (projection.type == ProjectionDescription::Type::Aggregate && analysis_result.need_aggregate && can_use_aggregate_projection)
         {
-            // std::cerr << fmt::format("====== aggregate projection analysis: {} ======", projection.name) << std::endl;
             bool match = true;
             Block aggregates;
             // Let's first check if all aggregates are provided by current projection
@@ -4081,11 +4075,8 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
             // needs to provide aggregation keys, and certain children DAG might be substituted by
             // some keys in projection.
             candidate.before_aggregation = analysis_result.before_aggregation->clone();
-            // std::cerr << fmt::format("keys = {}", fmt::join(keys, ", ")) << std::endl;
-            // std::cerr << fmt::format("before_aggregation = \n{}", candidate.before_aggregation->dumpDAG()) << std::endl;
             auto required_columns = candidate.before_aggregation->foldActionsByProjection(keys, projection.sample_block_for_keys);
-            // std::cerr << fmt::format("before_aggregation = \n{}", candidate.before_aggregation->dumpDAG()) << std::endl;
-            // std::cerr << fmt::format("aggregate_required_columns = \n{}", fmt::join(required_columns, ", ")) << std::endl;
+
             if (required_columns.empty() && !keys.empty())
                 continue;
 
@@ -4110,12 +4101,10 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
                     candidate.required_columns.push_back(aggregate.name);
                 candidates.push_back(std::move(candidate));
             }
-            // std::cerr << fmt::format("====== aggregate projection analysis end: {} ======", projection.name) << std::endl;
         }
 
         if (projection.type == ProjectionDescription::Type::Normal && (analysis_result.hasWhere() || analysis_result.hasPrewhere()))
         {
-            // std::cerr << fmt::format("====== normal projection analysis: {} ======", projection.name) << std::endl;
             const auto & actions
                 = analysis_result.before_aggregation ? analysis_result.before_aggregation : analysis_result.before_order_by;
             NameSet required_columns;
@@ -4127,16 +4116,12 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection(
                 candidate.required_columns = {required_columns.begin(), required_columns.end()};
                 candidates.push_back(std::move(candidate));
             }
-            // std::cerr << fmt::format("====== normal projection analysis end: {} ======", projection.name) << std::endl;
         }
     }
 
     // Let's select the best projection to execute the query.
     if (!candidates.empty())
     {
-        // First build a MergeTreeDataSelectCache to check if a projection is indeed better than base
-        // query_info.merge_tree_data_select_cache = std::make_unique();
-
         std::shared_ptr max_added_blocks;
         if (settings.select_sequential_consistency)
         {
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index cffedf44823..0a05eeb966e 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -201,7 +201,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
             // NOTE: prewhere is executed inside readFromParts
             if (query_info.projection->before_where)
             {
-                // std::cerr << fmt::format("projection before_where: {}", query_info.projection->before_where->dumpDAG());
                 auto where_step = std::make_unique(
                     plan->getCurrentDataStream(),
                     query_info.projection->before_where,
@@ -214,7 +213,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
 
             if (query_info.projection->before_aggregation)
             {
-                // std::cerr << fmt::format("projection before_aggregation: {}", query_info.projection->before_aggregation->dumpDAG());
                 auto expression_before_aggregation
                     = std::make_unique(plan->getCurrentDataStream(), query_info.projection->before_aggregation);
                 expression_before_aggregation->setStepDescription("Before GROUP BY");
@@ -268,9 +266,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
         {
             const auto & header_before_aggregation = pipe.getHeader();
 
-            // std::cerr << "============ header before aggregation" << std::endl;
-            // std::cerr << header_before_aggregation.dumpStructure() << std::endl;
-
             ColumnNumbers keys;
             for (const auto & key : query_info.projection->aggregation_keys)
                 keys.push_back(header_before_aggregation.getPositionByName(key.name));
@@ -350,9 +345,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read(
                 return std::make_shared(
                     header, transform_params, many_data, counter++, merge_threads, temporary_data_merge_threads);
             });
-
-            // std::cerr << "============ header after aggregation" << std::endl;
-            // std::cerr << pipe.getHeader().dumpStructure() << std::endl;
         };
 
         if (!projection_pipe.empty())
diff --git a/src/Storages/MergeTree/StorageFromBasePartsOfProjection.h b/src/Storages/MergeTree/StorageFromBasePartsOfProjection.h
deleted file mode 100644
index 5d82716af11..00000000000
--- a/src/Storages/MergeTree/StorageFromBasePartsOfProjection.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#pragma once
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-
-
-namespace DB
-{
-/// A Storage that allows reading from a single MergeTree data part.
-class StorageFromBasePartsOfProjection final : public shared_ptr_helper, public IStorage
-{
-    friend struct shared_ptr_helper;
-
-public:
-    String getName() const override { return "FromBasePartsOfProjection"; }
-
-    Pipe read(
-        const Names & column_names,
-        const StorageMetadataPtr & metadata_snapshot,
-        SelectQueryInfo & query_info,
-        ContextPtr context,
-        QueryProcessingStage::Enum /*processed_stage*/,
-        size_t max_block_size,
-        unsigned num_streams) override
-    {
-        // NOTE: It's used to read normal parts only
-        QueryPlan query_plan = std::move(*MergeTreeDataSelectExecutor(storage).readFromParts(
-            {},
-            column_names,
-            metadata_snapshot,
-            metadata_snapshot,
-            query_info,
-            context,
-            max_block_size,
-            num_streams,
-            nullptr,
-            query_info.projection ? query_info.projection->merge_tree_data_select_base_cache.get()
-                                  : query_info.merge_tree_data_select_cache.get()));
-
-        return query_plan.convertToPipe(
-            QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context));
-    }
-
-
-    bool supportsIndexForIn() const override { return true; }
-
-    bool mayBenefitFromIndexForIn(
-        const ASTPtr & left_in_operand, ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot) const override
-    {
-        return storage.mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot);
-    }
-
-    NamesAndTypesList getVirtuals() const override { return storage.getVirtuals(); }
-
-protected:
-    StorageFromBasePartsOfProjection(const MergeTreeData & storage_, const StorageMetadataPtr & metadata_snapshot)
-        : IStorage(storage_.getStorageID()), storage(storage_)
-    {
-        setInMemoryMetadata(*metadata_snapshot);
-    }
-
-
-private:
-    const MergeTreeData & storage;
-};
-
-}
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index a7d2ae3e7dd..fc308667db9 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -99,8 +99,6 @@ class IMergeTreeDataPart;
 
 using ManyExpressionActions = std::vector;
 
-struct MergeTreeDataSelectCache;
-
 // The projection selected to execute current query
 struct ProjectionCandidate
 {
@@ -119,8 +117,6 @@ struct ProjectionCandidate
     ReadInOrderOptimizerPtr order_optimizer;
     InputOrderInfoPtr input_order_info;
     ManyExpressionActions group_by_elements_actions;
-    // std::shared_ptr merge_tree_data_select_base_cache;
-    // std::shared_ptr merge_tree_data_select_projection_cache;
 };
 
 /** Query along with some additional data,
@@ -160,7 +156,6 @@ struct SelectQueryInfo
     /// If not null, it means we choose a projection to execute current query.
     std::optional projection;
     bool ignore_projections = false;
-    std::shared_ptr merge_tree_data_select_cache;
 };
 
 }

From 02977007dc7559c126e23a004eef86c717dd19a4 Mon Sep 17 00:00:00 2001
From: Vladimir 
Date: Thu, 8 Jul 2021 15:45:23 +0300
Subject: [PATCH 246/290] Remove queries with syntax error from
 01917_distinct_on.sql

---
 tests/queries/0_stateless/01917_distinct_on.sql | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/01917_distinct_on.sql b/tests/queries/0_stateless/01917_distinct_on.sql
index 75dd8c0b7b8..ae528b6e838 100644
--- a/tests/queries/0_stateless/01917_distinct_on.sql
+++ b/tests/queries/0_stateless/01917_distinct_on.sql
@@ -7,16 +7,17 @@ SELECT DISTINCT ON (a, b) a, b, c FROM t1;
 SELECT DISTINCT ON (a, b) * FROM t1;
 SELECT DISTINCT ON (a) * FROM t1;
 
-SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 62 }
+-- fuzzer will fail, enable when fixed
+-- SELECT DISTINCT ON (a, b) a, b, c FROM t1 LIMIT 1 BY a, b; -- { clientError 62 }
 
-SELECT DISTINCT ON a, b a, b FROM t1; -- { clientError 62 }
-SELECT DISTINCT ON a a, b FROM t1; -- { clientError 62 }
+-- SELECT DISTINCT ON a, b a, b FROM t1; -- { clientError 62 }
+-- SELECT DISTINCT ON a a, b FROM t1; -- { clientError 62 }
 
 -- "Code: 47. DB::Exception: Missing columns: 'DISTINCT'" - error can be better
-SELECT DISTINCT ON (a, b) DISTINCT a, b FROM t1; -- { serverError 47 }
-SELECT DISTINCT DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 }
+-- SELECT DISTINCT ON (a, b) DISTINCT a, b FROM t1; -- { serverError 47 }
+-- SELECT DISTINCT DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 }
 
-SELECT ALL DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 }
-SELECT DISTINCT ON (a, b) ALL a, b FROM t1; -- { clientError 62 }
+-- SELECT ALL DISTINCT ON (a, b) a, b FROM t1; -- { clientError 62 }
+-- SELECT DISTINCT ON (a, b) ALL a, b FROM t1; -- { clientError 62 }
 
 DROP TABLE IF EXISTS t1;

From e621fb644d33bad4196e54b3c58a2542d5366fdc Mon Sep 17 00:00:00 2001
From: alesapin 
Date: Thu, 8 Jul 2021 16:57:55 +0300
Subject: [PATCH 247/290] Fix abort in ZooKeeper client

---
 src/Common/ZooKeeper/ZooKeeperImpl.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
index a717052a1ba..b0fcafff752 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
@@ -852,7 +852,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
             }
 
             /// Send thread will exit after sending close request or on expired flag
-            send_thread.join();
+            if (send_thread.joinable())
+                send_thread.join();
         }
 
         /// Set expired flag after we sent close event
@@ -869,7 +870,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive)
             tryLogCurrentException(__PRETTY_FUNCTION__);
         }
 
-        if (!error_receive)
+        if (!error_receive && receive_thread.joinable())
             receive_thread.join();
 
         {

From 818bbd653943cae56ad182ab0eb593cbd4278819 Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Thu, 8 Jul 2021 17:06:37 +0300
Subject: [PATCH 248/290] Update programs/server/play.html

Co-authored-by: Vladimir 
---
 programs/server/play.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index c3e8708f20b..4165a2829bd 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -613,10 +613,10 @@
         if (load_dagre_promise) { return load_dagre_promise; }
 
         load_dagre_promise = Promise.all([
-            loadJS('https://dagrejs.github.io/project/dagre/latest/dagre.min.js'),
-            loadJS('https://dagrejs.github.io/project/graphlib-dot/latest/graphlib-dot.min.js'),
-            loadJS('https://dagrejs.github.io/project/dagre-d3/latest/dagre-d3.min.js'),
-            loadJS('https://cdn.jsdelivr.net/npm/d3@7'),
+            loadJS('https://dagrejs.github.io/project/dagre/v0.8.5/dagre.min.js'),
+            loadJS('https://dagrejs.github.io/project/graphlib-dot/v0.6.4/graphlib-dot.min.js'),
+            loadJS('https://dagrejs.github.io/project/dagre-d3/v0.6.4/dagre-d3.min.js'),
+            loadJS('https://cdn.jsdelivr.net/npm/d3@7.0.0'),
         ]);
 
         return load_dagre_promise;

From 88bc3995e0509a3c67074a90dfc6cb52c9a15cb5 Mon Sep 17 00:00:00 2001
From: Anton Ivashkin 
Date: Thu, 8 Jul 2021 17:41:18 +0300
Subject: [PATCH 249/290] Fix throwing exception when iterate over non existing
 remote directory

---
 src/Disks/IDiskRemote.cpp | 6 +++++-
 src/Disks/IDiskRemote.h   | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp
index b30e9613ed8..f4c207d9cbd 100644
--- a/src/Disks/IDiskRemote.cpp
+++ b/src/Disks/IDiskRemote.cpp
@@ -417,7 +417,11 @@ void IDiskRemote::removeDirectory(const String & path)
 
 DiskDirectoryIteratorPtr IDiskRemote::iterateDirectory(const String & path)
 {
-    return std::make_unique(metadata_path + path, path);
+    String meta_path = metadata_path + path;
+    if (fs::exists(meta_path) && fs::is_directory(meta_path))
+        return std::make_unique(meta_path, path);
+    else
+        return std::make_unique();
 }
 
 
diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h
index e725e0ed744..360d4e2de33 100644
--- a/src/Disks/IDiskRemote.h
+++ b/src/Disks/IDiskRemote.h
@@ -193,6 +193,7 @@ struct IDiskRemote::Metadata
 class RemoteDiskDirectoryIterator final : public IDiskDirectoryIterator
 {
 public:
+    RemoteDiskDirectoryIterator() {}
     RemoteDiskDirectoryIterator(const String & full_path, const String & folder_path_) : iter(full_path), folder_path(folder_path_) {}
 
     void next() override { ++iter; }

From ac68d5ea7130429fad3b6fda62021bd8e3ab79b2 Mon Sep 17 00:00:00 2001
From: Anton Ivashkin 
Date: Thu, 8 Jul 2021 18:39:41 +0300
Subject: [PATCH 250/290] Fix path concatenation

---
 src/Disks/IDiskRemote.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp
index f4c207d9cbd..a4dcc8037bc 100644
--- a/src/Disks/IDiskRemote.cpp
+++ b/src/Disks/IDiskRemote.cpp
@@ -417,7 +417,7 @@ void IDiskRemote::removeDirectory(const String & path)
 
 DiskDirectoryIteratorPtr IDiskRemote::iterateDirectory(const String & path)
 {
-    String meta_path = metadata_path + path;
+    fs::path meta_path = fs::path(metadata_path) / path;
     if (fs::exists(meta_path) && fs::is_directory(meta_path))
         return std::make_unique(meta_path, path);
     else

From e4b1e0619c3ff9b38653bbe556265606e61bb3bf Mon Sep 17 00:00:00 2001
From: zxc111 
Date: Fri, 9 Jul 2021 00:19:42 +0800
Subject: [PATCH 251/290] hex/bin functions support AggregateFunction states.

---
 src/Functions/FunctionsCoding.h                | 18 ++++++++++++++++--
 .../0_stateless/01926_bin_unbin.reference      |  4 ++++
 tests/queries/0_stateless/01926_bin_unbin.sql  |  6 ++++++
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h
index 72f2aa1be1c..4db138a12a2 100644
--- a/src/Functions/FunctionsCoding.h
+++ b/src/Functions/FunctionsCoding.h
@@ -25,6 +25,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -954,6 +955,8 @@ public:
 template 
 class EncodeToBinaryRepr : public IFunction
 {
+private:
+    ContextPtr context;
 public:
     static constexpr auto name = Impl::name;
     static constexpr size_t word_size = Impl::word_size;
@@ -978,18 +981,29 @@ public:
             !which.isDateTime64() &&
             !which.isUInt() &&
             !which.isFloat() &&
-            !which.isDecimal())
+            !which.isDecimal() &&
+            !which.isAggregateFunction())
             throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
                 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
         return std::make_shared();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const IColumn * column = arguments[0].column.get();
         ColumnPtr res_column;
 
+        WhichDataType which(column->getDataType());
+        if (which.isAggregateFunction())
+        {
+            auto to_string = FunctionFactory::instance().get("toString", context);
+            const ColumnPtr col = to_string->build(arguments)->execute(arguments, result_type, input_rows_count);
+            const auto * name_col = checkAndGetColumn(col.get());
+            tryExecuteString(name_col, res_column);
+            return res_column;
+        }
+
         if (tryExecuteUInt(column, res_column) ||
             tryExecuteUInt(column, res_column) ||
             tryExecuteUInt(column, res_column) ||
diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference
index f84a858e449..731d0223bb9 100644
--- a/tests/queries/0_stateless/01926_bin_unbin.reference
+++ b/tests/queries/0_stateless/01926_bin_unbin.reference
@@ -33,3 +33,7 @@
 1
 1
 1
+1
+1
+2D000000000000000A
+001011010000000000000000000000000000000000000000000000000000000000001010
diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql
index 555770d09c6..e112f8bd8a4 100644
--- a/tests/queries/0_stateless/01926_bin_unbin.sql
+++ b/tests/queries/0_stateless/01926_bin_unbin.sql
@@ -37,3 +37,9 @@ select bin(unbin('0')) == '00000000';
 select hex('') == bin('');
 select unhex('') == unbin('');
 select unhex('0') == unbin('0');
+
+-- hex and bin support AggregateFunction
+select hex(sumState(number)) == hex(toString(sumState(number))) from numbers(10);
+select hex(avgState(number)) == hex(toString(avgState(number))) from numbers(99);
+select hex(avgState(number)) from numbers(10);
+select bin(avgState(number)) from numbers(10);

From 334d5439c87c17d0a94e1c935ea52fc9bf06a4a9 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov 
Date: Thu, 8 Jul 2021 20:29:25 +0000
Subject: [PATCH 252/290] done

---
 tests/integration/test_cluster_copier/test_two_nodes.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py
index 817c3571833..255af13213a 100644
--- a/tests/integration/test_cluster_copier/test_two_nodes.py
+++ b/tests/integration/test_cluster_copier/test_two_nodes.py
@@ -473,17 +473,17 @@ def execute_task(started_cluster, task, cmd_options):
 
 
 # Tests
-@pytest.mark.timeout(600)
+@pytest.mark.skip(reason="Too flaky :(")
 def test_different_schema(started_cluster):
     execute_task(started_cluster, TaskWithDifferentSchema(started_cluster), [])
 
 
-@pytest.mark.timeout(600)
+@pytest.mark.skip(reason="Too flaky :(")
 def test_ttl_columns(started_cluster):
     execute_task(started_cluster, TaskTTL(started_cluster), [])
 
 
-@pytest.mark.timeout(600)
+@pytest.mark.skip(reason="Too flaky :(")
 def test_skip_index(started_cluster):
     execute_task(started_cluster, TaskSkipIndex(started_cluster), [])
 

From 3dfdcf4604e4c13e3b90743cede89cc5ed942b5c Mon Sep 17 00:00:00 2001
From: Vitaly Baranov 
Date: Thu, 8 Jul 2021 12:21:57 +0300
Subject: [PATCH 253/290] Improve implementation of leftPadString(), rename to
 leftPad(). Add new functions rightPad(), leftPadUTF8(), rightPadUTF8(). Add a
 test.

---
 src/Functions/GatherUtils/Sources.h           |   1 +
 src/Functions/leftPadString.cpp               | 194 -----------
 src/Functions/padString.cpp                   | 308 ++++++++++++++++++
 src/Functions/registerFunctionsString.cpp     |   6 +-
 src/Functions/ya.make                         |   2 +-
 .../0_stateless/01940_pad_string.reference    |  54 +++
 .../queries/0_stateless/01940_pad_string.sql  |  54 +++
 7 files changed, 420 insertions(+), 199 deletions(-)
 delete mode 100644 src/Functions/leftPadString.cpp
 create mode 100644 src/Functions/padString.cpp
 create mode 100644 tests/queries/0_stateless/01940_pad_string.reference
 create mode 100644 tests/queries/0_stateless/01940_pad_string.sql

diff --git a/src/Functions/GatherUtils/Sources.h b/src/Functions/GatherUtils/Sources.h
index 4dbaff9f567..9a459860a68 100644
--- a/src/Functions/GatherUtils/Sources.h
+++ b/src/Functions/GatherUtils/Sources.h
@@ -755,6 +755,7 @@ struct GenericValueSource : public ValueSourceImpl
 {
     using Slice = GenericValueSlice;
     using SinkType = GenericArraySink;
+    using Column = IColumn;
 
     const IColumn * column;
     size_t total_rows;
diff --git a/src/Functions/leftPadString.cpp b/src/Functions/leftPadString.cpp
deleted file mode 100644
index cdcfb46eb73..00000000000
--- a/src/Functions/leftPadString.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-#include 
-#include 
-#include 
-
-#include 
-#include 
-#include 
-
-#include 
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int ILLEGAL_COLUMN;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int BAD_ARGUMENTS;
-}
-
-namespace
-{
-    struct LeftPadStringImpl
-    {
-        static void vector(
-            const ColumnString::Chars & data,
-            const ColumnString::Offsets & offsets,
-            const size_t length,
-            const String & padstr,
-            ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
-        {
-            size_t size = offsets.size();
-            res_data.resize((length + 1 /* zero terminator */) * size);
-            res_offsets.resize(size);
-
-            const size_t padstr_size = padstr.size();
-
-            ColumnString::Offset prev_offset = 0;
-            ColumnString::Offset res_prev_offset = 0;
-            for (size_t i = 0; i < size; ++i)
-            {
-                size_t data_length = offsets[i] - prev_offset - 1 /* zero terminator */;
-                if (data_length < length)
-                {
-                    for (size_t j = 0; j < length - data_length; ++j)
-                        res_data[res_prev_offset + j] = padstr[j % padstr_size];
-                    memcpy(&res_data[res_prev_offset + length - data_length], &data[prev_offset], data_length);
-                }
-                else
-                {
-                    memcpy(&res_data[res_prev_offset], &data[prev_offset], length);
-                }
-                res_data[res_prev_offset + length] = 0;
-                res_prev_offset += length + 1;
-                res_offsets[i] = res_prev_offset;
-            }
-        }
-
-        static void vectorFixed(
-            const ColumnFixedString::Chars & data,
-            const size_t n,
-            const size_t length,
-            const String & padstr,
-            ColumnFixedString::Chars & res_data)
-        {
-            const size_t padstr_size = padstr.size();
-            const size_t size = data.size() / n;
-            res_data.resize(length * size);
-            for (size_t i = 0; i < size; ++i)
-            {
-                if (length < n)
-                {
-                    memcpy(&res_data[i * length], &data[i * n], length);
-                }
-                else
-                {
-                    for (size_t j = 0; j < length - n; ++j)
-                        res_data[i * length + j] = padstr[j % padstr_size];
-                    memcpy(&res_data[i * length + length - n], &data[i * n], n);
-                }
-            }
-        }
-    };
-
-    class FunctionLeftPadString : public IFunction
-    {
-    public:
-        static constexpr auto name = "leftPadString";
-        static FunctionPtr create(const ContextPtr) { return std::make_shared(); }
-
-        String getName() const override { return name; }
-
-        bool isVariadic() const override { return true; }
-        size_t getNumberOfArguments() const override { return 0; }
-
-        bool useDefaultImplementationForConstants() const override { return true; }
-
-        DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
-        {
-            size_t number_of_arguments = arguments.size();
-
-            if (number_of_arguments != 2 && number_of_arguments != 3)
-                throw Exception(
-                    ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                    "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3",
-                    getName(),
-                    toString(number_of_arguments));
-
-            if (!isStringOrFixedString(arguments[0]))
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName());
-
-            if (!isNativeNumber(arguments[1]))
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Illegal type {} of second argument of function {}",
-                    arguments[1]->getName(),
-                    getName());
-
-            if (number_of_arguments == 3 && !isStringOrFixedString(arguments[2]))
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Illegal type {} of third argument of function {}",
-                    arguments[2]->getName(),
-                    getName());
-
-            return arguments[0];
-        }
-
-        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
-        {
-            const ColumnPtr str_column = arguments[0].column;
-            String padstr = " ";
-            if (arguments.size() == 3)
-            {
-                const ColumnConst * pad_column = checkAndGetColumnConst(arguments[2].column.get());
-                if (!pad_column)
-                    throw Exception(
-                        ErrorCodes::ILLEGAL_COLUMN,
-                        "Illegal column {} of third ('pad') argument of function {}. Must be constant string.",
-                        arguments[2].column->getName(),
-                        getName());
-
-                padstr = pad_column->getValue();
-            }
-
-            const ColumnConst * len_column = checkAndGetColumnConst(arguments[1].column.get());
-            if (!len_column)
-                throw Exception(
-                    ErrorCodes::ILLEGAL_COLUMN,
-                    "Illegal column {} of second ('len') argument of function {}. Must be a positive integer.",
-                    arguments[1].column->getName(),
-                    getName());
-            Int64 len = len_column->getInt(0);
-            if (len <= 0)
-                throw Exception(
-                    ErrorCodes::BAD_ARGUMENTS,
-                    "Illegal value {} of second ('len') argument of function {}. Must be a positive integer.",
-                    arguments[1].column->getName(),
-                    getName());
-
-            if (const ColumnString * strings = checkAndGetColumn(str_column.get()))
-            {
-                auto col_res = ColumnString::create();
-                LeftPadStringImpl::vector(
-                    strings->getChars(), strings->getOffsets(), len, padstr, col_res->getChars(), col_res->getOffsets());
-                return col_res;
-            }
-            else if (const ColumnFixedString * strings_fixed = checkAndGetColumn(str_column.get()))
-            {
-                auto col_res = ColumnFixedString::create(len);
-                LeftPadStringImpl::vectorFixed(strings_fixed->getChars(), strings_fixed->getN(), len, padstr, col_res->getChars());
-                return col_res;
-            }
-            else
-            {
-                throw Exception(
-                    ErrorCodes::ILLEGAL_COLUMN,
-                    "Illegal column {} of first ('str') argument of function {}. Must be a string or fixed string.",
-                    arguments[0].column->getName(),
-                    getName());
-            }
-        }
-    };
-}
-
-void registerFunctionLeftPadString(FunctionFactory & factory)
-{
-    factory.registerFunction(FunctionFactory::CaseInsensitive);
-    factory.registerAlias("lpad", "leftPadString", FunctionFactory::CaseInsensitive);
-}
-
-}
diff --git a/src/Functions/padString.cpp b/src/Functions/padString.cpp
new file mode 100644
index 00000000000..7711ab1a056
--- /dev/null
+++ b/src/Functions/padString.cpp
@@ -0,0 +1,308 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+namespace DB
+{
+using namespace GatherUtils;
+
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int TOO_LARGE_STRING_SIZE;
+}
+
+namespace
+{
+    /// The maximum new padded length.
+    constexpr size_t MAX_NEW_LENGTH = 1000000;
+
+    /// Appends padding characters to a sink based on a pad string.
+    /// Depending on how many padding characters are required to add
+    /// the pad string can be copied only partly or be repeated multiple times.
+    template 
+    class PaddingChars
+    {
+    public:
+        explicit PaddingChars(const String & pad_string_) : pad_string(pad_string_) { init(); }
+
+        ALWAYS_INLINE size_t numCharsInPadString() const
+        {
+            if constexpr (is_utf8)
+                return utf8_offsets.size() - 1;
+            else
+                return pad_string.length();
+        }
+
+        ALWAYS_INLINE size_t numCharsToNumBytes(size_t count) const
+        {
+            if constexpr (is_utf8)
+                return utf8_offsets[count];
+            else
+                return count;
+        }
+
+        void appendTo(StringSink & res_sink, size_t num_chars) const
+        {
+            if (!num_chars)
+                return;
+
+            const size_t step = numCharsInPadString();
+            while (true)
+            {
+                if (num_chars <= step)
+                {
+                    writeSlice(StringSource::Slice{bit_cast(pad_string.data()), numCharsToNumBytes(num_chars)}, res_sink);
+                    break;
+                }
+                writeSlice(StringSource::Slice{bit_cast(pad_string.data()), numCharsToNumBytes(step)}, res_sink);
+                num_chars -= step;
+            }
+        }
+
+    private:
+        void init()
+        {
+            if (pad_string.empty())
+                pad_string = " ";
+
+            if constexpr (is_utf8)
+            {
+                size_t offset = 0;
+                utf8_offsets.reserve(pad_string.length() + 1);
+                while (true)
+                {
+                    utf8_offsets.push_back(offset);
+                    if (offset == pad_string.length())
+                        break;
+                    offset += UTF8::seqLength(pad_string[offset]);
+                    if (offset > pad_string.length())
+                        offset = pad_string.length();
+                }
+            }
+
+            /// Not necessary, but good for performance.
+            while (numCharsInPadString() < 16)
+            {
+                pad_string += pad_string;
+                if constexpr (is_utf8)
+                {
+                    size_t old_size = utf8_offsets.size();
+                    utf8_offsets.reserve((old_size - 1) * 2);
+                    size_t base = utf8_offsets.back();
+                    for (size_t i = 1; i != old_size; ++i)
+                        utf8_offsets.push_back(utf8_offsets[i] + base);
+                }
+            }
+        }
+
+        String pad_string;
+        std::vector utf8_offsets;
+    };
+
+    /// Returns the number of characters in a slice.
+    template 
+    inline ALWAYS_INLINE size_t getLengthOfSlice(const StringSource::Slice & slice)
+    {
+        if constexpr (is_utf8)
+            return UTF8::countCodePoints(slice.data, slice.size);
+        else
+            return slice.size;
+    }
+
+    /// Moves the end of a slice back by n characters.
+    template 
+    inline ALWAYS_INLINE StringSource::Slice removeSuffixFromSlice(const StringSource::Slice & slice, size_t suffix_length)
+    {
+        StringSource::Slice res = slice;
+        if constexpr (is_utf8)
+            res.size = UTF8StringSource::skipCodePointsBackward(slice.data + slice.size, suffix_length, slice.data) - res.data;
+        else
+            res.size -= std::min(suffix_length, res.size);
+        return res;
+    }
+
+    /// If `is_right_pad` - it's the rightPad() function instead of leftPad().
+    /// If `is_utf8` - lengths are measured in code points instead of bytes.
+    template 
+    class FunctionPadString : public IFunction
+    {
+    public:
+        static constexpr auto name = is_right_pad ? (is_utf8 ? "rightPadUTF8" : "rightPad") : (is_utf8 ? "leftPadUTF8" : "leftPad");
+        static FunctionPtr create(const ContextPtr) { return std::make_shared(); }
+
+        String getName() const override { return name; }
+
+        bool isVariadic() const override { return true; }
+        size_t getNumberOfArguments() const override { return 0; }
+
+        bool useDefaultImplementationForConstants() const override { return false; }
+
+        DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+        {
+            size_t number_of_arguments = arguments.size();
+
+            if (number_of_arguments != 2 && number_of_arguments != 3)
+                throw Exception(
+                    ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                    "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3",
+                    getName(),
+                    std::to_string(number_of_arguments));
+
+            if (!isStringOrFixedString(arguments[0]))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of the first argument of function {}, should be string",
+                    arguments[0]->getName(),
+                    getName());
+
+            if (!isUnsignedInteger(arguments[1]))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of the second argument of function {}, should be unsigned integer",
+                    arguments[1]->getName(),
+                    getName());
+
+            if (number_of_arguments == 3 && !isStringOrFixedString(arguments[2]))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Illegal type {} of the third argument of function {}, should be const string",
+                    arguments[2]->getName(),
+                    getName());
+
+            return arguments[0];
+        }
+
+        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+        {
+            auto column_string = arguments[0].column;
+            auto column_length = arguments[1].column;
+
+            String pad_string;
+            if (arguments.size() == 3)
+            {
+                auto column_pad = arguments[2].column;
+                const ColumnConst * column_pad_const = checkAndGetColumnConst(column_pad.get());
+                if (!column_pad_const)
+                    throw Exception(
+                        ErrorCodes::ILLEGAL_COLUMN,
+                        "Illegal column {}, third argument of function {} must be a constant string",
+                        column_pad->getName(),
+                        getName());
+
+                pad_string = column_pad_const->getValue();
+            }
+            PaddingChars padding_chars{pad_string};
+
+            auto col_res = ColumnString::create();
+            StringSink res_sink{*col_res, input_rows_count};
+
+            if (const ColumnString * col = checkAndGetColumn(column_string.get()))
+                executeForSource(StringSource{*col}, column_length, padding_chars, res_sink);
+            else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_string.get()))
+                executeForSource(FixedStringSource{*col_fixed}, column_length, padding_chars, res_sink);
+            else if (const ColumnConst * col_const = checkAndGetColumnConst(column_string.get()))
+                executeForSource(ConstSource{*col_const}, column_length, padding_chars, res_sink);
+            else if (const ColumnConst * col_const_fixed = checkAndGetColumnConst(column_string.get()))
+                executeForSource(ConstSource{*col_const_fixed}, column_length, padding_chars, res_sink);
+            else
+                throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Illegal column {}, first argument of function {} must be a string",
+                    arguments[0].column->getName(),
+                    getName());
+
+            return col_res;
+        }
+
+    private:
+        template 
+        void executeForSource(
+            SourceStrings && strings,
+            const ColumnPtr & column_length,
+            const PaddingChars & padding_chars,
+            StringSink & res_sink) const
+        {
+            if (const auto * col_const = checkAndGetColumn(column_length.get()))
+                executeForSourceAndLength(std::forward(strings), ConstSource{*col_const}, padding_chars, res_sink);
+            else
+                executeForSourceAndLength(std::forward(strings), GenericValueSource{*column_length}, padding_chars, res_sink);
+        }
+
+        template 
+        void executeForSourceAndLength(
+            SourceStrings && strings,
+            SourceLengths && lengths,
+            const PaddingChars & padding_chars,
+            StringSink & res_sink) const
+        {
+            bool is_const_length = lengths.isConst();
+            bool need_check_length = true;
+
+            for (; !res_sink.isEnd(); res_sink.next(), strings.next(), lengths.next())
+            {
+                auto str = strings.getWhole();
+                size_t current_length = getLengthOfSlice(str);
+
+                auto new_length_slice = lengths.getWhole();
+                size_t new_length = new_length_slice.elements->getUInt(new_length_slice.position);
+
+                if (need_check_length)
+                {
+                    if (new_length > MAX_NEW_LENGTH)
+                    {
+                        throw Exception(
+                            "New padded length (" + std::to_string(new_length) + ") is too big, maximum is: " + std::to_string(MAX_NEW_LENGTH),
+                            ErrorCodes::TOO_LARGE_STRING_SIZE);
+                    }
+                    if (is_const_length)
+                    {
+                        size_t rows_count = res_sink.offsets.size();
+                        res_sink.reserve((new_length + 1 /* zero terminator */) * rows_count);
+                        need_check_length = false;
+                    }
+                }
+
+                if (new_length == current_length)
+                {
+                    writeSlice(str, res_sink);
+                }
+                else if (new_length < current_length)
+                {
+                    str = removeSuffixFromSlice(str, current_length - new_length);
+                    writeSlice(str, res_sink);
+                }
+                else if (new_length > current_length)
+                {
+                    if constexpr (!is_right_pad)
+                        padding_chars.appendTo(res_sink, new_length - current_length);
+
+                    writeSlice(str, res_sink);
+
+                    if constexpr (is_right_pad)
+                        padding_chars.appendTo(res_sink, new_length - current_length);
+                }
+            }
+        }
+    };
+}
+
+void registerFunctionPadString(FunctionFactory & factory)
+{
+    factory.registerFunction>(); /// leftPad
+    factory.registerFunction>();  /// leftPadUTF8
+    factory.registerFunction>();  /// rightPad
+    factory.registerFunction>();   /// rightPadUTF8
+
+    factory.registerAlias("lpad", "leftPad", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("rpad", "rightPad", FunctionFactory::CaseInsensitive);
+}
+
+}
diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp
index 1c487981844..18a30469386 100644
--- a/src/Functions/registerFunctionsString.cpp
+++ b/src/Functions/registerFunctionsString.cpp
@@ -29,13 +29,11 @@ void registerFunctionAppendTrailingCharIfAbsent(FunctionFactory &);
 void registerFunctionStartsWith(FunctionFactory &);
 void registerFunctionEndsWith(FunctionFactory &);
 void registerFunctionTrim(FunctionFactory &);
+void registerFunctionPadString(FunctionFactory &);
 void registerFunctionRegexpQuoteMeta(FunctionFactory &);
 void registerFunctionNormalizeQuery(FunctionFactory &);
 void registerFunctionNormalizedQueryHash(FunctionFactory &);
 void registerFunctionCountMatches(FunctionFactory &);
-void registerFunctionEncodeXMLComponent(FunctionFactory & factory);
-void registerFunctionDecodeXMLComponent(FunctionFactory & factory);
-void registerFunctionLeftPadString(FunctionFactory & factory);
 void registerFunctionEncodeXMLComponent(FunctionFactory &);
 void registerFunctionDecodeXMLComponent(FunctionFactory &);
 void registerFunctionExtractTextFromHTML(FunctionFactory &);
@@ -71,13 +69,13 @@ void registerFunctionsString(FunctionFactory & factory)
     registerFunctionStartsWith(factory);
     registerFunctionEndsWith(factory);
     registerFunctionTrim(factory);
+    registerFunctionPadString(factory);
     registerFunctionRegexpQuoteMeta(factory);
     registerFunctionNormalizeQuery(factory);
     registerFunctionNormalizedQueryHash(factory);
     registerFunctionCountMatches(factory);
     registerFunctionEncodeXMLComponent(factory);
     registerFunctionDecodeXMLComponent(factory);
-    registerFunctionLeftPadString(factory);
     registerFunctionExtractTextFromHTML(factory);
 #if USE_BASE64
     registerFunctionBase64Encode(factory);
diff --git a/src/Functions/ya.make b/src/Functions/ya.make
index ba14e9a3e02..5f84511aa52 100644
--- a/src/Functions/ya.make
+++ b/src/Functions/ya.make
@@ -332,7 +332,6 @@ SRCS(
     jumpConsistentHash.cpp
     lcm.cpp
     least.cpp
-    leftPadString.cpp
     lengthUTF8.cpp
     less.cpp
     lessOrEquals.cpp
@@ -388,6 +387,7 @@ SRCS(
     now.cpp
     now64.cpp
     nullIf.cpp
+    padString.cpp
     partitionId.cpp
     pi.cpp
     plus.cpp
diff --git a/tests/queries/0_stateless/01940_pad_string.reference b/tests/queries/0_stateless/01940_pad_string.reference
new file mode 100644
index 00000000000..22cd3f9be07
--- /dev/null
+++ b/tests/queries/0_stateless/01940_pad_string.reference
@@ -0,0 +1,54 @@
+leftPad
+
+a
+ab
+abc
+ abc
+  abc
+       abc
+ab
+*abc
+**abc
+*******abc
+ab
+*abc
+*.abc
+*.*.*.*abc
+leftPadUTF8
+а
+аб
+аб
+абвг
+ЧАабвг
+ЧАСЧАСЧАабвг
+rightPad
+
+a
+ab
+abc
+abc 
+abc  
+abc       
+ab
+abc*
+abc**
+abc*******
+ab
+abc*
+abc*.
+abc*.*.*.*
+rightPadUTF8
+а
+аб
+аб
+абвг
+абвгЧА
+абвгЧАСЧАСЧА
+numbers
+
+1^
+_2^^
+__3^^^
+___4^^^^
+____5^^^^^
+_____6^^^^^^
diff --git a/tests/queries/0_stateless/01940_pad_string.sql b/tests/queries/0_stateless/01940_pad_string.sql
new file mode 100644
index 00000000000..e4ba0aec6d2
--- /dev/null
+++ b/tests/queries/0_stateless/01940_pad_string.sql
@@ -0,0 +1,54 @@
+SELECT 'leftPad';
+SELECT leftPad('abc', 0);
+SELECT leftPad('abc', 1);
+SELECT leftPad('abc', 2);
+SELECT leftPad('abc', 3);
+SELECT leftPad('abc', 4);
+SELECT leftPad('abc', 5);
+SELECT leftPad('abc', 10);
+
+SELECT leftPad('abc', 2, '*');
+SELECT leftPad('abc', 4, '*');
+SELECT leftPad('abc', 5, '*');
+SELECT leftPad('abc', 10, '*');
+SELECT leftPad('abc', 2, '*.');
+SELECT leftPad('abc', 4, '*.');
+SELECT leftPad('abc', 5, '*.');
+SELECT leftPad('abc', 10, '*.');
+
+SELECT 'leftPadUTF8';
+SELECT leftPad('абвг', 2);
+SELECT leftPadUTF8('абвг', 2);
+SELECT leftPad('абвг', 4);
+SELECT leftPadUTF8('абвг', 4);
+SELECT leftPad('абвг', 12, 'ЧАС');
+SELECT leftPadUTF8('абвг', 12, 'ЧАС');
+
+SELECT 'rightPad';
+SELECT rightPad('abc', 0);
+SELECT rightPad('abc', 1);
+SELECT rightPad('abc', 2);
+SELECT rightPad('abc', 3);
+SELECT rightPad('abc', 4);
+SELECT rightPad('abc', 5);
+SELECT rightPad('abc', 10);
+
+SELECT rightPad('abc', 2, '*');
+SELECT rightPad('abc', 4, '*');
+SELECT rightPad('abc', 5, '*');
+SELECT rightPad('abc', 10, '*');
+SELECT rightPad('abc', 2, '*.');
+SELECT rightPad('abc', 4, '*.');
+SELECT rightPad('abc', 5, '*.');
+SELECT rightPad('abc', 10, '*.');
+
+SELECT 'rightPadUTF8';
+SELECT rightPad('абвг', 2);
+SELECT rightPadUTF8('абвг', 2);
+SELECT rightPad('абвг', 4);
+SELECT rightPadUTF8('абвг', 4);
+SELECT rightPad('абвг', 12, 'ЧАС');
+SELECT rightPadUTF8('абвг', 12, 'ЧАС');
+
+SELECT 'numbers';
+SELECT rightPad(leftPad(toString(number), number, '_'), number*2, '^') FROM numbers(7);

From df90e43c3dfaf19d83e7ccabb6ed9983b6667398 Mon Sep 17 00:00:00 2001
From: Maksim Kita 
Date: Fri, 9 Jul 2021 02:09:51 +0300
Subject: [PATCH 254/290] Add changelog for 21.7

---
 CHANGELOG.md | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 167 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8987082db30..5e0a0f30804 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,170 @@
+### ClickHouse release v21.7, 2021-07-09
+
+#### Backward Incompatible Change
+
+* Forward/backward incompatible change of maximum buffer size in clickhouse-keeper. Better to do it now (before production), than later. [#25421](https://github.com/ClickHouse/ClickHouse/pull/25421) ([alesapin](https://github.com/alesapin)).
+* Improved performance of queries with explicitly defined large sets. Added compatibility setting `legacy_column_name_of_tuple_literal`. It makes sense to set it to `true`, while doing rolling update of cluster from version lower than 21.7 to any higher version. Otherwise distributed queries with explicitly defined sets at `IN` clause may fail during update. [#25371](https://github.com/ClickHouse/ClickHouse/pull/25371) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### New Feature
+
+* Added YAML configuration support to configuration loader. This closes [#3607](https://github.com/ClickHouse/ClickHouse/issues/3607). [#21858](https://github.com/ClickHouse/ClickHouse/pull/21858) ([BoloniniD](https://github.com/BoloniniD)).
+* Provides a way to restore replicated table when the data is (possibly) present, but the ZooKeeper metadata is lost. Resolves [#13458](https://github.com/ClickHouse/ClickHouse/issues/13458). [#13652](https://github.com/ClickHouse/ClickHouse/pull/13652) ([Mike Kot](https://github.com/myrrc)).
+* Now clickhouse-keeper supports ZooKeeper-like `digest` ACLs. [#24448](https://github.com/ClickHouse/ClickHouse/pull/24448) ([alesapin](https://github.com/alesapin)).
+* Support structs and maps in Arrow/Parquet/ORC and dictionaries in Arrow input/output formats. Present new setting `output_format_arrow_low_cardinality_as_dictionary`. [#24341](https://github.com/ClickHouse/ClickHouse/pull/24341) ([Kruglov Pavel](https://github.com/Avogar)).
+* Dictionaries added support for Array type. [#25119](https://github.com/ClickHouse/ClickHouse/pull/25119) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added function `bitPositionsToArray`. Closes [#23792](https://github.com/ClickHouse/ClickHouse/issues/23792). Author [Kevin Wan] (@MaxWk). [#25394](https://github.com/ClickHouse/ClickHouse/pull/25394) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added function `dateName`. Author [Daniil Kondratyev] (@dankondr). [#25372](https://github.com/ClickHouse/ClickHouse/pull/25372) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add `toJSONString` function to serialize columns to their JSON representations. [#25164](https://github.com/ClickHouse/ClickHouse/pull/25164) ([Amos Bird](https://github.com/amosbird)).
+* Now query_log has two new columns: `initial_query_start_time`, `initial_query_start_time_microsecond` that record the starting time of a distributed query if any. [#25022](https://github.com/ClickHouse/ClickHouse/pull/25022) ([Amos Bird](https://github.com/amosbird)).
+* Add aggregate function `segmentLengthSum`. [#24250](https://github.com/ClickHouse/ClickHouse/pull/24250) ([flynn](https://github.com/ucasfl)).
+* Add a new boolean setting `prefer_global_in_and_join` which defaults all IN/JOIN as GLOBAL IN/JOIN. [#23434](https://github.com/ClickHouse/ClickHouse/pull/23434) ([Amos Bird](https://github.com/amosbird)).
+* Support `ALTER DELETE` queries for `Join` table engine. [#23260](https://github.com/ClickHouse/ClickHouse/pull/23260) ([foolchi](https://github.com/foolchi)).
+* Add `quantileBFloat16` aggregate function as well as the corresponding `quantilesBFloat16` and `medianBFloat16`. It is very simple and fast quantile estimator with relative error not more than 0.390625%. This closes [#16641](https://github.com/ClickHouse/ClickHouse/issues/16641). [#23204](https://github.com/ClickHouse/ClickHouse/pull/23204) ([Ivan Novitskiy](https://github.com/RedClusive)).
+* Implement `sequenceNextNode()` function useful for `flow analysis`. [#19766](https://github.com/ClickHouse/ClickHouse/pull/19766) ([achimbab](https://github.com/achimbab)).
+
+#### Experimental Feature
+
+* Add support for VFS over HDFS. [#11058](https://github.com/ClickHouse/ClickHouse/pull/11058) ([overshov](https://github.com/overshov)).
+
+#### Performance Improvement
+
+* Added optimization, that transforms some functions to reading of subcolumns to reduce amount of read data. E.g., statement `col IS NULL` is transformed to reading of subcolumn `col.null`. Optimization can be enabled by setting `optimize_functions_to_subcolumns`. [#24406](https://github.com/ClickHouse/ClickHouse/pull/24406) ([Anton Popov](https://github.com/CurtizJ)).
+* Rewrite more columns to possible alias expressions. This may enable better optimization, such as projections. [#24405](https://github.com/ClickHouse/ClickHouse/pull/24405) ([Amos Bird](https://github.com/amosbird)).
+* Index of type bloom_filter can be used for expressions with `hasAny` function with constant arrays. This closes: [#24291](https://github.com/ClickHouse/ClickHouse/issues/24291). [#24900](https://github.com/ClickHouse/ClickHouse/pull/24900) ([Vasily Nemkov](https://github.com/Enmk)).
+* Add exponential backoff to reschedule read attempt in case RabbitMQ queues are empty. Closes [#24340](https://github.com/ClickHouse/ClickHouse/issues/24340). [#24415](https://github.com/ClickHouse/ClickHouse/pull/24415) ([Kseniia Sumarokova](https://github.com/kssenii)).
+
+#### Improvement
+
+* Add standalone `clickhouse-keeper` symlink to the main `clickhouse` binary. Now it's possible to run coordination without the main clickhouse server. [#24059](https://github.com/ClickHouse/ClickHouse/pull/24059) ([alesapin](https://github.com/alesapin)).
+* Use global settings for query to `VIEW`. Fixed the behavior when queries to `VIEW` use local settings, that leads to errors if setting on `CREATE VIEW` and `SELECT` were different. As for now, `VIEW` won't use these modified settings, but you can still pass additional settings in `SETTINGS` section of `CREATE VIEW` query. Close [#20551](https://github.com/ClickHouse/ClickHouse/issues/20551). [#24095](https://github.com/ClickHouse/ClickHouse/pull/24095) ([Vladimir](https://github.com/vdimir)).
+* Add two Replicated*MergeTree settings: `max_replicated_fetches_network_bandwidth` and `max_replicated_sends_network_bandwidth` which allows to limit maximum speed of replicated fetches/sends for table. Add two server-wide settings (in `default` user profile): `max_replicated_fetches_network_bandwidth_for_server` and `max_replicated_sends_network_bandwidth_for_server` which limit maximum speed of replication for all tables. The settings are not followed perfectly accurately. Turned off by default. Fixes [#1821](https://github.com/ClickHouse/ClickHouse/issues/1821). [#24573](https://github.com/ClickHouse/ClickHouse/pull/24573) ([alesapin](https://github.com/alesapin)).
+* On server start, parts with incorrect partition ID would not be ever removed, but always detached. [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25166](https://github.com/ClickHouse/ClickHouse/pull/25166) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Use separate `clickhouse-bridge` group and user for bridge processes. Set oom_score_adj so the bridges will be first subjects for OOM killer. Set set maximum RSS to 1 GiB. Closes [#23861](https://github.com/ClickHouse/ClickHouse/issues/23861). [#25280](https://github.com/ClickHouse/ClickHouse/pull/25280) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Increase size of background schedule pool to 128 (`background_schedule_pool_size` setting). It allows avoiding replication queue hung on slow zookeeper connection. [#25072](https://github.com/ClickHouse/ClickHouse/pull/25072) ([alesapin](https://github.com/alesapin)).
+* Add merge tree setting `max_parts_to_merge_at_once` which limits the number of parts that can be merged in the background at once. Doesn't affect `OPTIMIZE FINAL` query. Fixes [#1820](https://github.com/ClickHouse/ClickHouse/issues/1820). [#24496](https://github.com/ClickHouse/ClickHouse/pull/24496) ([alesapin](https://github.com/alesapin)).
+* Allow `not in` operator to be used in partition pruning. [#24894](https://github.com/ClickHouse/ClickHouse/pull/24894) ([Amos Bird](https://github.com/amosbird)).
+* Recognize IPv4 addresses like `127.0.1.1` as local. This is controversial and closes [#23504](https://github.com/ClickHouse/ClickHouse/issues/23504). Michael Filimonov will test this feature. [#24316](https://github.com/ClickHouse/ClickHouse/pull/24316) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* ClickHouse database created with MaterializeMySQL now contains all column comments from the MySQL database that materialized. [#25199](https://github.com/ClickHouse/ClickHouse/pull/25199) ([Storozhuk Kostiantyn](https://github.com/sand6255)).
+* Add settings (`connection_auto_close`/`connection_max_tries`/`connection_pool_size`) for MySQL storage engine. [#24146](https://github.com/ClickHouse/ClickHouse/pull/24146) ([Azat Khuzhin](https://github.com/azat)).
+* Improve startup time of Distributed engine. [#25663](https://github.com/ClickHouse/ClickHouse/pull/25663) ([Azat Khuzhin](https://github.com/azat)).
+* Drop replicas from dirname for internal_replication=true (allows INSERT into Distributed with cluster from any number of replicas, before only 15 replicas was supported, everything more will fail with ENAMETOOLONG while creating directory for async blocks). [#25513](https://github.com/ClickHouse/ClickHouse/pull/25513) ([Azat Khuzhin](https://github.com/azat)).
+* Added support Interval type for LowCardinality. Closes [#21730](https://github.com/ClickHouse/ClickHouse/issues/21730). [#25410](https://github.com/ClickHouse/ClickHouse/pull/25410) ([Vladimir](https://github.com/vdimir)).
+* Add == operator on time conditions for sequenceMatch and sequenceCount functions. For eg: sequenceMatch('(?1)(?t==1)(?2)')(time, data = 1, data = 2). [#25299](https://github.com/ClickHouse/ClickHouse/pull/25299) ([Christophe Kalenzaga](https://github.com/mga-chka)).
+* Add settings `http_max_fields`, `http_max_field_name_size`, `http_max_field_value_size`. [#25296](https://github.com/ClickHouse/ClickHouse/pull/25296) ([Ivan](https://github.com/abyss7)).
+* Add support for function `if` with Decimal and Int types on its branches. This closes [#20549](https://github.com/ClickHouse/ClickHouse/issues/20549). This closes [#10142](https://github.com/ClickHouse/ClickHouse/issues/10142). [#25283](https://github.com/ClickHouse/ClickHouse/pull/25283) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Update prompt in `clickhouse-client` and display a message when reconnecting. This closes [#10577](https://github.com/ClickHouse/ClickHouse/issues/10577). [#25281](https://github.com/ClickHouse/ClickHouse/pull/25281) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Correct memory tracking in aggregate function `topK`. This closes [#25259](https://github.com/ClickHouse/ClickHouse/issues/25259). [#25260](https://github.com/ClickHouse/ClickHouse/pull/25260) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix topLevelDomain() for IDN hosts (i.e. `example.рф`), before it returns empty string for such hosts. [#25103](https://github.com/ClickHouse/ClickHouse/pull/25103) ([Azat Khuzhin](https://github.com/azat)).
+* Detect linux version at runtime (for worked nested epoll, that is required for `async_socket_for_remote`/`use_hedged_requests`, otherwise remote queries may stuck). [#25067](https://github.com/ClickHouse/ClickHouse/pull/25067) ([Azat Khuzhin](https://github.com/azat)).
+* For distributed query, when `optimize_skip_unused_shards=1`, allow to skip shard with condition like `(sharding key) IN (one-element-tuple)`. (Tuples with many elements were supported. Tuple with single element did not work because it is parsed as literal). [#24930](https://github.com/ClickHouse/ClickHouse/pull/24930) ([Amos Bird](https://github.com/amosbird)).
+* Improved logging of S3 errors, no more double spaces in case of empty keys and buckets. [#24897](https://github.com/ClickHouse/ClickHouse/pull/24897) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Some queries require multi-pass semantic analysis. Try reusing built sets for `IN` in this case. [#24874](https://github.com/ClickHouse/ClickHouse/pull/24874) ([Amos Bird](https://github.com/amosbird)).
+* Respect `max_distributed_connections` for `insert_distributed_sync` (otherwise for huge clusters and sync insert it may run out of `max_thread_pool_size`). [#24754](https://github.com/ClickHouse/ClickHouse/pull/24754) ([Azat Khuzhin](https://github.com/azat)).
+* Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)).
+* Make String-to-Int parser stricter so that `toInt64('+')` will throw. [#24475](https://github.com/ClickHouse/ClickHouse/pull/24475) ([Amos Bird](https://github.com/amosbird)).
+* If SSDDictionary is created with DDL query, it can be created only inside user_files directory. [#24466](https://github.com/ClickHouse/ClickHouse/pull/24466) ([Maksim Kita](https://github.com/kitaisreal)).
+* PostgreSQL support specifying non default schema for insert queries. Closes [#24149](https://github.com/ClickHouse/ClickHouse/issues/24149). [#24413](https://github.com/ClickHouse/ClickHouse/pull/24413) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix IPv6 addresses resolving (i.e. fixes `select * from remote('[::1]', system.one)`). [#24319](https://github.com/ClickHouse/ClickHouse/pull/24319) ([Azat Khuzhin](https://github.com/azat)).
+* Fix trailing whitespaces in FROM clause with subqueries in multiline mode, and also changes the output of the queries slightly in a more human friendly way. [#24151](https://github.com/ClickHouse/ClickHouse/pull/24151) ([Azat Khuzhin](https://github.com/azat)).
+* Suppress exceptions from logger code. [#24069](https://github.com/ClickHouse/ClickHouse/pull/24069) ([Azat Khuzhin](https://github.com/azat)).
+* Add ability to split distributed batch on failures (i.e. due to memory limits, corruptions), under `distributed_directory_monitor_split_batch_on_failure` (OFF by default). [#23864](https://github.com/ClickHouse/ClickHouse/pull/23864) ([Azat Khuzhin](https://github.com/azat)).
+* Handle column name clashes for storage join. Closes [#20309](https://github.com/ClickHouse/ClickHouse/issues/20309). [#23769](https://github.com/ClickHouse/ClickHouse/pull/23769) ([Vladimir](https://github.com/vdimir)).
+* Display progress for File table engine in clickhouse-local and on INSERT query in clickhouse-client when data is passed to stdin. Closes [#18209](https://github.com/ClickHouse/ClickHouse/issues/18209). [#23656](https://github.com/ClickHouse/ClickHouse/pull/23656) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Bugfixes and improvements of clickhouse-copier. Allow to copy tables with different (but compatible schemas). Closes [#9159](https://github.com/ClickHouse/ClickHouse/issues/9159). Added test to copy ReplacingMergeTree. Closes [#22711](https://github.com/ClickHouse/ClickHouse/issues/22711). Support TTL on columns and Data Skipping Indices. It simply removes it to create internal Distributed table (underlying table will have TTL and skipping indices). Closes [#19384](https://github.com/ClickHouse/ClickHouse/issues/19384). Allow to copy MATERIALIZED and ALIAS columns. There are some cases in which it could be helpful (e.g. if this column is in PRIMARY KEY). Now it could be allowed by setting `allow_to_copy_alias_and_materialized_columns` property to true in task configuration. Closes [#9177](https://github.com/ClickHouse/ClickHouse/issues/9177). Closes [#11007] (https://github.com/ClickHouse/ClickHouse/issues/11007). Closes [#9514](https://github.com/ClickHouse/ClickHouse/issues/9514). Added a property `allow_to_drop_target_partitions` in task configuration to drop partition in original table before moving helping tables. Closes [#20957](https://github.com/ClickHouse/ClickHouse/issues/20957). Get rid of `OPTIMIZE DEDUPLICATE` query. This hack was needed, because `ALTER TABLE MOVE PARTITION` was retried many times and plain MergeTree tables don't have deduplication. Closes [#17966](https://github.com/ClickHouse/ClickHouse/issues/17966). Write progress to ZooKeeper node on path `task_path + /status` in JSON format. Closes [#20955](https://github.com/ClickHouse/ClickHouse/issues/20955). Support for ReplicatedTables without arguments. Closes [#24834](https://github.com/ClickHouse/ClickHouse/issues/24834) .[#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Added sleep with backoff between read retries from S3. [#23461](https://github.com/ClickHouse/ClickHouse/pull/23461) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Respect `insert_allow_materialized_columns` (allows materialized columns) for INSERT into `Distributed` table. [#23349](https://github.com/ClickHouse/ClickHouse/pull/23349) ([Azat Khuzhin](https://github.com/azat)).
+* Add ability to push down LIMIT for distributed queries. [#23027](https://github.com/ClickHouse/ClickHouse/pull/23027) ([Azat Khuzhin](https://github.com/azat)).
+* Fix Zero-Copy replication with several S3 volumes (Fixes [#22679](https://github.com/ClickHouse/ClickHouse/issues/22679)). [#22864](https://github.com/ClickHouse/ClickHouse/pull/22864) ([ianton-ru](https://github.com/ianton-ru)).
+* Resolve the actual port number bound when a user requests any available port from the operating system. [#25569](https://github.com/ClickHouse/ClickHouse/pull/25569) ([bnaecker](https://github.com/bnaecker)).
+
+#### Bug Fix
+
+* Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)).
+* If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([tavplubix](https://github.com/tavplubix)).
+* Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix bug which can lead to intersecting parts after merges with TTL: `Part all_40_40_0 is covered by all_40_40_1 but should be merged into all_40_41_1. This shouldn't happen often.`. [#25549](https://github.com/ClickHouse/ClickHouse/pull/25549) ([alesapin](https://github.com/alesapin)).
+* Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)).
+* Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)).
+* On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([tavplubix](https://github.com/tavplubix)).
+* Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix error `Key expression contains comparison between inconvertible types` for queries with `ARRAY JOIN` in case if array is used in primary key. Fixes [#8247](https://github.com/ClickHouse/ClickHouse/issues/8247). [#25546](https://github.com/ClickHouse/ClickHouse/pull/25546) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed case, when sometimes conversion of postgres arrays resulted in String data type, not n-dimensional array, because `attndims` works incorrectly in some cases. Closes [#24804](https://github.com/ClickHouse/ClickHouse/issues/24804). [#25538](https://github.com/ClickHouse/ClickHouse/pull/25538) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)).
+* Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([tavplubix](https://github.com/tavplubix)).
+* `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([tavplubix](https://github.com/tavplubix)).
+* Fixed a bug in `Replicated` database engine that might rarely cause some replica to skip enqueued DDL query. [#24805](https://github.com/ClickHouse/ClickHouse/pull/24805) ([tavplubix](https://github.com/tavplubix)).
+* Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix waiting of automatic dropping of empty parts. It could lead to full filling of background pool and stuck of replication. [#23315](https://github.com/ClickHouse/ClickHouse/pull/23315) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix restore S3 table. [#25601](https://github.com/ClickHouse/ClickHouse/pull/25601) ([ianton-ru](https://github.com/ianton-ru)).
+* Fix segfault in `Arrow` format when using `Decimal256`. Add arrow `Decimal256` support. [#25531](https://github.com/ClickHouse/ClickHouse/pull/25531) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix convertion of datetime with timezone for MySQL, PostgreSQL, ODBC. Closes [#5057](https://github.com/ClickHouse/ClickHouse/issues/5057). [#25528](https://github.com/ClickHouse/ClickHouse/pull/25528) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix excessive underscore before the names of the preprocessed configuration files. [#25431](https://github.com/ClickHouse/ClickHouse/pull/25431) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
+* Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix error `Bad cast from type DB::ColumnLowCardinality to DB::ColumnVector` for queries where `LowCardinality` argument was used for IN (this bug appeared in 21.6). Fixes [#25187](https://github.com/ClickHouse/ClickHouse/issues/25187). [#25290](https://github.com/ClickHouse/ClickHouse/pull/25290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)).
+* Fix incorrect behaviour and UBSan report in big integers. In previous versions `CAST(1e19 AS UInt128)` returned zero. [#25279](https://github.com/ClickHouse/ClickHouse/pull/25279) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed an error which occurred while inserting a subset of columns using CSVWithNames format. Fixes [#25129](https://github.com/ClickHouse/ClickHouse/issues/25129). [#25169](https://github.com/ClickHouse/ClickHouse/pull/25169) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)).
+* Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)).
+* Fix wrong result when using aggregate projection with **not empty** `GROUP BY` key to execute query with `GROUP BY` by **empty** key. [#25055](https://github.com/ClickHouse/ClickHouse/pull/25055) ([Amos Bird](https://github.com/amosbird)).
+* Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)).
+* Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix limit/offset settings for distributed queries (ignore on the remote nodes). [#24940](https://github.com/ClickHouse/ClickHouse/pull/24940) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fixed bug with declaring S3 disk at root of bucket. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Fixed possible error 'Cannot read from istream at offset 0' when reading a file from DiskS3. [#24885](https://github.com/ClickHouse/ClickHouse/pull/24885) ([Pavel Kovalenko](https://github.com/Jokser)).
+* Fix "Missing columns" exception when joining Distributed Materialized View. [#24870](https://github.com/ClickHouse/ClickHouse/pull/24870) ([Azat Khuzhin](https://github.com/azat)).
+* Allow NULL values in postgresql protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)).
+* Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([tavplubix](https://github.com/tavplubix)).
+* Disallow building uniqXXXXStates of other aggregation states. [#24523](https://github.com/ClickHouse/ClickHouse/pull/24523) ([Raúl Marín](https://github.com/Algunenano)).
+* Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)).
+* Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix totalBytes computation in StorageBuffer. In current CH version total_writes.bytes counter decreases too much during the buffer flush. It leads to counter overflow and totalBytes return something around 17.44 EB some time after the flush. [#24450](https://github.com/ClickHouse/ClickHouse/pull/24450) ([DimasKovas](https://github.com/DimasKovas)).
+* Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)).
+* Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)).
+* In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)).
+* Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)).
+* Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([tavplubix](https://github.com/tavplubix)).
+* Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)).
+* Set `max_threads = 1` to fix mutation fail of StorageMemory. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)).
+* Column cardinality in join output same as at the input, close [#23351](https://github.com/ClickHouse/ClickHouse/issues/23351), close [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315). [#24061](https://github.com/ClickHouse/ClickHouse/pull/24061) ([Vladimir](https://github.com/vdimir)).
+* Fix the bug in failover behavior when Engine=Kafka was not able to start consumption if the same consumer had an empty assignment previously. Closes [#21118](https://github.com/ClickHouse/ClickHouse/issues/21118). [#21267](https://github.com/ClickHouse/ClickHouse/pull/21267) ([filimonov](https://github.com/filimonov)).
+* Fix MySQL select user() return empty. Closes [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697). [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697) ([sundyli](https://github.com/sundy-li)).
+
+#### Build/Testing/Packaging Improvement
+
+* Adds cross-platform embedding of binary resources into executables. [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)).
+* Flatbuffers library updated to v.2.0.0. Improvements list https://github.com/google/flatbuffers/releases/tag/v2.0.0. [#25474](https://github.com/ClickHouse/ClickHouse/pull/25474) ([Ilya Yatsishin](https://github.com/qoega)).
+* Add CI check for darwin-aarch64 cross-compilation. [#25560](https://github.com/ClickHouse/ClickHouse/pull/25560) ([Ivan](https://github.com/abyss7)).
+* Ubuntu 20.04 is now used to run integration tests, docker-compose version used to run integration tests is updated to 1.28.2. Environment variables now take effect on docker-compose. Rework test_dictionaries_all_layouts_separate_sources to allow parallel run. [#20393](https://github.com/ClickHouse/ClickHouse/pull/20393) ([Ilya Yatsishin](https://github.com/qoega)).
+* Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir](https://github.com/vdimir)).
+* Enabling TestFlows RBAC tests. [#25498](https://github.com/ClickHouse/ClickHouse/pull/25498) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Increase LDAP verification cooldown performance tests timeout to 600 sec. [#25374](https://github.com/ClickHouse/ClickHouse/pull/25374) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Added rounding to mathematical and arithmetic function tests for consistent snapshot comparison. Cleaned up test names so they're more uniform. [#25297](https://github.com/ClickHouse/ClickHouse/pull/25297) ([MyroTk](https://github.com/MyroTk)).
+* Enable build with s3 module in osx [#25217](https://github.com/ClickHouse/ClickHouse/issues/25217). [#25218](https://github.com/ClickHouse/ClickHouse/pull/25218) ([kevin wan](https://github.com/MaxWk)).
+* Adding `leadInFrame` and `lagInFrame` window functions TestFlows tests. [#25144](https://github.com/ClickHouse/ClickHouse/pull/25144) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Fix using Yandex dockerhub registries for TestFlows. [#25133](https://github.com/ClickHouse/ClickHouse/pull/25133) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Disabling extended precision data types TestFlows tests. [#25125](https://github.com/ClickHouse/ClickHouse/pull/25125) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Add integration test cases to cover JDBC bridge. [#25047](https://github.com/ClickHouse/ClickHouse/pull/25047) ([Zhichun Wu](https://github.com/zhicwu)).
+* Integration tests configuration has special treatment for dictionaries. Removed remaining dictionaries manual setup. [#24728](https://github.com/ClickHouse/ClickHouse/pull/24728) ([Ilya Yatsishin](https://github.com/qoega)).
+* Adding support to save clickhouse server logs in TestFlows check. [#24504](https://github.com/ClickHouse/ClickHouse/pull/24504) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Add libfuzzer tests for YAMLParser class. [#24480](https://github.com/ClickHouse/ClickHouse/pull/24480) ([BoloniniD](https://github.com/BoloniniD)).
+* Testing for big ints using the following functions: * Arithmetic * Array, tuple, and map * Bit * Comparison * Conversion * Logical * Mathematical * Null * Rounding - Creating a table with columns that use the data types. [#24350](https://github.com/ClickHouse/ClickHouse/pull/24350) ([MyroTk](https://github.com/MyroTk)).
+* Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Changed CSS theme to dark for better code highlighting. [#25682](https://github.com/ClickHouse/ClickHouse/pull/25682) ([Mike Kot](https://github.com/myrrc)).
+
+
 ### ClickHouse release 21.6, 2021-06-05
 
 #### Upgrade Notes

From de90cc0e8ffcba4e7e4d44ed4fd01cbb0dde6232 Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Fri, 9 Jul 2021 03:00:49 +0300
Subject: [PATCH 255/290] Update CHANGELOG.md

---
 CHANGELOG.md | 139 ++++++++++++++++++++++++---------------------------
 1 file changed, 64 insertions(+), 75 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e0a0f30804..34d11c6a2cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,20 +2,19 @@
 
 #### Backward Incompatible Change
 
-* Forward/backward incompatible change of maximum buffer size in clickhouse-keeper. Better to do it now (before production), than later. [#25421](https://github.com/ClickHouse/ClickHouse/pull/25421) ([alesapin](https://github.com/alesapin)).
 * Improved performance of queries with explicitly defined large sets. Added compatibility setting `legacy_column_name_of_tuple_literal`. It makes sense to set it to `true`, while doing rolling update of cluster from version lower than 21.7 to any higher version. Otherwise distributed queries with explicitly defined sets at `IN` clause may fail during update. [#25371](https://github.com/ClickHouse/ClickHouse/pull/25371) ([Anton Popov](https://github.com/CurtizJ)).
+* Forward/backward incompatible change of maximum buffer size in clickhouse-keeper (an experimental alternative to ZooKeeper). Better to do it now (before production), than later. [#25421](https://github.com/ClickHouse/ClickHouse/pull/25421) ([alesapin](https://github.com/alesapin)).
 
 #### New Feature
 
-* Added YAML configuration support to configuration loader. This closes [#3607](https://github.com/ClickHouse/ClickHouse/issues/3607). [#21858](https://github.com/ClickHouse/ClickHouse/pull/21858) ([BoloniniD](https://github.com/BoloniniD)).
+* Support configuration in YAML format as alternative to XML. This closes [#3607](https://github.com/ClickHouse/ClickHouse/issues/3607). [#21858](https://github.com/ClickHouse/ClickHouse/pull/21858) ([BoloniniD](https://github.com/BoloniniD)).
 * Provides a way to restore replicated table when the data is (possibly) present, but the ZooKeeper metadata is lost. Resolves [#13458](https://github.com/ClickHouse/ClickHouse/issues/13458). [#13652](https://github.com/ClickHouse/ClickHouse/pull/13652) ([Mike Kot](https://github.com/myrrc)).
-* Now clickhouse-keeper supports ZooKeeper-like `digest` ACLs. [#24448](https://github.com/ClickHouse/ClickHouse/pull/24448) ([alesapin](https://github.com/alesapin)).
 * Support structs and maps in Arrow/Parquet/ORC and dictionaries in Arrow input/output formats. Present new setting `output_format_arrow_low_cardinality_as_dictionary`. [#24341](https://github.com/ClickHouse/ClickHouse/pull/24341) ([Kruglov Pavel](https://github.com/Avogar)).
-* Dictionaries added support for Array type. [#25119](https://github.com/ClickHouse/ClickHouse/pull/25119) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added support for `Array` type in dictionaries. [#25119](https://github.com/ClickHouse/ClickHouse/pull/25119) ([Maksim Kita](https://github.com/kitaisreal)).
 * Added function `bitPositionsToArray`. Closes [#23792](https://github.com/ClickHouse/ClickHouse/issues/23792). Author [Kevin Wan] (@MaxWk). [#25394](https://github.com/ClickHouse/ClickHouse/pull/25394) ([Maksim Kita](https://github.com/kitaisreal)).
-* Added function `dateName`. Author [Daniil Kondratyev] (@dankondr). [#25372](https://github.com/ClickHouse/ClickHouse/pull/25372) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added function `dateName` to return names like 'Friday' or 'April'. Author [Daniil Kondratyev] (@dankondr). [#25372](https://github.com/ClickHouse/ClickHouse/pull/25372) ([Maksim Kita](https://github.com/kitaisreal)).
 * Add `toJSONString` function to serialize columns to their JSON representations. [#25164](https://github.com/ClickHouse/ClickHouse/pull/25164) ([Amos Bird](https://github.com/amosbird)).
-* Now query_log has two new columns: `initial_query_start_time`, `initial_query_start_time_microsecond` that record the starting time of a distributed query if any. [#25022](https://github.com/ClickHouse/ClickHouse/pull/25022) ([Amos Bird](https://github.com/amosbird)).
+* Now `query_log` has two new columns: `initial_query_start_time`, `initial_query_start_time_microsecond` that record the starting time of a distributed query if any. [#25022](https://github.com/ClickHouse/ClickHouse/pull/25022) ([Amos Bird](https://github.com/amosbird)).
 * Add aggregate function `segmentLengthSum`. [#24250](https://github.com/ClickHouse/ClickHouse/pull/24250) ([flynn](https://github.com/ucasfl)).
 * Add a new boolean setting `prefer_global_in_and_join` which defaults all IN/JOIN as GLOBAL IN/JOIN. [#23434](https://github.com/ClickHouse/ClickHouse/pull/23434) ([Amos Bird](https://github.com/amosbird)).
 * Support `ALTER DELETE` queries for `Join` table engine. [#23260](https://github.com/ClickHouse/ClickHouse/pull/23260) ([foolchi](https://github.com/foolchi)).
@@ -24,145 +23,135 @@
 
 #### Experimental Feature
 
-* Add support for VFS over HDFS. [#11058](https://github.com/ClickHouse/ClickHouse/pull/11058) ([overshov](https://github.com/overshov)).
+* Add support for virtual filesystem over HDFS. [#11058](https://github.com/ClickHouse/ClickHouse/pull/11058) ([overshov](https://github.com/overshov)) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Now clickhouse-keeper (an experimental alternative to ZooKeeper) supports ZooKeeper-like `digest` ACLs. [#24448](https://github.com/ClickHouse/ClickHouse/pull/24448) ([alesapin](https://github.com/alesapin)).
 
 #### Performance Improvement
 
-* Added optimization, that transforms some functions to reading of subcolumns to reduce amount of read data. E.g., statement `col IS NULL` is transformed to reading of subcolumn `col.null`. Optimization can be enabled by setting `optimize_functions_to_subcolumns`. [#24406](https://github.com/ClickHouse/ClickHouse/pull/24406) ([Anton Popov](https://github.com/CurtizJ)).
+* Added optimization that transforms some functions to reading of subcolumns to reduce amount of read data. E.g., statement `col IS NULL` is transformed to reading of subcolumn `col.null`. Optimization can be enabled by setting `optimize_functions_to_subcolumns` which is currently off by default. [#24406](https://github.com/ClickHouse/ClickHouse/pull/24406) ([Anton Popov](https://github.com/CurtizJ)).
 * Rewrite more columns to possible alias expressions. This may enable better optimization, such as projections. [#24405](https://github.com/ClickHouse/ClickHouse/pull/24405) ([Amos Bird](https://github.com/amosbird)).
-* Index of type bloom_filter can be used for expressions with `hasAny` function with constant arrays. This closes: [#24291](https://github.com/ClickHouse/ClickHouse/issues/24291). [#24900](https://github.com/ClickHouse/ClickHouse/pull/24900) ([Vasily Nemkov](https://github.com/Enmk)).
-* Add exponential backoff to reschedule read attempt in case RabbitMQ queues are empty. Closes [#24340](https://github.com/ClickHouse/ClickHouse/issues/24340). [#24415](https://github.com/ClickHouse/ClickHouse/pull/24415) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Index of type `bloom_filter` can be used for expressions with `hasAny` function with constant arrays. This closes: [#24291](https://github.com/ClickHouse/ClickHouse/issues/24291). [#24900](https://github.com/ClickHouse/ClickHouse/pull/24900) ([Vasily Nemkov](https://github.com/Enmk)).
+* Add exponential backoff to reschedule read attempt in case RabbitMQ queues are empty. (ClickHouse has support for importing data from RabbitMQ). Closes [#24340](https://github.com/ClickHouse/ClickHouse/issues/24340). [#24415](https://github.com/ClickHouse/ClickHouse/pull/24415) ([Kseniia Sumarokova](https://github.com/kssenii)).
 
 #### Improvement
 
+* Allow to limit bandwidth for replication. Add two Replicated\*MergeTree settings: `max_replicated_fetches_network_bandwidth` and `max_replicated_sends_network_bandwidth` which allows to limit maximum speed of replicated fetches/sends for table. Add two server-wide settings (in `default` user profile): `max_replicated_fetches_network_bandwidth_for_server` and `max_replicated_sends_network_bandwidth_for_server` which limit maximum speed of replication for all tables. The settings are not followed perfectly accurately. Turned off by default. Fixes [#1821](https://github.com/ClickHouse/ClickHouse/issues/1821). [#24573](https://github.com/ClickHouse/ClickHouse/pull/24573) ([alesapin](https://github.com/alesapin)).
+* Resource constraints and isolation for ODBC and Library bridges. Use separate `clickhouse-bridge` group and user for bridge processes. Set oom_score_adj so the bridges will be first subjects for OOM killer. Set set maximum RSS to 1 GiB. Closes [#23861](https://github.com/ClickHouse/ClickHouse/issues/23861). [#25280](https://github.com/ClickHouse/ClickHouse/pull/25280) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add standalone `clickhouse-keeper` symlink to the main `clickhouse` binary. Now it's possible to run coordination without the main clickhouse server. [#24059](https://github.com/ClickHouse/ClickHouse/pull/24059) ([alesapin](https://github.com/alesapin)).
 * Use global settings for query to `VIEW`. Fixed the behavior when queries to `VIEW` use local settings, that leads to errors if setting on `CREATE VIEW` and `SELECT` were different. As for now, `VIEW` won't use these modified settings, but you can still pass additional settings in `SETTINGS` section of `CREATE VIEW` query. Close [#20551](https://github.com/ClickHouse/ClickHouse/issues/20551). [#24095](https://github.com/ClickHouse/ClickHouse/pull/24095) ([Vladimir](https://github.com/vdimir)).
-* Add two Replicated*MergeTree settings: `max_replicated_fetches_network_bandwidth` and `max_replicated_sends_network_bandwidth` which allows to limit maximum speed of replicated fetches/sends for table. Add two server-wide settings (in `default` user profile): `max_replicated_fetches_network_bandwidth_for_server` and `max_replicated_sends_network_bandwidth_for_server` which limit maximum speed of replication for all tables. The settings are not followed perfectly accurately. Turned off by default. Fixes [#1821](https://github.com/ClickHouse/ClickHouse/issues/1821). [#24573](https://github.com/ClickHouse/ClickHouse/pull/24573) ([alesapin](https://github.com/alesapin)).
 * On server start, parts with incorrect partition ID would not be ever removed, but always detached. [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25166](https://github.com/ClickHouse/ClickHouse/pull/25166) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Use separate `clickhouse-bridge` group and user for bridge processes. Set oom_score_adj so the bridges will be first subjects for OOM killer. Set set maximum RSS to 1 GiB. Closes [#23861](https://github.com/ClickHouse/ClickHouse/issues/23861). [#25280](https://github.com/ClickHouse/ClickHouse/pull/25280) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Increase size of background schedule pool to 128 (`background_schedule_pool_size` setting). It allows avoiding replication queue hung on slow zookeeper connection. [#25072](https://github.com/ClickHouse/ClickHouse/pull/25072) ([alesapin](https://github.com/alesapin)).
 * Add merge tree setting `max_parts_to_merge_at_once` which limits the number of parts that can be merged in the background at once. Doesn't affect `OPTIMIZE FINAL` query. Fixes [#1820](https://github.com/ClickHouse/ClickHouse/issues/1820). [#24496](https://github.com/ClickHouse/ClickHouse/pull/24496) ([alesapin](https://github.com/alesapin)).
-* Allow `not in` operator to be used in partition pruning. [#24894](https://github.com/ClickHouse/ClickHouse/pull/24894) ([Amos Bird](https://github.com/amosbird)).
+* Allow `NOT IN` operator to be used in partition pruning. [#24894](https://github.com/ClickHouse/ClickHouse/pull/24894) ([Amos Bird](https://github.com/amosbird)).
 * Recognize IPv4 addresses like `127.0.1.1` as local. This is controversial and closes [#23504](https://github.com/ClickHouse/ClickHouse/issues/23504). Michael Filimonov will test this feature. [#24316](https://github.com/ClickHouse/ClickHouse/pull/24316) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* ClickHouse database created with MaterializeMySQL now contains all column comments from the MySQL database that materialized. [#25199](https://github.com/ClickHouse/ClickHouse/pull/25199) ([Storozhuk Kostiantyn](https://github.com/sand6255)).
+* ClickHouse database created with MaterializeMySQL (it is an experimental feature) now contains all column comments from the MySQL database that materialized. [#25199](https://github.com/ClickHouse/ClickHouse/pull/25199) ([Storozhuk Kostiantyn](https://github.com/sand6255)).
 * Add settings (`connection_auto_close`/`connection_max_tries`/`connection_pool_size`) for MySQL storage engine. [#24146](https://github.com/ClickHouse/ClickHouse/pull/24146) ([Azat Khuzhin](https://github.com/azat)).
 * Improve startup time of Distributed engine. [#25663](https://github.com/ClickHouse/ClickHouse/pull/25663) ([Azat Khuzhin](https://github.com/azat)).
-* Drop replicas from dirname for internal_replication=true (allows INSERT into Distributed with cluster from any number of replicas, before only 15 replicas was supported, everything more will fail with ENAMETOOLONG while creating directory for async blocks). [#25513](https://github.com/ClickHouse/ClickHouse/pull/25513) ([Azat Khuzhin](https://github.com/azat)).
-* Added support Interval type for LowCardinality. Closes [#21730](https://github.com/ClickHouse/ClickHouse/issues/21730). [#25410](https://github.com/ClickHouse/ClickHouse/pull/25410) ([Vladimir](https://github.com/vdimir)).
-* Add == operator on time conditions for sequenceMatch and sequenceCount functions. For eg: sequenceMatch('(?1)(?t==1)(?2)')(time, data = 1, data = 2). [#25299](https://github.com/ClickHouse/ClickHouse/pull/25299) ([Christophe Kalenzaga](https://github.com/mga-chka)).
+* Improvement for Distributed tables. Drop replicas from dirname for internal_replication=true (allows INSERT into Distributed with cluster from any number of replicas, before only 15 replicas was supported, everything more will fail with ENAMETOOLONG while creating directory for async blocks). [#25513](https://github.com/ClickHouse/ClickHouse/pull/25513) ([Azat Khuzhin](https://github.com/azat)).
+* Added support `Interval` type for `LowCardinality`. It is needed for intermediate values of some expressions. Closes [#21730](https://github.com/ClickHouse/ClickHouse/issues/21730). [#25410](https://github.com/ClickHouse/ClickHouse/pull/25410) ([Vladimir](https://github.com/vdimir)).
+* Add `==` operator on time conditions for `sequenceMatch` and `sequenceCount` functions. For eg: sequenceMatch('(?1)(?t==1)(?2)')(time, data = 1, data = 2). [#25299](https://github.com/ClickHouse/ClickHouse/pull/25299) ([Christophe Kalenzaga](https://github.com/mga-chka)).
 * Add settings `http_max_fields`, `http_max_field_name_size`, `http_max_field_value_size`. [#25296](https://github.com/ClickHouse/ClickHouse/pull/25296) ([Ivan](https://github.com/abyss7)).
-* Add support for function `if` with Decimal and Int types on its branches. This closes [#20549](https://github.com/ClickHouse/ClickHouse/issues/20549). This closes [#10142](https://github.com/ClickHouse/ClickHouse/issues/10142). [#25283](https://github.com/ClickHouse/ClickHouse/pull/25283) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add support for function `if` with `Decimal` and `Int` types on its branches. This closes [#20549](https://github.com/ClickHouse/ClickHouse/issues/20549). This closes [#10142](https://github.com/ClickHouse/ClickHouse/issues/10142). [#25283](https://github.com/ClickHouse/ClickHouse/pull/25283) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Update prompt in `clickhouse-client` and display a message when reconnecting. This closes [#10577](https://github.com/ClickHouse/ClickHouse/issues/10577). [#25281](https://github.com/ClickHouse/ClickHouse/pull/25281) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Correct memory tracking in aggregate function `topK`. This closes [#25259](https://github.com/ClickHouse/ClickHouse/issues/25259). [#25260](https://github.com/ClickHouse/ClickHouse/pull/25260) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Fix topLevelDomain() for IDN hosts (i.e. `example.рф`), before it returns empty string for such hosts. [#25103](https://github.com/ClickHouse/ClickHouse/pull/25103) ([Azat Khuzhin](https://github.com/azat)).
-* Detect linux version at runtime (for worked nested epoll, that is required for `async_socket_for_remote`/`use_hedged_requests`, otherwise remote queries may stuck). [#25067](https://github.com/ClickHouse/ClickHouse/pull/25067) ([Azat Khuzhin](https://github.com/azat)).
+* Fix `topLevelDomain` for IDN hosts (i.e. `example.рф`), before it returns empty string for such hosts. [#25103](https://github.com/ClickHouse/ClickHouse/pull/25103) ([Azat Khuzhin](https://github.com/azat)).
+* Detect Linux kernel version at runtime (for worked nested epoll, that is required for `async_socket_for_remote`/`use_hedged_requests`, otherwise remote queries may stuck). [#25067](https://github.com/ClickHouse/ClickHouse/pull/25067) ([Azat Khuzhin](https://github.com/azat)).
 * For distributed query, when `optimize_skip_unused_shards=1`, allow to skip shard with condition like `(sharding key) IN (one-element-tuple)`. (Tuples with many elements were supported. Tuple with single element did not work because it is parsed as literal). [#24930](https://github.com/ClickHouse/ClickHouse/pull/24930) ([Amos Bird](https://github.com/amosbird)).
-* Improved logging of S3 errors, no more double spaces in case of empty keys and buckets. [#24897](https://github.com/ClickHouse/ClickHouse/pull/24897) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Improved log messages of S3 errors, no more double whitespaces in case of empty keys and buckets. [#24897](https://github.com/ClickHouse/ClickHouse/pull/24897) ([Vladimir Chebotarev](https://github.com/excitoon)).
 * Some queries require multi-pass semantic analysis. Try reusing built sets for `IN` in this case. [#24874](https://github.com/ClickHouse/ClickHouse/pull/24874) ([Amos Bird](https://github.com/amosbird)).
 * Respect `max_distributed_connections` for `insert_distributed_sync` (otherwise for huge clusters and sync insert it may run out of `max_thread_pool_size`). [#24754](https://github.com/ClickHouse/ClickHouse/pull/24754) ([Azat Khuzhin](https://github.com/azat)).
 * Avoid hiding errors like `Limit for rows or bytes to read exceeded` for scalar subqueries. [#24545](https://github.com/ClickHouse/ClickHouse/pull/24545) ([nvartolomei](https://github.com/nvartolomei)).
 * Make String-to-Int parser stricter so that `toInt64('+')` will throw. [#24475](https://github.com/ClickHouse/ClickHouse/pull/24475) ([Amos Bird](https://github.com/amosbird)).
-* If SSDDictionary is created with DDL query, it can be created only inside user_files directory. [#24466](https://github.com/ClickHouse/ClickHouse/pull/24466) ([Maksim Kita](https://github.com/kitaisreal)).
-* PostgreSQL support specifying non default schema for insert queries. Closes [#24149](https://github.com/ClickHouse/ClickHouse/issues/24149). [#24413](https://github.com/ClickHouse/ClickHouse/pull/24413) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* If `SSD_CACHE` is created with DDL query, it can be created only inside `user_files` directory. [#24466](https://github.com/ClickHouse/ClickHouse/pull/24466) ([Maksim Kita](https://github.com/kitaisreal)).
+* PostgreSQL support for specifying non default schema for insert queries. Closes [#24149](https://github.com/ClickHouse/ClickHouse/issues/24149). [#24413](https://github.com/ClickHouse/ClickHouse/pull/24413) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix IPv6 addresses resolving (i.e. fixes `select * from remote('[::1]', system.one)`). [#24319](https://github.com/ClickHouse/ClickHouse/pull/24319) ([Azat Khuzhin](https://github.com/azat)).
 * Fix trailing whitespaces in FROM clause with subqueries in multiline mode, and also changes the output of the queries slightly in a more human friendly way. [#24151](https://github.com/ClickHouse/ClickHouse/pull/24151) ([Azat Khuzhin](https://github.com/azat)).
-* Suppress exceptions from logger code. [#24069](https://github.com/ClickHouse/ClickHouse/pull/24069) ([Azat Khuzhin](https://github.com/azat)).
-* Add ability to split distributed batch on failures (i.e. due to memory limits, corruptions), under `distributed_directory_monitor_split_batch_on_failure` (OFF by default). [#23864](https://github.com/ClickHouse/ClickHouse/pull/23864) ([Azat Khuzhin](https://github.com/azat)).
-* Handle column name clashes for storage join. Closes [#20309](https://github.com/ClickHouse/ClickHouse/issues/20309). [#23769](https://github.com/ClickHouse/ClickHouse/pull/23769) ([Vladimir](https://github.com/vdimir)).
-* Display progress for File table engine in clickhouse-local and on INSERT query in clickhouse-client when data is passed to stdin. Closes [#18209](https://github.com/ClickHouse/ClickHouse/issues/18209). [#23656](https://github.com/ClickHouse/ClickHouse/pull/23656) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Bugfixes and improvements of clickhouse-copier. Allow to copy tables with different (but compatible schemas). Closes [#9159](https://github.com/ClickHouse/ClickHouse/issues/9159). Added test to copy ReplacingMergeTree. Closes [#22711](https://github.com/ClickHouse/ClickHouse/issues/22711). Support TTL on columns and Data Skipping Indices. It simply removes it to create internal Distributed table (underlying table will have TTL and skipping indices). Closes [#19384](https://github.com/ClickHouse/ClickHouse/issues/19384). Allow to copy MATERIALIZED and ALIAS columns. There are some cases in which it could be helpful (e.g. if this column is in PRIMARY KEY). Now it could be allowed by setting `allow_to_copy_alias_and_materialized_columns` property to true in task configuration. Closes [#9177](https://github.com/ClickHouse/ClickHouse/issues/9177). Closes [#11007] (https://github.com/ClickHouse/ClickHouse/issues/11007). Closes [#9514](https://github.com/ClickHouse/ClickHouse/issues/9514). Added a property `allow_to_drop_target_partitions` in task configuration to drop partition in original table before moving helping tables. Closes [#20957](https://github.com/ClickHouse/ClickHouse/issues/20957). Get rid of `OPTIMIZE DEDUPLICATE` query. This hack was needed, because `ALTER TABLE MOVE PARTITION` was retried many times and plain MergeTree tables don't have deduplication. Closes [#17966](https://github.com/ClickHouse/ClickHouse/issues/17966). Write progress to ZooKeeper node on path `task_path + /status` in JSON format. Closes [#20955](https://github.com/ClickHouse/ClickHouse/issues/20955). Support for ReplicatedTables without arguments. Closes [#24834](https://github.com/ClickHouse/ClickHouse/issues/24834) .[#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Improvement for Distributed tables. Add ability to split distributed batch on failures (i.e. due to memory limits, corruptions), under `distributed_directory_monitor_split_batch_on_failure` (OFF by default). [#23864](https://github.com/ClickHouse/ClickHouse/pull/23864) ([Azat Khuzhin](https://github.com/azat)).
+* Handle column name clashes for `Join` table engine. Closes [#20309](https://github.com/ClickHouse/ClickHouse/issues/20309). [#23769](https://github.com/ClickHouse/ClickHouse/pull/23769) ([Vladimir](https://github.com/vdimir)).
+* Display progress for `File` table engine in `clickhouse-local` and on INSERT query in `clickhouse-client` when data is passed to stdin. Closes [#18209](https://github.com/ClickHouse/ClickHouse/issues/18209). [#23656](https://github.com/ClickHouse/ClickHouse/pull/23656) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Bugfixes and improvements of `clickhouse-copier`. Allow to copy tables with different (but compatible schemas). Closes [#9159](https://github.com/ClickHouse/ClickHouse/issues/9159). Added test to copy ReplacingMergeTree. Closes [#22711](https://github.com/ClickHouse/ClickHouse/issues/22711). Support TTL on columns and Data Skipping Indices. It simply removes it to create internal Distributed table (underlying table will have TTL and skipping indices). Closes [#19384](https://github.com/ClickHouse/ClickHouse/issues/19384). Allow to copy MATERIALIZED and ALIAS columns. There are some cases in which it could be helpful (e.g. if this column is in PRIMARY KEY). Now it could be allowed by setting `allow_to_copy_alias_and_materialized_columns` property to true in task configuration. Closes [#9177](https://github.com/ClickHouse/ClickHouse/issues/9177). Closes [#11007] (https://github.com/ClickHouse/ClickHouse/issues/11007). Closes [#9514](https://github.com/ClickHouse/ClickHouse/issues/9514). Added a property `allow_to_drop_target_partitions` in task configuration to drop partition in original table before moving helping tables. Closes [#20957](https://github.com/ClickHouse/ClickHouse/issues/20957). Get rid of `OPTIMIZE DEDUPLICATE` query. This hack was needed, because `ALTER TABLE MOVE PARTITION` was retried many times and plain MergeTree tables don't have deduplication. Closes [#17966](https://github.com/ClickHouse/ClickHouse/issues/17966). Write progress to ZooKeeper node on path `task_path + /status` in JSON format. Closes [#20955](https://github.com/ClickHouse/ClickHouse/issues/20955). Support for ReplicatedTables without arguments. Closes [#24834](https://github.com/ClickHouse/ClickHouse/issues/24834) .[#23518](https://github.com/ClickHouse/ClickHouse/pull/23518) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Added sleep with backoff between read retries from S3. [#23461](https://github.com/ClickHouse/ClickHouse/pull/23461) ([Vladimir Chebotarev](https://github.com/excitoon)).
 * Respect `insert_allow_materialized_columns` (allows materialized columns) for INSERT into `Distributed` table. [#23349](https://github.com/ClickHouse/ClickHouse/pull/23349) ([Azat Khuzhin](https://github.com/azat)).
 * Add ability to push down LIMIT for distributed queries. [#23027](https://github.com/ClickHouse/ClickHouse/pull/23027) ([Azat Khuzhin](https://github.com/azat)).
-* Fix Zero-Copy replication with several S3 volumes (Fixes [#22679](https://github.com/ClickHouse/ClickHouse/issues/22679)). [#22864](https://github.com/ClickHouse/ClickHouse/pull/22864) ([ianton-ru](https://github.com/ianton-ru)).
-* Resolve the actual port number bound when a user requests any available port from the operating system. [#25569](https://github.com/ClickHouse/ClickHouse/pull/25569) ([bnaecker](https://github.com/bnaecker)).
+* Fix zero-copy replication with several S3 volumes (Fixes [#22679](https://github.com/ClickHouse/ClickHouse/issues/22679)). [#22864](https://github.com/ClickHouse/ClickHouse/pull/22864) ([ianton-ru](https://github.com/ianton-ru)).
+* Resolve the actual port number bound when a user requests any available port from the operating system to show it in the log message. [#25569](https://github.com/ClickHouse/ClickHouse/pull/25569) ([bnaecker](https://github.com/bnaecker)).
+* Fixed case, when sometimes conversion of postgres arrays resulted in String data type, not n-dimensional array, because `attndims` works incorrectly in some cases. Closes [#24804](https://github.com/ClickHouse/ClickHouse/issues/24804). [#25538](https://github.com/ClickHouse/ClickHouse/pull/25538) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix convertion of DateTime with timezone for MySQL, PostgreSQL, ODBC. Closes [#5057](https://github.com/ClickHouse/ClickHouse/issues/5057). [#25528](https://github.com/ClickHouse/ClickHouse/pull/25528) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)).
+* Allow to declare S3 disk at root of bucket (S3 virtual filesystem is an experimental feature under development). [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)).
+* Enable reading of subcolumns (e.g. components of Tuples) for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)).
+* A feature for MySQL compatibility protocol: make `user` function to return correct output. Closes [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697). [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697) ([sundyli](https://github.com/sundy-li)).
 
 #### Bug Fix
 
-* Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)).
-* If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. Fixed crash when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([tavplubix](https://github.com/tavplubix)).
-* Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix bug which can lead to intersecting parts after merges with TTL: `Part all_40_40_0 is covered by all_40_40_1 but should be merged into all_40_41_1. This shouldn't happen often.`. [#25549](https://github.com/ClickHouse/ClickHouse/pull/25549) ([alesapin](https://github.com/alesapin)).
+* Improvement for backward compatibility. Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix extremely rare bug on low-memory servers which can lead to the inability to perform merges without restart. Possibly fixes [#24603](https://github.com/ClickHouse/ClickHouse/issues/24603). [#24872](https://github.com/ClickHouse/ClickHouse/pull/24872) ([alesapin](https://github.com/alesapin)).
 * Fix extremely rare error `Tagging already tagged part` in replication queue during concurrent `alter move/replace partition`. Possibly fixes [#22142](https://github.com/ClickHouse/ClickHouse/issues/22142). [#24961](https://github.com/ClickHouse/ClickHouse/pull/24961) ([alesapin](https://github.com/alesapin)).
+* Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` does not finish. This was detected on server with extremely low amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix bug which can lead to ZooKeeper client hung inside clickhouse-server. [#24721](https://github.com/ClickHouse/ClickHouse/pull/24721) ([alesapin](https://github.com/alesapin)).
+* If ZooKeeper connection was lost and replica was cloned after restoring the connection, its replication queue might contain outdated entries. Fixed failed assertion when replication queue contains intersecting virtual parts. It may rarely happen if some data part was lost. Print error in log instead of terminating. [#24777](https://github.com/ClickHouse/ClickHouse/pull/24777) ([tavplubix](https://github.com/tavplubix)).
+* Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix bug which can lead to intersecting parts after merges with TTL: `Part all_40_40_0 is covered by all_40_40_1 but should be merged into all_40_41_1. This shouldn't happen often.`. [#25549](https://github.com/ClickHouse/ClickHouse/pull/25549) ([alesapin](https://github.com/alesapin)).
 * On ZooKeeper connection loss `ReplicatedMergeTree` table might wait for background operations to complete before trying to reconnect. It's fixed, now background operations are stopped forcefully. [#25306](https://github.com/ClickHouse/ClickHouse/pull/25306) ([tavplubix](https://github.com/tavplubix)).
-* Use old modulo function version when used in partition key. Closes [#23508](https://github.com/ClickHouse/ClickHouse/issues/23508). [#24157](https://github.com/ClickHouse/ClickHouse/pull/24157) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix error `Key expression contains comparison between inconvertible types` for queries with `ARRAY JOIN` in case if array is used in primary key. Fixes [#8247](https://github.com/ClickHouse/ClickHouse/issues/8247). [#25546](https://github.com/ClickHouse/ClickHouse/pull/25546) ([Anton Popov](https://github.com/CurtizJ)).
 * Fix wrong totals for query `WITH TOTALS` and `WITH FILL`. Fixes [#20872](https://github.com/ClickHouse/ClickHouse/issues/20872). [#25539](https://github.com/ClickHouse/ClickHouse/pull/25539) ([Anton Popov](https://github.com/CurtizJ)).
-* Fixed case, when sometimes conversion of postgres arrays resulted in String data type, not n-dimensional array, because `attndims` works incorrectly in some cases. Closes [#24804](https://github.com/ClickHouse/ClickHouse/issues/24804). [#25538](https://github.com/ClickHouse/ClickHouse/pull/25538) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)).
 * Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([tavplubix](https://github.com/tavplubix)).
 * `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([tavplubix](https://github.com/tavplubix)).
 * Fixed a bug in `Replicated` database engine that might rarely cause some replica to skip enqueued DDL query. [#24805](https://github.com/ClickHouse/ClickHouse/pull/24805) ([tavplubix](https://github.com/tavplubix)).
 * Fix null pointer dereference in `EXPLAIN AST` without query. [#25631](https://github.com/ClickHouse/ClickHouse/pull/25631) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix potential crash when calculating aggregate function states by aggregation of aggregate function states of other aggregate functions (not a practical use case). See [#24523](https://github.com/ClickHouse/ClickHouse/issues/24523). [#25015](https://github.com/ClickHouse/ClickHouse/pull/25015) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fix waiting of automatic dropping of empty parts. It could lead to full filling of background pool and stuck of replication. [#23315](https://github.com/ClickHouse/ClickHouse/pull/23315) ([Anton Popov](https://github.com/CurtizJ)).
-* Fix restore S3 table. [#25601](https://github.com/ClickHouse/ClickHouse/pull/25601) ([ianton-ru](https://github.com/ianton-ru)).
-* Fix segfault in `Arrow` format when using `Decimal256`. Add arrow `Decimal256` support. [#25531](https://github.com/ClickHouse/ClickHouse/pull/25531) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix convertion of datetime with timezone for MySQL, PostgreSQL, ODBC. Closes [#5057](https://github.com/ClickHouse/ClickHouse/issues/5057). [#25528](https://github.com/ClickHouse/ClickHouse/pull/25528) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix restore of a table stored in S3 virtual filesystem (it is an experimental feature not ready for production). [#25601](https://github.com/ClickHouse/ClickHouse/pull/25601) ([ianton-ru](https://github.com/ianton-ru)).
+* Fix nullptr dereference in `Arrow` format when using `Decimal256`. Add `Decimal256` support for `Arrow` format. [#25531](https://github.com/ClickHouse/ClickHouse/pull/25531) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fix excessive underscore before the names of the preprocessed configuration files. [#25431](https://github.com/ClickHouse/ClickHouse/pull/25431) ([Vitaly Baranov](https://github.com/vitlibar)).
-* Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* A fix for `clickhouse-copier` tool: Fix segfault when sharding_key is absent in task config for copier. [#25419](https://github.com/ClickHouse/ClickHouse/pull/25419) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)).
 * Fix the possibility of non-deterministic behaviour of the `quantileDeterministic` function and similar. This closes [#20480](https://github.com/ClickHouse/ClickHouse/issues/20480). [#25313](https://github.com/ClickHouse/ClickHouse/pull/25313) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Support `SimpleAggregateFunction(LowCardinality)` for `SummingMergeTree`. Fixes [#25134](https://github.com/ClickHouse/ClickHouse/issues/25134). [#25300](https://github.com/ClickHouse/ClickHouse/pull/25300) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix Logical Error Cannot sum Array/Tuple in min/maxMap. [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix logical error with exception message "Cannot sum Array/Tuple in min/maxMap". [#25298](https://github.com/ClickHouse/ClickHouse/pull/25298) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fix error `Bad cast from type DB::ColumnLowCardinality to DB::ColumnVector` for queries where `LowCardinality` argument was used for IN (this bug appeared in 21.6). Fixes [#25187](https://github.com/ClickHouse/ClickHouse/issues/25187). [#25290](https://github.com/ClickHouse/ClickHouse/pull/25290) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix joinGetOrNull with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)).
+* Fix incorrect behaviour of `joinGetOrNull` with not-nullable columns. This fixes [#24261](https://github.com/ClickHouse/ClickHouse/issues/24261). [#25288](https://github.com/ClickHouse/ClickHouse/pull/25288) ([Amos Bird](https://github.com/amosbird)).
 * Fix incorrect behaviour and UBSan report in big integers. In previous versions `CAST(1e19 AS UInt128)` returned zero. [#25279](https://github.com/ClickHouse/ClickHouse/pull/25279) ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fixed an error which occurred while inserting a subset of columns using CSVWithNames format. Fixes [#25129](https://github.com/ClickHouse/ClickHouse/issues/25129). [#25169](https://github.com/ClickHouse/ClickHouse/pull/25169) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)).
 * Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix crash in query with cross join and `joined_subquery_requires_alias = 0`. Fixes [#24011](https://github.com/ClickHouse/ClickHouse/issues/24011). [#25082](https://github.com/ClickHouse/ClickHouse/pull/25082) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix bug with constant maps in mapContains that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix bug which allows creating tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)).
-* Fix wrong result when using aggregate projection with **not empty** `GROUP BY` key to execute query with `GROUP BY` by **empty** key. [#25055](https://github.com/ClickHouse/ClickHouse/pull/25055) ([Amos Bird](https://github.com/amosbird)).
-* Distinguish KILL MUTATION for different tables (fixes unexpected `Cancelled mutating parts` error). [#25025](https://github.com/ClickHouse/ClickHouse/pull/25025) ([Azat Khuzhin](https://github.com/azat)).
+* Fix bug with constant maps in mapContains function that lead to error `empty column was returned by function mapContains`. Closes [#25077](https://github.com/ClickHouse/ClickHouse/issues/25077). [#25080](https://github.com/ClickHouse/ClickHouse/pull/25080) ([Kruglov Pavel](https://github.com/Avogar)).
+* Remove possibility to create tables with columns referencing themselves like `a UInt32 ALIAS a + 1` or `b UInt32 MATERIALIZED b`. Fixes [#24910](https://github.com/ClickHouse/ClickHouse/issues/24910), [#24292](https://github.com/ClickHouse/ClickHouse/issues/24292). [#25059](https://github.com/ClickHouse/ClickHouse/pull/25059) ([alesapin](https://github.com/alesapin)).
+* Fix wrong result when using aggregate projection with *not empty* `GROUP BY` key to execute query with `GROUP BY` by *empty* key. [#25055](https://github.com/ClickHouse/ClickHouse/pull/25055) ([Amos Bird](https://github.com/amosbird)).
 * Fix serialization of splitted nested messages in Protobuf format. This PR fixes [#24647](https://github.com/ClickHouse/ClickHouse/issues/24647). [#25000](https://github.com/ClickHouse/ClickHouse/pull/25000) ([Vitaly Baranov](https://github.com/vitlibar)).
 * Fix limit/offset settings for distributed queries (ignore on the remote nodes). [#24940](https://github.com/ClickHouse/ClickHouse/pull/24940) ([Azat Khuzhin](https://github.com/azat)).
-* Fix possible heap-buffer-overflow in Arrow. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fixed bug with declaring S3 disk at root of bucket. [#24898](https://github.com/ClickHouse/ClickHouse/pull/24898) ([Vladimir Chebotarev](https://github.com/excitoon)).
-* Fixed possible error 'Cannot read from istream at offset 0' when reading a file from DiskS3. [#24885](https://github.com/ClickHouse/ClickHouse/pull/24885) ([Pavel Kovalenko](https://github.com/Jokser)).
+* Fix possible heap-buffer-overflow in `Arrow` format. [#24922](https://github.com/ClickHouse/ClickHouse/pull/24922) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fixed possible error 'Cannot read from istream at offset 0' when reading a file from DiskS3 (S3 virtual filesystem is an experimental feature under development that should not be used in production). [#24885](https://github.com/ClickHouse/ClickHouse/pull/24885) ([Pavel Kovalenko](https://github.com/Jokser)).
 * Fix "Missing columns" exception when joining Distributed Materialized View. [#24870](https://github.com/ClickHouse/ClickHouse/pull/24870) ([Azat Khuzhin](https://github.com/azat)).
-* Allow NULL values in postgresql protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Allow `NULL` values in postgresql compatibility protocol. Closes [#22622](https://github.com/ClickHouse/ClickHouse/issues/22622). [#24857](https://github.com/ClickHouse/ClickHouse/pull/24857) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix bug when exception `Mutation was killed` can be thrown to the client on mutation wait when mutation not loaded into memory yet. [#24809](https://github.com/ClickHouse/ClickHouse/pull/24809) ([alesapin](https://github.com/alesapin)).
 * Fixed bug in deserialization of random generator state with might cause some data types such as `AggregateFunction(groupArraySample(N), T))` to behave in a non-deterministic way. [#24538](https://github.com/ClickHouse/ClickHouse/pull/24538) ([tavplubix](https://github.com/tavplubix)).
-* Disallow building uniqXXXXStates of other aggregation states. [#24523](https://github.com/ClickHouse/ClickHouse/pull/24523) ([Raúl Marín](https://github.com/Algunenano)).
-* Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)).
+* Disallow building uniqXXXXStates of other aggregation states. [#24523](https://github.com/ClickHouse/ClickHouse/pull/24523) ([Raúl Marín](https://github.com/Algunenano)). Then allow it back by actually eliminating the root cause of the related issue. ([alexey-milovidov](https://github.com/alexey-milovidov)).
 * Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)).
-* Fixed the behavior when query `SYSTEM RESTART REPLICA` or `SYSTEM SYNC REPLICA` is being processed infinitely. This was detected on server with extremely little amount of RAM. [#24457](https://github.com/ClickHouse/ClickHouse/pull/24457) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
-* Fix totalBytes computation in StorageBuffer. In current CH version total_writes.bytes counter decreases too much during the buffer flush. It leads to counter overflow and totalBytes return something around 17.44 EB some time after the flush. [#24450](https://github.com/ClickHouse/ClickHouse/pull/24450) ([DimasKovas](https://github.com/DimasKovas)).
-* Fix incorrect monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)).
-* Fixed the deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)).
+* Fix computation of total bytes in `Buffer` table. In current ClickHouse version total_writes.bytes counter decreases too much during the buffer flush. It leads to counter overflow and totalBytes return something around 17.44 EB some time after the flush. [#24450](https://github.com/ClickHouse/ClickHouse/pull/24450) ([DimasKovas](https://github.com/DimasKovas)).
+* Fix incorrect information about the monotonicity of toWeek function. This fixes [#24422](https://github.com/ClickHouse/ClickHouse/issues/24422) . This bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/5212 , and was exposed later by smarter partition pruner. [#24446](https://github.com/ClickHouse/ClickHouse/pull/24446) ([Amos Bird](https://github.com/amosbird)).
+* When user authentication is managed by LDAP. Fixed potential deadlock that can happen during LDAP role (re)mapping, when LDAP group is mapped to a nonexistent local role. [#24431](https://github.com/ClickHouse/ClickHouse/pull/24431) ([Denis Glazachev](https://github.com/traceon)).
 * In "multipart/form-data" message consider the CRLF preceding a boundary as part of it. Fixes [#23905](https://github.com/ClickHouse/ClickHouse/issues/23905). [#24399](https://github.com/ClickHouse/ClickHouse/pull/24399) ([Ivan](https://github.com/abyss7)).
 * Fix drop partition with intersect fake parts. In rare cases there might be parts with mutation version greater than current block number. [#24321](https://github.com/ClickHouse/ClickHouse/pull/24321) ([Amos Bird](https://github.com/amosbird)).
 * Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([tavplubix](https://github.com/tavplubix)).
 * Allow empty HTTP headers. Fixes [#23901](https://github.com/ClickHouse/ClickHouse/issues/23901). [#24285](https://github.com/ClickHouse/ClickHouse/pull/24285) ([Ivan](https://github.com/abyss7)).
-* Set `max_threads = 1` to fix mutation fail of StorageMemory. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)).
-* Column cardinality in join output same as at the input, close [#23351](https://github.com/ClickHouse/ClickHouse/issues/23351), close [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315). [#24061](https://github.com/ClickHouse/ClickHouse/pull/24061) ([Vladimir](https://github.com/vdimir)).
-* Fix the bug in failover behavior when Engine=Kafka was not able to start consumption if the same consumer had an empty assignment previously. Closes [#21118](https://github.com/ClickHouse/ClickHouse/issues/21118). [#21267](https://github.com/ClickHouse/ClickHouse/pull/21267) ([filimonov](https://github.com/filimonov)).
-* Fix MySQL select user() return empty. Closes [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697). [#25697](https://github.com/ClickHouse/ClickHouse/pull/25697) ([sundyli](https://github.com/sundy-li)).
+* Correct processing of mutations (ALTER UPDATE/DELETE) in Memory tables. Closes [#24274](https://github.com/ClickHouse/ClickHouse/issues/24274). [#24275](https://github.com/ClickHouse/ClickHouse/pull/24275) ([flynn](https://github.com/ucasfl)).
+* Make column LowCardinality property in JOIN output the same as in the input, close [#23351](https://github.com/ClickHouse/ClickHouse/issues/23351), close [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315). [#24061](https://github.com/ClickHouse/ClickHouse/pull/24061) ([Vladimir](https://github.com/vdimir)).
+* A fix for Kafka tables. Fix the bug in failover behavior when Engine = Kafka was not able to start consumption if the same consumer had an empty assignment previously. Closes [#21118](https://github.com/ClickHouse/ClickHouse/issues/21118). [#21267](https://github.com/ClickHouse/ClickHouse/pull/21267) ([filimonov](https://github.com/filimonov)).
 
 #### Build/Testing/Packaging Improvement
 
-* Adds cross-platform embedding of binary resources into executables. [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)).
-* Flatbuffers library updated to v.2.0.0. Improvements list https://github.com/google/flatbuffers/releases/tag/v2.0.0. [#25474](https://github.com/ClickHouse/ClickHouse/pull/25474) ([Ilya Yatsishin](https://github.com/qoega)).
-* Add CI check for darwin-aarch64 cross-compilation. [#25560](https://github.com/ClickHouse/ClickHouse/pull/25560) ([Ivan](https://github.com/abyss7)).
-* Ubuntu 20.04 is now used to run integration tests, docker-compose version used to run integration tests is updated to 1.28.2. Environment variables now take effect on docker-compose. Rework test_dictionaries_all_layouts_separate_sources to allow parallel run. [#20393](https://github.com/ClickHouse/ClickHouse/pull/20393) ([Ilya Yatsishin](https://github.com/qoega)).
-* Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir](https://github.com/vdimir)).
-* Enabling TestFlows RBAC tests. [#25498](https://github.com/ClickHouse/ClickHouse/pull/25498) ([vzakaznikov](https://github.com/vzakaznikov)).
-* Increase LDAP verification cooldown performance tests timeout to 600 sec. [#25374](https://github.com/ClickHouse/ClickHouse/pull/25374) ([vzakaznikov](https://github.com/vzakaznikov)).
-* Added rounding to mathematical and arithmetic function tests for consistent snapshot comparison. Cleaned up test names so they're more uniform. [#25297](https://github.com/ClickHouse/ClickHouse/pull/25297) ([MyroTk](https://github.com/MyroTk)).
+* Add `darwin-aarch64` (Mac M1 / Apple Silicon) builds in CI [#25560](https://github.com/ClickHouse/ClickHouse/pull/25560) ([Ivan](https://github.com/abyss7)) and put the links to the docs and website ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Adds cross-platform embedding of binary resources into executables. It works on Illumos. [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)).
+* Add join related options to stress tests to improve fuzzing. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir](https://github.com/vdimir)).
 * Enable build with s3 module in osx [#25217](https://github.com/ClickHouse/ClickHouse/issues/25217). [#25218](https://github.com/ClickHouse/ClickHouse/pull/25218) ([kevin wan](https://github.com/MaxWk)).
-* Adding `leadInFrame` and `lagInFrame` window functions TestFlows tests. [#25144](https://github.com/ClickHouse/ClickHouse/pull/25144) ([vzakaznikov](https://github.com/vzakaznikov)).
-* Fix using Yandex dockerhub registries for TestFlows. [#25133](https://github.com/ClickHouse/ClickHouse/pull/25133) ([vzakaznikov](https://github.com/vzakaznikov)).
-* Disabling extended precision data types TestFlows tests. [#25125](https://github.com/ClickHouse/ClickHouse/pull/25125) ([vzakaznikov](https://github.com/vzakaznikov)).
 * Add integration test cases to cover JDBC bridge. [#25047](https://github.com/ClickHouse/ClickHouse/pull/25047) ([Zhichun Wu](https://github.com/zhicwu)).
 * Integration tests configuration has special treatment for dictionaries. Removed remaining dictionaries manual setup. [#24728](https://github.com/ClickHouse/ClickHouse/pull/24728) ([Ilya Yatsishin](https://github.com/qoega)).
-* Adding support to save clickhouse server logs in TestFlows check. [#24504](https://github.com/ClickHouse/ClickHouse/pull/24504) ([vzakaznikov](https://github.com/vzakaznikov)).
 * Add libfuzzer tests for YAMLParser class. [#24480](https://github.com/ClickHouse/ClickHouse/pull/24480) ([BoloniniD](https://github.com/BoloniniD)).
-* Testing for big ints using the following functions: * Arithmetic * Array, tuple, and map * Bit * Comparison * Conversion * Logical * Mathematical * Null * Rounding - Creating a table with columns that use the data types. [#24350](https://github.com/ClickHouse/ClickHouse/pull/24350) ([MyroTk](https://github.com/MyroTk)).
+* Ubuntu 20.04 is now used to run integration tests, docker-compose version used to run integration tests is updated to 1.28.2. Environment variables now take effect on docker-compose. Rework test_dictionaries_all_layouts_separate_sources to allow parallel run. [#20393](https://github.com/ClickHouse/ClickHouse/pull/20393) ([Ilya Yatsishin](https://github.com/qoega)).
 * Fix TOCTOU error in installation script. [#25277](https://github.com/ClickHouse/ClickHouse/pull/25277) ([alexey-milovidov](https://github.com/alexey-milovidov)).
-* Changed CSS theme to dark for better code highlighting. [#25682](https://github.com/ClickHouse/ClickHouse/pull/25682) ([Mike Kot](https://github.com/myrrc)).
 
 
 ### ClickHouse release 21.6, 2021-06-05

From 325e0764fe3e42560f8ccbed22c4d392f343cf28 Mon Sep 17 00:00:00 2001
From: ywill3 <87159180+ywill3@users.noreply.github.com>
Date: Fri, 9 Jul 2021 10:05:27 +0800
Subject: [PATCH 256/290] Update bitmap-functions.md

bitmapSubsetLimit example sql is wrong
---
 docs/zh/sql-reference/functions/bitmap-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/sql-reference/functions/bitmap-functions.md b/docs/zh/sql-reference/functions/bitmap-functions.md
index 5a6baf2f217..57bf69cb5c3 100644
--- a/docs/zh/sql-reference/functions/bitmap-functions.md
+++ b/docs/zh/sql-reference/functions/bitmap-functions.md
@@ -81,7 +81,7 @@ SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,
 **示例**
 
 ``` sql
-SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
+SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res
 ```
 
     ┌─res───────────────────────┐

From 89c72da7f3a5ff03464a8be8e749d75507c50867 Mon Sep 17 00:00:00 2001
From: ywill3 <87159180+ywill3@users.noreply.github.com>
Date: Fri, 9 Jul 2021 10:14:10 +0800
Subject: [PATCH 257/290] Update bitmap-functions.md

may be  inappropriate sub-title
---
 docs/zh/sql-reference/functions/bitmap-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/sql-reference/functions/bitmap-functions.md b/docs/zh/sql-reference/functions/bitmap-functions.md
index 5a6baf2f217..ee3cb712f38 100644
--- a/docs/zh/sql-reference/functions/bitmap-functions.md
+++ b/docs/zh/sql-reference/functions/bitmap-functions.md
@@ -174,7 +174,7 @@ SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS re
     │ [3] │
     └─────┘
 
-## 位图 {#bitmapor}
+## 位图或 {#bitmapor}
 
 为两个位图对象进行或操作，返回一个新的位图对象。
 

From 9290d6f112a47e923616ce104d5e3295016d90ad Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Fri, 9 Jul 2021 06:06:21 +0300
Subject: [PATCH 258/290] Fix UBSan report in pointInPolygon

---
 src/Functions/PolygonUtils.h                           | 10 +++++++++-
 .../0_stateless/01940_point_in_polygon_ubsan.reference |  1 +
 .../0_stateless/01940_point_in_polygon_ubsan.sql       |  2 ++
 3 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference
 create mode 100644 tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql

diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h
index 3367b52cc36..ea91a187229 100644
--- a/src/Functions/PolygonUtils.h
+++ b/src/Functions/PolygonUtils.h
@@ -6,6 +6,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 
@@ -304,6 +305,13 @@ void PointInPolygonWithGrid::calcGridAttributes(
     y_scale = 1 / cell_height;
     x_shift = -min_corner.x();
     y_shift = -min_corner.y();
+
+    if (!(isFinite(x_scale)
+        && isFinite(y_scale)
+        && isFinite(x_shift)
+        && isFinite(y_shift)
+        && isFinite(grid_size)))
+        throw Exception("Polygon is not valid: bounding box is unbounded", ErrorCodes::BAD_ARGUMENTS);
 }
 
 template 
@@ -358,7 +366,7 @@ bool PointInPolygonWithGrid::contains(CoordinateType x, Coordina
     if (has_empty_bound)
         return false;
 
-    if (std::isnan(x) || std::isnan(y))
+    if (!isFinite(x) || !isFinite(y))
         return false;
 
     CoordinateType float_row = (y + y_shift) * y_scale;
diff --git a/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference
new file mode 100644
index 00000000000..573541ac970
--- /dev/null
+++ b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference
@@ -0,0 +1 @@
+0
diff --git a/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql
new file mode 100644
index 00000000000..d011725691f
--- /dev/null
+++ b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql
@@ -0,0 +1,2 @@
+SET validate_polygons = 0;
+SELECT pointInPolygon((-inf, 1023), [(10.000100135803223, 10000000000.), (inf, 0.9998999834060669), (1.1920928955078125e-7, 100.0000991821289), (1.000100016593933, 100.0000991821289)]);

From 09c20d5d0854693fc556b127a7a4ff2456ac9cc2 Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Fri, 9 Jul 2021 06:15:53 +0300
Subject: [PATCH 259/290] Update geoToH3.cpp

---
 src/Functions/geoToH3.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp
index 57973ab94fe..16f8de72eb0 100644
--- a/src/Functions/geoToH3.cpp
+++ b/src/Functions/geoToH3.cpp
@@ -84,14 +84,10 @@ public:
             coord.lng = degsToRads(lon);
             coord.lat = degsToRads(lat);
  
-	    H3Index hindex;
-	    H3Error err = latLngToCell(&coord, res, &hindex);
-	    if (err) 
-	    {
-                throw Exception(
-                    "Incorrect coordinates lat:" + std::to_string(coord.lat) + " lng:" + std::to_string(coord.lng) + " err:" + std::to_string(err),
-                    ErrorCodes::INCORRECT_DATA);
-	    }
+            H3Index hindex;
+            H3Error err = latLngToCell(&coord, res, &hindex);
+            if (err)
+	            throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lon, err);
 
             dst_data[row] = hindex;
         }

From b264c3f192c16f48fd5ad095fd67b307eb4749cf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Fri, 9 Jul 2021 06:26:35 +0300
Subject: [PATCH 260/290] Make graph pipeline rendering compatible with
 Dagre.JS

---
 src/Processors/printPipeline.cpp | 2 +-
 src/Processors/printPipeline.h   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/printPipeline.cpp b/src/Processors/printPipeline.cpp
index 5cdab1ed3ff..cbf8cb3a77d 100644
--- a/src/Processors/printPipeline.cpp
+++ b/src/Processors/printPipeline.cpp
@@ -103,7 +103,7 @@ void printPipelineCompact(const Processors & processors, WriteBuffer & out, bool
 
     out << "digraph\n{\n";
     out << "  rankdir=\"LR\";\n";
-    out << "  { node [shape = box]\n";
+    out << "  { node [shape = rect]\n";
 
     /// Nodes // TODO quoting and escaping
     size_t next_step = 0;
diff --git a/src/Processors/printPipeline.h b/src/Processors/printPipeline.h
index 9497bc3cc3c..6ff5fb24c37 100644
--- a/src/Processors/printPipeline.h
+++ b/src/Processors/printPipeline.h
@@ -16,7 +16,7 @@ void printPipeline(const Processors & processors, const Statuses & statuses, Wri
 {
     out << "digraph\n{\n";
     out << "  rankdir=\"LR\";\n";
-    out << "  { node [shape = box]\n";
+    out << "  { node [shape = rect]\n";
 
     auto get_proc_id = [](const IProcessor & proc) -> UInt64
     {

From 0bad9453924dedade34329493e64e7703e06f1fb Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Fri, 9 Jul 2021 06:49:08 +0300
Subject: [PATCH 261/290] Update PolygonUtils.h

---
 src/Functions/PolygonUtils.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h
index ea91a187229..a050de2edb6 100644
--- a/src/Functions/PolygonUtils.h
+++ b/src/Functions/PolygonUtils.h
@@ -41,6 +41,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int BAD_ARGUMENTS;
 }
 
 

From 95f8ca4e037acd7dacb1667edddb096f2ee28d99 Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Fri, 9 Jul 2021 12:16:03 +0300
Subject: [PATCH 262/290] Do not use default impl for low cardinality for
 joinGet

---
 src/Functions/FunctionJoinGet.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h
index 2250fa3ccf0..c701625e9cd 100644
--- a/src/Functions/FunctionJoinGet.h
+++ b/src/Functions/FunctionJoinGet.h
@@ -28,7 +28,7 @@ public:
     static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet";
 
     bool useDefaultImplementationForNulls() const override { return false; }
-    bool useDefaultImplementationForLowCardinalityColumns() const override { return true; }
+    bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
     bool useDefaultImplementationForConstants() const override { return true; }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override;

From 43880af722d9a136ace02bf8b4e5f68b32ae3f42 Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Fri, 9 Jul 2021 12:20:36 +0300
Subject: [PATCH 263/290] Add test to join_get_low_card_fix

---
 .../01735_join_get_low_card_fix.reference         |  7 ++++++-
 .../0_stateless/01735_join_get_low_card_fix.sql   | 15 ++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/01735_join_get_low_card_fix.reference b/tests/queries/0_stateless/01735_join_get_low_card_fix.reference
index 0b20aead00e..a9e2f17562a 100644
--- a/tests/queries/0_stateless/01735_join_get_low_card_fix.reference
+++ b/tests/queries/0_stateless/01735_join_get_low_card_fix.reference
@@ -1 +1,6 @@
-yyy
+1
+1
+1
+1
+1
+1
diff --git a/tests/queries/0_stateless/01735_join_get_low_card_fix.sql b/tests/queries/0_stateless/01735_join_get_low_card_fix.sql
index bdc979bc11e..e2002112360 100644
--- a/tests/queries/0_stateless/01735_join_get_low_card_fix.sql
+++ b/tests/queries/0_stateless/01735_join_get_low_card_fix.sql
@@ -1,9 +1,14 @@
-drop table if exists join_tbl;
+DROP TABLE IF EXISTS join_tbl;
 
-create table join_tbl (`id` String, `name` String) engine Join(any, left, id);
+CREATE TABLE join_tbl (`id` String, `name` String, lcname LowCardinality(String)) ENGINE = Join(any, left, id);
 
-insert into join_tbl values ('xxx', 'yyy');
+INSERT INTO join_tbl VALUES ('xxx', 'yyy', 'yyy');
 
-select joinGet('join_tbl', 'name', toLowCardinality('xxx'));
+SELECT joinGet('join_tbl', 'name', 'xxx') == 'yyy';
+SELECT joinGet('join_tbl', 'name', toLowCardinality('xxx')) == 'yyy';
+SELECT joinGet('join_tbl', 'name', toLowCardinality(materialize('xxx'))) == 'yyy';
+SELECT joinGet('join_tbl', 'lcname', 'xxx') == 'yyy';
+SELECT joinGet('join_tbl', 'lcname', toLowCardinality('xxx')) == 'yyy';
+SELECT joinGet('join_tbl', 'lcname', toLowCardinality(materialize('xxx'))) == 'yyy';
 
-drop table if exists join_tbl;
+DROP TABLE IF EXISTS join_tbl;

From 0ec402ff64a4b3d02b3d03bbe056f0b05bc80a9b Mon Sep 17 00:00:00 2001
From: zxc111 
Date: Fri, 9 Jul 2021 18:35:26 +0800
Subject: [PATCH 264/290] castColumn instead of execute toString function

---
 src/Functions/FunctionsCoding.h | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h
index 4db138a12a2..00b09acea1f 100644
--- a/src/Functions/FunctionsCoding.h
+++ b/src/Functions/FunctionsCoding.h
@@ -19,13 +19,13 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
 #include 
 #include 
 #include 
-#include 
 
 #include 
 #include 
@@ -955,8 +955,6 @@ public:
 template 
 class EncodeToBinaryRepr : public IFunction
 {
-private:
-    ContextPtr context;
 public:
     static constexpr auto name = Impl::name;
     static constexpr size_t word_size = Impl::word_size;
@@ -989,7 +987,7 @@ public:
         return std::make_shared();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const IColumn * column = arguments[0].column.get();
         ColumnPtr res_column;
@@ -997,10 +995,9 @@ public:
         WhichDataType which(column->getDataType());
         if (which.isAggregateFunction())
         {
-            auto to_string = FunctionFactory::instance().get("toString", context);
-            const ColumnPtr col = to_string->build(arguments)->execute(arguments, result_type, input_rows_count);
-            const auto * name_col = checkAndGetColumn(col.get());
-            tryExecuteString(name_col, res_column);
+            const ColumnPtr to_string = castColumn(arguments[0], std::make_shared());
+            const auto * str_column = checkAndGetColumn(to_string.get());
+            tryExecuteString(str_column, res_column);
             return res_column;
         }
 

From 947bb4a9421cf8932b66c2126423a855f7abfea1 Mon Sep 17 00:00:00 2001
From: kssenii 
Date: Fri, 9 Jul 2021 16:43:00 +0300
Subject: [PATCH 265/290] Fix progress bar for local

---
 programs/local/LocalServer.cpp           | 18 +++++++++++++-----
 src/Common/ProgressIndication.cpp        |  3 ---
 src/Interpreters/executeQuery.cpp        |  5 ++++-
 src/Interpreters/executeQuery.h          |  3 ++-
 src/Processors/Formats/IOutputFormat.cpp |  4 +++-
 src/Processors/Formats/IOutputFormat.h   |  6 +++++-
 6 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 2633f0e9426..6be7ba1ad73 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -388,24 +388,32 @@ void LocalServer::processQueries()
     /// Use the same query_id (and thread group) for all queries
     CurrentThread::QueryScope query_scope_holder(context);
 
-    ///Set progress show
+    /// Set progress show
     need_render_progress = config().getBool("progress", false);
 
+    std::function finalize_progress;
     if (need_render_progress)
     {
+        /// Set progress callback, which can be run from multiple threads.
         context->setProgressCallback([&](const Progress & value)
         {
             /// Write progress only if progress was updated
             if (progress_indication.updateProgress(value))
                 progress_indication.writeProgress();
         });
+
+        /// Set finalizing callback for progress, which is called right before finalizing query output.
+        finalize_progress = [&]()
+        {
+            progress_indication.clearProgressOutput();
+        };
+
+        /// Set callback for file processing progress.
+        progress_indication.setFileProgressCallback(context);
     }
 
     bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
 
-    if (need_render_progress)
-        progress_indication.setFileProgressCallback(context);
-
     std::exception_ptr exception;
 
     for (const auto & query : queries)
@@ -425,7 +433,7 @@ void LocalServer::processQueries()
 
         try
         {
-            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {});
+            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {}, finalize_progress);
         }
         catch (...)
         {
diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp
index e1a7c420c54..0d65eaece86 100644
--- a/src/Common/ProgressIndication.cpp
+++ b/src/Common/ProgressIndication.cpp
@@ -4,9 +4,6 @@
 #include 
 #include 
 
-/// FIXME: progress bar in clickhouse-local needs to be cleared after query execution
-///        - same as it is now in clickhouse-client. Also there is no writeFinalProgress call
-///        in clickhouse-local.
 
 namespace DB
 {
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 5b55754f00a..99c08c70b7c 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -948,7 +948,8 @@ void executeQuery(
     WriteBuffer & ostr,
     bool allow_into_outfile,
     ContextMutablePtr context,
-    std::function set_result_details)
+    std::function set_result_details,
+    std::function before_finalize_callback)
 {
     PODArray parse_buf;
     const char * begin;
@@ -1079,6 +1080,8 @@ void executeQuery(
                     out->onProgress(progress);
                 });
 
+                out->setBeforeFinalizeCallback(before_finalize_callback);
+
                 if (set_result_details)
                     set_result_details(
                         context->getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone());
diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h
index 6448b26a652..77f142de121 100644
--- a/src/Interpreters/executeQuery.h
+++ b/src/Interpreters/executeQuery.h
@@ -17,7 +17,8 @@ void executeQuery(
     WriteBuffer & ostr,                 /// Where to write query output to.
     bool allow_into_outfile,            /// If true and the query contains INTO OUTFILE section, redirect output to that file.
     ContextMutablePtr context,                 /// DB, tables, data types, storage engines, functions, aggregate functions...
-    std::function set_result_details /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone.
+    std::function set_result_details, /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone.
+    std::function before_finalize_callback = {} /// Will be set in output format to be called before finalize.
 );
 
 
diff --git a/src/Processors/Formats/IOutputFormat.cpp b/src/Processors/Formats/IOutputFormat.cpp
index 88649d9ca25..7d82c267f36 100644
--- a/src/Processors/Formats/IOutputFormat.cpp
+++ b/src/Processors/Formats/IOutputFormat.cpp
@@ -76,6 +76,9 @@ void IOutputFormat::work()
         if (rows_before_limit_counter && rows_before_limit_counter->hasAppliedLimit())
             setRowsBeforeLimit(rows_before_limit_counter->get());
 
+        if (before_finalize_callback)
+            before_finalize_callback();
+
         finalize();
         finalized = true;
         return;
@@ -117,4 +120,3 @@ void IOutputFormat::write(const Block & block)
 }
 
 }
-
diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h
index 4c2b3f30070..4d86d18f70e 100644
--- a/src/Processors/Formats/IOutputFormat.h
+++ b/src/Processors/Formats/IOutputFormat.h
@@ -67,6 +67,9 @@ public:
     /// Passed value are delta, that must be summarized.
     virtual void onProgress(const Progress & /*progress*/) {}
 
+    /// Set callback, which will be called before call to finalize().
+    void setBeforeFinalizeCallback(std::function callback) { before_finalize_callback = callback; }
+
     /// Content-Type to set when sending HTTP response.
     virtual std::string getContentType() const { return "text/plain; charset=UTF-8"; }
 
@@ -91,6 +94,7 @@ private:
     size_t result_bytes = 0;
 
     bool prefix_written = false;
+
+    std::function before_finalize_callback;
 };
 }
-

From 03c4853451664c07ecb1e88adcd75038ab5f16be Mon Sep 17 00:00:00 2001
From: Maksim Kita 
Date: Fri, 9 Jul 2021 17:20:02 +0300
Subject: [PATCH 266/290] Functions dictGet, dictHas complex key dictionary key
 argument tuple fix

---
 .../functions/ext-dict-functions.md           | 22 ++---
 src/Functions/FunctionsExternalDictionaries.h | 94 +++++++++++--------
 ..._dict_get_has_complex_single_key.reference | 10 ++
 .../01941_dict_get_has_complex_single_key.sql | 26 +++++
 4 files changed, 103 insertions(+), 49 deletions(-)
 create mode 100644 tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference
 create mode 100644 tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql

diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md
index 7c0fe11ae64..d7f142dd8b1 100644
--- a/docs/en/sql-reference/functions/ext-dict-functions.md
+++ b/docs/en/sql-reference/functions/ext-dict-functions.md
@@ -12,7 +12,7 @@ For information on connecting and configuring external dictionaries, see [Extern
 
 ## dictGet, dictGetOrDefault, dictGetOrNull {#dictget}
 
-Retrieves values from an external dictionary. 
+Retrieves values from an external dictionary.
 
 ``` sql
 dictGet('dict_name', attr_names, id_expr)
@@ -24,7 +24,7 @@ dictGetOrNull('dict_name', attr_name, id_expr)
 
 -   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
 -   `attr_names` — Name of the column of the dictionary, [String literal](../../sql-reference/syntax.md#syntax-string-literal), or tuple of column names, [Tuple](../../sql-reference/data-types/tuple.md)([String literal](../../sql-reference/syntax.md#syntax-string-literal)).
--   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
+-   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
 -   `default_value_expr` — Values returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute.
 
 **Returned value**
@@ -138,7 +138,7 @@ Configure the external dictionary:
                 c2
                 String
                 
-                        
+            
         
         0
     
@@ -237,7 +237,7 @@ dictHas('dict_name', id_expr)
 **Arguments**
 
 -   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
--   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
+-   `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration.
 
 **Returned value**
 
@@ -292,16 +292,16 @@ Type: `UInt8`.
 
 Returns first-level children as an array of indexes. It is the inverse transformation for [dictGetHierarchy](#dictgethierarchy).
 
-**Syntax** 
+**Syntax**
 
 ``` sql
 dictGetChildren(dict_name, key)
 ```
 
-**Arguments** 
+**Arguments**
 
--   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). 
--   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. 
+-   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
+-   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
 
 **Returned values**
 
@@ -339,7 +339,7 @@ SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers
 
 ## dictGetDescendant {#dictgetdescendant}
 
-Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.  
+Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively.
 
 **Syntax**
 
@@ -347,9 +347,9 @@ Returns all descendants as if [dictGetChildren](#dictgetchildren) function was a
 dictGetDescendants(dict_name, key, level)
 ```
 
-**Arguments** 
+**Arguments**
 
--   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). 
+-   `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal).
 -   `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value.
 -   `level` — Hierarchy level. If `level = 0` returns all descendants to the end. [UInt8](../../sql-reference/data-types/int-uint.md).
 
diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h
index 381401be2c5..118855b4bf8 100644
--- a/src/Functions/FunctionsExternalDictionaries.h
+++ b/src/Functions/FunctionsExternalDictionaries.h
@@ -163,13 +163,6 @@ public:
                 arguments[0]->getName(),
                 getName());
 
-        if (!WhichDataType(arguments[1]).isUInt64() &&
-            !isTuple(arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Illegal type {} of second argument of function {} must be UInt64 or tuple(...)",
-                arguments[1]->getName(),
-                getName());
-
         return std::make_shared();
     }
 
@@ -189,8 +182,8 @@ public:
         auto dictionary_key_type = dictionary->getKeyType();
 
         const ColumnWithTypeAndName & key_column_with_type = arguments[1];
-        const auto key_column = key_column_with_type.column;
-        const auto key_column_type = WhichDataType(key_column_with_type.type);
+        auto key_column = key_column_with_type.column;
+        auto key_column_type = key_column_with_type.type;
 
         ColumnPtr range_col = nullptr;
         DataTypePtr range_col_type = nullptr;
@@ -214,7 +207,7 @@ public:
 
         if (dictionary_key_type == DictionaryKeyType::simple)
         {
-            if (!key_column_type.isUInt64())
+            if (!WhichDataType(key_column_type).isUInt64())
                  throw Exception(
                      ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
                      "Second argument of function {} must be UInt64 when dictionary is simple. Actual type {}.",
@@ -225,24 +218,39 @@ public:
         }
         else if (dictionary_key_type == DictionaryKeyType::complex)
         {
-            if (!key_column_type.isTuple())
-                throw Exception(
-                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Second argument of function {} must be tuple when dictionary is complex. Actual type {}.",
-                    getName(),
-                    key_column_with_type.type->getName());
-
             /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
-            ColumnPtr key_column_full = key_column->convertToFullColumnIfConst();
+            key_column = key_column->convertToFullColumnIfConst();
+            size_t keys_size = dictionary->getStructure().getKeysSize();
 
-            const auto & key_columns = typeid_cast(*key_column_full).getColumnsCopy();
-            const auto & key_types = static_cast(*key_column_with_type.type).getElements();
+            if (!isTuple(key_column_type))
+            {
+                if (keys_size > 1)
+                {
+                    throw Exception(
+                        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                        "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
+                        "Actual type {}.",
+                        getName(),
+                        key_column_type->getName());
+                }
+                else
+                {
+                    Columns tuple_columns = {std::move(key_column)};
+                    key_column = ColumnTuple::create(tuple_columns);
+
+                    DataTypes tuple_types = {key_column_type};
+                    key_column_type = std::make_shared(tuple_types);
+                }
+            }
+
+            const auto & key_columns = assert_cast(*key_column).getColumnsCopy();
+            const auto & key_types = assert_cast(*key_column_type).getElements();
 
             return dictionary->hasKeys(key_columns, key_types);
         }
         else
         {
-            if (!key_column_type.isUInt64())
+            if (!WhichDataType(key_column_type).isUInt64())
                 throw Exception(
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
                     "Second argument of function {} must be UInt64 when dictionary is range. Actual type {}.",
@@ -346,13 +354,6 @@ public:
         Strings attribute_names = getAttributeNamesFromColumn(arguments[1].column, arguments[1].type);
 
         auto dictionary = helper.getDictionary(dictionary_name);
-
-        if (!WhichDataType(arguments[2].type).isUInt64() && !isTuple(arguments[2].type))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Illegal type {} of third argument of function {}, must be UInt64 or tuple(...).",
-                arguments[2].type->getName(),
-                getName());
-
         auto dictionary_key_type = dictionary->getKeyType();
 
         size_t current_arguments_index = 3;
@@ -446,18 +447,35 @@ public:
         }
         else if (dictionary_key_type == DictionaryKeyType::complex)
         {
-            if (!isTuple(key_col_with_type.type))
-                 throw Exception(
-                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                     "Third argument of function {} must be tuple when dictionary is complex. Actual type {}.",
-                     getName(),
-                     key_col_with_type.type->getName());
-
             /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys.
-            ColumnPtr key_column_full = key_col_with_type.column->convertToFullColumnIfConst();
+            ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst();
+            DataTypePtr key_column_type = key_col_with_type.type;
 
-            const auto & key_columns = typeid_cast(*key_column_full).getColumnsCopy();
-            const auto & key_types = static_cast(*key_col_with_type.type).getElements();
+            size_t keys_size = dictionary->getStructure().getKeysSize();
+
+            if (!isTuple(key_column_type))
+            {
+                if (keys_size > 1)
+                {
+                    throw Exception(
+                         ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                         "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute."
+                         "Actual type {}.",
+                         getName(),
+                         key_col_with_type.type->getName());
+                }
+                else
+                {
+                    Columns tuple_columns = {std::move(key_column)};
+                    key_column = ColumnTuple::create(tuple_columns);
+
+                    DataTypes tuple_types = {key_column_type};
+                    key_column_type = std::make_shared(tuple_types);
+                }
+            }
+
+            const auto & key_columns = assert_cast(*key_column).getColumnsCopy();
+            const auto & key_types = assert_cast(*key_column_type).getElements();
 
             result = executeDictionaryRequest(
                 dictionary,
diff --git a/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference
new file mode 100644
index 00000000000..c7e9cb788cb
--- /dev/null
+++ b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference
@@ -0,0 +1,10 @@
+dictGet
+Value
+Value
+Value
+Value
+dictHas
+1
+1
+1
+1
diff --git a/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql
new file mode 100644
index 00000000000..a44107d6882
--- /dev/null
+++ b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql
@@ -0,0 +1,26 @@
+DROP TABLE IF EXISTS test_dictionary_source;
+CREATE TABLE test_dictionary_source (key String, value String) ENGINE=TinyLog;
+
+INSERT INTO test_dictionary_source VALUES ('Key', 'Value');
+
+DROP DICTIONARY IF EXISTS test_dictionary;
+CREATE DICTIONARY test_dictionary(key String, value String)
+PRIMARY KEY key
+LAYOUT(COMPLEX_KEY_HASHED())
+SOURCE(CLICKHOUSE(TABLE 'test_dictionary_source'))
+LIFETIME(0);
+
+SELECT 'dictGet';
+SELECT dictGet('test_dictionary', 'value', tuple('Key'));
+SELECT dictGet('test_dictionary', 'value', tuple(materialize('Key')));
+SELECT dictGet('test_dictionary', 'value', 'Key');
+SELECT dictGet('test_dictionary', 'value', materialize('Key'));
+
+SELECT 'dictHas';
+SELECT dictHas('test_dictionary', tuple('Key'));
+SELECT dictHas('test_dictionary', tuple(materialize('Key')));
+SELECT dictHas('test_dictionary', 'Key');
+SELECT dictHas('test_dictionary', materialize('Key'));
+
+DROP DICTIONARY test_dictionary;
+DROP TABLE test_dictionary_source;

From 19a83b75b751395ace20eea49f9d605f62721cb5 Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Fri, 9 Jul 2021 18:20:45 +0300
Subject: [PATCH 267/290] Update version_date.tsv after release 21.7.2.7

---
 utils/list-versions/version_date.tsv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 541dea23698..3b12363712a 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,3 +1,4 @@
+v21.7.2.7-stable	2021-07-09
 v21.6.6.51-stable	2021-07-02
 v21.6.5.37-stable	2021-06-19
 v21.6.4.26-stable	2021-06-11

From 2fc16dd69280532b22b0347ff4987aa78ccbc43e Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Fri, 9 Jul 2021 18:53:32 +0300
Subject: [PATCH 268/290] Add minus sign in prometheus metric name in test

---
 tests/integration/test_prometheus_endpoint/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_prometheus_endpoint/test.py b/tests/integration/test_prometheus_endpoint/test.py
index 06276803c3d..c2b5a57218b 100644
--- a/tests/integration/test_prometheus_endpoint/test.py
+++ b/tests/integration/test_prometheus_endpoint/test.py
@@ -30,7 +30,7 @@ def parse_response_line(line):
 
     if line.startswith("#"):
         return {}
-    match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? (\d)', line)
+    match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:-]+)(\{.*\})? (\d)', line)
     assert match, line
     name, _, val = match.groups()
     return {name: int(val)}

From c01f4588b4647068a085cf58aab31d1f57785fa2 Mon Sep 17 00:00:00 2001
From: vdimir 
Date: Fri, 9 Jul 2021 19:12:26 +0300
Subject: [PATCH 269/290] Validate prometheus metric name with regex

---
 src/Server/PrometheusMetricsWriter.cpp        | 22 +++++++++++++------
 .../test_prometheus_endpoint/test.py          |  2 +-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/Server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp
index 787f0fcd95e..30ae6f6fe42 100644
--- a/src/Server/PrometheusMetricsWriter.cpp
+++ b/src/Server/PrometheusMetricsWriter.cpp
@@ -4,7 +4,7 @@
 
 #include 
 #include 
-#include 
+#include 
 
 namespace
 {
@@ -24,9 +24,13 @@ void writeOutLine(DB::WriteBuffer & wb, T && val, TArgs &&... args)
     writeOutLine(wb, std::forward(args)...);
 }
 
-void replaceInvalidChars(std::string & metric_name)
+/// Returns false if name is not valid
+bool replaceInvalidChars(std::string & metric_name)
 {
-    std::replace(metric_name.begin(), metric_name.end(), '.', '_');
+    /// dirty solution
+    metric_name = std::regex_replace(metric_name, std::regex("[^a-zA-Z0-9_:]"), "_");
+    metric_name = std::regex_replace(metric_name, std::regex("^[^a-zA-Z]*"), "");
+    return !metric_name.empty();
 }
 
 }
@@ -57,7 +61,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const
             std::string metric_name{ProfileEvents::getName(static_cast(i))};
             std::string metric_doc{ProfileEvents::getDocumentation(static_cast(i))};
 
-            replaceInvalidChars(metric_name);
+            if (!replaceInvalidChars(metric_name))
+                continue;
             std::string key{profile_events_prefix + metric_name};
 
             writeOutLine(wb, "# HELP", key, metric_doc);
@@ -75,7 +80,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const
             std::string metric_name{CurrentMetrics::getName(static_cast(i))};
             std::string metric_doc{CurrentMetrics::getDocumentation(static_cast(i))};
 
-            replaceInvalidChars(metric_name);
+            if (!replaceInvalidChars(metric_name))
+                continue;
             std::string key{current_metrics_prefix + metric_name};
 
             writeOutLine(wb, "# HELP", key, metric_doc);
@@ -91,7 +97,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const
         {
             std::string key{asynchronous_metrics_prefix + name_value.first};
 
-            replaceInvalidChars(key);
+            if (!replaceInvalidChars(key))
+                continue;
             auto value = name_value.second;
 
             // TODO: add HELP section? asynchronous_metrics contains only key and value
@@ -108,7 +115,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const
             std::string metric_name{CurrentStatusInfo::getName(static_cast(i))};
             std::string metric_doc{CurrentStatusInfo::getDocumentation(static_cast(i))};
 
-            replaceInvalidChars(metric_name);
+            if (!replaceInvalidChars(metric_name))
+                continue;
             std::string key{current_status_prefix + metric_name};
 
             writeOutLine(wb, "# HELP", key, metric_doc);
diff --git a/tests/integration/test_prometheus_endpoint/test.py b/tests/integration/test_prometheus_endpoint/test.py
index c2b5a57218b..06276803c3d 100644
--- a/tests/integration/test_prometheus_endpoint/test.py
+++ b/tests/integration/test_prometheus_endpoint/test.py
@@ -30,7 +30,7 @@ def parse_response_line(line):
 
     if line.startswith("#"):
         return {}
-    match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:-]+)(\{.*\})? (\d)', line)
+    match = re.match('^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? (\d)', line)
     assert match, line
     name, _, val = match.groups()
     return {name: int(val)}

From b7cc1904ccd6856e14be8aafcb0d02ce2202047a Mon Sep 17 00:00:00 2001
From: Vitaly Baranov 
Date: Fri, 9 Jul 2021 21:04:31 +0300
Subject: [PATCH 270/290] Add comments to the implementations of the pad
 functions

---
 src/Functions/padString.cpp | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/Functions/padString.cpp b/src/Functions/padString.cpp
index 7711ab1a056..c03733a1198 100644
--- a/src/Functions/padString.cpp
+++ b/src/Functions/padString.cpp
@@ -89,6 +89,9 @@ namespace
             }
 
             /// Not necessary, but good for performance.
+            /// We repeat `pad_string` multiple times until it's length becomes 16 or more.
+            /// It speeds up the function appendTo() because it allows to copy padding characters by portions of at least
+            /// 16 bytes instead of single bytes.
             while (numCharsInPadString() < 16)
             {
                 pad_string += pad_string;
@@ -104,6 +107,12 @@ namespace
         }
 
         String pad_string;
+
+        /// Offsets of code points in `pad_string`:
+        /// utf8_offsets[0] is the offset of the first code point in `pad_string`, it's always 0;
+        /// utf8_offsets[1] is the offset of the second code point in `pad_string`;
+        /// utf8_offsets[2] is the offset of the third code point in `pad_string`;
+        /// ...
         std::vector utf8_offsets;
     };
 
@@ -243,30 +252,32 @@ namespace
             const PaddingChars & padding_chars,
             StringSink & res_sink) const
         {
-            bool is_const_length = lengths.isConst();
-            bool need_check_length = true;
+            bool is_const_new_length = lengths.isConst();
+            size_t new_length = 0;
 
+            /// Insert padding characters to each string from `strings`, write the result strings into `res_sink`.
+            /// If for some input string its current length is greater than the specified new length then that string
+            /// will be trimmed to the specified new length instead of padding.
             for (; !res_sink.isEnd(); res_sink.next(), strings.next(), lengths.next())
             {
                 auto str = strings.getWhole();
                 size_t current_length = getLengthOfSlice(str);
 
-                auto new_length_slice = lengths.getWhole();
-                size_t new_length = new_length_slice.elements->getUInt(new_length_slice.position);
-
-                if (need_check_length)
+                if (!res_sink.rowNum() || !is_const_new_length)
                 {
+                    /// If `is_const_new_length` is true we can get and check the new length only once.
+                    auto new_length_slice = lengths.getWhole();
+                    new_length = new_length_slice.elements->getUInt(new_length_slice.position);
                     if (new_length > MAX_NEW_LENGTH)
                     {
                         throw Exception(
                             "New padded length (" + std::to_string(new_length) + ") is too big, maximum is: " + std::to_string(MAX_NEW_LENGTH),
                             ErrorCodes::TOO_LARGE_STRING_SIZE);
                     }
-                    if (is_const_length)
+                    if (is_const_new_length)
                     {
                         size_t rows_count = res_sink.offsets.size();
                         res_sink.reserve((new_length + 1 /* zero terminator */) * rows_count);
-                        need_check_length = false;
                     }
                 }
 

From 53f5c63e2cd991658b52da34bdecf5aef4a6719e Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Fri, 9 Jul 2021 22:16:57 +0300
Subject: [PATCH 271/290] Update geoToH3.cpp

---
 src/Functions/geoToH3.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp
index 16f8de72eb0..d269f9a3a24 100644
--- a/src/Functions/geoToH3.cpp
+++ b/src/Functions/geoToH3.cpp
@@ -87,7 +87,7 @@ public:
             H3Index hindex;
             H3Error err = latLngToCell(&coord, res, &hindex);
             if (err)
-	            throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lon, err);
+                throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lng, err);
 
             dst_data[row] = hindex;
         }

From 623b368a707cf17713cf38079161439b9b0c7a6e Mon Sep 17 00:00:00 2001
From: alesapin 
Date: Fri, 9 Jul 2021 22:58:21 +0300
Subject: [PATCH 272/290] Add draft for clickhouse-keeper docs

---
 docs/en/operations/clickhouse-keeper.md | 114 ++++++++++++++++++++++++
 1 file changed, 114 insertions(+)
 create mode 100644 docs/en/operations/clickhouse-keeper.md

diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md
new file mode 100644
index 00000000000..3dec4c74088
--- /dev/null
+++ b/docs/en/operations/clickhouse-keeper.md
@@ -0,0 +1,114 @@
+---
+toc_priority: 66
+toc_title: ClickHouse Keeper
+---
+
+# [pre-production] clickhouse-keeper
+
+ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../../engines/table-engines/mergetree-family/replication/) and [distributed DDL](../../sql-reference/distributed-ddl/) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
+
+!!! warning "Warning"
+    This feature currently in pre-production stage. We test it in our CI and on small internal installations.
+
+## Implemetation details
+
+ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages.
+
+By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible.
+
+## Configuration
+
+`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is ``. Keeper configuration has the following parameters:
+
+-    `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`)
+-    `tcp_port_secure` — the secure port for a client to connect
+-    `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on)
+-    `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper)
+-    `snapshot_storage_path` — path to coordination snapshots
+
+Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on).
+
+Internal coordination settings are located in `.` section:
+
+-    `operation_timeout_ms` — timeout for a single client operation
+-    `session_timeout_ms` — timeout for client session
+-    `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them
+-    `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers
+-    `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election
+-    `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election
+-    `rotate_log_storage_interval` — how many logs to store in a single file
+-    `reserved_log_items` — how many coordination logs to store before compaction
+-    `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of logs)
+-    `snapshots_to_keep` — how many snapshots to keep
+-    `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs
+-    `force_sync` — call `fsync` on each write to coordination log
+-    `raft_logs_level` — text logging level about coordination (trace, debug, and so on)
+-    `shutdown_timeout` — wait to finish internal connections and shutdown
+-    `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate
+
+Quorum configuration is located in `.` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `` are:
+
+-    `id` — server_id in quorum
+-    `hostname` — hostname where this server placed
+-    `port` — port where this server listen for connections
+
+
+Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1:
+
+```xml
+
+    2181
+    1
+    /var/lib/clickhouse/coordination/log
+    /var/lib/clickhouse/coordination/snapshots
+
+    
+        10000
+        30000
+        trace
+    
+
+    
+        
+            1
+            zoo1
+            9444
+        
+        
+            2
+            zoo2
+            9444
+        
+        
+            3
+            zoo3
+            9444
+        
+    
+
+```
+
+## How to run
+
+`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with:
+
+```bash
+clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
+```
+
+## [experimental] Migration from ZooKeeper
+
+Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration:
+
+1. Stop all ZooKeeper nodes.
+
+2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot.
+
+3. Run `clickhouse-keeper-converter` on leader, example
+
+```bash
+clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots
+```
+
+4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist only on leader node, leader will sync it automatically to other nodes.
+

From 3588c5ad1722e378f07a8d9c03e191dbf7ba8311 Mon Sep 17 00:00:00 2001
From: Anton Popov 
Date: Fri, 9 Jul 2021 23:32:14 +0300
Subject: [PATCH 273/290] add color for double colon

---
 programs/client/Client.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index c4aef014971..9c1c8338321 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -430,6 +430,7 @@ private:
                {TokenType::ClosingRoundBracket, Replxx::Color::BROWN},
                {TokenType::OpeningSquareBracket, Replxx::Color::BROWN},
                {TokenType::ClosingSquareBracket, Replxx::Color::BROWN},
+               {TokenType::DoubleColon, Replxx::Color::BROWN},
                {TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE},
                {TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE},
 

From 56c04c604e72d7b9ce3a8e7c020d08ac5c0bb830 Mon Sep 17 00:00:00 2001
From: kssenii 
Date: Fri, 9 Jul 2021 23:44:57 +0300
Subject: [PATCH 274/290] Remove misleading stderr

---
 src/Common/Config/configReadClient.cpp | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp
index cbe5b3f7bc2..e7bc0b72814 100644
--- a/src/Common/Config/configReadClient.cpp
+++ b/src/Common/Config/configReadClient.cpp
@@ -10,16 +10,10 @@ namespace fs = std::filesystem;
 namespace DB
 {
 
-/// Checks if file exists without throwing an exception but with message in console.
-bool safeFsExists(const auto & path)
+bool safeFsExists(const String & path)
 {
     std::error_code ec;
-    bool res = fs::exists(path, ec);
-    if (ec)
-    {
-        std::cerr << "Can't check '" << path << "': [" << ec.value() << "] "  << ec.message() << std::endl;
-    }
-    return res;
+    return fs::exists(path, ec);
 };
 
 bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path)

From 856892a81754b73ad54030860f24d6060e3f7fb6 Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Sat, 10 Jul 2021 05:46:11 +0300
Subject: [PATCH 275/290] Update geoToH3.cpp

---
 src/Functions/geoToH3.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp
index d269f9a3a24..2dad8fc13f2 100644
--- a/src/Functions/geoToH3.cpp
+++ b/src/Functions/geoToH3.cpp
@@ -83,7 +83,7 @@ public:
             LatLng coord;
             coord.lng = degsToRads(lon);
             coord.lat = degsToRads(lat);
- 
+
             H3Index hindex;
             H3Error err = latLngToCell(&coord, res, &hindex);
             if (err)

From 354a57aea87fe57627e53571d89a180909657972 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 05:49:36 +0300
Subject: [PATCH 276/290] Drop Arcadia

---
 src/Functions/ya.make    | 3 +--
 src/Functions/ya.make.in | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Functions/ya.make b/src/Functions/ya.make
index d6da7eadd35..c78ef1908d1 100644
--- a/src/Functions/ya.make
+++ b/src/Functions/ya.make
@@ -4,13 +4,12 @@ OWNER(g:clickhouse)
 LIBRARY()
 
 CFLAGS(
-    -DUSE_H3 -DUSE_SSL -DUSE_XXHASH
+    -DUSE_SSL -DUSE_XXHASH
 )
 
 ADDINCL(
     library/cpp/consistent_hashing
     contrib/libs/farmhash
-    contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
     contrib/libs/libdivide
     contrib/libs/rapidjson/include
diff --git a/src/Functions/ya.make.in b/src/Functions/ya.make.in
index f75773fb47e..cfc58b7bf5d 100644
--- a/src/Functions/ya.make.in
+++ b/src/Functions/ya.make.in
@@ -3,13 +3,12 @@ OWNER(g:clickhouse)
 LIBRARY()
 
 CFLAGS(
-    -DUSE_H3 -DUSE_SSL -DUSE_XXHASH
+    -DUSE_SSL -DUSE_XXHASH
 )
 
 ADDINCL(
     library/cpp/consistent_hashing
     contrib/libs/farmhash
-    contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
     contrib/libs/libdivide
     contrib/libs/rapidjson/include

From ed3b30115845052c799f7d81b08d787ad3ac1e3c Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Sat, 10 Jul 2021 08:18:57 +0300
Subject: [PATCH 277/290] Auto version update to [21.8.1.7409] [54453]

---
 cmake/autogenerated_versions.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 49cf30d2556..3249eb765c5 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -6,7 +6,7 @@ SET(VERSION_REVISION 54453)
 SET(VERSION_MAJOR 21)
 SET(VERSION_MINOR 8)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH fb895056568e26200629c7d19626e92d2dedc70d)
-SET(VERSION_DESCRIBE v21.8.1.1-prestable)
-SET(VERSION_STRING 21.8.1.1)
+SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238)
+SET(VERSION_DESCRIBE v21.8.1.7409-prestable)
+SET(VERSION_STRING 21.8.1.7409)
 # end of autochange

From 8ae8b26954d07ec1bdadaeae73b2aaf0e19c2ea0 Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Sat, 10 Jul 2021 08:22:52 +0300
Subject: [PATCH 278/290] Auto version update to [21.9.1.1] [54454]

---
 cmake/autogenerated_versions.txt                    |  8 ++++----
 debian/changelog                                    |  4 ++--
 docker/client/Dockerfile                            |  2 +-
 docker/server/Dockerfile                            |  2 +-
 docker/test/Dockerfile                              |  2 +-
 .../System/StorageSystemContributors.generated.cpp  | 13 +++++++++++++
 6 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 3249eb765c5..18072566d04 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -2,11 +2,11 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54453)
+SET(VERSION_REVISION 54454)
 SET(VERSION_MAJOR 21)
-SET(VERSION_MINOR 8)
+SET(VERSION_MINOR 9)
 SET(VERSION_PATCH 1)
 SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238)
-SET(VERSION_DESCRIBE v21.8.1.7409-prestable)
-SET(VERSION_STRING 21.8.1.7409)
+SET(VERSION_DESCRIBE v21.9.1.1-prestable)
+SET(VERSION_STRING 21.9.1.1)
 # end of autochange
diff --git a/debian/changelog b/debian/changelog
index 36c29fce1d0..38f740ae062 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (21.8.1.1) unstable; urgency=low
+clickhouse (21.9.1.1) unstable; urgency=low
 
   * Modified source code
 
- -- clickhouse-release   Mon, 28 Jun 2021 00:50:15 +0300
+ -- clickhouse-release   Sat, 10 Jul 2021 08:22:49 +0300
diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
index 19cadccb926..f17fa8ade16 100644
--- a/docker/client/Dockerfile
+++ b/docker/client/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index 65d90bf52ce..5da9e703f4d 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 ARG gosu_ver=1.10
 
 # set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile
index 687393025f0..5768753cd7c 100644
--- a/docker/test/Dockerfile
+++ b/docker/test/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04
 
 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.8.1.*
+ARG version=21.9.1.*
 
 RUN apt-get update && \
     apt-get install -y apt-transport-https dirmngr && \
diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp
index f45acb0efd9..bed8eadc19c 100644
--- a/src/Storages/System/StorageSystemContributors.generated.cpp
+++ b/src/Storages/System/StorageSystemContributors.generated.cpp
@@ -95,6 +95,7 @@ const char * auto_contributors[] {
     "Anatoly Pugachev",
     "ana-uvarova",
     "AnaUvarova",
+    "Andreas Hunkeler",
     "AndreevDm",
     "Andrei Bodrov",
     "Andrei Chulkov",
@@ -280,6 +281,7 @@ const char * auto_contributors[] {
     "Dongdong Yang",
     "DoomzD",
     "Dr. Strange Looker",
+    "d.v.semenov",
     "eaxdev",
     "eejoin",
     "egatov",
@@ -290,6 +292,7 @@ const char * auto_contributors[] {
     "Eldar Zaitov",
     "Elena Baskakova",
     "elenaspb2019",
+    "elevankoff",
     "Elghazal Ahmed",
     "Elizaveta Mironyuk",
     "emakarov",
@@ -434,6 +437,7 @@ const char * auto_contributors[] {
     "Ivan Starkov",
     "ivanzhukov",
     "Ivan Zhukov",
+    "Jack Song",
     "JackyWoo",
     "Jacob Hayes",
     "jakalletti",
@@ -476,6 +480,7 @@ const char * auto_contributors[] {
     "Konstantin Lebedev",
     "Konstantin Malanchev",
     "Konstantin Podshumok",
+    "Konstantin Rudenskii",
     "Korenevskiy Denis",
     "Korviakov Andrey",
     "koshachy",
@@ -488,6 +493,7 @@ const char * auto_contributors[] {
     "kshvakov",
     "kssenii",
     "l",
+    "l1tsolaiki",
     "lalex",
     "Latysheva Alexandra",
     "lehasm",
@@ -515,6 +521,7 @@ const char * auto_contributors[] {
     "long2ice",
     "Lopatin Konstantin",
     "Loud_Scream",
+    "ltybc-coder",
     "luc1ph3r",
     "Lucid Dreams",
     "Luis Bosque",
@@ -633,6 +640,7 @@ const char * auto_contributors[] {
     "nicelulu",
     "Nickita",
     "Nickolay Yastrebov",
+    "nickzhwang",
     "Nicolae Vartolomei",
     "Nico Mandery",
     "Nico Piderman",
@@ -871,6 +879,7 @@ const char * auto_contributors[] {
     "Veselkov Konstantin",
     "vic",
     "vicdashkov",
+    "Victor",
     "Victor Tarnavsky",
     "Viktor Taranenko",
     "vinity",
@@ -947,6 +956,7 @@ const char * auto_contributors[] {
     "Yuriy Korzhenevskiy",
     "Yury Karpovich",
     "Yury Stankevich",
+    "ywill3",
     "zamulla",
     "zhang2014",
     "zhangshengyu",
@@ -957,11 +967,13 @@ const char * auto_contributors[] {
     "Zhichun Wu",
     "Zhipeng",
     "zhukai",
+    "Zijie Lu",
     "zlx19950903",
     "Zoran Pandovski",
     "zvonand",
     "zvrr",
     "zvvr",
+    "zxc111",
     "zzsmdfj",
     "Артем Стрельцов",
     "Владислав Тихонов",
@@ -980,6 +992,7 @@ const char * auto_contributors[] {
     "张风啸",
     "徐炘",
     "曲正鹏",
+    "未来星___费",
     "极客青年",
     "谢磊",
     "贾顺名(Jarvis)",

From a6d0cda7c19f5284c26535eb5ed5efa26f9a95d6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 10:12:18 +0300
Subject: [PATCH 279/290] Merging #24404

---
 docs/en/operations/configuration-files.md | 2 +-
 src/Common/Config/ConfigProcessor.cpp     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md
index 28e520fece5..5c942efc77f 100644
--- a/docs/en/operations/configuration-files.md
+++ b/docs/en/operations/configuration-files.md
@@ -32,7 +32,7 @@ XML substitution example:
     
 
     
-        
+        
         
         
     
diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp
index 346abce8600..03ee76240cb 100644
--- a/src/Common/Config/ConfigProcessor.cpp
+++ b/src/Common/Config/ConfigProcessor.cpp
@@ -302,14 +302,14 @@ void ConfigProcessor::doIncludesRecursive(
     }
 
     if (substs_count > 1) /// only one substitution is allowed
-        throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">");
+        throw Poco::Exception("More than one substitution attribute is set for element <" + node->nodeName() + ">");
 
     if (node->nodeName() == "include")
     {
         if (node->hasChildNodes())
             throw Poco::Exception(" element must have no children");
         if (substs_count == 0)
-            throw Poco::Exception("no substitution attributes set for element , must have one");
+            throw Poco::Exception("No substitution attributes set for element , must have exactly one");
     }
 
     /// Replace the original contents, not add to it.

From 4ed170a652c3eb0d470c061b4385e8fad2a505d7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 10:14:21 +0300
Subject: [PATCH 280/290] Added failing test #24404

---
 .../0_stateless/01942_untuple_transformers_msan.reference        | 0
 tests/queries/0_stateless/01942_untuple_transformers_msan.sql    | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 tests/queries/0_stateless/01942_untuple_transformers_msan.reference
 create mode 100644 tests/queries/0_stateless/01942_untuple_transformers_msan.sql

diff --git a/tests/queries/0_stateless/01942_untuple_transformers_msan.reference b/tests/queries/0_stateless/01942_untuple_transformers_msan.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/01942_untuple_transformers_msan.sql b/tests/queries/0_stateless/01942_untuple_transformers_msan.sql
new file mode 100644
index 00000000000..c1be25d34ac
--- /dev/null
+++ b/tests/queries/0_stateless/01942_untuple_transformers_msan.sql
@@ -0,0 +1 @@
+SELECT untuple(tuple(100.0000991821289)), NULL, untuple((toDateTime(9223372036854775806, -1, NULL, NULL, toDateTime(NULL, NULL)), * EXCEPT b)), NULL FROM (SELECT 1 AS a, 1024, NULL AS b);

From efbc3087394ae82b38a41e797ecfd6cc2275b1fc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 10:27:48 +0300
Subject: [PATCH 281/290] Remove harmful code and fix crash

---
 src/Interpreters/ActionsVisitor.cpp                        | 7 +------
 .../0_stateless/01942_untuple_transformers_msan.reference  | 1 +
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 7aad11252cb..03fa756276e 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -686,7 +686,7 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat
 
     ASTs columns;
     size_t tid = 0;
-    for (const auto & name : tuple_type->getElementNames())
+    for (const auto & name [[maybe_unused]] : tuple_type->getElementNames())
     {
         auto tuple_ast = function->arguments->children[0];
         if (tid != 0)
@@ -697,11 +697,6 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat
 
         auto func = makeASTFunction("tupleElement", tuple_ast, literal);
 
-        if (tuple_type->haveExplicitNames())
-            func->setAlias(name);
-        else
-            func->setAlias(data.getUniqueName("_ut_" + name));
-
         auto function_builder = FunctionFactory::instance().get(func->name, data.getContext());
         data.addFunction(function_builder, {tuple_name_type->name, literal->getColumnName(data.getContext()->getSettingsRef())}, func->getColumnName(data.getContext()->getSettingsRef()));
 
diff --git a/tests/queries/0_stateless/01942_untuple_transformers_msan.reference b/tests/queries/0_stateless/01942_untuple_transformers_msan.reference
index e69de29bb2d..82dea36febd 100644
--- a/tests/queries/0_stateless/01942_untuple_transformers_msan.reference
+++ b/tests/queries/0_stateless/01942_untuple_transformers_msan.reference
@@ -0,0 +1 @@
+100.0000991821289	\N	\N	1	1024	\N

From e95d67b8e23e2906c18895a8593b89df20b417bf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 10:29:30 +0300
Subject: [PATCH 282/290] Update tests after removing harmful code

---
 tests/queries/0_stateless/01232_untuple.reference              | 2 +-
 tests/queries/0_stateless/01616_untuple_access_field.reference | 2 +-
 tests/queries/0_stateless/01616_untuple_access_field.sql       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01232_untuple.reference b/tests/queries/0_stateless/01232_untuple.reference
index 44f96e1decd..21fd0c4a8a5 100644
--- a/tests/queries/0_stateless/01232_untuple.reference
+++ b/tests/queries/0_stateless/01232_untuple.reference
@@ -2,7 +2,7 @@
 hello	1	3	world
 9
 9	(0,1)
-key	v1	v2	v3	v4	v5
+key	tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 1)	tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 2)	tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 3)	tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 4)	tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 5)
 4	10	20	10	20	30
 3	70	20	10	20	30
 2	11	20	10	20	30
diff --git a/tests/queries/0_stateless/01616_untuple_access_field.reference b/tests/queries/0_stateless/01616_untuple_access_field.reference
index d00491fd7e5..9874d6464ab 100644
--- a/tests/queries/0_stateless/01616_untuple_access_field.reference
+++ b/tests/queries/0_stateless/01616_untuple_access_field.reference
@@ -1 +1 @@
-1
+1	2
diff --git a/tests/queries/0_stateless/01616_untuple_access_field.sql b/tests/queries/0_stateless/01616_untuple_access_field.sql
index 569efca5349..82cdf80c8bc 100644
--- a/tests/queries/0_stateless/01616_untuple_access_field.sql
+++ b/tests/queries/0_stateless/01616_untuple_access_field.sql
@@ -1 +1 @@
-select _ut_1 from (select untuple((1,2)));
+select * from (select untuple((1,2)));

From 9ca38235aa2f0e1d6b552625da40f4ee3d5e5ff7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 11:29:08 +0300
Subject: [PATCH 283/290] Correct fix for #26041

---
 src/Functions/URL/FirstSignificantSubdomainCustomImpl.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
index 4670d610725..ba39eeb5e69 100644
--- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
+++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
@@ -41,6 +41,9 @@ public:
     String getName() const override { return name; }
     size_t getNumberOfArguments() const override { return 2; }
 
+    bool useDefaultImplementationForConstants() const override { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
+
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         if (!isString(arguments[0].type))
@@ -65,8 +68,6 @@ public:
         const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
         FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue());
 
-        /// FIXME: convertToFullColumnIfConst() is suboptimal
-        auto column = arguments[0].column->convertToFullColumnIfConst();
         if (const ColumnString * col = checkAndGetColumn(*column))
         {
             auto col_res = ColumnString::create();

From ba1442532b4e59007ecda55785aeadaa5ab3eb5a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 11:43:28 +0300
Subject: [PATCH 284/290] Fix build

---
 src/Functions/URL/FirstSignificantSubdomainCustomImpl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
index ba39eeb5e69..08576fe59ec 100644
--- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
+++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
@@ -68,7 +68,7 @@ public:
         const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
         FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue());
 
-        if (const ColumnString * col = checkAndGetColumn(*column))
+        if (const ColumnString * col = checkAndGetColumn(*arguments[0].column))
         {
             auto col_res = ColumnString::create();
             vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());

From 0d4e0bb8fd8429c01e19c121bdc18c521d4a1ee5 Mon Sep 17 00:00:00 2001
From: alesapin 
Date: Sat, 10 Jul 2021 11:50:03 +0300
Subject: [PATCH 285/290] Fix links to nowhere

---
 docs/en/operations/clickhouse-keeper.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md
index 3dec4c74088..6af12eb9b01 100644
--- a/docs/en/operations/clickhouse-keeper.md
+++ b/docs/en/operations/clickhouse-keeper.md
@@ -5,7 +5,7 @@ toc_title: ClickHouse Keeper
 
 # [pre-production] clickhouse-keeper
 
-ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../../engines/table-engines/mergetree-family/replication/) and [distributed DDL](../../sql-reference/distributed-ddl/) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
+ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.
 
 !!! warning "Warning"
     This feature currently in pre-production stage. We test it in our CI and on small internal installations.

From d93fb6c93fa7e818685788cff85f8fab78f3b8d9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov 
Date: Sat, 10 Jul 2021 11:54:43 +0300
Subject: [PATCH 286/290] Drop Arcadia

---
 src/Functions/ya.make    | 1 +
 src/Functions/ya.make.in | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/Functions/ya.make b/src/Functions/ya.make
index 600b34e4bbf..2db4a7645a1 100644
--- a/src/Functions/ya.make
+++ b/src/Functions/ya.make
@@ -10,6 +10,7 @@ CFLAGS(
 ADDINCL(
     library/cpp/consistent_hashing
     contrib/libs/farmhash
+    contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
     contrib/libs/libdivide
     contrib/libs/rapidjson/include
diff --git a/src/Functions/ya.make.in b/src/Functions/ya.make.in
index cfc58b7bf5d..b21bf64304a 100644
--- a/src/Functions/ya.make.in
+++ b/src/Functions/ya.make.in
@@ -9,6 +9,7 @@ CFLAGS(
 ADDINCL(
     library/cpp/consistent_hashing
     contrib/libs/farmhash
+    contrib/libs/h3/h3lib/include
     contrib/libs/hyperscan/src
     contrib/libs/libdivide
     contrib/libs/rapidjson/include

From eb5ca241ecaf1ae6ccdb4394dda64d9e1d858c2d Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Sat, 10 Jul 2021 13:51:03 +0300
Subject: [PATCH 287/290] Update version_date.tsv after release 21.6.7.57

---
 utils/list-versions/version_date.tsv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 3b12363712a..1cb9d58ec2d 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,4 +1,5 @@
 v21.7.2.7-stable	2021-07-09
+v21.6.7.57-stable	2021-07-09
 v21.6.6.51-stable	2021-07-02
 v21.6.5.37-stable	2021-06-19
 v21.6.4.26-stable	2021-06-11

From f45869ab44cf8c93dd012f35fb6edc224b4d0cc2 Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Sat, 10 Jul 2021 15:04:34 +0300
Subject: [PATCH 288/290] Update version_date.tsv after release 21.3.15.4

---
 utils/list-versions/version_date.tsv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 1cb9d58ec2d..a8079ec89f6 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -13,6 +13,7 @@ v21.4.6.55-stable	2021-04-30
 v21.4.5.46-stable	2021-04-24
 v21.4.4.30-stable	2021-04-16
 v21.4.3.21-stable	2021-04-12
+v21.3.15.4-stable	2021-07-10
 v21.3.14.1-lts	2021-07-01
 v21.3.13.9-lts	2021-06-22
 v21.3.12.2-lts	2021-05-25

From f0cd4dd467a5c721954a2e6bc95f28171ab02b40 Mon Sep 17 00:00:00 2001
From: robot-clickhouse 
Date: Sat, 10 Jul 2021 17:01:18 +0300
Subject: [PATCH 289/290] Update version_date.tsv after release 21.5.9.4

---
 utils/list-versions/version_date.tsv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index a8079ec89f6..c46c393c630 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -4,6 +4,7 @@ v21.6.6.51-stable	2021-07-02
 v21.6.5.37-stable	2021-06-19
 v21.6.4.26-stable	2021-06-11
 v21.6.3.14-stable	2021-06-04
+v21.5.9.4-stable	2021-07-10
 v21.5.8.21-stable	2021-07-02
 v21.5.7.9-stable	2021-06-22
 v21.5.6.6-stable	2021-05-29

From 103b860555d52787d90da2e8f07a87bde2c6e6fb Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Sat, 10 Jul 2021 23:12:32 +0300
Subject: [PATCH 290/290] Update tuple-functions.md

---
 docs/en/sql-reference/functions/tuple-functions.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md
index 4189d0feeb5..39e59ae2ba9 100644
--- a/docs/en/sql-reference/functions/tuple-functions.md
+++ b/docs/en/sql-reference/functions/tuple-functions.md
@@ -87,6 +87,8 @@ Result:
 └───────┴───────┘
 ```
 
+Note: the names are implementation specific and are subject to change. You should not assume specific names of the columns after application of the `untuple`.
+
 Example of using an `EXCEPT` expression:
 
 Query: