diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index a78b183e6e1..d97fe4aff2c 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -273,17 +273,13 @@ int Server::main(const std::vector & /*args*/) * table engines could use Context on destroy. */ LOG_INFO(log, "Shutting down storages."); - if (text_log) - text_log->shutdown(); global_context->shutdown(); LOG_DEBUG(log, "Shutted down storages."); /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. * At this moment, no one could own shared part of Context. */ - text_log.reset(); global_context.reset(); - LOG_DEBUG(log, "Destroyed global context."); }); @@ -413,7 +409,7 @@ int Server::main(const std::vector & /*args*/) main_config_zk_changed_event, [&](ConfigurationPtr config) { - setTextLog(text_log); + setTextLog(global_context->getTextLog()); buildLoggers(*config, logger()); global_context->setClustersConfig(config); global_context->setMacros(std::make_unique(*config, "macros")); @@ -500,14 +496,11 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Loading metadata from " + path); - /// Create text_log instance - text_log = createSystemLog(*global_context, "system", "text_log", global_context->getConfigRef(), "text_log"); - try { loadMetadataSystem(*global_context); /// After attaching system databases we can initialize system log. - global_context->initializeSystemLogs(text_log); + global_context->initializeSystemLogs(); /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper); /// Then, load remaining databases diff --git a/dbms/programs/server/Server.h b/dbms/programs/server/Server.h index 5fc5f16b550..337d1551b70 100644 --- a/dbms/programs/server/Server.h +++ b/dbms/programs/server/Server.h @@ -57,7 +57,6 @@ protected: private: std::unique_ptr global_context; - std::shared_ptr text_log; }; } diff --git a/dbms/src/Functions/FunctionsConsistentHashing.cpp b/dbms/src/Functions/FunctionsConsistentHashing.cpp deleted file mode 100644 index 7f93257774b..00000000000 --- a/dbms/src/Functions/FunctionsConsistentHashing.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "FunctionsConsistentHashing.h" -#include - - -namespace DB -{ - -void registerFunctionsConsistentHashing(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/dbms/src/Functions/FunctionsConsistentHashing.h b/dbms/src/Functions/FunctionsConsistentHashing.h index 04640e0ec57..25c6c868306 100644 --- a/dbms/src/Functions/FunctionsConsistentHashing.h +++ b/dbms/src/Functions/FunctionsConsistentHashing.h @@ -8,9 +8,6 @@ #include #include -#include -#include - namespace DB { @@ -23,69 +20,6 @@ namespace ErrorCodes } -/// An O(1) time and space consistent hash algorithm by Konstantin Oblakov -struct YandexConsistentHashImpl -{ - static constexpr auto name = "yandexConsistentHash"; - - using HashType = UInt64; - /// Actually it supports UInt64, but it is efficient only if n <= 32768 - using ResultType = UInt16; - using BucketsType = ResultType; - static constexpr auto max_buckets = 32768; - - static inline ResultType apply(UInt64 hash, BucketsType n) - { - return ConsistentHashing(hash, n); - } -}; - - -/// Code from https://arxiv.org/pdf/1406.2294.pdf -static inline int32_t JumpConsistentHash(uint64_t key, int32_t num_buckets) -{ - int64_t b = -1, j = 0; - while (j < num_buckets) - { - b = j; - key = key * 2862933555777941757ULL + 1; - j = static_cast((b + 1) * (double(1LL << 31) / double((key >> 33) + 1))); - } - return static_cast(b); -} - -struct JumpConsistentHashImpl -{ - static constexpr auto name = "jumpConsistentHash"; - - using HashType = UInt64; - using ResultType = Int32; - using BucketsType = ResultType; - static constexpr auto max_buckets = static_cast(std::numeric_limits::max()); - - static inline ResultType apply(UInt64 hash, BucketsType n) - { - return JumpConsistentHash(hash, n); - } -}; - - -struct SumburConsistentHashImpl -{ - static constexpr auto name = "sumburConsistentHash"; - - using HashType = UInt32; - using ResultType = UInt16; - using BucketsType = ResultType; - static constexpr auto max_buckets = static_cast(std::numeric_limits::max()); - - static inline ResultType apply(HashType hash, BucketsType n) - { - return static_cast(sumburConsistentHash(hash, n)); - } -}; - - template class FunctionConsistentHashImpl : public IFunction { @@ -221,10 +155,4 @@ private: } }; - -using FunctionYandexConsistentHash = FunctionConsistentHashImpl; -using FunctionJumpConsistentHash = FunctionConsistentHashImpl; -using FunctionSumburConsistentHash = FunctionConsistentHashImpl; - - } diff --git a/dbms/src/Functions/FunctionsVisitParam.cpp b/dbms/src/Functions/FunctionsVisitParam.cpp deleted file mode 100644 index 8d82f93a3db..00000000000 --- a/dbms/src/Functions/FunctionsVisitParam.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include - - -namespace DB -{ - -struct NameVisitParamHas { static constexpr auto name = "visitParamHas"; }; -struct NameVisitParamExtractUInt { static constexpr auto name = "visitParamExtractUInt"; }; -struct NameVisitParamExtractInt { static constexpr auto name = "visitParamExtractInt"; }; -struct NameVisitParamExtractFloat { static constexpr auto name = "visitParamExtractFloat"; }; -struct NameVisitParamExtractBool { static constexpr auto name = "visitParamExtractBool"; }; -struct NameVisitParamExtractRaw { static constexpr auto name = "visitParamExtractRaw"; }; -struct NameVisitParamExtractString { static constexpr auto name = "visitParamExtractString"; }; - - -using FunctionVisitParamHas = FunctionsStringSearch, NameVisitParamHas>; -using FunctionVisitParamExtractUInt = FunctionsStringSearch>, NameVisitParamExtractUInt>; -using FunctionVisitParamExtractInt = FunctionsStringSearch>, NameVisitParamExtractInt>; -using FunctionVisitParamExtractFloat = FunctionsStringSearch>, NameVisitParamExtractFloat>; -using FunctionVisitParamExtractBool = FunctionsStringSearch, NameVisitParamExtractBool>; -using FunctionVisitParamExtractRaw = FunctionsStringSearchToString, NameVisitParamExtractRaw>; -using FunctionVisitParamExtractString = FunctionsStringSearchToString, NameVisitParamExtractString>; - - -void registerFunctionsVisitParam(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index 353dda930ef..41bef3947ec 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -1,14 +1,9 @@ #pragma once -#include -#include #include #include #include #include -#include -#include -#include #include #include #include @@ -43,15 +38,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -struct HasParam -{ - using ResultType = UInt8; - - static UInt8 extract(const UInt8 *, const UInt8 *) - { - return true; - } -}; template struct ExtractNumericType @@ -78,77 +64,6 @@ struct ExtractNumericType } }; -struct ExtractBool -{ - using ResultType = UInt8; - - static UInt8 extract(const UInt8 * begin, const UInt8 * end) - { - return begin + 4 <= end && 0 == strncmp(reinterpret_cast(begin), "true", 4); - } -}; - - -struct ExtractRaw -{ - using ExpectChars = PODArrayWithStackMemory; - - static void extract(const UInt8 * pos, const UInt8 * end, ColumnString::Chars & res_data) - { - ExpectChars expects_end; - UInt8 current_expect_end = 0; - - for (auto extract_begin = pos; pos != end; ++pos) - { - if (current_expect_end && *pos == current_expect_end) - { - expects_end.pop_back(); - current_expect_end = expects_end.empty() ? 0 : expects_end.back(); - } - else - { - switch (*pos) - { - case '[': - current_expect_end = ']'; - expects_end.push_back(current_expect_end); - break; - case '{': - current_expect_end = '}'; - expects_end.push_back(current_expect_end); - break; - case '"' : - current_expect_end = '"'; - expects_end.push_back(current_expect_end); - break; - case '\\': - /// skip backslash - if (pos + 1 < end && pos[1] == '"') - pos++; - break; - default: - if (!current_expect_end && (*pos == ',' || *pos == '}')) - { - res_data.insert(extract_begin, pos); - return; - } - } - } - } - } -}; - -struct ExtractString -{ - static void extract(const UInt8 * pos, const UInt8 * end, ColumnString::Chars & res_data) - { - size_t old_size = res_data.size(); - ReadBufferFromMemory in(pos, end - pos); - if (!tryReadJSONStringInto(res_data, in)) - res_data.resize(old_size); - } -}; - /** Searches for occurrences of a field in the visit parameter and calls ParamExtractor * for each occurrence of the field, passing it a pointer to the part of the string, @@ -285,6 +200,4 @@ struct ExtractParamToStringImpl } }; - - } diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index c05ece54e11..f4dbac7c638 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -232,10 +232,9 @@ bool PreparedFunctionImpl::defaultImplementationForConstantArguments(Block & blo const ColumnWithTypeAndName & column = block.getByPosition(args[arg_num]); if (arguments_to_remain_constants.end() != std::find(arguments_to_remain_constants.begin(), arguments_to_remain_constants.end(), arg_num)) - if (column.column->empty()) - temporary_block.insert({column.column->cloneResized(1), column.type, column.name}); - else - temporary_block.insert(column); + { + temporary_block.insert({column.column->cloneResized(1), column.type, column.name}); + } else { have_converted_columns = true; diff --git a/dbms/src/Functions/array/arrayReduce.cpp b/dbms/src/Functions/array/arrayReduce.cpp index ffab005e949..516449a4872 100644 --- a/dbms/src/Functions/array/arrayReduce.cpp +++ b/dbms/src/Functions/array/arrayReduce.cpp @@ -111,8 +111,6 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum std::unique_ptr arena = agg_func.allocatesMemoryInArena() ? std::make_unique() : nullptr; - size_t rows = input_rows_count; - /// Aggregate functions do not support constant columns. Therefore, we materialize them. std::vector materialized_columns; @@ -124,6 +122,7 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum for (size_t i = 0; i < num_arguments_columns; ++i) { const IColumn * col = block.getByPosition(arguments[i + 1]).column.get(); + const ColumnArray::Offsets * offsets_i = nullptr; if (const ColumnArray * arr = checkAndGetColumn(col)) { @@ -159,7 +158,7 @@ void FunctionArrayReduce::executeImpl(Block & block, const ColumnNumbers & argum + block.getByPosition(result).type->getName(), ErrorCodes::ILLEGAL_COLUMN); ColumnArray::Offset current_offset = 0; - for (size_t i = 0; i < rows; ++i) + for (size_t i = 0; i < input_rows_count; ++i) { agg_func.create(place); ColumnArray::Offset next_offset = (*offsets)[i]; diff --git a/dbms/src/Functions/jumpConsistentHash.cpp b/dbms/src/Functions/jumpConsistentHash.cpp new file mode 100644 index 00000000000..b1a3109c066 --- /dev/null +++ b/dbms/src/Functions/jumpConsistentHash.cpp @@ -0,0 +1,44 @@ +#include "FunctionsConsistentHashing.h" +#include + + +namespace DB +{ + +/// Code from https://arxiv.org/pdf/1406.2294.pdf +static inline int32_t JumpConsistentHash(uint64_t key, int32_t num_buckets) +{ + int64_t b = -1, j = 0; + while (j < num_buckets) + { + b = j; + key = key * 2862933555777941757ULL + 1; + j = static_cast((b + 1) * (double(1LL << 31) / double((key >> 33) + 1))); + } + return static_cast(b); +} + +struct JumpConsistentHashImpl +{ + static constexpr auto name = "jumpConsistentHash"; + + using HashType = UInt64; + using ResultType = Int32; + using BucketsType = ResultType; + static constexpr auto max_buckets = static_cast(std::numeric_limits::max()); + + static inline ResultType apply(UInt64 hash, BucketsType n) + { + return JumpConsistentHash(hash, n); + } +}; + +using FunctionJumpConsistentHash = FunctionConsistentHashImpl; + +void registerFunctionJumpConsistentHash(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index eba9a96e5e0..501f8e7f90a 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -20,7 +20,6 @@ void registerFunctionsExternalDictionaries(FunctionFactory &); void registerFunctionsExternalModels(FunctionFactory &); void registerFunctionsFormatting(FunctionFactory &); void registerFunctionsHashing(FunctionFactory &); -void registerFunctionsConsistentHashing(FunctionFactory &); void registerFunctionsHigherOrder(FunctionFactory &); void registerFunctionsLogical(FunctionFactory &); void registerFunctionsMiscellaneous(FunctionFactory &); @@ -41,6 +40,7 @@ void registerFunctionsNull(FunctionFactory &); void registerFunctionsFindCluster(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); void registerFunctionsIntrospection(FunctionFactory &); +void registerFunctionsConsistentHashing(FunctionFactory & factory); void registerFunctions() { @@ -60,7 +60,6 @@ void registerFunctions() registerFunctionsExternalModels(factory); registerFunctionsFormatting(factory); registerFunctionsHashing(factory); - registerFunctionsConsistentHashing(factory); registerFunctionsHigherOrder(factory); registerFunctionsLogical(factory); registerFunctionsMiscellaneous(factory); @@ -80,6 +79,7 @@ void registerFunctions() registerFunctionsFindCluster(factory); registerFunctionsJSON(factory); registerFunctionsIntrospection(factory); + registerFunctionsConsistentHashing(factory); } } diff --git a/dbms/src/Functions/registerFunctionsConsistentHashing.cpp b/dbms/src/Functions/registerFunctionsConsistentHashing.cpp new file mode 100644 index 00000000000..dc1e90826bf --- /dev/null +++ b/dbms/src/Functions/registerFunctionsConsistentHashing.cpp @@ -0,0 +1,18 @@ +namespace DB +{ + +class FunctionFactory; + +void registerFunctionYandexConsistentHash(FunctionFactory & factory); +void registerFunctionJumpConsistentHash(FunctionFactory & factory); +void registerFunctionSumburConsistentHash(FunctionFactory & factory); + +void registerFunctionsConsistentHashing(FunctionFactory & factory) +{ + registerFunctionYandexConsistentHash(factory); + registerFunctionJumpConsistentHash(factory); + registerFunctionSumburConsistentHash(factory); +} + +} + diff --git a/dbms/src/Functions/registerFunctionsVisitParam.cpp b/dbms/src/Functions/registerFunctionsVisitParam.cpp new file mode 100644 index 00000000000..01084594f08 --- /dev/null +++ b/dbms/src/Functions/registerFunctionsVisitParam.cpp @@ -0,0 +1,25 @@ +namespace DB +{ + +class FunctionFactory; + +void registerFunctionVisitParamHas(FunctionFactory & factory); +void registerFunctionVisitParamExtractUInt(FunctionFactory & factory); +void registerFunctionVisitParamExtractInt(FunctionFactory & factory); +void registerFunctionVisitParamExtractFloat(FunctionFactory & factory); +void registerFunctionVisitParamExtractBool(FunctionFactory & factory); +void registerFunctionVisitParamExtractRaw(FunctionFactory & factory); +void registerFunctionVisitParamExtractString(FunctionFactory & factory); + +void registerFunctionsVisitParam(FunctionFactory & factory) +{ + registerFunctionVisitParamHas(factory); + registerFunctionVisitParamExtractUInt(factory); + registerFunctionVisitParamExtractInt(factory); + registerFunctionVisitParamExtractFloat(factory); + registerFunctionVisitParamExtractBool(factory); + registerFunctionVisitParamExtractRaw(factory); + registerFunctionVisitParamExtractString(factory); +} + +} diff --git a/dbms/src/Functions/sumburConsistentHash.cpp b/dbms/src/Functions/sumburConsistentHash.cpp new file mode 100644 index 00000000000..1fc26502355 --- /dev/null +++ b/dbms/src/Functions/sumburConsistentHash.cpp @@ -0,0 +1,34 @@ +#include "FunctionsConsistentHashing.h" +#include + +#include + + +namespace DB +{ + +struct SumburConsistentHashImpl +{ + static constexpr auto name = "sumburConsistentHash"; + + using HashType = UInt32; + using ResultType = UInt16; + using BucketsType = ResultType; + static constexpr auto max_buckets = static_cast(std::numeric_limits::max()); + + static inline ResultType apply(HashType hash, BucketsType n) + { + return static_cast(sumburConsistentHash(hash, n)); + } +}; + +using FunctionSumburConsistentHash = FunctionConsistentHashImpl; + +void registerFunctionSumburConsistentHash(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + + diff --git a/dbms/src/Functions/visitParamExtractBool.cpp b/dbms/src/Functions/visitParamExtractBool.cpp new file mode 100644 index 00000000000..7f989ccbb9e --- /dev/null +++ b/dbms/src/Functions/visitParamExtractBool.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + + +namespace DB +{ + +struct ExtractBool +{ + using ResultType = UInt8; + + static UInt8 extract(const UInt8 * begin, const UInt8 * end) + { + return begin + 4 <= end && 0 == strncmp(reinterpret_cast(begin), "true", 4); + } +}; + +struct NameVisitParamExtractBool { static constexpr auto name = "visitParamExtractBool"; }; +using FunctionVisitParamExtractBool = FunctionsStringSearch, NameVisitParamExtractBool>; + + +void registerFunctionVisitParamExtractBool(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamExtractFloat.cpp b/dbms/src/Functions/visitParamExtractFloat.cpp new file mode 100644 index 00000000000..b02b0209daf --- /dev/null +++ b/dbms/src/Functions/visitParamExtractFloat.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + + +namespace DB +{ + +struct NameVisitParamExtractFloat { static constexpr auto name = "visitParamExtractFloat"; }; +using FunctionVisitParamExtractFloat = FunctionsStringSearch>, NameVisitParamExtractFloat>; + + +void registerFunctionVisitParamExtractFloat(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamExtractInt.cpp b/dbms/src/Functions/visitParamExtractInt.cpp new file mode 100644 index 00000000000..f3f30f566e6 --- /dev/null +++ b/dbms/src/Functions/visitParamExtractInt.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + + +namespace DB +{ + +struct NameVisitParamExtractInt { static constexpr auto name = "visitParamExtractInt"; }; +using FunctionVisitParamExtractInt = FunctionsStringSearch>, NameVisitParamExtractInt>; + + +void registerFunctionVisitParamExtractInt(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamExtractRaw.cpp b/dbms/src/Functions/visitParamExtractRaw.cpp new file mode 100644 index 00000000000..5eeb36286a5 --- /dev/null +++ b/dbms/src/Functions/visitParamExtractRaw.cpp @@ -0,0 +1,67 @@ +#include +#include +#include + + +namespace DB +{ + +struct ExtractRaw +{ + using ExpectChars = PODArrayWithStackMemory; + + static void extract(const UInt8 * pos, const UInt8 * end, ColumnString::Chars & res_data) + { + ExpectChars expects_end; + UInt8 current_expect_end = 0; + + for (auto extract_begin = pos; pos != end; ++pos) + { + if (current_expect_end && *pos == current_expect_end) + { + expects_end.pop_back(); + current_expect_end = expects_end.empty() ? 0 : expects_end.back(); + } + else + { + switch (*pos) + { + case '[': + current_expect_end = ']'; + expects_end.push_back(current_expect_end); + break; + case '{': + current_expect_end = '}'; + expects_end.push_back(current_expect_end); + break; + case '"' : + current_expect_end = '"'; + expects_end.push_back(current_expect_end); + break; + case '\\': + /// skip backslash + if (pos + 1 < end && pos[1] == '"') + pos++; + break; + default: + if (!current_expect_end && (*pos == ',' || *pos == '}')) + { + res_data.insert(extract_begin, pos); + return; + } + } + } + } + } +}; + +struct NameVisitParamExtractRaw { static constexpr auto name = "visitParamExtractRaw"; }; +using FunctionVisitParamExtractRaw = FunctionsStringSearchToString, NameVisitParamExtractRaw>; + + +void registerFunctionVisitParamExtractRaw(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamExtractString.cpp b/dbms/src/Functions/visitParamExtractString.cpp new file mode 100644 index 00000000000..a6f4b98145d --- /dev/null +++ b/dbms/src/Functions/visitParamExtractString.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + + +namespace DB +{ + +struct ExtractString +{ + static void extract(const UInt8 * pos, const UInt8 * end, ColumnString::Chars & res_data) + { + size_t old_size = res_data.size(); + ReadBufferFromMemory in(pos, end - pos); + if (!tryReadJSONStringInto(res_data, in)) + res_data.resize(old_size); + } +}; + +struct NameVisitParamExtractString { static constexpr auto name = "visitParamExtractString"; }; +using FunctionVisitParamExtractString = FunctionsStringSearchToString, NameVisitParamExtractString>; + + +void registerFunctionVisitParamExtractString(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamExtractUInt.cpp b/dbms/src/Functions/visitParamExtractUInt.cpp new file mode 100644 index 00000000000..5e70eed8253 --- /dev/null +++ b/dbms/src/Functions/visitParamExtractUInt.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + + +namespace DB +{ + +struct NameVisitParamExtractUInt { static constexpr auto name = "visitParamExtractUInt"; }; +using FunctionVisitParamExtractUInt = FunctionsStringSearch>, NameVisitParamExtractUInt>; + + +void registerFunctionVisitParamExtractUInt(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/visitParamHas.cpp b/dbms/src/Functions/visitParamHas.cpp new file mode 100644 index 00000000000..5fbedfb4995 --- /dev/null +++ b/dbms/src/Functions/visitParamHas.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + + +namespace DB +{ + +struct HasParam +{ + using ResultType = UInt8; + + static UInt8 extract(const UInt8 *, const UInt8 *) + { + return true; + } +}; + +struct NameVisitParamHas { static constexpr auto name = "visitParamHas"; }; +using FunctionVisitParamHas = FunctionsStringSearch, NameVisitParamHas>; + + +void registerFunctionVisitParamHas(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/yandexConsistentHash.cpp b/dbms/src/Functions/yandexConsistentHash.cpp new file mode 100644 index 00000000000..58617e29af7 --- /dev/null +++ b/dbms/src/Functions/yandexConsistentHash.cpp @@ -0,0 +1,34 @@ +#include "FunctionsConsistentHashing.h" +#include + +#include + +namespace DB +{ + +/// An O(1) time and space consistent hash algorithm by Konstantin Oblakov +struct YandexConsistentHashImpl +{ + static constexpr auto name = "yandexConsistentHash"; + + using HashType = UInt64; + /// Actually it supports UInt64, but it is efficient only if n <= 32768 + using ResultType = UInt16; + using BucketsType = ResultType; + static constexpr auto max_buckets = 32768; + + static inline ResultType apply(UInt64 hash, BucketsType n) + { + return ConsistentHashing(hash, n); + } +}; + +using FunctionYandexConsistentHash = FunctionConsistentHashImpl; + +void registerFunctionYandexConsistentHash(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 59c88ed5ed8..992593d852c 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1645,11 +1645,10 @@ Compiler & Context::getCompiler() } -void Context::initializeSystemLogs(std::shared_ptr text_log) +void Context::initializeSystemLogs() { auto lock = getLock(); shared->system_logs.emplace(*global_context, getConfigRef()); - shared->system_logs->text_log = text_log; } bool Context::hasTraceCollector() @@ -1716,11 +1715,10 @@ std::shared_ptr Context::getTextLog() { auto lock = getLock(); - if (!shared->system_logs) - if (auto log = shared->system_logs->text_log.lock()) - return log; + if (!shared->system_logs || !shared->system_logs->text_log) + return {}; - return {}; + return shared->system_logs->text_log; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index f6998c77824..50b7ab3eba2 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -424,7 +424,7 @@ public: Compiler & getCompiler(); /// Call after initialization before using system logs. Call for global context. - void initializeSystemLogs(std::shared_ptr text_log); + void initializeSystemLogs(); void initializeTraceCollector(); bool hasTraceCollector(); diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index 4b456bc2542..f1f65dfe883 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -47,6 +47,7 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi query_thread_log = createSystemLog(global_context, "system", "query_thread_log", config, "query_thread_log"); part_log = createSystemLog(global_context, "system", "part_log", config, "part_log"); trace_log = createSystemLog(global_context, "system", "trace_log", config, "trace_log"); + text_log = createSystemLog(global_context, "system", "text_log", config, "text_log"); part_log_database = config.getString("part_log.database", "system"); } @@ -67,6 +68,8 @@ void SystemLogs::shutdown() part_log->shutdown(); if (trace_log) trace_log->shutdown(); + if (text_log) + text_log->shutdown(); } } diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 7b1192ac970..3dd329d577b 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -75,8 +75,7 @@ struct SystemLogs std::shared_ptr query_thread_log; /// Used to log query threads. std::shared_ptr part_log; /// Used to log operations with parts std::shared_ptr trace_log; /// Used to log traces from query profiler - std::weak_ptr text_log; /// Used to log all text. We use weak_ptr, because this log is - /// a server's field. + std::shared_ptr text_log; /// Used to log all text messages. String part_log_database; }; diff --git a/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.reference b/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.reference new file mode 100644 index 00000000000..135fc035446 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.reference @@ -0,0 +1,43 @@ +0 ([100,200],[30,30]) +0 ([100,200],[30,30]) +1 ([100,200],[30,30]) +2 ([100,200],[30,30]) +3 ([100,200],[30,30]) +4 ([100,200],[30,30]) +5 ([100,200],[30,30]) +6 ([100,200],[30,30]) +7 ([100,200],[30,30]) +8 ([100,200],[30,30]) +9 ([100,200],[30,30]) +0 ([100,200],[30,30]) +1 ([100,200],[30,30]) +2 ([100,200],[30,30]) +3 ([100,200],[30,30]) +4 ([100,200],[30,30]) +5 ([100,200],[30,30]) +6 ([100,200],[30,30]) +7 ([100,200],[30,30]) +8 ([100,200],[30,30]) +9 ([100,200],[30,30]) +0 400 +1 400 +2 400 +3 400 +4 400 +5 400 +6 400 +7 400 +8 400 +9 400 +0 [100,100,200] +1 [100,100,200] +2 [100,100,200] +3 [100,100,200] +4 [100,100,200] +5 [100,100,200] +6 [100,100,200] +7 [100,100,200] +8 [100,100,200] +9 [100,100,200] +Array(Array(UInt8)), Const(size = 2, Array(size = 1, UInt64(size = 1), Array(size = 1, UInt64(size = 1), UInt8(size = 2)))) ([1,2],[1,2]) +Array(Array(UInt8)), Const(size = 2, Array(size = 1, UInt64(size = 1), Array(size = 1, UInt64(size = 1), UInt8(size = 2)))) ([1,2],[1,2]) diff --git a/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.sql b/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.sql new file mode 100644 index 00000000000..091c56c6868 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00978_sum_map_bugfix.sql @@ -0,0 +1,7 @@ +select number, arrayReduce( 'sumMap', [a],[b] ) from (select [100,100,200] a,[10,20,30] b, number from numbers(1)); +select number, arrayReduce( 'sumMap', [a],[b] ) from (select materialize([100,100,200]) a,materialize([10,20,30]) b, number from numbers(10)); +select number, arrayReduce( 'sumMap', [a],[b] ) from (select [100,100,200] a,[10,20,30] b, number from numbers(10)); +select number, arrayReduce( 'sum', a) from (select materialize([100,100,200]) a, number from numbers(10)); +select number, arrayReduce( 'max', [a] ) from (select materialize([100,100,200]) a, number from numbers(10)); + +select dumpColumnStructure([a]), arrayReduce('sumMap', [a], [a]) from (select [1, 2] a FROM numbers(2)); diff --git a/docs/en/operations/table_engines/replacingmergetree.md b/docs/en/operations/table_engines/replacingmergetree.md index 53b9a47c5c2..d630e3b2e70 100644 --- a/docs/en/operations/table_engines/replacingmergetree.md +++ b/docs/en/operations/table_engines/replacingmergetree.md @@ -17,6 +17,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = ReplacingMergeTree([ver]) [PARTITION BY expr] [ORDER BY expr] +[PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name=value, ...] ``` diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index e5f9ea2b996..d7466273320 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -20,23 +20,25 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_SIMPLE(logger, message, priority, PRIORITY) do \ -{ \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ - (CurrentThread::getGroup()->client_logs_level >= (priority)); \ - if ((logger)->is((PRIORITY)) || is_clients_log) { \ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (auto channel = (logger)->getChannel()) { \ - std::string file_function; \ - file_function += __FILE__; \ - file_function += ", "; \ - file_function += __PRETTY_FUNCTION__; \ - Message poco_message((logger)->name(), oss_internal_rare.str(), \ - (PRIORITY), file_function.c_str(), __LINE__); \ - channel->log(poco_message); \ - } \ - } \ +#define LOG_SIMPLE(logger, message, priority, PRIORITY) do \ +{ \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ + (CurrentThread::getGroup()->client_logs_level >= (priority)); \ + if ((logger)->is((PRIORITY)) || is_clients_log) \ + { \ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + if (auto channel = (logger)->getChannel()) \ + { \ + std::string file_function; \ + file_function += __FILE__; \ + file_function += "; "; \ + file_function += __PRETTY_FUNCTION__; \ + Message poco_message((logger)->name(), oss_internal_rare.str(), \ + (PRIORITY), file_function.c_str(), __LINE__); \ + channel->log(poco_message); \ + } \ + } \ } while (false)