From 499262dba9fd8b41ee77271f31039cb156c9bc08 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 7 Jul 2022 22:16:01 +0000 Subject: [PATCH 001/439] allow to randomize merge tree settings --- programs/client/Client.cpp | 2 + src/Client/ClientBase.cpp | 65 ++++++++++++++--- src/Client/ClientBase.h | 5 ++ src/Core/BaseSettings.h | 52 +++++++++++++- src/Core/Settings.cpp | 33 --------- src/Core/Settings.h | 19 ----- src/Parsers/ASTCreateQuery.cpp | 5 ++ src/Parsers/ASTCreateQuery.h | 2 + .../MergeTree/registerStorageMergeTree.cpp | 6 +- tests/clickhouse-test | 69 ++++++++++++++++--- 10 files changed, 184 insertions(+), 74 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 4e3aa701d95..38af518a32d 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1183,6 +1183,8 @@ void Client::readArguments( } else if (arg == "--allow_repeated_settings") allow_repeated_settings = true; + else if (arg == "--allow_merge_tree_settings") + allow_merge_tree_settings = true; else common_arguments.emplace_back(arg); } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 8230c97f49c..4b2e3a2967f 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -52,6 +52,7 @@ #include #include #include +#include #include #include @@ -699,16 +700,21 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr parsed_query) { - if (fake_drop) + if (fake_drop && parsed_query->as()) + return; + + auto query = query_to_execute; + + if (allow_merge_tree_settings && parsed_query->as()) { - if (parsed_query->as()) - return; + /// Rewrite query if new settings were added. + if (addMergeTreeSettings(*parsed_query->as())) + query = serializeAST(*parsed_query); } /// Rewrite query only when we have query parameters. /// Note that if query is rewritten, comments in query are lost. /// But the user often wants to see comments in server logs, query log, processlist, etc. - auto query = query_to_execute; if (!query_parameters.empty()) { /// Replace ASTQueryParameter with ASTLiteral for prepared statements. @@ -1892,6 +1898,41 @@ void ClientBase::initQueryIdFormats() } +bool ClientBase::addMergeTreeSettings(ASTCreateQuery & ast_create) +{ + if (ast_create.attach + || !ast_create.storage + || !ast_create.storage->isExtendedStorageDefinition() + || !ast_create.storage->engine + || ast_create.storage->engine->name.find("MergeTree") == std::string::npos) + return false; + + auto all_changed = cmd_merge_tree_settings.allChanged(); + if (all_changed.begin() == all_changed.end()) + return false; + + if (!ast_create.storage->settings) + { + auto settings_ast = std::make_shared(); + settings_ast->is_standalone = false; + ast_create.storage->set(ast_create.storage->settings, settings_ast); + } + + auto & storage_settings = *ast_create.storage->settings; + bool added_new_setting = false; + + for (const auto & setting : all_changed) + { + if (!storage_settings.changes.tryGet(setting.getName())) + { + storage_settings.changes.emplace_back(setting.getName(), setting.getValue()); + added_new_setting = true; + } + } + + return added_new_setting; +} + void ClientBase::runInteractive() { if (config().has("query_id")) @@ -2100,10 +2141,18 @@ void ClientBase::showClientVersion() void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { - if (allow_repeated_settings) - cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); - else - cmd_settings.addProgramOptions(options_description.main_description.value()); + auto add_program_options = [&](auto & to_settings) + { + if (allow_repeated_settings) + to_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); + else + to_settings.addProgramOptions(options_description.main_description.value()); + }; + + add_program_options(cmd_settings); + if (allow_merge_tree_settings) + add_program_options(cmd_merge_tree_settings); + /// Parse main commandline options. auto parser = po::command_line_parser(arguments).options(options_description.main_description.value()).allow_unregistered(); po::parsed_options parsed = parser.run(); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index ec2267a3be6..9f72525e3cd 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -1,6 +1,7 @@ #pragma once #include "Common/NamePrompter.h" +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include namespace po = boost::program_options; @@ -151,6 +153,7 @@ private: void updateSuggest(const ASTPtr & ast); void initQueryIdFormats(); + bool addMergeTreeSettings(ASTCreateQuery & ast_create); protected: static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); @@ -193,6 +196,7 @@ protected: /// Settings specified via command line args Settings cmd_settings; + MergeTreeSettings cmd_merge_tree_settings; SharedContextHolder shared_context; ContextMutablePtr global_context; @@ -270,6 +274,7 @@ protected: std::vector hosts_and_ports{}; bool allow_repeated_settings = false; + bool allow_merge_tree_settings = false; bool cancelled = false; diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index a4ddc6571ed..e2cf9579cbc 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -5,10 +5,17 @@ #include #include #include +#include +namespace boost::program_options +{ + class options_description; +} + namespace DB { + class ReadBuffer; class WriteBuffer; @@ -19,7 +26,6 @@ enum class SettingsWriteFormat DEFAULT = STRINGS_WITH_FLAGS, }; - /** Template class to define collections of settings. * Example of usage: * @@ -110,6 +116,18 @@ public: std::conditional_t custom_setting; }; + /// Adds program options to set the settings from a command line. + /// (Don't forget to call notify() on the `variables_map` after parsing it!) + void addProgramOptions(boost::program_options::options_description & options); + + /// Adds program options as to set the settings from a command line. + /// Allows to set one setting multiple times, the last value will be used. + /// (Don't forget to call notify() on the `variables_map` after parsing it!) + void addProgramOptionsAsMultitokens(boost::program_options::options_description & options); + + void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field); + void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field); + enum SkipFlags { SKIP_NONE = 0, @@ -501,6 +519,38 @@ String BaseSettings::toString() const return res; } +template +void BaseSettings::addProgramOptions(boost::program_options::options_description & options) +{ + for (const auto & field : all()) + addProgramOption(options, field); +} + +template +void BaseSettings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options) +{ + for (const auto & field : all()) + addProgramOptionAsMultitoken(options, field); +} + +template +void BaseSettings::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field) +{ + const std::string_view name = field.getName(); + auto on_program_option = boost::function1([this, name](const std::string & value) { set(name, value); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); +} + +template +void BaseSettings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field) +{ + const std::string_view name = field.getName(); + auto on_program_option = boost::function1([this, name](const Strings & values) { set(name, values.back()); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); +} + template bool operator==(const BaseSettings & left, const BaseSettings & right) { diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 5251569505e..b44e981852f 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB { @@ -81,38 +80,6 @@ void Settings::dumpToMapColumn(IColumn * column, bool changed_only) offsets.push_back(offsets.back() + size); } -void Settings::addProgramOptions(boost::program_options::options_description & options) -{ - for (const auto & field : all()) - { - addProgramOption(options, field); - } -} - -void Settings::addProgramOptionsAsMultitokens(boost::program_options::options_description & options) -{ - for (const auto & field : all()) - { - addProgramOptionAsMultitoken(options, field); - } -} - -void Settings::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field) -{ - const std::string_view name = field.getName(); - auto on_program_option = boost::function1([this, name](const std::string & value) { set(name, value); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); -} - -void Settings::addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field) -{ - const std::string_view name = field.getName(); - auto on_program_option = boost::function1([this, name](const Strings & values) { set(name, values.back()); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), boost::program_options::value()->multitoken()->composing()->notifier(on_program_option), field.getDescription()))); -} - void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) { if (config.getBool("skip_check_for_incorrect_settings", false)) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5597d9076a4..61e06bd241a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -12,12 +12,6 @@ namespace Poco::Util class AbstractConfiguration; } -namespace boost::program_options -{ - class options_description; -} - - namespace DB { class IColumn; @@ -803,24 +797,11 @@ struct Settings : public BaseSettings, public IHints<2, Settings /// Dumps profile events to column of type Map(String, String) void dumpToMapColumn(IColumn * column, bool changed_only = true); - /// Adds program options to set the settings from a command line. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptions(boost::program_options::options_description & options); - - /// Adds program options as to set the settings from a command line. - /// Allows to set one setting multiple times, the last value will be used. - /// (Don't forget to call notify() on the `variables_map` after parsing it!) - void addProgramOptionsAsMultitokens(boost::program_options::options_description & options); - /// Check that there is no user-level settings at the top level in config. /// This is a common source of mistake (user don't know where to write user-level setting). static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); std::vector getAllRegisteredNames() const override; - - void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field); - - void addProgramOptionAsMultitoken(boost::program_options::options_description & options, const SettingFieldRef & field); }; /* diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index e676be63921..1268d35798c 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -74,6 +74,11 @@ void ASTStorage::formatImpl(const FormatSettings & s, FormatState & state, Forma } } +bool ASTStorage::isExtendedStorageDefinition() const +{ + return partition_by || primary_key || order_by || sample_by || settings; +} + class ASTColumnsElement : public IAST { diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index f3729b1523f..ae87cb24f49 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -31,6 +31,8 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + bool isExtendedStorageDefinition() const; }; diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e52a0fed674..7341ba1d491 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -154,9 +154,9 @@ static StoragePtr create(const StorageFactory::Arguments & args) * - Additional MergeTreeSettings in the SETTINGS clause; */ - bool is_extended_storage_def = args.storage_def->partition_by || args.storage_def->primary_key || args.storage_def->order_by - || args.storage_def->sample_by || (args.query.columns_list->indices && !args.query.columns_list->indices->children.empty()) - || (args.query.columns_list->projections && !args.query.columns_list->projections->children.empty()) || args.storage_def->settings; + bool is_extended_storage_def = args.storage_def->isExtendedStorageDefinition() + || (args.query.columns_list->indices && !args.query.columns_list->indices->children.empty()) + || (args.query.columns_list->projections && !args.query.columns_list->projections->children.empty()); String name_part = args.engine_name.substr(0, args.engine_name.size() - strlen("MergeTree")); diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cab6daf3a50..7d672098f71 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -436,6 +436,18 @@ class SettingsRandomizer: return random_settings +class MergeTreeSettingsRandomizer: + settings = {"index_granularity": lambda: random.randint(1, 65536)} + + @staticmethod + def get_random_settings(args): + random_settings = [] + for setting, generator in MergeTreeSettingsRandomizer.settings.items(): + if setting not in args.changed_merge_tree_settings: + random_settings.append(f"{setting}={generator()}") + return random_settings + + class TestResult: def __init__( self, @@ -542,8 +554,8 @@ class TestCase: return testcase_args - def cli_random_settings(self) -> str: - return " ".join([f"--{setting}" for setting in self.random_settings]) + def cli_format_settings(self, settings_list) -> str: + return " ".join([f"--{setting}" for setting in settings_list]) def add_random_settings(self, args, client_options): if self.tags and "no-random-settings" in self.tags: @@ -558,7 +570,10 @@ class TestCase: self.base_url_params + "&" + "&".join(self.random_settings) ) - new_options = f" --allow_repeated_settings {self.cli_random_settings()}" + new_options = f" --allow_repeated_settings {self.cli_format_settings(self.random_settings)}" + if args.randomize_merge_tree_settings: + new_options += f" --allow_merge_tree_settings {self.cli_format_settings(self.merge_tree_random_settings)}" + os.environ["CLICKHOUSE_CLIENT_OPT"] = ( self.base_client_options + new_options + " " ) @@ -574,9 +589,11 @@ class TestCase: if args.no_random_settings: return description - return ( - f"{description}\nSettings used in the test: {self.cli_random_settings()}\n" - ) + description += f"\nSettings used in the test: {self.cli_format_settings(self.random_settings)}" + if args.randomize_merge_tree_settings: + description += f"\nMergeTreeSettings used in test: {self.cli_format_settings(self.merge_tree_random_settings)}" + + return description + "\n" def __init__(self, suite, case: str, args, is_concurrent: bool): self.case: str = case # case file name @@ -601,11 +618,16 @@ class TestCase: self.runs_count = 0 self.random_settings = SettingsRandomizer.get_random_settings() + self.merge_tree_random_settings = ( + MergeTreeSettingsRandomizer.get_random_settings(args) + ) + self.base_url_params = ( os.environ["CLICKHOUSE_URL_PARAMS"] if "CLICKHOUSE_URL_PARAMS" in os.environ else "" ) + self.base_client_options = ( os.environ["CLICKHOUSE_CLIENT_OPT"] if "CLICKHOUSE_CLIENT_OPT" in os.environ @@ -710,7 +732,9 @@ class TestCase: return None - def process_result_impl(self, proc, stdout: str, stderr: str, debug_log: str, total_time: float): + def process_result_impl( + self, proc, stdout: str, stderr: str, debug_log: str, total_time: float + ): description = "" if proc: @@ -1038,7 +1062,9 @@ class TestCase: server_logs_level, client_options ) - result = self.process_result_impl(proc, stdout, stderr, debug_log, total_time) + result = self.process_result_impl( + proc, stdout, stderr, debug_log, total_time + ) result.check_if_need_retry(args, stdout, stderr, self.runs_count) if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings( @@ -1530,7 +1556,8 @@ def collect_build_flags(args): result.append(BuildFlags.RELEASE) value = clickhouse_execute( - args, "SELECT value FROM system.settings WHERE name = 'allow_deprecated_database_ordinary'" + args, + "SELECT value FROM system.settings WHERE name = 'allow_deprecated_database_ordinary'", ) if value == b"1": result.append(BuildFlags.ORDINARY_DATABASE) @@ -1562,6 +1589,19 @@ def collect_build_flags(args): return result +def collect_changed_merge_tree_settings(args): + changed_settings = ( + clickhouse_execute( + args, + "SELECT name FROM system.merge_tree_settings WHERE changed", + ) + .strip() + .splitlines() + ) + + return list(map(lambda s: s.decode(), changed_settings)) + + def check_table_column(args, database, table, column): return ( int( @@ -1634,7 +1674,9 @@ def do_run_tests(jobs, test_suite: TestSuite, parallel): queue.close() except Full: - print("Couldn't put test to the queue within timeout. Server probably hung.") + print( + "Couldn't put test to the queue within timeout. Server probably hung." + ) print_stacktraces() queue.close() @@ -1691,6 +1733,7 @@ def main(args): raise Exception(msg) args.build_flags = collect_build_flags(args) + args.changed_merge_tree_settings = collect_changed_merge_tree_settings(args) args.suppport_system_processes_is_all_data_sent = check_table_column( args, "system", "processes", "is_all_data_sent" ) @@ -2034,6 +2077,12 @@ if __name__ == "__main__": type=int, help="Total test groups for crc32(test_name) % run_by_hash_total == run_by_hash_num", ) + parser.add_argument( + "--randomize-merge-tree-settings", + action="store_true", + default=False, + help="Enable MergeTree settings randomization", + ) group = parser.add_mutually_exclusive_group(required=False) group.add_argument( From 53bb42d9ba646c86c0cc4e072e2c3e40695ad1bb Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 8 Jul 2022 14:47:55 +0000 Subject: [PATCH 002/439] add some merge tree settings for randomization --- src/Client/ClientBase.cpp | 29 ++++++++++++++++++++--------- tests/clickhouse-test | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 4b2e3a2967f..4c588a00ff3 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2141,17 +2141,28 @@ void ClientBase::showClientVersion() void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { - auto add_program_options = [&](auto & to_settings) - { - if (allow_repeated_settings) - to_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); - else - to_settings.addProgramOptions(options_description.main_description.value()); - }; + if (allow_repeated_settings) + cmd_settings.addProgramOptionsAsMultitokens(options_description.main_description.value()); + else + cmd_settings.addProgramOptions(options_description.main_description.value()); - add_program_options(cmd_settings); if (allow_merge_tree_settings) - add_program_options(cmd_merge_tree_settings); + { + /// Add merge tree settings manually, because names of some settings + /// may clash. Query settings have higher priority and we just + /// skip ambigous merge tree settings. + auto & main_options = options_description.main_description.value(); + for (const auto & setting : cmd_merge_tree_settings.all()) + { + if (main_options.find_nothrow(setting.getName(), false)) + continue; + + if (allow_repeated_settings) + cmd_merge_tree_settings.addProgramOptionAsMultitoken(main_options, setting); + else + cmd_merge_tree_settings.addProgramOption(main_options, setting); + } + } /// Parse main commandline options. auto parser = po::command_line_parser(arguments).options(options_description.main_description.value()).allow_unregistered(); diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 7d672098f71..35eb799f886 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -426,6 +426,8 @@ class SettingsRandomizer: "read_in_order_two_level_merge_threshold": lambda: random.randint(0, 100), "optimize_aggregation_in_order": lambda: random.randint(0, 1), "aggregation_in_order_max_block_bytes": lambda: random.randint(0, 50000000), + "min_compress_block_size": lambda: random.randint(1, 1048576 * 3), + "max_compress_block_size": lambda: random.randint(1, 1048576 * 3), } @staticmethod @@ -437,7 +439,36 @@ class SettingsRandomizer: class MergeTreeSettingsRandomizer: - settings = {"index_granularity": lambda: random.randint(1, 65536)} + settings = { + "index_granularity": lambda: random.randint(1, 65536), + "min_bytes_for_wide_part": lambda: 0 + if random.random() < 0.3 + else random.randint(0, 1024 * 1024 * 1024), + "min_rows_for_wide_part": lambda: 0 + if random.random() < 0.3 + else random.randint(0, 10000000), + "ratio_of_defaults_for_sparse_serialization": lambda: 0.0 + if random.random() < 0.1 + else 1.0 + if random.random() < 4 / 9 + else random.random(), + "merge_max_block_size": lambda: random.randint(1, 8192 * 3), + "prefer_fetch_merged_part_size_threshold": lambda: random.randint( + 1, 10 * 1024 * 1024 * 1024 + ), + "always_fetch_merged_part": lambda: random.randint(0, 1), + "vertical_merge_algorithm_min_rows_to_activate": lambda: random.randint( + 1, 32 * 8192 + ), + "vertical_merge_algorithm_min_columns_to_activate": lambda: random.randint( + 1, 20 + ), + "min_merge_bytes_to_use_direct_io": lambda: 1 + if random.random() < 0.2 + else random.randint(1, 10 * 1024 * 1024 * 1024), + "index_granularity_bytes": lambda: random.randint(1024, 30 * 1024 * 1024), + "write_final_mark": lambda: random.randint(0, 1), + } @staticmethod def get_random_settings(args): @@ -591,7 +622,7 @@ class TestCase: description += f"\nSettings used in the test: {self.cli_format_settings(self.random_settings)}" if args.randomize_merge_tree_settings: - description += f"\nMergeTreeSettings used in test: {self.cli_format_settings(self.merge_tree_random_settings)}" + description += f"\n\nMergeTreeSettings used in test: {self.cli_format_settings(self.merge_tree_random_settings)}" return description + "\n" From 14a3c7b8cf1e6247b02483dc4aa8737d4e8883ce Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 13 Jul 2022 20:57:24 +0000 Subject: [PATCH 003/439] print changed MergeTree settings on fail --- programs/client/Client.cpp | 31 ++++++++++++++++++------------- src/Client/ClientBase.cpp | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 38af518a32d..fb2a1b0043b 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -458,24 +459,28 @@ void Client::connect() // Prints changed settings to stderr. Useful for debugging fuzzing failures. void Client::printChangedSettings() const { - const auto & changes = global_context->getSettingsRef().changes(); - if (!changes.empty()) + auto print_changes = [](const auto & changes, std::string_view settings_name) { - fmt::print(stderr, "Changed settings: "); - for (size_t i = 0; i < changes.size(); ++i) + if (!changes.empty()) { - if (i) + fmt::print(stderr, "Changed {}: ", settings_name); + for (size_t i = 0; i < changes.size(); ++i) { - fmt::print(stderr, ", "); + if (i) + fmt::print(stderr, ", "); + fmt::print(stderr, "{} = '{}'", changes[i].name, toString(changes[i].value)); } - fmt::print(stderr, "{} = '{}'", changes[i].name, toString(changes[i].value)); + + fmt::print(stderr, "\n"); } - fmt::print(stderr, "\n"); - } - else - { - fmt::print(stderr, "No changed settings.\n"); - } + else + { + fmt::print(stderr, "No changed {}.\n", settings_name); + } + }; + + print_changes(global_context->getSettingsRef().changes(), "settings"); + print_changes(cmd_merge_tree_settings.changes(), "MergeTree settings"); } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 4c588a00ff3..d397b08acbe 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2150,7 +2150,7 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, { /// Add merge tree settings manually, because names of some settings /// may clash. Query settings have higher priority and we just - /// skip ambigous merge tree settings. + /// skip ambiguous merge tree settings. auto & main_options = options_description.main_description.value(); for (const auto & setting : cmd_merge_tree_settings.all()) { From 97425bb1672d1438b0879ada451b7274efb60e6e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 13 Jul 2022 21:07:12 +0000 Subject: [PATCH 004/439] turn on merge tree settings randomization --- docker/test/stateless/run.sh | 2 ++ docker/test/stress/stress | 3 +++ tests/clickhouse-test | 1 + 3 files changed, 6 insertions(+) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 075f588cae3..4cbda33ed2f 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -119,6 +119,8 @@ function run_tests() ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') fi + ADDITIONAL_OPTIONS+=("--randomize-merge-tree-settings") + set +e clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \ --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ diff --git a/docker/test/stress/stress b/docker/test/stress/stress index e195f81b551..f43a1a4a992 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -11,6 +11,9 @@ import time def get_options(i, backward_compatibility_check): options = [] client_options = [] + + options.append("--randomize-merge-tree-settings") + if 0 < i: options.append("--order=random") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 35eb799f886..948711d8d96 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -585,6 +585,7 @@ class TestCase: return testcase_args + @staticmethod def cli_format_settings(self, settings_list) -> str: return " ".join([f"--{setting}" for setting in settings_list]) From 90779941cd4f10c81323ea4d02fd9f9f5d39a2f4 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 14 Jul 2022 11:42:12 +0000 Subject: [PATCH 005/439] fix clickhouse-test --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 948711d8d96..3c36ebc04b4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -586,7 +586,7 @@ class TestCase: return testcase_args @staticmethod - def cli_format_settings(self, settings_list) -> str: + def cli_format_settings(settings_list) -> str: return " ".join([f"--{setting}" for setting in settings_list]) def add_random_settings(self, args, client_options): From 6a9f195390bb639a909241913011fda36895c334 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 28 Oct 2022 15:10:48 +0000 Subject: [PATCH 006/439] do not randomize mt settings with 'show create table' --- docker/test/stateless/run.sh | 2 - docker/test/stress/stress | 3 - tests/clickhouse-test | 155 ++++++++++++++++++++--------------- 3 files changed, 90 insertions(+), 70 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 1db4e5cf428..8151ec9d5e4 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -126,8 +126,6 @@ function run_tests() ADDITIONAL_OPTIONS+=('--db-engine=Ordinary') fi - ADDITIONAL_OPTIONS+=("--randomize-merge-tree-settings") - if [[ "${HIGH_LEVEL_COVERAGE}" = "YES" ]]; then ADDITIONAL_OPTIONS+=('--report-coverage') fi diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 8b7d8d8bf73..a0ec86f7fbe 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -11,9 +11,6 @@ import time def get_options(i, backward_compatibility_check): options = [] client_options = [] - - options.append("--randomize-merge-tree-settings") - if 0 < i: options.append("--order=random") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 5d71488cba0..b6b0b6e02db 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -617,41 +617,42 @@ class TestCase: def cli_format_settings(settings_list) -> str: return " ".join([f"--{setting}" for setting in settings_list]) - def add_random_settings(self, args, client_options): - if self.tags and "no-random-settings" in self.tags: - return client_options - if args.no_random_settings: - return client_options + def has_show_create_table_in_test(self): + return not subprocess.call(["grep", "-iq", "show create table", self.case_file]) - if len(self.base_url_params) == 0: - os.environ["CLICKHOUSE_URL_PARAMS"] = "&".join(self.random_settings) - else: - os.environ["CLICKHOUSE_URL_PARAMS"] = ( - self.base_url_params + "&" + "&".join(self.random_settings) - ) + def add_random_settings(self, client_options): + new_options = "" + if self.randomize_settings: + if len(self.base_url_params) == 0: + os.environ["CLICKHOUSE_URL_PARAMS"] = "&".join(self.random_settings) + else: + os.environ["CLICKHOUSE_URL_PARAMS"] = ( + self.base_url_params + "&" + "&".join(self.random_settings) + ) - new_options = f" --allow_repeated_settings {self.cli_format_settings(self.random_settings)}" - if args.randomize_merge_tree_settings: + new_options += f" {self.cli_format_settings(self.random_settings)}" + + if self.randomize_merge_tree_settings: new_options += f" --allow_merge_tree_settings {self.cli_format_settings(self.merge_tree_random_settings)}" - os.environ["CLICKHOUSE_CLIENT_OPT"] = ( - self.base_client_options + new_options + " " - ) + if new_options != "": + new_options += " --allow_repeated_settings" + + os.environ["CLICKHOUSE_CLIENT_OPT"] = ( + self.base_client_options + new_options + " " + ) + return client_options + new_options def remove_random_settings_from_env(self): os.environ["CLICKHOUSE_URL_PARAMS"] = self.base_url_params os.environ["CLICKHOUSE_CLIENT_OPT"] = self.base_client_options - def add_info_about_settings(self, args, description): - if self.tags and "no-random-settings" in self.tags: - return description - if args.no_random_settings: - return description - - description += f"\nSettings used in the test: {self.cli_format_settings(self.random_settings)}" - if args.randomize_merge_tree_settings: - description += f"\n\nMergeTreeSettings used in test: {self.cli_format_settings(self.merge_tree_random_settings)}" + def add_info_about_settings(self, description): + if self.randomize_settings: + description += f"\nSettings used in the test: {self.cli_format_settings(self.random_settings)}" + if self.randomize_merge_tree_settings: + description += f"\n\nMergeTree settings used in test: {self.cli_format_settings(self.merge_tree_random_settings)}" return description + "\n" @@ -677,11 +678,29 @@ class TestCase: self.testcase_args = None self.runs_count = 0 - self.random_settings = SettingsRandomizer.get_random_settings() - self.merge_tree_random_settings = ( - MergeTreeSettingsRandomizer.get_random_settings(args) + has_no_random_settings_tag = self.tags and "no-random-settings" in self.tags + + self.randomize_settings = not ( + args.no_random_settings or has_no_random_settings_tag ) + # If test contains SHOW CREATE TABLE do not + # randomize merge tree settings, because + # they are added to table definition and test will fail + self.randomize_merge_tree_settings = not ( + args.no_random_merge_tree_settings + or has_no_random_settings_tag + or self.has_show_create_table_in_test() + ) + + if self.randomize_settings: + self.random_settings = SettingsRandomizer.get_random_settings() + + if self.randomize_merge_tree_settings: + self.merge_tree_random_settings = ( + MergeTreeSettingsRandomizer.get_random_settings(args) + ) + self.base_url_params = ( os.environ["CLICKHOUSE_URL_PARAMS"] if "CLICKHOUSE_URL_PARAMS" in os.environ @@ -1057,7 +1076,9 @@ class TestCase: try: drop_database_query = "DROP DATABASE " + database if args.replicated_database: - drop_database_query += " ON CLUSTER test_cluster_database_replicated" + drop_database_query += ( + " ON CLUSTER test_cluster_database_replicated" + ) clickhouse_execute( args, drop_database_query, @@ -1126,7 +1147,7 @@ class TestCase: self.testcase_args = self.configure_testcase_args( args, self.case_file, suite.suite_tmp_path ) - client_options = self.add_random_settings(args, client_options) + client_options = self.add_random_settings(client_options) proc, stdout, stderr, debug_log, total_time = self.run_single_test( server_logs_level, client_options ) @@ -1136,9 +1157,7 @@ class TestCase: ) result.check_if_need_retry(args, stdout, stderr, self.runs_count) if result.status == TestStatus.FAIL: - result.description = self.add_info_about_settings( - args, result.description - ) + result.description = self.add_info_about_settings(result.description) return result except KeyboardInterrupt as e: raise e @@ -1149,7 +1168,7 @@ class TestCase: FailureReason.INTERNAL_QUERY_FAIL, 0.0, self.add_info_about_settings( - args, self.get_description_from_exception_info(sys.exc_info()) + self.get_description_from_exception_info(sys.exc_info()) ), ) except (ConnectionRefusedError, ConnectionResetError): @@ -1159,7 +1178,7 @@ class TestCase: FailureReason.SERVER_DIED, 0.0, self.add_info_about_settings( - args, self.get_description_from_exception_info(sys.exc_info()) + self.get_description_from_exception_info(sys.exc_info()) ), ) except Exception: @@ -1253,7 +1272,7 @@ class TestSuite: line = line.strip() if line and not is_shebang(line): return line - return '' + return "" def load_tags_from_file(filepath): comment_sign = get_comment_sign(filepath) @@ -1790,7 +1809,7 @@ def removesuffix(text, *suffixes): return text -def reportCoverageFor(args, what, query, permissive = False): +def reportCoverageFor(args, what, query, permissive=False): value = clickhouse_execute(args, query).decode() if value != "": @@ -1803,10 +1822,11 @@ def reportCoverageFor(args, what, query, permissive = False): def reportCoverage(args): - return reportCoverageFor( - args, - "functions", - """ + return ( + reportCoverageFor( + args, + "functions", + """ SELECT name FROM system.functions WHERE NOT is_aggregate AND origin = 'System' AND alias_to = '' @@ -1816,11 +1836,12 @@ def reportCoverage(args): ) ORDER BY name """, - True - ) and reportCoverageFor( - args, - "aggregate functions", - """ + True, + ) + and reportCoverageFor( + args, + "aggregate functions", + """ SELECT name FROM system.functions WHERE is_aggregate AND origin = 'System' AND alias_to = '' @@ -1829,11 +1850,12 @@ def reportCoverage(args): SELECT arrayJoin(used_aggregate_functions) FROM system.query_log WHERE event_date >= yesterday() ) ORDER BY name - """ - ) and reportCoverageFor( - args, - "aggregate function combinators", - """ + """, + ) + and reportCoverageFor( + args, + "aggregate function combinators", + """ SELECT name FROM system.aggregate_function_combinators WHERE NOT is_internal @@ -1842,11 +1864,12 @@ def reportCoverage(args): SELECT arrayJoin(used_aggregate_function_combinators) FROM system.query_log WHERE event_date >= yesterday() ) ORDER BY name - """ - ) and reportCoverageFor( - args, - "data type families", - """ + """, + ) + and reportCoverageFor( + args, + "data type families", + """ SELECT name FROM system.data_type_families WHERE alias_to = '' AND name NOT LIKE 'Interval%' @@ -1855,7 +1878,8 @@ def reportCoverage(args): SELECT arrayJoin(used_data_type_families) FROM system.query_log WHERE event_date >= yesterday() ) ORDER BY name - """ + """, + ) ) @@ -1882,7 +1906,9 @@ def main(args): args, "system", "processes", "is_all_data_sent" ) - if args.s3_storage and (BuildFlags.THREAD in args.build_flags or BuildFlags.DEBUG in args.build_flags): + if args.s3_storage and ( + BuildFlags.THREAD in args.build_flags or BuildFlags.DEBUG in args.build_flags + ): args.no_random_settings = True if args.skip: @@ -2214,7 +2240,12 @@ if __name__ == "__main__": default=False, help="Disable settings randomization", ) - + parser.add_argument( + "--no-random-merge-tree-settings", + action="store_true", + default=False, + help="Disable MergeTree settings randomization", + ) parser.add_argument( "--run-by-hash-num", type=int, @@ -2225,12 +2256,6 @@ if __name__ == "__main__": type=int, help="Total test groups for crc32(test_name) % run_by_hash_total == run_by_hash_num", ) - parser.add_argument( - "--randomize-merge-tree-settings", - action="store_true", - default=False, - help="Enable MergeTree settings randomization", - ) group = parser.add_mutually_exclusive_group(required=False) group.add_argument( From c7f6a410ac9c19b1d0a4b5fedb27a44cdc013bf1 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 28 Oct 2022 23:26:06 +0000 Subject: [PATCH 007/439] fix some tests --- docker/test/fasttest/run.sh | 1 + tests/clickhouse-test | 10 +++++--- .../00443_preferred_block_size_bytes.sh | 6 ++--- .../00653_verification_monotonic_data_load.sh | 1 + ...ndex_granularity_collapsing_merge_tree.sql | 2 ++ ..._adaptive_index_granularity_merge_tree.sql | 25 +++++++++++-------- .../00926_adaptive_index_granularity_pk.sql | 2 +- ...index_granularity_replacing_merge_tree.sql | 20 +++++++++++---- ...larity_versioned_collapsing_merge_tree.sql | 12 ++++++--- ...test_fix_extra_seek_on_compressed_cache.sh | 4 +-- ...hecksums_in_system_parts_columns_table.sql | 3 ++- .../00974_primary_key_for_lowCardinality.sh | 1 + .../0_stateless/00997_set_index_array.sql | 2 ++ .../0_stateless/01039_mergetree_exec_time.sql | 2 +- ...1042_check_query_and_last_granule_size.sql | 4 +-- .../01045_order_by_pk_special_storages.sh | 6 ++--- .../01055_compact_parts_granularity.sh | 1 + ...ion_collapsing_attach_detach_zookeeper.sql | 2 +- .../0_stateless/01533_multiple_nested.sql | 2 +- .../01600_log_queries_with_extensive_info.sh | 2 +- ...605_adaptive_granularity_block_borders.sql | 5 +++- .../01606_merge_from_wide_to_compact.sql | 3 ++- .../01623_constraints_column_swap.sql | 2 ++ .../01785_parallel_formatting_memory.sh | 4 +-- .../0_stateless/01786_explain_merge_tree.sh | 1 + .../01824_move_to_prewhere_many_columns.sql | 2 ++ .../0_stateless/02286_parallel_final.sh | 1 + ..._ignore_auto_increment_in_create_table.sql | 4 +++ .../0_stateless/02361_fsync_profile_events.sh | 2 +- .../02381_compress_marks_and_primary_key.sql | 2 +- 30 files changed, 91 insertions(+), 43 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index de9125d565b..e01d9c5c528 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -226,6 +226,7 @@ function run_tests --hung-check --fast-tests-only --no-random-settings + --no-random-merge-tree-settings --no-long --testname --shard diff --git a/tests/clickhouse-test b/tests/clickhouse-test index b6b0b6e02db..d0537037e76 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -486,7 +486,6 @@ class MergeTreeSettingsRandomizer: "prefer_fetch_merged_part_size_threshold": lambda: random.randint( 1, 10 * 1024 * 1024 * 1024 ), - "always_fetch_merged_part": lambda: random.randint(0, 1), "vertical_merge_algorithm_min_rows_to_activate": lambda: random.randint( 1, 32 * 8192 ), @@ -618,7 +617,7 @@ class TestCase: return " ".join([f"--{setting}" for setting in settings_list]) def has_show_create_table_in_test(self): - return not subprocess.call(["grep", "-iq", "show create table", self.case_file]) + return not subprocess.call(["grep", "-iq", "show create", self.case_file]) def add_random_settings(self, client_options): new_options = "" @@ -684,12 +683,17 @@ class TestCase: args.no_random_settings or has_no_random_settings_tag ) + has_no_random_merge_tree_settings_tag = ( + self.tags and "no-random-merge-tree-settings" in self.tags + ) + # If test contains SHOW CREATE TABLE do not # randomize merge tree settings, because - # they are added to table definition and test will fail + # they will be added to table definition and test will fail self.randomize_merge_tree_settings = not ( args.no_random_merge_tree_settings or has_no_random_settings_tag + or has_no_random_merge_tree_settings_tag or self.has_show_create_table_in_test() ) diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 399a4677a44..4f442241559 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" -$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE preferred_block_size_bytes (p Date, s String) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=1, index_granularity_bytes=0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO preferred_block_size_bytes (s) SELECT '16_bytes_-_-_-_' AS s FROM system.numbers LIMIT 10, 90" $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE preferred_block_size_bytes" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=26 -q "SELECT DISTINCT blockSize(), ignore(p, s) FROM preferred_block_size_bytes" @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS preferred_block_size_bytes" # PREWHERE using empty column $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS pbs" -$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE pbs (p Date, i UInt64, sa Array(String)) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=100, index_granularity_bytes=0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO pbs (p, i, sa) SELECT toDate(i % 30) AS p, number AS i, ['a'] AS sa FROM system.numbers LIMIT 1000" $CLICKHOUSE_CLIENT -q "ALTER TABLE pbs ADD COLUMN s UInt8 DEFAULT 0" $CLICKHOUSE_CLIENT --preferred_block_size_bytes=100000 -q "SELECT count() FROM pbs PREWHERE s = 0" @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE pbs" # Nullable PREWHERE $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" -$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE nullable_prewhere (p Date, f Nullable(UInt64), d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY p SETTINGS index_granularity=8, index_granularity_bytes=0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO nullable_prewhere SELECT toDate(0) AS p, if(number % 2 = 0, CAST(number AS Nullable(UInt64)), CAST(NULL AS Nullable(UInt64))) AS f, number as d FROM system.numbers LIMIT 1001" $CLICKHOUSE_CLIENT -q "SELECT sum(d), sum(f), max(d) FROM nullable_prewhere PREWHERE NOT isNull(f)" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS nullable_prewhere" diff --git a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh index 00a7e3c5232..e10b2f86145 100755 --- a/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh +++ b/tests/queries/0_stateless/00653_verification_monotonic_data_load.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings #-------------------------------------------- # Description of test result: diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql index 58b266f106f..81820d2c60a 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql @@ -10,6 +10,7 @@ CREATE TABLE zero_rows_per_granule ( ) ENGINE CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes=20, min_index_granularity_bytes=10, write_final_mark = 0, min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0; @@ -42,6 +43,7 @@ CREATE TABLE four_rows_per_granule ( ) ENGINE CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes=110, min_index_granularity_bytes=100, write_final_mark = 0, min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql index a29a1bf9d1c..d07323bb3d6 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql @@ -9,7 +9,7 @@ CREATE TABLE zero_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 20, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -36,7 +36,7 @@ CREATE TABLE two_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 40, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 40, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -63,7 +63,7 @@ CREATE TABLE four_rows_per_granule ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 10, write_final_mark = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -97,7 +97,7 @@ CREATE TABLE huge_granularity_small_blocks ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 1000000, write_final_mark = 0, min_bytes_for_wide_part = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 1000000, write_final_mark = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -128,7 +128,7 @@ CREATE TABLE adaptive_granularity_alter ( k UInt64, v1 UInt64, v2 Int64 -) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 100, write_final_mark = 0, min_bytes_for_wide_part = 0; +) ENGINE MergeTree() PARTITION BY toYYYYMM(p) ORDER BY k SETTINGS index_granularity_bytes = 110, min_index_granularity_bytes = 100, write_final_mark = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -188,7 +188,8 @@ CREATE TABLE zero_rows_per_granule ( enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -223,7 +224,8 @@ CREATE TABLE two_rows_per_granule ( enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -257,7 +259,8 @@ CREATE TABLE four_rows_per_granule ( enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -296,7 +299,8 @@ CREATE TABLE huge_granularity_small_blocks ( enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -334,7 +338,8 @@ CREATE TABLE adaptive_granularity_alter ( enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, vertical_merge_algorithm_min_columns_to_activate=0, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql index 47e1d3fea0f..ed9be1170ec 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql @@ -77,7 +77,7 @@ CREATE TABLE large_alter_table_00926 ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) -) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS min_index_granularity_bytes=30, write_final_mark = 0, min_bytes_for_wide_part = '10M'; +) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS min_index_granularity_bytes=30, write_final_mark = 0, min_bytes_for_wide_part = '10M', min_rows_for_wide_part = 0; INSERT INTO large_alter_table_00926 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql index 075548df6c3..7f53ab8b329 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql @@ -14,7 +14,9 @@ CREATE TABLE zero_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -47,7 +49,9 @@ CREATE TABLE two_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO two_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -80,7 +84,9 @@ CREATE TABLE four_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -126,7 +132,9 @@ CREATE TABLE huge_granularity_small_blocks ( SETTINGS index_granularity_bytes=1000000, write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO huge_granularity_small_blocks (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); @@ -164,7 +172,9 @@ CREATE TABLE adaptive_granularity_alter ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO adaptive_granularity_alter (p, k, v1, v2) VALUES ('2018-05-15', 1, 1000, 2000), ('2018-05-16', 2, 3000, 4000), ('2018-05-17', 3, 5000, 6000), ('2018-05-18', 4, 7000, 8000); diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql index 44dd0412aea..0fa20c863c8 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql @@ -14,7 +14,9 @@ CREATE TABLE zero_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO zero_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 2, 3000, 4000, 1, 1), ('2018-05-17', 3, 5000, 6000, 1, 1), ('2018-05-18', 4, 7000, 8000, 1, 1); @@ -48,7 +50,9 @@ CREATE TABLE four_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO four_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 2, 3000, 4000, 1, 1), ('2018-05-17', 3, 5000, 6000, 1, 1), ('2018-05-18', 4, 7000, 8000, 1, 1); @@ -99,7 +103,9 @@ CREATE TABLE six_rows_per_granule ( write_final_mark = 0, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=0, - vertical_merge_algorithm_min_columns_to_activate=0, min_bytes_for_wide_part = 0; + vertical_merge_algorithm_min_columns_to_activate=0, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; INSERT INTO six_rows_per_granule (p, k, v1, v2, Sign, Version) VALUES ('2018-05-15', 1, 1000, 2000, 1, 1), ('2018-05-16', 1, 1000, 2000, -1, 2); diff --git a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh index df4c5e233e4..b7c2045c83d 100755 --- a/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh +++ b/tests/queries/0_stateless/00933_test_fix_extra_seek_on_compressed_cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS small_table" -$CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt64) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY (a) SETTINGS min_bytes_for_wide_part = 0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE small_table (a UInt64 default 0, n UInt64) ENGINE = MergeTree() PARTITION BY tuple() ORDER BY (a) SETTINGS min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT --query="INSERT INTO small_table (n) SELECT * from system.numbers limit 100000;" $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE small_table FINAL;" diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index 792bf62f9b1..34540262b47 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,7 +1,8 @@ DROP TABLE IF EXISTS test_00961; CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) - ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0; + ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) + SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); diff --git a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh index a817acd88a6..f8527cd491e 100755 --- a/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh +++ b/tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/00997_set_index_array.sql b/tests/queries/0_stateless/00997_set_index_array.sql index a0cb4ec0547..34d0f0b71ec 100644 --- a/tests/queries/0_stateless/00997_set_index_array.sql +++ b/tests/queries/0_stateless/00997_set_index_array.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + DROP TABLE IF EXISTS set_array; CREATE TABLE set_array diff --git a/tests/queries/0_stateless/01039_mergetree_exec_time.sql b/tests/queries/0_stateless/01039_mergetree_exec_time.sql index d3aade41cea..bb114c41ec8 100644 --- a/tests/queries/0_stateless/01039_mergetree_exec_time.sql +++ b/tests/queries/0_stateless/01039_mergetree_exec_time.sql @@ -1,5 +1,5 @@ DROP TABLE IF EXISTS tab; -create table tab (A Int64) Engine=MergeTree order by tuple() SETTINGS min_bytes_for_wide_part = 0; +create table tab (A Int64) Engine=MergeTree order by tuple() SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; insert into tab select cityHash64(number) from numbers(1000); select sum(sleep(0.1)) from tab settings max_block_size = 1, max_execution_time=1; -- { serverError 159 } DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql index c62fe25a041..b66aff8384d 100644 --- a/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql +++ b/tests/queries/0_stateless/01042_check_query_and_last_granule_size.sql @@ -1,7 +1,7 @@ SET check_query_single_value_result = 0; DROP TABLE IF EXISTS check_query_test; -CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS min_bytes_for_wide_part = 0; +CREATE TABLE check_query_test (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; -- Number of rows in last granule should be equals to granularity. -- Rows in this table are short, so granularity will be 8192. @@ -17,7 +17,7 @@ DROP TABLE IF EXISTS check_query_test; DROP TABLE IF EXISTS check_query_test_non_adaptive; -CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS index_granularity_bytes = 0, min_bytes_for_wide_part = 0; +CREATE TABLE check_query_test_non_adaptive (SomeKey UInt64, SomeValue String) ENGINE = MergeTree() ORDER BY SomeKey SETTINGS index_granularity_bytes = 0, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; INSERT INTO check_query_test_non_adaptive SELECT number, toString(number) FROM system.numbers LIMIT 81920; diff --git a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh index bb76f3978cc..8bd3d5291e4 100755 --- a/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh +++ b/tests/queries/0_stateless/01045_order_by_pk_special_storages.sh @@ -12,8 +12,8 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS m" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS buf" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv" -$CLICKHOUSE_CLIENT -q "CREATE TABLE s1 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" -$CLICKHOUSE_CLIENT -q "CREATE TABLE s2 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE s1 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE s2 (a UInt32, s String) ENGINE = MergeTree ORDER BY a PARTITION BY a % 3 SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "CREATE TABLE m (a UInt32, s String) engine = Merge('$CLICKHOUSE_DATABASE', 's[1,2]')" $CLICKHOUSE_CLIENT -q "INSERT INTO s1 select (number % 20) * 2 as n, toString(number * number) from numbers(100000)" @@ -46,7 +46,7 @@ else fi $CLICKHOUSE_CLIENT -q "SELECT '---MaterializedView---'" -$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv (a UInt32, s String) engine = MergeTree ORDER BY s SETTINGS min_bytes_for_wide_part = 0 POPULATE AS SELECT a, s FROM s1 WHERE a % 7 = 0" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv (a UInt32, s String) engine = MergeTree ORDER BY s SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0 POPULATE AS SELECT a, s FROM s1 WHERE a % 7 = 0" $CLICKHOUSE_CLIENT -q "SELECT a, s FROM mv ORDER BY s LIMIT 10" rows_read=$($CLICKHOUSE_CLIENT -q "SELECT a, s FROM mv ORDER BY s LIMIT 10 FORMAT JSON" --max_threads=1 --max_block_size=20 --optimize_read_in_order=1 | grep "rows_read" | sed 's/[^0-9]*//g') diff --git a/tests/queries/0_stateless/01055_compact_parts_granularity.sh b/tests/queries/0_stateless/01055_compact_parts_granularity.sh index b5c6609bed8..f3da33f6ccf 100755 --- a/tests/queries/0_stateless/01055_compact_parts_granularity.sh +++ b/tests/queries/0_stateless/01055_compact_parts_granularity.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01357_version_collapsing_attach_detach_zookeeper.sql b/tests/queries/0_stateless/01357_version_collapsing_attach_detach_zookeeper.sql index c76e2650ae9..8dcb9319cce 100644 --- a/tests/queries/0_stateless/01357_version_collapsing_attach_detach_zookeeper.sql +++ b/tests/queries/0_stateless/01357_version_collapsing_attach_detach_zookeeper.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper +-- Tags: zookeeper, no-random-merge-tree-settings DROP TABLE IF EXISTS versioned_collapsing_table; diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql index 94d81c110cb..f39b56bbaa2 100644 --- a/tests/queries/0_stateless/01533_multiple_nested.sql +++ b/tests/queries/0_stateless/01533_multiple_nested.sql @@ -1,4 +1,4 @@ --- Tags: no-s3-storage +-- Tags: no-s3-storage, no-random-merge-tree-settings -- no-s3 because read FileOpen metric DROP TABLE IF EXISTS nested; diff --git a/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh b/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh index 46fea5001a1..50a16250ea8 100755 --- a/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh +++ b/tests/queries/0_stateless/01600_log_queries_with_extensive_info.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-random-merge-tree-settings set -ue diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql index 7654be4eb29..030e0c5574c 100644 --- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql +++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql @@ -11,7 +11,10 @@ CREATE TABLE adaptive_table( value String ) ENGINE MergeTree() ORDER BY key -SETTINGS index_granularity_bytes=1048576, min_bytes_for_wide_part = 0, enable_vertical_merge_algorithm = 0; +SETTINGS index_granularity_bytes=1048576, +min_bytes_for_wide_part = 0, +min_rows_for_wide_part = 0, +enable_vertical_merge_algorithm = 0; SET max_block_size=900; diff --git a/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql index 15596d3f6db..0f2fbcaa76d 100644 --- a/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql +++ b/tests/queries/0_stateless/01606_merge_from_wide_to_compact.sql @@ -4,7 +4,8 @@ CREATE TABLE wide_to_comp (a Int, b Int, c Int) ENGINE = MergeTree ORDER BY a settings vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1, - min_bytes_for_wide_part = 0; + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 0; SYSTEM STOP merges wide_to_comp; diff --git a/tests/queries/0_stateless/01623_constraints_column_swap.sql b/tests/queries/0_stateless/01623_constraints_column_swap.sql index 0fb0d417a43..c81b37c8428 100644 --- a/tests/queries/0_stateless/01623_constraints_column_swap.sql +++ b/tests/queries/0_stateless/01623_constraints_column_swap.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; diff --git a/tests/queries/0_stateless/01785_parallel_formatting_memory.sh b/tests/queries/0_stateless/01785_parallel_formatting_memory.sh index 6d081c61fd3..1cfe89d191f 100755 --- a/tests/queries/0_stateless/01785_parallel_formatting_memory.sh +++ b/tests/queries/0_stateless/01785_parallel_formatting_memory.sh @@ -4,5 +4,5 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --compress 0 --max_memory_usage 1G --query "SELECT range(65535) FROM system.one ARRAY JOIN range(65536) AS number" 2>&1 | grep -oF 'Code: 241' | head -n1 -$CLICKHOUSE_CLIENT --compress 1 --max_memory_usage 1G --query "SELECT range(65535) FROM system.one ARRAY JOIN range(65536) AS number" 2>&1 | grep -oF 'Code: 241' | head -n1 +$CLICKHOUSE_CLIENT --compression 0 --max_memory_usage 1G --query "SELECT range(65535) FROM system.one ARRAY JOIN range(65536) AS number" 2>&1 | grep -oF 'Code: 241' | head -n1 +$CLICKHOUSE_CLIENT --compression 1 --max_memory_usage 1G --query "SELECT range(65535) FROM system.one ARRAY JOIN range(65536) AS number" 2>&1 | grep -oF 'Code: 241' | head -n1 diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.sh b/tests/queries/0_stateless/01786_explain_merge_tree.sh index 138905c65e7..15f8821d80d 100755 --- a/tests/queries/0_stateless/01786_explain_merge_tree.sh +++ b/tests/queries/0_stateless/01786_explain_merge_tree.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql b/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql index c4ef5516fc8..2987c541aef 100644 --- a/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql +++ b/tests/queries/0_stateless/01824_move_to_prewhere_many_columns.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + SET optimize_move_to_prewhere = 1; SET convert_query_to_cnf = 0; diff --git a/tests/queries/0_stateless/02286_parallel_final.sh b/tests/queries/0_stateless/02286_parallel_final.sh index d31450a0482..de0cca0e966 100755 --- a/tests/queries/0_stateless/02286_parallel_final.sh +++ b/tests/queries/0_stateless/02286_parallel_final.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02293_compatibility_ignore_auto_increment_in_create_table.sql b/tests/queries/0_stateless/02293_compatibility_ignore_auto_increment_in_create_table.sql index a671826ccf5..5d9b711ae9c 100644 --- a/tests/queries/0_stateless/02293_compatibility_ignore_auto_increment_in_create_table.sql +++ b/tests/queries/0_stateless/02293_compatibility_ignore_auto_increment_in_create_table.sql @@ -1,3 +1,7 @@ +-- Tags: no-random-merge-tree-settings +-- Tag no-random-merge-tree-settings: query is rewritten in parser +-- while adding merge tree settings + select 'disable AUTO_INCREMENT compatibility mode'; set compatibility_ignore_auto_increment_in_create_table=false; diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.sh b/tests/queries/0_stateless/02361_fsync_profile_events.sh index d54da9a49e5..eef0f57de7d 100755 --- a/tests/queries/0_stateless/02361_fsync_profile_events.sh +++ b/tests/queries/0_stateless/02361_fsync_profile_events.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-s3-storage +# Tags: no-s3-storage, no-random-merge-tree-settings # Tag no-s3-storage: s3 does not have fsync CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql index cf5ca15adeb..1a839b62642 100644 --- a/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql +++ b/tests/queries/0_stateless/02381_compress_marks_and_primary_key.sql @@ -1,4 +1,4 @@ --- Tags: no-backward-compatibility-check +-- Tags: no-backward-compatibility-check, no-random-merge-tree-settings drop table if exists test_02381; create table test_02381(a UInt64, b UInt64) ENGINE = MergeTree order by (a, b); From b069259125a3b5d1c9aff3dc67629a7106223aee Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 28 Oct 2022 23:54:37 +0000 Subject: [PATCH 008/439] better random settings --- tests/clickhouse-test | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index d0537037e76..641bb46b23b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -486,17 +486,20 @@ class MergeTreeSettingsRandomizer: "prefer_fetch_merged_part_size_threshold": lambda: random.randint( 1, 10 * 1024 * 1024 * 1024 ), - "vertical_merge_algorithm_min_rows_to_activate": lambda: random.randint( - 1, 32 * 8192 - ), - "vertical_merge_algorithm_min_columns_to_activate": lambda: random.randint( - 1, 20 - ), + "vertical_merge_algorithm_min_rows_to_activate": lambda: 1 + if random.random() < 0.2 + else 100000000 + if random.random() < 0.25 + else random.randint(1, 32 * 8192), + "vertical_merge_algorithm_min_columns_to_activate": lambda: 1 + if random.random() < 0.2 + else 100000000 + if random.random() < 0.25 + else random.randint(1, 20), "min_merge_bytes_to_use_direct_io": lambda: 1 if random.random() < 0.2 else random.randint(1, 10 * 1024 * 1024 * 1024), "index_granularity_bytes": lambda: random.randint(1024, 30 * 1024 * 1024), - "write_final_mark": lambda: random.randint(0, 1), } @staticmethod From 0a951db1d034079fe9c331f4b249292e43abde7c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 2 Dec 2022 14:16:28 +0000 Subject: [PATCH 009/439] fix more tests with random merge tree settings --- src/Client/ClientBase.cpp | 23 +++-- src/Storages/MergeTree/MutateTask.cpp | 3 + tests/clickhouse-test | 91 +++++++++---------- .../00443_preferred_block_size_bytes.sh | 1 + ..._system_columns_and_system_tables_long.sql | 2 +- .../00804_test_delta_codec_compression.sql | 2 +- .../00950_test_double_delta_codec.sql | 2 + ...hecksums_in_system_parts_columns_table.sql | 4 +- .../01508_partition_pruning_long.sh | 3 +- .../01640_marks_corruption_regression.sql | 2 + .../01913_exact_rows_before_limit.sql | 3 +- .../02293_selected_rows_and_merges.sh | 1 + 12 files changed, 76 insertions(+), 61 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index b823be86726..cd03d3e0f84 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -822,13 +822,6 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa auto query = query_to_execute; - if (allow_merge_tree_settings && parsed_query->as()) - { - /// Rewrite query if new settings were added. - if (addMergeTreeSettings(*parsed_query->as())) - query = serializeAST(*parsed_query); - } - /// Rewrite query only when we have query parameters. /// Note that if query is rewritten, comments in query are lost. /// But the user often wants to see comments in server logs, query log, processlist, etc. @@ -844,6 +837,22 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa query = serializeAST(*parsed_query); } + if (allow_merge_tree_settings && parsed_query->as()) + { + /// Rewrite query if new settings were added. + if (addMergeTreeSettings(*parsed_query->as())) + { + /// Replace query parameters because AST cannot be serialized otherwise. + if (!query_parameters.empty()) + { + ReplaceQueryParameterVisitor visitor(query_parameters); + visitor.visit(parsed_query); + } + + query = serializeAST(*parsed_query); + } + } + int retries_left = 10; while (retries_left) { diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index f6befe67fd4..a5fcb98abb3 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -488,6 +488,9 @@ static NameSet collectFilesToSkip( { NameSet files_to_skip = source_part->getFileNamesWithoutChecksums(); + /// Do not hardlink this file because it's always rewritten at the end of mutation. + files_to_skip.insert(IMergeTreeDataPart::SERIALIZATION_FILE_NAME); + auto new_stream_counts = getStreamCounts(new_part, new_part->getColumns().getNames()); auto source_updated_stream_counts = getStreamCounts(source_part, updated_header.getNames()); auto new_updated_stream_counts = getStreamCounts(new_part, updated_header.getNames()); diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 641bb46b23b..addfcfc8d93 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -402,21 +402,31 @@ class FailureReason(enum.Enum): INTERNAL_ERROR = "Test internal error: " +def threshold_generator(always_on_prob, always_off_prob, min_val, max_val): + def gen(): + tmp = random.random() + if tmp <= always_on_prob: + return min_val + if tmp <= always_on_prob + always_off_prob: + return max_val + + if isinstance(min_val, int) and isinstance(max_val, int): + return random.randint(min_val, max_val) + else: + return random.uniform(min_val, max_val) + + return gen + + class SettingsRandomizer: settings = { "max_insert_threads": lambda: 0 if random.random() < 0.5 else random.randint(1, 16), - "group_by_two_level_threshold": lambda: 1 - if random.random() < 0.1 - else 2**60 - if random.random() < 0.11 - else 100000, - "group_by_two_level_threshold_bytes": lambda: 1 - if random.random() < 0.1 - else 2**60 - if random.random() < 0.11 - else 50000000, + "group_by_two_level_threshold": threshold_generator(0.2, 0.2, 1, 1000000), + "group_by_two_level_threshold_bytes": threshold_generator( + 0.2, 0.2, 1, 50000000 + ), "distributed_aggregation_memory_efficient": lambda: random.randint(0, 1), "fsync_metadata": lambda: random.randint(0, 1), "output_format_parallel_formatting": lambda: random.randint(0, 1), @@ -436,16 +446,12 @@ class SettingsRandomizer: "min_compress_block_size": lambda: random.randint(1, 1048576 * 3), "max_compress_block_size": lambda: random.randint(1, 1048576 * 3), "use_uncompressed_cache": lambda: random.randint(0, 1), - "min_bytes_to_use_direct_io": lambda: 0 - if random.random() < 0.5 - else 1 - if random.random() < 0.2 - else random.randint(1, 1024 * 1024 * 1024), - "min_bytes_to_use_mmap_io": lambda: 0 - if random.random() < 0.5 - else 1 - if random.random() < 0.2 - else random.randint(1, 1024 * 1024 * 1024), + "min_bytes_to_use_direct_io": threshold_generator( + 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 + ), + "min_bytes_to_use_mmap_io": threshold_generator( + 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 + ), "local_filesystem_read_method": lambda: random.choice( ["read", "pread", "mmap", "pread_threadpool"] ), @@ -470,36 +476,25 @@ class SettingsRandomizer: class MergeTreeSettingsRandomizer: settings = { - "index_granularity": lambda: random.randint(1, 65536), - "min_bytes_for_wide_part": lambda: 0 - if random.random() < 0.3 - else random.randint(0, 1024 * 1024 * 1024), - "min_rows_for_wide_part": lambda: 0 - if random.random() < 0.3 - else random.randint(0, 10000000), - "ratio_of_defaults_for_sparse_serialization": lambda: 0.0 - if random.random() < 0.1 - else 1.0 - if random.random() < 4 / 9 - else random.random(), - "merge_max_block_size": lambda: random.randint(1, 8192 * 3), - "prefer_fetch_merged_part_size_threshold": lambda: random.randint( - 1, 10 * 1024 * 1024 * 1024 + "ratio_of_defaults_for_sparse_serialization": threshold_generator( + 0.1, 0.6, 0.0, 1.0 + ), + "prefer_fetch_merged_part_size_threshold": threshold_generator( + 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 + ), + "vertical_merge_algorithm_min_rows_to_activate": threshold_generator( + 0.25, 0.25, 1, 100000000 + ), + "vertical_merge_algorithm_min_columns_to_activate": threshold_generator( + 0.25, 0.25, 1, 1000 + ), + "min_merge_bytes_to_use_direct_io": threshold_generator( + 0.25, 0.25, 1, 10 * 1024 * 1024 * 1024 ), - "vertical_merge_algorithm_min_rows_to_activate": lambda: 1 - if random.random() < 0.2 - else 100000000 - if random.random() < 0.25 - else random.randint(1, 32 * 8192), - "vertical_merge_algorithm_min_columns_to_activate": lambda: 1 - if random.random() < 0.2 - else 100000000 - if random.random() < 0.25 - else random.randint(1, 20), - "min_merge_bytes_to_use_direct_io": lambda: 1 - if random.random() < 0.2 - else random.randint(1, 10 * 1024 * 1024 * 1024), "index_granularity_bytes": lambda: random.randint(1024, 30 * 1024 * 1024), + "merge_max_block_size": lambda: random.randint(1, 8192 * 3), + "index_granularity": lambda: random.randint(1, 65536), + "min_bytes_for_wide_part": threshold_generator(0.3, 0.3, 0, 1024 * 1024 * 1024), } @staticmethod diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 4f442241559..c184b58bf53 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings set -e diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index bbc7bedcb4f..f4c4110cd5b 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -1,4 +1,4 @@ --- Tags: long, no-s3-storage +-- Tags: long, no-s3-storage, no-random-merge-tree-settings DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql index ca9bb1b177e..4afd8e6d860 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings SET send_logs_level = 'fatal'; SET joined_subquery_requires_alias = 0; diff --git a/tests/queries/0_stateless/00950_test_double_delta_codec.sql b/tests/queries/0_stateless/00950_test_double_delta_codec.sql index 6bf9b2628ad..f6199a6e4ec 100644 --- a/tests/queries/0_stateless/00950_test_double_delta_codec.sql +++ b/tests/queries/0_stateless/00950_test_double_delta_codec.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + DROP TABLE IF EXISTS codecTest; CREATE TABLE codecTest ( diff --git a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql index a1d7b05cef3..40da12baddc 100644 --- a/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql +++ b/tests/queries/0_stateless/00961_checksums_in_system_parts_columns_table.sql @@ -1,8 +1,10 @@ +-- Tags: no-random-merge-tree-settings + DROP TABLE IF EXISTS test_00961; CREATE TABLE test_00961 (d Date, a String, b UInt8, x String, y Int8, z UInt32) ENGINE = MergeTree PARTITION BY d ORDER BY (a, b) - SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, compress_marks = 0, compress_primary_key = 0; + SETTINGS index_granularity = 111, min_bytes_for_wide_part = 0, compress_marks = 0, compress_primary_key = 0; INSERT INTO test_00961 VALUES ('2000-01-01', 'Hello, world!', 123, 'xxx yyy', -123, 123456789); diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh index ddf63bf4808..4c22586d03a 100755 --- a/tests/queries/0_stateless/01508_partition_pruning_long.sh +++ b/tests/queries/0_stateless/01508_partition_pruning_long.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-polymorphic-parts -# Tag no-polymorphic-parts: bug, shoud be fixed +# Tags: long, no-random-merge-tree-settings # Description of test result: # Test the correctness of the partition pruning diff --git a/tests/queries/0_stateless/01640_marks_corruption_regression.sql b/tests/queries/0_stateless/01640_marks_corruption_regression.sql index 7ccd8741dda..c75c26f3165 100644 --- a/tests/queries/0_stateless/01640_marks_corruption_regression.sql +++ b/tests/queries/0_stateless/01640_marks_corruption_regression.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + DROP TABLE IF EXISTS adaptive_table; CREATE TABLE adaptive_table( diff --git a/tests/queries/0_stateless/01913_exact_rows_before_limit.sql b/tests/queries/0_stateless/01913_exact_rows_before_limit.sql index f6b02d0a510..6a354f575be 100644 --- a/tests/queries/0_stateless/01913_exact_rows_before_limit.sql +++ b/tests/queries/0_stateless/01913_exact_rows_before_limit.sql @@ -1,4 +1,5 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings + drop table if exists test_rows_compact_part; create table test_rows_compact_part(f1 int,f2 int) engine=MergeTree partition by f1 order by f2 settings min_bytes_for_wide_part=10485760; insert into test_rows_compact_part select 0,arrayJoin(range(10000)) ; diff --git a/tests/queries/0_stateless/02293_selected_rows_and_merges.sh b/tests/queries/0_stateless/02293_selected_rows_and_merges.sh index 33bfdd45810..9d1483f5bf7 100755 --- a/tests/queries/0_stateless/02293_selected_rows_and_merges.sh +++ b/tests/queries/0_stateless/02293_selected_rows_and_merges.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings set -ue CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) From 898cf867afbe982afa01562a4b6b67e16102e0ac Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 13 Dec 2022 11:08:59 +0000 Subject: [PATCH 010/439] Try with fallocate --- src/Coordination/Changelog.cpp | 93 ++++++++++++++----- src/Coordination/Changelog.h | 5 +- src/Coordination/KeeperLogStore.cpp | 3 +- src/IO/ZstdDeflatingAppendableWriteBuffer.cpp | 52 +++++------ src/IO/ZstdDeflatingAppendableWriteBuffer.h | 9 +- 5 files changed, 105 insertions(+), 57 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index c85caa43f19..1632e96a63f 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -66,6 +66,8 @@ Checksum computeRecordChecksum(const ChangelogRecord & record) return hash.get64(); } +constexpr size_t log_size = 50 * 1024 * 1024; + } class ChangelogWriter @@ -74,6 +76,7 @@ public: ChangelogWriter(const std::string & filepath_, WriteMode mode, uint64_t start_index_) : filepath(filepath_) , file_buf(std::make_unique(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY))) + , memory_buf(std::make_unique(memory)) , start_index(start_index_) { auto compression_method = chooseCompressionMethod(filepath_, ""); @@ -84,7 +87,7 @@ public: else if (compression_method == CompressionMethod::Zstd) { compressed_buffer = std::make_unique( - std::move(file_buf), /* compression level = */ 3, /* append_to_existing_file_ = */ mode == WriteMode::Append); + std::move(memory_buf), /* compression level = */ 3); } else { @@ -93,8 +96,17 @@ public: } - void appendRecord(ChangelogRecord && record) + bool appendRecord(ChangelogRecord && record) { + if (!prealloc_done) [[unlikely]] + { + int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_size); + if (res == ENOSPC) + return false; + + prealloc_done = true; + } + writeIntBinary(computeRecordChecksum(record), getBuffer()); writeIntBinary(record.header.version, getBuffer()); @@ -105,6 +117,9 @@ public: if (record.header.blob_size != 0) getBuffer().write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); + + // TODO: Maybe try writing to file after each record + return true; } void flush(bool force_fsync) @@ -115,14 +130,18 @@ public: compressed_buffer->next(); } - WriteBuffer * working_buf = compressed_buffer ? compressed_buffer->getNestedBuffer() : file_buf.get(); + WriteBufferMemory & cur_memory_buf = compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; - /// Flush working buffer to file system - working_buf->next(); + cur_memory_buf.finalize(); + // TODO: rotate if it's larger than max log size + + file_buf->write(memory.raw_data(), memory.size()); /// Fsync file system if needed if (force_fsync) - working_buf->sync(); + file_buf->sync(); + + cur_memory_buf.restart(); } uint64_t getStartIndex() const @@ -135,12 +154,22 @@ private: { if (compressed_buffer) return *compressed_buffer; - return *file_buf; + return *memory_buf; } + using MemoryBuffer = PODArray; + using WriteBufferMemory = WriteBufferFromVector; + MemoryBuffer memory; + std::string filepath; std::unique_ptr file_buf; + std::unique_ptr memory_buf; std::unique_ptr compressed_buffer; + + bool prealloc_done{false}; + + //size_t written{0}; + uint64_t start_index; }; @@ -550,12 +579,16 @@ ChangelogRecord Changelog::buildRecord(uint64_t index, const LogEntryPtr & log_e void Changelog::writeThread() { WriteOperation write_operation; + bool batch_append_ok = true; while (write_operations.pop(write_operation)) { assert(initialized); if (auto * append_log = std::get_if(&write_operation)) { + if (!batch_append_ok) + continue; + std::lock_guard writer_lock(writer_mutex); assert(current_writer); @@ -565,30 +598,40 @@ void Changelog::writeThread() if (log_is_complete) rotate(append_log->index, writer_lock); - current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); + + batch_append_ok &= current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); } else { const auto & flush = std::get(write_operation); + if (batch_append_ok) { - std::lock_guard writer_lock(writer_mutex); - if (current_writer) - current_writer->flush(force_sync); - } + { + std::lock_guard writer_lock(writer_mutex); + if (current_writer) + current_writer->flush(force_sync); + } + { + std::lock_guard lock{durable_idx_mutex}; + last_durable_idx = flush.index; + } + } + else { - std::lock_guard lock{durable_idx_mutex}; - last_durable_idx = flush.index; + *flush.failed = true; } durable_idx_cv.notify_all(); // we shouldn't start the raft_server before sending it here if (auto raft_server_locked = raft_server.lock()) - raft_server_locked->notify_log_append_completion(true); + raft_server_locked->notify_log_append_completion(batch_append_ok); else LOG_WARNING(log, "Raft server is not set in LogStore."); + + batch_append_ok = true; } } } @@ -799,21 +842,29 @@ void Changelog::applyEntriesFromBuffer(uint64_t index, nuraft::buffer & buffer) } } -void Changelog::flush() +bool Changelog::flush() { - if (flushAsync()) + if (auto failed_ptr = flushAsync()) { std::unique_lock lock{durable_idx_mutex}; - durable_idx_cv.wait(lock, [&] { return last_durable_idx == max_log_id; }); + durable_idx_cv.wait(lock, [&] { return failed_ptr || last_durable_idx == max_log_id; }); + + return !*failed_ptr; } + + return true; } -bool Changelog::flushAsync() +std::shared_ptr Changelog::flushAsync() { - bool pushed = write_operations.push(Flush{max_log_id}); + auto failed = std::make_shared(false); + bool pushed = write_operations.push(Flush{max_log_id, failed}); if (!pushed) + { LOG_WARNING(log, "Changelog is shut down"); - return pushed; + return nullptr; + } + return failed; } void Changelog::shutdown() diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index a9464a59003..157ab68df69 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -121,9 +121,9 @@ public: void applyEntriesFromBuffer(uint64_t index, nuraft::buffer & buffer); /// Fsync latest log to disk and flush buffer - void flush(); + bool flush(); - bool flushAsync(); + std::shared_ptr flushAsync(); void shutdown(); @@ -197,6 +197,7 @@ private: struct Flush { uint64_t index; + std::shared_ptr failed; }; using WriteOperation = std::variant; diff --git a/src/Coordination/KeeperLogStore.cpp b/src/Coordination/KeeperLogStore.cpp index ea72022af09..073889ff107 100644 --- a/src/Coordination/KeeperLogStore.cpp +++ b/src/Coordination/KeeperLogStore.cpp @@ -90,8 +90,7 @@ bool KeeperLogStore::compact(uint64_t last_log_index) bool KeeperLogStore::flush() { std::lock_guard lock(changelog_lock); - changelog.flush(); - return true; + return changelog.flush(); } void KeeperLogStore::apply_pack(uint64_t index, nuraft::buffer & pack) diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp index 79fb4ccead5..cbac8eee942 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp @@ -11,15 +11,13 @@ namespace ErrorCodes } ZstdDeflatingAppendableWriteBuffer::ZstdDeflatingAppendableWriteBuffer( - std::unique_ptr out_, + std::unique_ptr out_, int compression_level, - bool append_to_existing_file_, size_t buf_size, char * existing_memory, size_t alignment) : BufferWithOwnMemory(buf_size, existing_memory, alignment) , out(std::move(out_)) - , append_to_existing_file(append_to_existing_file_) { cctx = ZSTD_createCCtx(); if (cctx == nullptr) @@ -41,11 +39,11 @@ void ZstdDeflatingAppendableWriteBuffer::nextImpl() input.size = offset(); input.pos = 0; - if (first_write && append_to_existing_file && isNeedToAddEmptyBlock()) - { - addEmptyBlock(); - first_write = false; - } + //if (first_write && append_to_existing_file && isNeedToAddEmptyBlock()) + //{ + // addEmptyBlock(); + // first_write = false; + //} try { @@ -178,26 +176,26 @@ void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock() bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock() { - ReadBufferFromFile reader(out->getFileName()); - auto fsize = reader.getFileSize(); - if (fsize > 3) - { - std::array result; - reader.seek(fsize - 3, SEEK_SET); - reader.readStrict(result.data(), 3); + //ReadBufferFromFile reader(out->getFileName()); + //auto fsize = reader.getFileSize(); + //if (fsize > 3) + //{ + // std::array result; + // reader.seek(fsize - 3, SEEK_SET); + // reader.readStrict(result.data(), 3); - /// If we don't have correct block in the end, then we need to add it manually. - /// NOTE: maybe we can have the same bytes in case of data corruption/unfinished write. - /// But in this case file still corrupted and we have to remove it. - return result != ZSTD_CORRECT_TERMINATION_LAST_BLOCK; - } - else if (fsize > 0) - { - throw Exception( - ErrorCodes::ZSTD_ENCODER_FAILED, - "Trying to write to non-empty file '{}' with tiny size {}. It can lead to data corruption", - out->getFileName(), fsize); - } + // /// If we don't have correct block in the end, then we need to add it manually. + // /// NOTE: maybe we can have the same bytes in case of data corruption/unfinished write. + // /// But in this case file still corrupted and we have to remove it. + // return result != ZSTD_CORRECT_TERMINATION_LAST_BLOCK; + //} + //else if (fsize > 0) + //{ + // throw Exception( + // ErrorCodes::ZSTD_ENCODER_FAILED, + // "Trying to write to non-empty file '{}' with tiny size {}. It can lead to data corruption", + // out->getFileName(), fsize); + //} return false; } diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.h b/src/IO/ZstdDeflatingAppendableWriteBuffer.h index a0715480737..cf926e87d43 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.h +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.h @@ -29,9 +29,8 @@ public: static inline constexpr ZSTDLastBlock ZSTD_CORRECT_TERMINATION_LAST_BLOCK = {0x01, 0x00, 0x00}; ZstdDeflatingAppendableWriteBuffer( - std::unique_ptr out_, + std::unique_ptr out_, int compression_level, - bool append_to_existing_file_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0); @@ -63,14 +62,14 @@ private: /// Read three last bytes from non-empty compressed file and compares them with /// ZSTD_CORRECT_TERMINATION_LAST_BLOCK. - bool isNeedToAddEmptyBlock(); + static bool isNeedToAddEmptyBlock(); /// Adding zstd empty block (ZSTD_CORRECT_TERMINATION_LAST_BLOCK) to out.working_buffer void addEmptyBlock(); - std::unique_ptr out; + std::unique_ptr out; - bool append_to_existing_file = false; + //bool append_to_existing_file = false; ZSTD_CCtx * cctx; ZSTD_inBuffer input; ZSTD_outBuffer output; From 662c1f956c6824c19f887f52de141c1ef931f382 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 13 Dec 2022 14:00:27 +0000 Subject: [PATCH 011/439] Changelog refactoring --- src/Coordination/Changelog.cpp | 81 ++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 1632e96a63f..797958b0494 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -73,28 +73,32 @@ constexpr size_t log_size = 50 * 1024 * 1024; class ChangelogWriter { public: - ChangelogWriter(const std::string & filepath_, WriteMode mode, uint64_t start_index_) - : filepath(filepath_) - , file_buf(std::make_unique(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY))) + ChangelogWriter(std::map & existing_changelogs_, bool compress_logs_, bool force_fsync_, + const std::filesystem::path & changelogs_dir_) + : existing_changelogs(existing_changelogs_) , memory_buf(std::make_unique(memory)) - , start_index(start_index_) + , compress_logs(compress_logs_) + , force_fsync(force_fsync_) + , changelogs_dir(changelogs_dir_) { - auto compression_method = chooseCompressionMethod(filepath_, ""); - if (compression_method != CompressionMethod::Zstd && compression_method != CompressionMethod::None) - { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Unsupported coordination log serialization format {}", toContentEncodingName(compression_method)); - } - else if (compression_method == CompressionMethod::Zstd) + if (compress_logs) { compressed_buffer = std::make_unique( std::move(memory_buf), /* compression level = */ 3); } else { - /// no compression, only file buffer + /// no compression, only memory buffer } } + void setFile(ChangelogFileDescription & file_description, WriteMode mode) + { + file_buf = std::make_unique(file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); + start_index = file_description.from_log_index; + current_file_description = &file_description; + } + bool appendRecord(ChangelogRecord && record) { @@ -118,12 +122,6 @@ public: if (record.header.blob_size != 0) getBuffer().write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); - // TODO: Maybe try writing to file after each record - return true; - } - - void flush(bool force_fsync) - { if (compressed_buffer) { /// Flush compressed data to WriteBufferFromFile working_buffer @@ -137,11 +135,18 @@ public: // TODO: rotate if it's larger than max log size file_buf->write(memory.raw_data(), memory.size()); + + cur_memory_buf.restart(); + + current_file_description->to_log_index++; + return true; + } + + void flush() + { /// Fsync file system if needed if (force_fsync) file_buf->sync(); - - cur_memory_buf.restart(); } uint64_t getStartIndex() const @@ -149,6 +154,28 @@ public: return start_index; } + void rotate(uint64_t new_start_log_index) + { + flush(); + + /// Start new one + ChangelogFileDescription new_description; + new_description.prefix = DEFAULT_PREFIX; + new_description.from_log_index = new_start_log_index; + new_description.to_log_index = new_start_log_index; + new_description.extension = "bin"; + + if (compress_logs) + new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); + + new_description.path = formatChangelogPath(changelogs_dir, new_description); + + LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description.path); + auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); + + setFile(it->second, WriteMode::Rewrite); + } + private: WriteBuffer & getBuffer() { @@ -157,17 +184,26 @@ private: return *memory_buf; } + std::map & existing_changelogs; + using MemoryBuffer = PODArray; using WriteBufferMemory = WriteBufferFromVector; MemoryBuffer memory; - - std::string filepath; - std::unique_ptr file_buf; std::unique_ptr memory_buf; + + ChangelogFileDescription * current_file_description{nullptr}; + std::unique_ptr file_buf; + std::unique_ptr compressed_buffer; bool prealloc_done{false}; + bool compress_logs; + + bool force_fsync; + + const std::filesystem::path changelogs_dir; + //size_t written{0}; uint64_t start_index; @@ -598,7 +634,6 @@ void Changelog::writeThread() if (log_is_complete) rotate(append_log->index, writer_lock); - batch_append_ok &= current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); } else From b08be3e60446af42e35f163716a07768a8f7da55 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 14 Dec 2022 03:21:24 +0000 Subject: [PATCH 012/439] fix tests --- tests/clickhouse-test | 7 ++++--- .../01213_alter_rename_compact_part.sql | 3 ++- .../01375_compact_parts_codecs.sql | 20 +++++++++---------- .../02404_memory_bound_merging.sql | 2 +- ...493_max_streams_for_merge_tree_reading.sql | 2 ++ 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index addfcfc8d93..e5c8780b203 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -476,9 +476,10 @@ class SettingsRandomizer: class MergeTreeSettingsRandomizer: settings = { - "ratio_of_defaults_for_sparse_serialization": threshold_generator( - 0.1, 0.6, 0.0, 1.0 - ), + # Temporary disable due to large number of failures. TODO: fix. + # "ratio_of_defaults_for_sparse_serialization": threshold_generator( + # 0.1, 0.6, 0.0, 1.0 + # ), "prefer_fetch_merged_part_size_threshold": threshold_generator( 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), diff --git a/tests/queries/0_stateless/01213_alter_rename_compact_part.sql b/tests/queries/0_stateless/01213_alter_rename_compact_part.sql index 07188ece519..ebf93521d67 100644 --- a/tests/queries/0_stateless/01213_alter_rename_compact_part.sql +++ b/tests/queries/0_stateless/01213_alter_rename_compact_part.sql @@ -12,7 +12,8 @@ ENGINE = MergeTree() PARTITION BY date ORDER BY key settings index_granularity = 8, -min_rows_for_wide_part = 10; +min_rows_for_wide_part = 10, +min_bytes_for_wide_part = '10G'; INSERT INTO table_with_compact_parts SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number) from numbers(9); diff --git a/tests/queries/0_stateless/01375_compact_parts_codecs.sql b/tests/queries/0_stateless/01375_compact_parts_codecs.sql index 35f125515a2..1dd39e67876 100644 --- a/tests/queries/0_stateless/01375_compact_parts_codecs.sql +++ b/tests/queries/0_stateless/01375_compact_parts_codecs.sql @@ -1,13 +1,13 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings DROP TABLE IF EXISTS codecs; -CREATE TABLE codecs (id UInt32, val UInt32, s String) +CREATE TABLE codecs (id UInt32, val UInt32, s String) ENGINE = MergeTree ORDER BY id SETTINGS min_rows_for_wide_part = 10000; INSERT INTO codecs SELECT number, number, toString(number) FROM numbers(1000); -SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) - FROM system.parts +SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) + FROM system.parts WHERE table = 'codecs' AND database = currentDatabase(); SELECT sum(id), sum(val), max(s) FROM codecs; @@ -19,12 +19,12 @@ SELECT sum(id), sum(val), max(s) FROM codecs; DROP TABLE codecs; -CREATE TABLE codecs (id UInt32 CODEC(NONE), val UInt32 CODEC(NONE), s String CODEC(NONE)) +CREATE TABLE codecs (id UInt32 CODEC(NONE), val UInt32 CODEC(NONE), s String CODEC(NONE)) ENGINE = MergeTree ORDER BY id SETTINGS min_rows_for_wide_part = 10000; INSERT INTO codecs SELECT number, number, toString(number) FROM numbers(1000); -SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) - FROM system.parts +SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) + FROM system.parts WHERE table = 'codecs' AND database = currentDatabase(); SELECT sum(id), sum(val), max(s) FROM codecs; @@ -36,12 +36,12 @@ SELECT sum(id), sum(val), max(s) FROM codecs; DROP TABLE codecs; -CREATE TABLE codecs (id UInt32, val UInt32 CODEC(Delta, ZSTD), s String CODEC(ZSTD)) +CREATE TABLE codecs (id UInt32, val UInt32 CODEC(Delta, ZSTD), s String CODEC(ZSTD)) ENGINE = MergeTree ORDER BY id SETTINGS min_rows_for_wide_part = 10000; INSERT INTO codecs SELECT number, number, toString(number) FROM numbers(1000); -SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) - FROM system.parts +SELECT sum(data_compressed_bytes), sum(data_uncompressed_bytes) + FROM system.parts WHERE table = 'codecs' AND database = currentDatabase(); SELECT sum(id), sum(val), max(s) FROM codecs; diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql index c41e2d3abae..a0ac0342c34 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.sql +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings create table t(a UInt64, b UInt64) engine=MergeTree order by a; system stop merges t; diff --git a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql index 29fb6062a8e..12b56cfe674 100644 --- a/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql +++ b/tests/queries/0_stateless/02493_max_streams_for_merge_tree_reading.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + create table t (x UInt64) engine = MergeTree order by x; insert into t select number from numbers_mt(10000000) settings max_insert_threads=8; From d454e00016c64e312c935215fcbb61124ea4f554 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 15 Dec 2022 08:12:02 +0000 Subject: [PATCH 013/439] Changelog refactor --- src/Coordination/Changelog.cpp | 87 ++++++++++++++-------------------- src/Coordination/Changelog.h | 6 +-- 2 files changed, 38 insertions(+), 55 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 797958b0494..929d5e14d74 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -29,10 +29,10 @@ namespace constexpr auto DEFAULT_PREFIX = "changelog"; -std::string formatChangelogPath(const std::string & prefix, const ChangelogFileDescription & name) +std::string formatChangelogPath(const std::string & prefix, const ChangelogFileDescription & name, bool with_to_log) { std::filesystem::path path(prefix); - path /= std::filesystem::path(name.prefix + "_" + std::to_string(name.from_log_index) + "_" + std::to_string(name.to_log_index) + "." + name.extension); + path /= std::filesystem::path(name.prefix + "_" + std::to_string(name.from_log_index) + (with_to_log ? "_" + std::to_string(name.to_log_index) : "") + "." + name.extension); return path; } @@ -74,11 +74,12 @@ class ChangelogWriter { public: ChangelogWriter(std::map & existing_changelogs_, bool compress_logs_, bool force_fsync_, - const std::filesystem::path & changelogs_dir_) + const std::filesystem::path & changelogs_dir_, uint64_t rotate_interval_) : existing_changelogs(existing_changelogs_) , memory_buf(std::make_unique(memory)) , compress_logs(compress_logs_) , force_fsync(force_fsync_) + , rotate_interval(rotate_interval_) , changelogs_dir(changelogs_dir_) { if (compress_logs) @@ -97,11 +98,17 @@ public: file_buf = std::make_unique(file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); start_index = file_description.from_log_index; current_file_description = &file_description; + prealloc_done = false; } bool appendRecord(ChangelogRecord && record) { + const bool log_is_complete = record.header.index - getStartIndex() == rotate_interval; + + if (log_is_complete) + rotate(record.header.index); + if (!prealloc_done) [[unlikely]] { int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_size); @@ -145,7 +152,7 @@ public: void flush() { /// Fsync file system if needed - if (force_fsync) + if (force_fsync && file_buf) file_buf->sync(); } @@ -158,6 +165,15 @@ public: { flush(); + if (current_file_description) + { + const auto old_path = current_file_description->path; + auto new_path = formatChangelogPath(changelogs_dir, *current_file_description, true); + + std::filesystem::rename(old_path, new_path); + current_file_description->path = std::move(new_path); + } + /// Start new one ChangelogFileDescription new_description; new_description.prefix = DEFAULT_PREFIX; @@ -168,7 +184,7 @@ public: if (compress_logs) new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); - new_description.path = formatChangelogPath(changelogs_dir, new_description); + new_description.path = formatChangelogPath(changelogs_dir, new_description, false); LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description.path); auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); @@ -202,6 +218,8 @@ private: bool force_fsync; + uint64_t rotate_interval; + const std::filesystem::path changelogs_dir; //size_t written{0}; @@ -342,9 +360,7 @@ Changelog::Changelog( : changelogs_dir(changelogs_dir_) , changelogs_detached_dir(changelogs_dir / "detached") , rotate_interval(rotate_interval_) - , force_sync(force_sync_) , log(log_) - , compress_logs(compress_logs_) , write_operations(std::numeric_limits::max()) { /// Load all files in changelog directory @@ -367,6 +383,8 @@ Changelog::Changelog( clean_log_thread = ThreadFromGlobalPool([this] { cleanLogThread(); }); write_thread = ThreadFromGlobalPool([this] { writeThread(); }); + + current_writer = std::make_unique(existing_changelogs, compress_logs_, force_sync_, changelogs_dir, rotate_interval_); } void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uint64_t logs_to_keep) @@ -438,10 +456,10 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin max_log_id = last_log_read_result->last_read_index; /// How many entries we have in the last changelog - uint64_t expected_entries_in_log = changelog_description.expectedEntriesCountInLog(); + uint64_t log_count = changelog_description.storedLogCount(); /// Unfinished log - if (last_log_read_result->error || last_log_read_result->total_entries_read_from_log < expected_entries_in_log) + if (last_log_read_result->error || last_log_read_result->total_entries_read_from_log < log_count) { last_log_is_not_complete = true; break; @@ -479,7 +497,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin assert(existing_changelogs.find(last_log_read_result->log_start_index)->first == existing_changelogs.rbegin()->first); /// Continue to write into incomplete existing log if it doesn't finished with error - auto description = existing_changelogs[last_log_read_result->log_start_index]; + auto & description = existing_changelogs[last_log_read_result->log_start_index]; if (last_log_read_result->last_read_index == 0 || last_log_read_result->error) /// If it's broken log then remove it { @@ -502,13 +520,10 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin } -void Changelog::initWriter(const ChangelogFileDescription & description) +void Changelog::initWriter(ChangelogFileDescription & description) { - if (description.expectedEntriesCountInLog() != rotate_interval) - LOG_TRACE(log, "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, description.expectedEntriesCountInLog()); - LOG_TRACE(log, "Continue to write into {}", description.path); - current_writer = std::make_unique(description.path, WriteMode::Append, description.from_log_index); + current_writer->setFile(description, WriteMode::Append); } namespace @@ -571,29 +586,6 @@ void Changelog::removeAllLogs() logs.clear(); } -void Changelog::rotate(uint64_t new_start_log_index, std::lock_guard &) -{ - /// Flush previous log - if (current_writer) - current_writer->flush(force_sync); - - /// Start new one - ChangelogFileDescription new_description; - new_description.prefix = DEFAULT_PREFIX; - new_description.from_log_index = new_start_log_index; - new_description.to_log_index = new_start_log_index + rotate_interval - 1; - new_description.extension = "bin"; - - if (compress_logs) - new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); - - new_description.path = formatChangelogPath(changelogs_dir, new_description); - - LOG_TRACE(log, "Starting new changelog {}", new_description.path); - existing_changelogs[new_start_log_index] = new_description; - current_writer = std::make_unique(new_description.path, WriteMode::Rewrite, new_start_log_index); -} - ChangelogRecord Changelog::buildRecord(uint64_t index, const LogEntryPtr & log_entry) { ChangelogRecord record; @@ -628,12 +620,6 @@ void Changelog::writeThread() std::lock_guard writer_lock(writer_mutex); assert(current_writer); - const auto & current_changelog_description = existing_changelogs[current_writer->getStartIndex()]; - const bool log_is_complete = append_log->index - current_writer->getStartIndex() == current_changelog_description.expectedEntriesCountInLog(); - - if (log_is_complete) - rotate(append_log->index, writer_lock); - batch_append_ok &= current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); } else @@ -644,8 +630,7 @@ void Changelog::writeThread() { { std::lock_guard writer_lock(writer_mutex); - if (current_writer) - current_writer->flush(force_sync); + current_writer->flush(); } { @@ -698,15 +683,15 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) { auto index_changelog = existing_changelogs.lower_bound(index); - ChangelogFileDescription description; + ChangelogFileDescription * description; if (index_changelog->first == index) /// exactly this file starts from index - description = index_changelog->second; + description = &index_changelog->second; else - description = std::prev(index_changelog)->second; + description = &std::prev(index_changelog)->second; - /// Initialize writer from this log file - current_writer = std::make_unique(description.path, WriteMode::Append, index_changelog->first); + current_writer->flush(); + current_writer->setFile(*description, WriteMode::Append); /// Remove all subsequent files if overwritten something in previous one auto to_remove_itr = existing_changelogs.upper_bound(index); diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 157ab68df69..e8a72b4883e 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -61,7 +61,7 @@ struct ChangelogFileDescription std::string path; /// How many entries should be stored in this log - uint64_t expectedEntriesCountInLog() const + uint64_t storedLogCount() const { return to_log_index - from_log_index + 1; } @@ -162,7 +162,7 @@ private: /// Remove all logs from disk void removeAllLogs(); /// Init writer for existing log with some entries already written - void initWriter(const ChangelogFileDescription & description); + void initWriter(ChangelogFileDescription & description); /// Clean useless log files in a background thread void cleanLogThread(); @@ -170,9 +170,7 @@ private: const std::filesystem::path changelogs_dir; const std::filesystem::path changelogs_detached_dir; const uint64_t rotate_interval; - const bool force_sync; Poco::Logger * log; - bool compress_logs; std::mutex writer_mutex; /// Current writer for changelog file From ee900ca7dff6999eafddfb6b453fbe9679e93013 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 15 Dec 2022 13:09:52 +0000 Subject: [PATCH 014/439] Add truncate --- src/Coordination/Changelog.cpp | 101 +++++++++++++----- src/Coordination/Changelog.h | 3 - src/IO/ZstdDeflatingAppendableWriteBuffer.cpp | 50 ++++----- src/IO/ZstdDeflatingAppendableWriteBuffer.h | 14 +-- 4 files changed, 109 insertions(+), 59 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 929d5e14d74..40d4db79555 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -29,10 +29,10 @@ namespace constexpr auto DEFAULT_PREFIX = "changelog"; -std::string formatChangelogPath(const std::string & prefix, const ChangelogFileDescription & name, bool with_to_log) +std::string formatChangelogPath(const std::string & prefix, const std::string & name_prefix, uint64_t from_index, uint64_t to_index, const std::string & extension) { std::filesystem::path path(prefix); - path /= std::filesystem::path(name.prefix + "_" + std::to_string(name.from_log_index) + (with_to_log ? "_" + std::to_string(name.to_log_index) : "") + "." + name.extension); + path /= std::filesystem::path(name_prefix + "_" + std::to_string(from_index) + "_" + std::to_string(to_index) + "." + extension); return path; } @@ -66,7 +66,7 @@ Checksum computeRecordChecksum(const ChangelogRecord & record) return hash.get64(); } -constexpr size_t log_size = 50 * 1024 * 1024; +constexpr size_t log_size = 60 * 1024 * 1024; } @@ -97,10 +97,23 @@ public: { file_buf = std::make_unique(file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); start_index = file_description.from_log_index; + last_index = start_index; current_file_description = &file_description; + + if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(file_description.path)) + { + ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); + file_buf->sync(); + } + prealloc_done = false; } + bool isFileSet() const + { + return file_buf != nullptr; + } + bool appendRecord(ChangelogRecord && record) { @@ -111,11 +124,10 @@ public: if (!prealloc_done) [[unlikely]] { - int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_size); - if (res == ENOSPC) - return false; + tryPreallocateForFile(); - prealloc_done = true; + if (!prealloc_done) + return false; } writeIntBinary(computeRecordChecksum(record), getBuffer()); @@ -139,13 +151,22 @@ public: cur_memory_buf.finalize(); - // TODO: rotate if it's larger than max log size + if (written + memory.size() > total_bytes_available) + { + LOG_INFO(&Poco::Logger::get("LOGGER"), "Size of the file is too big, rotating"); + rotate(record.header.index); + + tryPreallocateForFile(); + if (!prealloc_done) + return false; + } file_buf->write(memory.raw_data(), memory.size()); - cur_memory_buf.restart(); + written += memory.size(); - current_file_description->to_log_index++; + cur_memory_buf.restart(); + last_index = record.header.index; return true; } @@ -163,14 +184,23 @@ public: void rotate(uint64_t new_start_log_index) { - flush(); - - if (current_file_description) + if (file_buf) { - const auto old_path = current_file_description->path; - auto new_path = formatChangelogPath(changelogs_dir, *current_file_description, true); + if (compress_logs && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) + { + ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); + file_buf->sync(); + written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + } - std::filesystem::rename(old_path, new_path); + flush(); + + ftruncate(file_buf->getFD(), written); + + LOG_INFO(&Poco::Logger::get("LOGGER"), "Truncated to {} bytes", written); + + auto new_path = formatChangelogPath(changelogs_dir, current_file_description->prefix, current_file_description->from_log_index, last_index, current_file_description->extension); + std::filesystem::rename(current_file_description->path, new_path); current_file_description->path = std::move(new_path); } @@ -178,13 +208,13 @@ public: ChangelogFileDescription new_description; new_description.prefix = DEFAULT_PREFIX; new_description.from_log_index = new_start_log_index; - new_description.to_log_index = new_start_log_index; + new_description.to_log_index = new_start_log_index + rotate_interval - 1; new_description.extension = "bin"; if (compress_logs) new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); - new_description.path = formatChangelogPath(changelogs_dir, new_description, false); + new_description.path = formatChangelogPath(changelogs_dir, new_description.prefix, new_start_log_index, new_start_log_index + rotate_interval - 1, new_description.extension); LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description.path); auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); @@ -200,6 +230,28 @@ private: return *memory_buf; } + void tryPreallocateForFile() + { + int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_size); + if (res == ENOSPC) + return; + + struct stat buf; + res = fstat(file_buf->getFD(), &buf); + assert(res == 0); + + LOG_INFO(&Poco::Logger::get("LOGGER"), "ALLOCATED BLOCKS {}, size {}", buf.st_blocks, buf.st_size); + + written = buf.st_size; + total_bytes_available = buf.st_blocks * 512; + + // always leave space for last termination block + if (compress_logs) + total_bytes_available -= ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + + prealloc_done = true; + } + std::map & existing_changelogs; using MemoryBuffer = PODArray; @@ -209,6 +261,8 @@ private: ChangelogFileDescription * current_file_description{nullptr}; std::unique_ptr file_buf; + size_t written{0}; + size_t total_bytes_available{0}; std::unique_ptr compressed_buffer; @@ -222,9 +276,8 @@ private: const std::filesystem::path changelogs_dir; - //size_t written{0}; - uint64_t start_index; + uint64_t last_index; }; struct ChangelogReadResult @@ -423,7 +476,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin removeAllLogs(); min_log_id = last_commited_log_index; max_log_id = last_commited_log_index == 0 ? 0 : last_commited_log_index - 1; - rotate(max_log_id + 1, writer_lock); + current_writer->rotate(max_log_id + 1); return; } else if (changelog_description.from_log_index > start_to_read_from) @@ -513,8 +566,8 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin } /// Start new log if we don't initialize writer from previous log. All logs can be "complete". - if (!current_writer) - rotate(max_log_id + 1, writer_lock); + if (!current_writer->isFileSet()) + current_writer->rotate(max_log_id + 1); initialized = true; } @@ -760,7 +813,7 @@ void Changelog::compact(uint64_t up_to_log_index) std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); if (need_rotate) - rotate(up_to_log_index + 1, lock); + current_writer->rotate(up_to_log_index + 1); LOG_INFO(log, "Compaction up to {} finished new min index {}, new max index {}", up_to_log_index, min_log_id, max_log_id); } diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index e8a72b4883e..fa67b902795 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -147,9 +147,6 @@ private: /// Pack log_entry into changelog record static ChangelogRecord buildRecord(uint64_t index, const LogEntryPtr & log_entry); - /// Starts new file [new_start_log_index, new_start_log_index + rotate_interval] - void rotate(uint64_t new_start_log_index, std::lock_guard & writer_lock); - /// Currently existing changelogs std::map existing_changelogs; diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp index cbac8eee942..ea2bb8f0c5a 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp @@ -160,42 +160,42 @@ void ZstdDeflatingAppendableWriteBuffer::finalizeZstd() } } -void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock() +void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(WriteBuffer & write_buffer) { /// HACK: https://github.com/facebook/zstd/issues/2090#issuecomment-620158967 - if (out->buffer().size() - out->offset() < ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()) - out->next(); + if (write_buffer.buffer().size() - write_buffer.offset() < ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()) + write_buffer.next(); - std::memcpy(out->buffer().begin() + out->offset(), + std::memcpy(write_buffer.buffer().begin() + write_buffer.offset(), ZSTD_CORRECT_TERMINATION_LAST_BLOCK.data(), ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()); - out->position() = out->buffer().begin() + out->offset() + ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + write_buffer.position() = write_buffer.buffer().begin() + write_buffer.offset() + ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); } -bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock() +bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(const std::string & file_name) { - //ReadBufferFromFile reader(out->getFileName()); - //auto fsize = reader.getFileSize(); - //if (fsize > 3) - //{ - // std::array result; - // reader.seek(fsize - 3, SEEK_SET); - // reader.readStrict(result.data(), 3); + ReadBufferFromFile reader(file_name); + auto fsize = reader.getFileSize(); + if (fsize > 3) + { + std::array result; + reader.seek(fsize - 3, SEEK_SET); + reader.readStrict(result.data(), 3); - // /// If we don't have correct block in the end, then we need to add it manually. - // /// NOTE: maybe we can have the same bytes in case of data corruption/unfinished write. - // /// But in this case file still corrupted and we have to remove it. - // return result != ZSTD_CORRECT_TERMINATION_LAST_BLOCK; - //} - //else if (fsize > 0) - //{ - // throw Exception( - // ErrorCodes::ZSTD_ENCODER_FAILED, - // "Trying to write to non-empty file '{}' with tiny size {}. It can lead to data corruption", - // out->getFileName(), fsize); - //} + /// If we don't have correct block in the end, then we need to add it manually. + /// NOTE: maybe we can have the same bytes in case of data corruption/unfinished write. + /// But in this case file still corrupted and we have to remove it. + return result != ZSTD_CORRECT_TERMINATION_LAST_BLOCK; + } + else if (fsize > 0) + { + throw Exception( + ErrorCodes::ZSTD_ENCODER_FAILED, + "Trying to write to non-empty file '{}' with tiny size {}. It can lead to data corruption", + file_name, fsize); + } return false; } diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.h b/src/IO/ZstdDeflatingAppendableWriteBuffer.h index cf926e87d43..4819efbc1ab 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.h +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.h @@ -45,6 +45,13 @@ public: WriteBuffer * getNestedBuffer() { return out.get(); } + /// Read three last bytes from non-empty compressed file and compares them with + /// ZSTD_CORRECT_TERMINATION_LAST_BLOCK. + static bool isNeedToAddEmptyBlock(const std::string & file_name); + + /// Adding zstd empty block (ZSTD_CORRECT_TERMINATION_LAST_BLOCK) to out.working_buffer + static void addEmptyBlock(WriteBuffer & write_buffer); + private: /// NOTE: will fill compressed data to the out.working_buffer, but will not call out.next method until the buffer is full void nextImpl() override; @@ -60,13 +67,6 @@ private: void finalizeAfter(); void finalizeZstd(); - /// Read three last bytes from non-empty compressed file and compares them with - /// ZSTD_CORRECT_TERMINATION_LAST_BLOCK. - static bool isNeedToAddEmptyBlock(); - - /// Adding zstd empty block (ZSTD_CORRECT_TERMINATION_LAST_BLOCK) to out.working_buffer - void addEmptyBlock(); - std::unique_ptr out; //bool append_to_existing_file = false; From 33dc0d8283bd0205e3597b2ce12a39a847795b3d Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 16 Dec 2022 17:50:01 +0000 Subject: [PATCH 015/439] fix more tests --- src/Client/ClientBase.cpp | 7 ++++++- tests/clickhouse-test | 4 ++-- tests/queries/0_stateless/00955_test_final_mark.sql | 2 +- .../0_stateless/01710_projection_group_by_order_by.sql | 4 ++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index a043f7da47d..7268dd0ce60 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2353,9 +2353,14 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, /// may clash. Query settings have higher priority and we just /// skip ambiguous merge tree settings. auto & main_options = options_description.main_description.value(); + + NameSet main_option_names; + for (const auto & option : main_options.options()) + main_option_names.insert(option->long_name()); + for (const auto & setting : cmd_merge_tree_settings.all()) { - if (main_options.find_nothrow(setting.getName(), false)) + if (main_option_names.contains(setting.getName())) continue; if (allow_repeated_settings) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e5c8780b203..fe717a20c00 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -484,10 +484,10 @@ class MergeTreeSettingsRandomizer: 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), "vertical_merge_algorithm_min_rows_to_activate": threshold_generator( - 0.25, 0.25, 1, 100000000 + 0.4, 0.4, 1, 1000000 ), "vertical_merge_algorithm_min_columns_to_activate": threshold_generator( - 0.25, 0.25, 1, 1000 + 0.4, 0.4, 1, 100 ), "min_merge_bytes_to_use_direct_io": threshold_generator( 0.25, 0.25, 1, 10 * 1024 * 1024 * 1024 diff --git a/tests/queries/0_stateless/00955_test_final_mark.sql b/tests/queries/0_stateless/00955_test_final_mark.sql index 6615c945ef8..44eb4a69c2d 100644 --- a/tests/queries/0_stateless/00955_test_final_mark.sql +++ b/tests/queries/0_stateless/00955_test_final_mark.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql index d93339aedef..780162e0284 100644 --- a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql +++ b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql @@ -1,3 +1,7 @@ +--Tags: no-random-merge-tree-settings +-- Tag no-random-merge-tree-settings: bug in formatting of projections. +-- https://github.com/ClickHouse/ClickHouse/issues/44318 + DROP TABLE IF EXISTS t; drop table if exists tp; From 0939ad4459d35f47c01bd6b264e94e5bb8c24108 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 19 Dec 2022 11:21:19 +0000 Subject: [PATCH 016/439] fix more tests --- .../00926_adaptive_index_granularity_collapsing_merge_tree.sql | 2 ++ .../0_stateless/00926_adaptive_index_granularity_merge_tree.sql | 2 +- .../queries/0_stateless/00926_adaptive_index_granularity_pk.sql | 2 +- .../00926_adaptive_index_granularity_replacing_merge_tree.sql | 2 +- ...aptive_index_granularity_versioned_collapsing_merge_tree.sql | 2 ++ tests/queries/0_stateless/01158_zookeeper_log_long.sql | 2 +- ...655_plan_optimizations_optimize_read_in_window_order_long.sh | 2 +- 7 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql index 81820d2c60a..42d5f60b8b2 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_collapsing_merge_tree.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + ----- Group of very similar simple tests ------ DROP TABLE IF EXISTS zero_rows_per_granule; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql index d07323bb3d6..e812a6cae5c 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_merge_tree.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings ----- Group of very similar simple tests ------ select '----HORIZONTAL MERGE TESTS----'; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql index ed9be1170ec..636cd6589ce 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_pk.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings SET send_logs_level = 'fatal'; SELECT '----00489----'; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql index 7f53ab8b329..a5170aa8058 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_replacing_merge_tree.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-random-merge-tree-settings ----- Group of very similar simple tests ------ DROP TABLE IF EXISTS zero_rows_per_granule; diff --git a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql index 0fa20c863c8..43495ea3697 100644 --- a/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql +++ b/tests/queries/0_stateless/00926_adaptive_index_granularity_versioned_collapsing_merge_tree.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + ----- Group of very similar simple tests ------ DROP TABLE IF EXISTS zero_rows_per_granule; diff --git a/tests/queries/0_stateless/01158_zookeeper_log_long.sql b/tests/queries/0_stateless/01158_zookeeper_log_long.sql index cabb0ffe294..45771494af6 100644 --- a/tests/queries/0_stateless/01158_zookeeper_log_long.sql +++ b/tests/queries/0_stateless/01158_zookeeper_log_long.sql @@ -1,4 +1,4 @@ --- Tags: long, zookeeper, no-replicated-database, no-polymorphic-parts +-- Tags: long, zookeeper, no-replicated-database, no-polymorphic-parts, no-random-merge-tree-settings -- Tag no-replicated-database: Fails due to additional replicas or shards SET insert_keeper_fault_injection_probability=0; -- disable fault injection; part ids are non-deterministic in case of insert retries diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh index bfb4601e62b..e5e57ddb78a 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From f241701c2a37c497a2a234c52b4fb49f32d1e2dc Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 19 Dec 2022 11:48:54 +0000 Subject: [PATCH 017/439] Add checks when fallocate is not supported --- src/Coordination/Changelog.cpp | 196 +++++++++++++++++++++------------ src/Coordination/Changelog.h | 44 +++----- 2 files changed, 145 insertions(+), 95 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 40d4db79555..dd677e1d413 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -1,11 +1,11 @@ -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include #include +#include #include #include #include @@ -29,7 +29,8 @@ namespace constexpr auto DEFAULT_PREFIX = "changelog"; -std::string formatChangelogPath(const std::string & prefix, const std::string & name_prefix, uint64_t from_index, uint64_t to_index, const std::string & extension) +std::string formatChangelogPath( + const std::string & prefix, const std::string & name_prefix, uint64_t from_index, uint64_t to_index, const std::string & extension) { std::filesystem::path path(prefix); path /= std::filesystem::path(name_prefix + "_" + std::to_string(from_index) + "_" + std::to_string(to_index) + "." + extension); @@ -66,26 +67,29 @@ Checksum computeRecordChecksum(const ChangelogRecord & record) return hash.get64(); } -constexpr size_t log_size = 60 * 1024 * 1024; - } class ChangelogWriter { public: - ChangelogWriter(std::map & existing_changelogs_, bool compress_logs_, bool force_fsync_, - const std::filesystem::path & changelogs_dir_, uint64_t rotate_interval_) + ChangelogWriter( + std::map & existing_changelogs_, + bool compress_logs_, + bool force_fsync_, + const std::filesystem::path & changelogs_dir_, + uint64_t rotate_interval_, + uint64_t max_log_file_size_) : existing_changelogs(existing_changelogs_) , memory_buf(std::make_unique(memory)) , compress_logs(compress_logs_) , force_fsync(force_fsync_) , rotate_interval(rotate_interval_) + , max_log_file_size(max_log_file_size_) , changelogs_dir(changelogs_dir_) { if (compress_logs) { - compressed_buffer = std::make_unique( - std::move(memory_buf), /* compression level = */ 3); + compressed_buffer = std::make_unique(std::move(memory_buf), /* compression level = */ 3); } else { @@ -95,7 +99,8 @@ public: void setFile(ChangelogFileDescription & file_description, WriteMode mode) { - file_buf = std::make_unique(file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); + file_buf = std::make_unique( + file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); start_index = file_description.from_log_index; last_index = start_index; current_file_description = &file_description; @@ -109,10 +114,7 @@ public: prealloc_done = false; } - bool isFileSet() const - { - return file_buf != nullptr; - } + bool isFileSet() const { return file_buf != nullptr; } bool appendRecord(ChangelogRecord && record) @@ -127,7 +129,9 @@ public: tryPreallocateForFile(); if (!prealloc_done) + { return false; + } } writeIntBinary(computeRecordChecksum(record), getBuffer()); @@ -147,13 +151,14 @@ public: compressed_buffer->next(); } - WriteBufferMemory & cur_memory_buf = compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; + WriteBufferMemory & cur_memory_buf + = compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; cur_memory_buf.finalize(); - if (written + memory.size() > total_bytes_available) + // if the file is too big, rotate + if (total_bytes_available != 0 && written + memory.size() > total_bytes_available) { - LOG_INFO(&Poco::Logger::get("LOGGER"), "Size of the file is too big, rotating"); rotate(record.header.index); tryPreallocateForFile(); @@ -177,10 +182,7 @@ public: file_buf->sync(); } - uint64_t getStartIndex() const - { - return start_index; - } + uint64_t getStartIndex() const { return start_index; } void rotate(uint64_t new_start_log_index) { @@ -197,9 +199,12 @@ public: ftruncate(file_buf->getFD(), written); - LOG_INFO(&Poco::Logger::get("LOGGER"), "Truncated to {} bytes", written); - - auto new_path = formatChangelogPath(changelogs_dir, current_file_description->prefix, current_file_description->from_log_index, last_index, current_file_description->extension); + auto new_path = formatChangelogPath( + changelogs_dir, + current_file_description->prefix, + current_file_description->from_log_index, + last_index, + current_file_description->extension); std::filesystem::rename(current_file_description->path, new_path); current_file_description->path = std::move(new_path); } @@ -214,7 +219,12 @@ public: if (compress_logs) new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); - new_description.path = formatChangelogPath(changelogs_dir, new_description.prefix, new_start_log_index, new_start_log_index + rotate_interval - 1, new_description.extension); + new_description.path = formatChangelogPath( + changelogs_dir, + new_description.prefix, + new_start_log_index, + new_start_log_index + rotate_interval - 1, + new_description.extension); LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description.path); auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); @@ -222,6 +232,12 @@ public: setFile(it->second, WriteMode::Rewrite); } + void finalize() + { + if (file_buf) + ftruncate(file_buf->getFD(), written); + } + private: WriteBuffer & getBuffer() { @@ -232,18 +248,28 @@ private: void tryPreallocateForFile() { - int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_size); - if (res == ENOSPC) + if (max_log_file_size == 0) + { + total_bytes_available = 0; + prealloc_done = true; return; + } + + int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, max_log_file_size); + if (res == ENOSPC) + { + LOG_FATAL(&Poco::Logger::get("Changelog"), "Failed to allocate enough space on disk for logs"); + return; + } + + bool fallocate_ok = res == 0; struct stat buf; res = fstat(file_buf->getFD(), &buf); assert(res == 0); - LOG_INFO(&Poco::Logger::get("LOGGER"), "ALLOCATED BLOCKS {}, size {}", buf.st_blocks, buf.st_size); - written = buf.st_size; - total_bytes_available = buf.st_blocks * 512; + total_bytes_available = fallocate_ok ? buf.st_blocks * 512 : max_log_file_size; // always leave space for last termination block if (compress_logs) @@ -269,11 +295,13 @@ private: bool prealloc_done{false}; bool compress_logs; - + bool force_fsync; uint64_t rotate_interval; + uint64_t max_log_file_size; + const std::filesystem::path changelogs_dir; uint64_t start_index; @@ -305,8 +333,7 @@ struct ChangelogReadResult class ChangelogReader { public: - explicit ChangelogReader(const std::string & filepath_) - : filepath(filepath_) + explicit ChangelogReader(const std::string & filepath_) : filepath(filepath_) { auto compression_method = chooseCompressionMethod(filepath, ""); auto read_buffer_from_file = std::make_unique(filepath); @@ -335,7 +362,8 @@ public: readIntBinary(record.header.blob_size, *read_buf); if (record.header.version > CURRENT_CHANGELOG_VERSION) - throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported changelog version {} on path {}", record.header.version, filepath); + throw Exception( + ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported changelog version {} on path {}", record.header.version, filepath); /// Read data if (record.header.blob_size != 0) @@ -352,14 +380,18 @@ public: Checksum checksum = computeRecordChecksum(record); if (checksum != record_checksum) { - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, - "Checksums doesn't match for log {} (version {}), index {}, blob_size {}", - filepath, record.header.version, record.header.index, record.header.blob_size); + throw Exception( + ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksums doesn't match for log {} (version {}), index {}, blob_size {}", + filepath, + record.header.version, + record.header.index, + record.header.blob_size); } /// Check for duplicated changelog ids if (logs.contains(record.header.index)) - std::erase_if(logs, [&record] (const auto & item) { return item.first >= record.header.index; }); + std::erase_if(logs, [&record](const auto & item) { return item.first >= record.header.index; }); result.total_entries_read_from_log += 1; @@ -409,7 +441,8 @@ Changelog::Changelog( uint64_t rotate_interval_, bool force_sync_, Poco::Logger * log_, - bool compress_logs_) + bool compress_logs_, + uint64_t max_log_file_size) : changelogs_dir(changelogs_dir_) , changelogs_detached_dir(changelogs_dir / "detached") , rotate_interval(rotate_interval_) @@ -437,7 +470,8 @@ Changelog::Changelog( write_thread = ThreadFromGlobalPool([this] { writeThread(); }); - current_writer = std::make_unique(existing_changelogs, compress_logs_, force_sync_, changelogs_dir, rotate_interval_); + current_writer = std::make_unique( + existing_changelogs, compress_logs_, force_sync_, changelogs_dir, rotate_interval_, max_log_file_size); } void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uint64_t logs_to_keep) @@ -460,7 +494,6 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin /// Got through changelog files in order of start_index for (const auto & [changelog_start_index, changelog_description] : existing_changelogs) { - /// [from_log_index.>=.......start_to_read_from.....<=.to_log_index] if (changelog_description.to_log_index >= start_to_read_from) { @@ -469,9 +502,15 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin /// Our first log starts from the more fresh log_id than we required to read and this changelog is not empty log. /// So we are missing something in our logs, but it's not dataloss, we will receive snapshot and required /// entries from leader. - if (changelog_description.from_log_index > last_commited_log_index && (changelog_description.from_log_index - last_commited_log_index) > 1) + if (changelog_description.from_log_index > last_commited_log_index + && (changelog_description.from_log_index - last_commited_log_index) > 1) { - LOG_ERROR(log, "Some records were lost, last committed log index {}, smallest available log index on disk {}. Hopefully will receive missing records from leader.", last_commited_log_index, changelog_description.from_log_index); + LOG_ERROR( + log, + "Some records were lost, last committed log index {}, smallest available log index on disk {}. Hopefully will " + "receive missing records from leader.", + last_commited_log_index, + changelog_description.from_log_index); /// Nothing to do with our more fresh log, leader will overwrite them, so remove everything and just start from last_commited_index removeAllLogs(); min_log_id = last_commited_log_index; @@ -482,12 +521,22 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin else if (changelog_description.from_log_index > start_to_read_from) { /// We don't have required amount of reserved logs, but nothing was lost. - LOG_WARNING(log, "Don't have required amount of reserved log records. Need to read from {}, smallest available log index on disk {}.", start_to_read_from, changelog_description.from_log_index); + LOG_WARNING( + log, + "Don't have required amount of reserved log records. Need to read from {}, smallest available log index on disk " + "{}.", + start_to_read_from, + changelog_description.from_log_index); } } else if ((changelog_description.from_log_index - last_log_read_result->last_read_index) > 1) { - LOG_ERROR(log, "Some records were lost, last found log index {}, while the next log index on disk is {}. Hopefully will receive missing records from leader.", last_log_read_result->last_read_index, changelog_description.from_log_index); + LOG_ERROR( + log, + "Some records were lost, last found log index {}, while the next log index on disk is {}. Hopefully will receive " + "missing records from leader.", + last_log_read_result->last_read_index, + changelog_description.from_log_index); removeAllLogsAfter(last_log_read_result->log_start_index); break; } @@ -509,7 +558,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin max_log_id = last_log_read_result->last_read_index; /// How many entries we have in the last changelog - uint64_t log_count = changelog_description.storedLogCount(); + uint64_t log_count = changelog_description.expectedEntriesCountInLog(); /// Unfinished log if (last_log_read_result->error || last_log_read_result->total_entries_read_from_log < log_count) @@ -531,7 +580,11 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin } else if (last_commited_log_index != 0 && max_log_id < last_commited_log_index - 1) /// If we have more fresh snapshot than our logs { - LOG_WARNING(log, "Our most fresh log_id {} is smaller than stored data in snapshot {}. It can indicate data loss. Removing outdated logs.", max_log_id, last_commited_log_index - 1); + LOG_WARNING( + log, + "Our most fresh log_id {} is smaller than stored data in snapshot {}. It can indicate data loss. Removing outdated logs.", + max_log_id, + last_commited_log_index - 1); removeAllLogs(); min_log_id = last_commited_log_index; @@ -557,7 +610,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin LOG_INFO(log, "Removing chagelog {} because it's empty or read finished with error", description.path); std::filesystem::remove(description.path); existing_changelogs.erase(last_log_read_result->log_start_index); - std::erase_if(logs, [last_log_read_result] (const auto & item) { return item.first >= last_log_read_result->log_start_index; }); + std::erase_if(logs, [last_log_read_result](const auto & item) { return item.first >= last_log_read_result->log_start_index; }); } else { @@ -582,18 +635,18 @@ void Changelog::initWriter(ChangelogFileDescription & description) namespace { -std::string getCurrentTimestampFolder() -{ - const auto timestamp = LocalDateTime{std::time(nullptr)}; - return fmt::format( - "{:02}{:02}{:02}T{:02}{:02}{:02}", - timestamp.year(), - timestamp.month(), - timestamp.day(), - timestamp.hour(), - timestamp.minute(), - timestamp.second()); -} + std::string getCurrentTimestampFolder() + { + const auto timestamp = LocalDateTime{std::time(nullptr)}; + return fmt::format( + "{:02}{:02}{:02}T{:02}{:02}{:02}", + timestamp.year(), + timestamp.month(), + timestamp.day(), + timestamp.hour(), + timestamp.minute(), + timestamp.second()); + } } @@ -629,7 +682,7 @@ void Changelog::removeAllLogsAfter(uint64_t remove_after_log_start_index) LOG_WARNING(log, "Removing changelogs that go after broken changelog entry"); removeExistingLogs(start_to_remove_from_itr, existing_changelogs.end()); - std::erase_if(logs, [start_to_remove_from_log_id] (const auto & item) { return item.first >= start_to_remove_from_log_id; }); + std::erase_if(logs, [start_to_remove_from_log_id](const auto & item) { return item.first >= start_to_remove_from_log_id; }); } void Changelog::removeAllLogs() @@ -673,7 +726,7 @@ void Changelog::writeThread() std::lock_guard writer_lock(writer_mutex); assert(current_writer); - batch_append_ok &= current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); + batch_append_ok = current_writer->appendRecord(buildRecord(append_log->index, append_log->log_entry)); } else { @@ -758,7 +811,7 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) /// Remove redundant logs from memory /// Everything >= index must be removed - std::erase_if(logs, [index] (const auto & item) { return item.first >= index; }); + std::erase_if(logs, [index](const auto & item) { return item.first >= index; }); /// Now we can actually override entry at index appendEntry(index, log_entry); @@ -786,7 +839,10 @@ void Changelog::compact(uint64_t up_to_log_index) { if (current_writer && itr->second.from_log_index == current_writer->getStartIndex()) { - LOG_INFO(log, "Trying to remove log {} which is current active log for write. Possibly this node recovers from snapshot", itr->second.path); + LOG_INFO( + log, + "Trying to remove log {} which is current active log for write. Possibly this node recovers from snapshot", + itr->second.path); need_rotate = true; current_writer.reset(); } @@ -810,7 +866,7 @@ void Changelog::compact(uint64_t up_to_log_index) } /// Compaction from the past is possible, so don't make our min_log_id smaller. min_log_id = std::max(min_log_id, up_to_log_index + 1); - std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); + std::erase_if(logs, [up_to_log_index](const auto & item) { return item.first <= up_to_log_index; }); if (need_rotate) current_writer->rotate(up_to_log_index + 1); @@ -953,6 +1009,8 @@ void Changelog::shutdown() if (write_thread.joinable()) write_thread.join(); + + current_writer->finalize(); } Changelog::~Changelog() diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index fa67b902795..99f936ba75a 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -1,14 +1,14 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include #include namespace DB @@ -61,10 +61,7 @@ struct ChangelogFileDescription std::string path; /// How many entries should be stored in this log - uint64_t storedLogCount() const - { - return to_log_index - from_log_index + 1; - } + uint64_t expectedEntriesCountInLog() const { return to_log_index - from_log_index + 1; } }; class ChangelogWriter; @@ -74,10 +71,14 @@ class ChangelogWriter; /// Able to read broken files/entries and discard them. Not thread safe. class Changelog { - public: - Changelog(const std::string & changelogs_dir_, uint64_t rotate_interval_, - bool force_sync_, Poco::Logger * log_, bool compress_logs_ = true); + Changelog( + const std::string & changelogs_dir_, + uint64_t rotate_interval_, + bool force_sync_, + Poco::Logger * log_, + bool compress_logs_ = true, + uint64_t max_log_file_size = 0); /// Read changelog from files on changelogs_dir_ skipping all entries before from_log_index /// Truncate broken entries, remove files after broken entries. @@ -92,15 +93,9 @@ public: /// Remove log files with to_log_index <= up_to_log_index. void compact(uint64_t up_to_log_index); - uint64_t getNextEntryIndex() const - { - return max_log_id + 1; - } + uint64_t getNextEntryIndex() const { return max_log_id + 1; } - uint64_t getStartIndex() const - { - return min_log_id; - } + uint64_t getStartIndex() const { return min_log_id; } /// Last entry in log, or fake entry with term 0 if log is empty LogEntryPtr getLastEntry() const; @@ -127,10 +122,7 @@ public: void shutdown(); - uint64_t size() const - { - return logs.size(); - } + uint64_t size() const { return logs.size(); } uint64_t lastDurableIndex() const { From c2c115d2ba991d61c055c45c6b4c1abdf653dd24 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 19 Dec 2022 11:50:18 +0000 Subject: [PATCH 018/439] Add setting for max_log_file_size --- src/Coordination/CoordinationSettings.h | 3 ++- src/Coordination/KeeperLogStore.cpp | 5 +++-- src/Coordination/KeeperLogStore.h | 7 ++++++- src/Coordination/KeeperStateManager.cpp | 3 ++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h index c59d2e78108..7242077ea3c 100644 --- a/src/Coordination/CoordinationSettings.h +++ b/src/Coordination/CoordinationSettings.h @@ -44,7 +44,8 @@ struct Settings; M(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0) \ M(Bool, compress_logs, true, "Write compressed coordination logs in ZSTD format", 0) \ M(Bool, compress_snapshots_with_zstd_format, true, "Write compressed snapshots in ZSTD format (instead of custom LZ4)", 0) \ - M(UInt64, configuration_change_tries_count, 20, "How many times we will try to apply configuration change (add/remove server) to the cluster", 0) + M(UInt64, configuration_change_tries_count, 20, "How many times we will try to apply configuration change (add/remove server) to the cluster", 0) \ + M(UInt64, max_log_file_size, 50 * 1024 * 1024, "Max size of the Raft log file. If possible, each created log file will preallocate this amount of bytes on disk. Set to 0 to disable the check", 0) DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS) diff --git a/src/Coordination/KeeperLogStore.cpp b/src/Coordination/KeeperLogStore.cpp index 073889ff107..d5fded540aa 100644 --- a/src/Coordination/KeeperLogStore.cpp +++ b/src/Coordination/KeeperLogStore.cpp @@ -4,9 +4,10 @@ namespace DB { -KeeperLogStore::KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_, bool compress_logs_) +KeeperLogStore::KeeperLogStore( + const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_, bool compress_logs_, uint64_t max_log_file_size) : log(&Poco::Logger::get("KeeperLogStore")) - , changelog(changelogs_path, rotate_interval_, force_sync_, log, compress_logs_) + , changelog(changelogs_path, rotate_interval_, force_sync_, log, compress_logs_, max_log_file_size) { if (force_sync_) LOG_INFO(log, "force_sync enabled"); diff --git a/src/Coordination/KeeperLogStore.h b/src/Coordination/KeeperLogStore.h index 260a6b29320..8de6064a067 100644 --- a/src/Coordination/KeeperLogStore.h +++ b/src/Coordination/KeeperLogStore.h @@ -14,7 +14,12 @@ namespace DB class KeeperLogStore : public nuraft::log_store { public: - KeeperLogStore(const std::string & changelogs_path, uint64_t rotate_interval_, bool force_sync_, bool compress_logs_); + KeeperLogStore( + const std::string & changelogs_path, + uint64_t rotate_interval_, + bool force_sync_, + bool compress_logs_, + uint64_t max_log_file_size = 0); /// Read log storage from filesystem starting from last_commited_log_index void init(uint64_t last_commited_log_index, uint64_t logs_to_keep); diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index 9a3b423d4ac..5d70ebce69f 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -240,7 +240,8 @@ KeeperStateManager::KeeperStateManager( log_storage_path, coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync, - coordination_settings->compress_logs)) + coordination_settings->compress_logs, + coordination_settings->max_log_file_size)) , server_state_path(state_file_path) , logger(&Poco::Logger::get("KeeperStateManager")) { From fcb78cd1d775257fcc6eda4d89515d06f9ff66a8 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 19 Dec 2022 12:05:15 +0000 Subject: [PATCH 019/439] Update NuRaft --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index afc36dfa9b0..ec266b904cc 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit afc36dfa9b0beb45bc4cd935060631cc80ba04a5 +Subproject commit ec266b904cc5a3260210a790043f388c49a5402d From 6d83221d4eab866b62e670edbb8709cbe0f72ea8 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 19 Dec 2022 12:20:26 +0000 Subject: [PATCH 020/439] Remove unused error code --- src/Coordination/Changelog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index dd677e1d413..793b540bbb2 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -19,7 +19,6 @@ namespace ErrorCodes { extern const int CHECKSUM_DOESNT_MATCH; extern const int CORRUPTED_DATA; - extern const int UNSUPPORTED_METHOD; extern const int UNKNOWN_FORMAT_VERSION; extern const int LOGICAL_ERROR; } From a0c8d34775d3d1d9862270f707cf6df4a5ea45e5 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 19 Dec 2022 21:07:44 +0100 Subject: [PATCH 021/439] RabbitMQ: respect flushing data by flush interval and by block size --- src/Storages/RabbitMQ/RabbitMQSettings.h | 2 +- src/Storages/RabbitMQ/RabbitMQSource.cpp | 41 ++-- src/Storages/RabbitMQ/RabbitMQSource.h | 11 +- .../ReadBufferFromRabbitMQConsumer.cpp | 2 +- .../RabbitMQ/ReadBufferFromRabbitMQConsumer.h | 8 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 19 +- .../integration/test_storage_rabbitmq/test.py | 221 +++++++++++++++++- 7 files changed, 271 insertions(+), 33 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index 38baeadad2a..e54445cf64e 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -24,7 +24,7 @@ namespace DB M(String, rabbitmq_address, "", "Address for connection", 0) \ M(UInt64, rabbitmq_skip_broken_messages, 0, "Skip at least this number of broken messages from RabbitMQ per block", 0) \ M(UInt64, rabbitmq_max_block_size, 0, "Number of row collected before flushing data from RabbitMQ.", 0) \ - M(Milliseconds, rabbitmq_flush_interval_ms, 0, "Timeout for flushing data from RabbitMQ.", 0) \ + M(UInt64, rabbitmq_flush_interval_ms, 0, "Timeout for flushing data from RabbitMQ.", 0) \ M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) \ M(String, rabbitmq_queue_settings_list, "", "A list of rabbitmq queue settings", 0) \ M(Bool, rabbitmq_queue_consume, false, "Use user-defined queues and do not make any RabbitMQ setup: declaring exchanges, queues, bindings", 0) \ diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index f6e5bb84037..b997e74abfc 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -61,6 +62,7 @@ RabbitMQSource::RabbitMQSource( , ack_in_suffix(ack_in_suffix_) , non_virtual_header(std::move(headers.first)) , virtual_header(std::move(headers.second)) + , log(&Poco::Logger::get("RabbitMQSource")) { storage.incrementReader(); } @@ -106,17 +108,15 @@ Chunk RabbitMQSource::generate() return chunk; } -bool RabbitMQSource::checkTimeLimit() const +bool RabbitMQSource::isTimeLimitExceeded() const { - if (max_execution_time != 0) + if (max_execution_time_ms != 0) { - auto elapsed_ns = total_stopwatch.elapsed(); - - if (elapsed_ns > static_cast(max_execution_time.totalMicroseconds()) * 1000) - return false; + uint64_t elapsed_time_ms = total_stopwatch.elapsedMilliseconds(); + return max_execution_time_ms <= elapsed_time_ms; } - return true; + return false; } Chunk RabbitMQSource::generateImpl() @@ -127,9 +127,11 @@ Chunk RabbitMQSource::generateImpl() buffer = storage.popReadBuffer(timeout); } - if (!buffer || is_finished) + if (is_finished || !buffer || buffer->isConsumerStopped()) return {}; + /// Currently it is one time usage source: to make sure data is flushed + /// strictly by timeout or by block size. is_finished = true; MutableColumns virtual_columns = virtual_header.cloneEmptyColumns(); @@ -137,15 +139,17 @@ Chunk RabbitMQSource::generateImpl() storage.getFormatName(), *buffer, non_virtual_header, context, max_block_size); StreamingFormatExecutor executor(non_virtual_header, input_format); - size_t total_rows = 0; + buffer->allowNext(); while (true) { - if (buffer->queueEmpty()) - break; + size_t new_rows = 0; - auto new_rows = executor.execute(); + if (!buffer->hasPendingMessages()) + { + new_rows = executor.execute(); + } if (new_rows) { @@ -168,15 +172,20 @@ Chunk RabbitMQSource::generateImpl() virtual_columns[5]->insert(timestamp); } - total_rows = total_rows + new_rows; + total_rows += new_rows; } - buffer->allowNext(); - - if (total_rows >= max_block_size || buffer->queueEmpty() || buffer->isConsumerStopped() || !checkTimeLimit()) + if (total_rows >= max_block_size || buffer->isConsumerStopped() || isTimeLimitExceeded()) break; + + buffer->allowNext(); } + LOG_TEST( + log, + "Flushing {} rows (max block size: {}, time: {} / {} ms)", + total_rows, max_block_size, total_stopwatch.elapsedMilliseconds(), max_execution_time_ms); + if (total_rows == 0) return {}; diff --git a/src/Storages/RabbitMQ/RabbitMQSource.h b/src/Storages/RabbitMQ/RabbitMQSource.h index bd2882d1938..721a98aa2b2 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.h +++ b/src/Storages/RabbitMQ/RabbitMQSource.h @@ -27,13 +27,12 @@ public: Chunk generate() override; - bool queueEmpty() const { return !buffer || buffer->queueEmpty(); } + bool queueEmpty() const { return !buffer || buffer->hasPendingMessages(); } bool needChannelUpdate(); void updateChannel(); bool sendAck(); - - void setTimeLimit(Poco::Timespan max_execution_time_) { max_execution_time = max_execution_time_; } + void setTimeLimit(uint64_t max_execution_time_ms_) { max_execution_time_ms = max_execution_time_ms_; } private: StorageRabbitMQ & storage; @@ -47,12 +46,14 @@ private: const Block non_virtual_header; const Block virtual_header; + Poco::Logger * log; + ConsumerBufferPtr buffer; - Poco::Timespan max_execution_time = 0; + uint64_t max_execution_time_ms = 0; Stopwatch total_stopwatch {CLOCK_MONOTONIC_COARSE}; - bool checkTimeLimit() const; + bool isTimeLimitExceeded() const; RabbitMQSource( StorageRabbitMQ & storage_, diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 3543085f5a0..ba69b66ee4a 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -183,8 +183,8 @@ bool ReadBufferFromRabbitMQConsumer::nextImpl() { auto * new_position = const_cast(current.message.data()); BufferBase::set(new_position, current.message.size(), 0); - allowed = false; + allowed = false; return true; } diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index bd55d169744..f157cef7a4d 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -67,8 +67,9 @@ public: bool ackMessages(); void updateAckTracker(AckTracker record = AckTracker()); - bool queueEmpty() { return received.empty(); } - void allowNext() { allowed = true; } // Allow to read next message. + bool hasPendingMessages() { return received.empty(); } + + void allowNext() { allowed = true; } auto getChannelID() const { return current.track.channel_id; } auto getDeliveryTag() const { return current.track.delivery_tag; } @@ -89,7 +90,6 @@ private: const size_t channel_id_base; Poco::Logger * log; char row_delimiter; - bool allowed = true; const std::atomic & stopped; String channel_id; @@ -100,6 +100,8 @@ private: AckTracker last_inserted_record_info; UInt64 prev_tag = 0, channel_id_counter = 0; + + bool allowed = false; }; } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index bce3fee71f7..07fa8bac189 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -715,6 +715,12 @@ void StorageRabbitMQ::read( auto rabbit_source = std::make_shared( *this, storage_snapshot, modified_context, column_names, 1, rabbitmq_settings->rabbitmq_commit_on_select); + uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed + ? rabbitmq_settings->rabbitmq_flush_interval_ms + : (static_cast(getContext()->getSettingsRef().stream_flush_interval_ms) * 1000); + + rabbit_source->setTimeLimit(max_execution_time_ms); + auto converting_dag = ActionsDAG::makeConvertingActions( rabbit_source->getPort().getHeader().getColumnsWithTypeAndName(), sample_block.getColumnsWithTypeAndName(), @@ -1075,14 +1081,15 @@ bool StorageRabbitMQ::streamToViews() { auto source = std::make_shared( *this, storage_snapshot, rabbitmq_context, column_names, block_size, false); + + uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed + ? rabbitmq_settings->rabbitmq_flush_interval_ms + : (static_cast(getContext()->getSettingsRef().stream_flush_interval_ms) * 1000); + + source->setTimeLimit(max_execution_time_ms); + sources.emplace_back(source); pipes.emplace_back(source); - - Poco::Timespan max_execution_time = rabbitmq_settings->rabbitmq_flush_interval_ms.changed - ? rabbitmq_settings->rabbitmq_flush_interval_ms - : getContext()->getSettingsRef().stream_flush_interval_ms; - - source->setTimeLimit(max_execution_time); } block_io.pipeline.complete(Pipe::unitePipes(std::move(pipes))); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index f5d216a8b92..ddb7a185c2f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -153,6 +153,7 @@ def test_rabbitmq_select_empty(rabbitmq_cluster): rabbitmq_exchange_name = 'empty', rabbitmq_commit_on_select = 1, rabbitmq_format = 'TSV', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; """.format( rabbitmq_cluster.rabbitmq_host @@ -169,6 +170,7 @@ def test_rabbitmq_json_without_delimiter(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{}:5672', rabbitmq_commit_on_select = 1, + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'json', rabbitmq_format = 'JSONEachRow' """.format( @@ -221,6 +223,7 @@ def test_rabbitmq_csv_with_delimiter(rabbitmq_cluster): rabbitmq_exchange_name = 'csv', rabbitmq_commit_on_select = 1, rabbitmq_format = 'CSV', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; """ ) @@ -262,6 +265,7 @@ def test_rabbitmq_tsv_with_delimiter(rabbitmq_cluster): rabbitmq_exchange_name = 'tsv', rabbitmq_format = 'TSV', rabbitmq_commit_on_select = 1, + rabbitmq_flush_interval_ms=1000, rabbitmq_queue_base = 'tsv', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) @@ -303,6 +307,7 @@ def test_rabbitmq_macros(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = '{rabbitmq_host}:{rabbitmq_port}', rabbitmq_commit_on_select = 1, + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = '{rabbitmq_exchange_name}', rabbitmq_format = '{rabbitmq_format}' """ @@ -342,6 +347,7 @@ def test_rabbitmq_materialized_view(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mv', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree() @@ -398,6 +404,7 @@ def test_rabbitmq_materialized_view_with_subquery(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mvsq', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) @@ -441,6 +448,7 @@ def test_rabbitmq_many_materialized_views(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mmv', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view1 (key UInt64, value UInt64) @@ -498,6 +506,7 @@ def test_rabbitmq_protobuf(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'pb', rabbitmq_format = 'Protobuf', + rabbitmq_flush_interval_ms=1000, rabbitmq_schema = 'rabbitmq.proto:KeyValueProto'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree() @@ -572,6 +581,7 @@ def test_rabbitmq_big_message(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'big', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow'; CREATE TABLE test.view (key UInt64, value String) ENGINE = MergeTree @@ -608,6 +618,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): rabbitmq_exchange_name = 'test_sharding', rabbitmq_num_queues = 10, rabbitmq_num_consumers = 10, + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64, channel_id String) @@ -685,6 +696,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'combo', rabbitmq_queue_base = 'combo', + rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = 2, rabbitmq_num_queues = 5, rabbitmq_format = 'JSONEachRow', @@ -778,6 +790,7 @@ def test_rabbitmq_insert(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'insert', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert1', rabbitmq_format = 'TSV', @@ -835,6 +848,7 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'insert_headers', rabbitmq_exchange_type = 'headers', + rabbitmq_flush_interval_ms=1000, rabbitmq_routing_key_list = 'test=insert,topic=headers', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; @@ -901,6 +915,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): rabbitmq_exchange_name = 'many_inserts', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert2', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.rabbitmq_consume (key UInt64, value UInt64) @@ -909,6 +924,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): rabbitmq_exchange_name = 'many_inserts', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert2', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; """ @@ -988,6 +1004,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 5, + rabbitmq_flush_interval_ms=1000, rabbitmq_num_queues = 10, rabbitmq_max_block_size = 10000, rabbitmq_routing_key_list = 'over', @@ -999,6 +1016,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_name = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'over', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view_overload (key UInt64, value UInt64) @@ -1084,6 +1102,7 @@ def test_rabbitmq_direct_exchange(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'direct_exchange_testing', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'direct_{0}', @@ -1175,6 +1194,7 @@ def test_rabbitmq_fanout_exchange(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_routing_key_list = 'key_{0}', rabbitmq_exchange_name = 'fanout_exchange_testing', rabbitmq_exchange_type = 'fanout', @@ -1261,6 +1281,7 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'topic_exchange_testing', rabbitmq_exchange_type = 'topic', rabbitmq_routing_key_list = '*.{0}', @@ -1284,6 +1305,7 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'topic_exchange_testing', rabbitmq_exchange_type = 'topic', rabbitmq_routing_key_list = '*.logs', @@ -1385,6 +1407,7 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster): rabbitmq_exchange_type = 'consistent_hash', rabbitmq_exchange_name = 'hash_exchange_testing', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.{0}_mv TO test.destination AS SELECT key, value, _channel_id AS channel_id FROM test.{0}; @@ -1482,6 +1505,7 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): rabbitmq_exchange_name = 'multiple_bindings_testing', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'key1,key2,key3,key4,key5', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.bindings_mv TO test.destination AS @@ -1571,6 +1595,7 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_num_consumers = 2, rabbitmq_exchange_name = 'headers_exchange_testing', rabbitmq_exchange_type = 'headers', + rabbitmq_flush_interval_ms=1000, rabbitmq_routing_key_list = 'x-match=all,format=logs,type=report,year=2020', rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -1595,6 +1620,7 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_exchange_type = 'headers', rabbitmq_routing_key_list = 'x-match=all,format=logs,type=report,year=2019', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.headers_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.headers_exchange_{0}; @@ -1667,6 +1693,7 @@ def test_rabbitmq_virtual_columns(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'virtuals', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow'; CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, key, _exchange_name, _channel_id, _delivery_tag, _redelivered FROM test.rabbitmq_virtuals; @@ -1735,6 +1762,7 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'virtuals_mv', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow'; CREATE TABLE test.view (key UInt64, value UInt64, exchange_name String, channel_id String, delivery_tag UInt64, redelivered UInt8) ENGINE = MergeTree() @@ -1820,6 +1848,7 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): rabbitmq_exchange_name = 'many_consumers', rabbitmq_num_queues = 2, rabbitmq_num_consumers = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_queue_base = 'many_consumers', rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -1909,6 +1938,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): CREATE TABLE test.consume (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'producer_reconnect', rabbitmq_format = 'JSONEachRow', rabbitmq_num_consumers = 2, @@ -1921,6 +1951,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'producer_reconnect', rabbitmq_persistent = '1', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; """ @@ -1984,6 +2015,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'consumer_reconnect', rabbitmq_num_consumers = 10, + rabbitmq_flush_interval_ms=1000, rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -2064,6 +2096,7 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): rabbitmq_exchange_name = 'block', rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'block', + rabbitmq_flush_interval_ms=1000, rabbitmq_max_block_size = 100, rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) @@ -2144,6 +2177,7 @@ def test_rabbitmq_no_connection_at_startup_1(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'cs', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = '5', rabbitmq_row_delimiter = '\\n'; """ @@ -2160,6 +2194,7 @@ def test_rabbitmq_no_connection_at_startup_2(rabbitmq_cluster): rabbitmq_exchange_name = 'cs', rabbitmq_format = 'JSONEachRow', rabbitmq_num_consumers = '5', + rabbitmq_flush_interval_ms=1000, rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree @@ -2216,6 +2251,7 @@ def test_rabbitmq_format_factory_settings(rabbitmq_cluster): ) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'format_settings', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', date_time_input_format = 'best_effort'; """ @@ -2278,6 +2314,7 @@ def test_rabbitmq_vhost(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'vhost', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_vhost = '/' """ ) @@ -2306,6 +2343,7 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster): CREATE TABLE test.rabbitmq_drop (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'drop', rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue_drop' @@ -2352,6 +2390,7 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'rabbit_exchange', + rabbitmq_flush_interval_ms=1000, rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue_settings', rabbitmq_queue_settings_list = 'x-max-length=10,x-overflow=reject-publish' @@ -2433,6 +2472,7 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue', + rabbitmq_flush_interval_ms=1000, rabbitmq_queue_consume = 1; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; @@ -2467,6 +2507,7 @@ def test_rabbitmq_produce_consume_avro(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_format = 'Avro', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'avro', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'avro'; @@ -2475,6 +2516,7 @@ def test_rabbitmq_produce_consume_avro(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_format = 'Avro', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'avro', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'avro'; @@ -2517,6 +2559,7 @@ def test_rabbitmq_bad_args(rabbitmq_cluster): CREATE TABLE test.drop (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = 'f', rabbitmq_format = 'JSONEachRow'; """ @@ -2529,6 +2572,7 @@ def test_rabbitmq_issue_30691(rabbitmq_cluster): CREATE TABLE test.rabbitmq_drop (json String) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_flush_interval_ms=1000, rabbitmq_exchange_name = '30691', rabbitmq_row_delimiter = '\\n', -- Works only if adding this setting rabbitmq_format = 'LineAsString', @@ -2589,6 +2633,7 @@ def test_rabbitmq_drop_mv(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mv', rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms=1000, rabbitmq_queue_base = 'drop_mv'; """ ) @@ -2679,6 +2724,7 @@ def test_rabbitmq_random_detach(rabbitmq_cluster): rabbitmq_exchange_name = 'random', rabbitmq_queue_base = 'random', rabbitmq_num_queues = 2, + rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = 2, rabbitmq_format = 'JSONEachRow'; CREATE TABLE test.view (key UInt64, value UInt64, channel_id String) @@ -2743,7 +2789,9 @@ def test_rabbitmq_predefined_configuration(rabbitmq_cluster): instance.query( """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) - ENGINE = RabbitMQ(rabbit1, rabbitmq_vhost = '/') """ + ENGINE = RabbitMQ(rabbit1, rabbitmq_vhost = '/') + SETTINGS rabbitmq_flush_interval_ms=1000; + """ ) channel.basic_publish( @@ -2779,6 +2827,7 @@ def test_rabbitmq_msgpack(rabbitmq_cluster): settings rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'xhep', rabbitmq_format = 'MsgPack', + rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = 1; create table rabbit_out (val String) @@ -2786,6 +2835,7 @@ def test_rabbitmq_msgpack(rabbitmq_cluster): settings rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'xhep', rabbitmq_format = 'MsgPack', + rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = 1; set stream_like_engine_allow_direct_select=1; insert into rabbit_out select 'kek'; @@ -2821,12 +2871,14 @@ def test_rabbitmq_address(rabbitmq_cluster): SETTINGS rabbitmq_exchange_name = 'rxhep', rabbitmq_format = 'CSV', rabbitmq_num_consumers = 1, + rabbitmq_flush_interval_ms=1000, rabbitmq_address='amqp://root:clickhouse@rabbitmq1:5672/'; create table rabbit_out (val String) engine=RabbitMQ SETTINGS rabbitmq_exchange_name = 'rxhep', rabbitmq_format = 'CSV', rabbitmq_num_consumers = 1, + rabbitmq_flush_interval_ms=1000, rabbitmq_address='amqp://root:clickhouse@rabbitmq1:5672/'; set stream_like_engine_allow_direct_select=1; insert into rabbit_out select 'kek'; @@ -2848,3 +2900,170 @@ def test_rabbitmq_address(rabbitmq_cluster): instance2.query("drop table rabbit_in sync") instance2.query("drop table rabbit_out sync") + + +def test_rabbitmq_flush_by_block_size(rabbitmq_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + + CREATE TABLE test.rabbitmq (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'flush_by_block', + rabbitmq_queue_base = 'flush_by_block', + rabbitmq_max_block_size = 100, + rabbitmq_flush_interval_ms = 640000, /* should not flush by time during test */ + rabbitmq_format = 'JSONEachRow'; + + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree() + ORDER BY key; + + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.rabbitmq; + """ + ) + + cancel = threading.Event() + + def produce(): + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, + rabbitmq_cluster.rabbitmq_port, + "/", + credentials, + ) + connection = pika.BlockingConnection(parameters) + + while not cancel.is_set(): + try: + channel = connection.channel() + channel.basic_publish( + exchange="flush_by_block", + routing_key="", + body=json.dumps({"key": 0, "value": 0}), + ) + except e: + print(f"Got error: {str(e)}") + + produce_thread = threading.Thread(target=produce) + produce_thread.start() + + while 0 == int( + instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" + ) + ): + time.sleep(0.5) + + cancel.set() + produce_thread.join() + + # more flushes can happens during test, we need to check only result of first flush (part named all_1_1_0). + result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") + # logging.debug(result) + + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.view; + DROP TABLE test.rabbitmq; + """ + ) + + # 100 = first poll should return 100 messages (and rows) + # not waiting for stream_flush_interval_ms + assert ( + int(result) == 100 + ), "Messages from rabbitmq should be flushed when block of size rabbitmq_max_block_size is formed!" + + +def test_rabbitmq_flush_by_time(rabbitmq_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + + CREATE TABLE test.rabbitmq (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'flush_by_time', + rabbitmq_queue_base = 'flush_by_time', + rabbitmq_max_block_size = 100, + rabbitmq_flush_interval_ms = 5000, + rabbitmq_format = 'JSONEachRow'; + + CREATE TABLE test.view (key UInt64, value UInt64, ts DateTime64(3) MATERIALIZED now64(3)) + ENGINE = MergeTree() + ORDER BY key; + """ + ) + + cancel = threading.Event() + + def produce(): + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, + rabbitmq_cluster.rabbitmq_port, + "/", + credentials, + ) + connection = pika.BlockingConnection(parameters) + + while not cancel.is_set(): + try: + channel = connection.channel() + channel.basic_publish( + exchange="flush_by_time", + routing_key="", + body=json.dumps({"key": 0, "value": 0}), + ) + print("Produced a message") + time.sleep(0.8) + except e: + print(f"Got error: {str(e)}") + + produce_thread = threading.Thread(target=produce) + produce_thread.start() + + instance.query( + """ + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.rabbitmq; + """ + ) + + while True: + time.sleep(0.2) + count = instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view'" + ) + print(f"kssenii total count: {count}") + count = int( + instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" + ) + ) + print(f"kssenii count: {count}") + if count > 0: + break + + time.sleep(12) + result = instance.query("SELECT uniqExact(ts) FROM test.view") + + cancel.set() + produce_thread.join() + + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.view; + DROP TABLE test.rabbitmq; + """ + ) + + assert int(result) == 3 From ef9870e4cbb5182f6f7279bcc63bf36b685b5b7e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 20 Dec 2022 13:36:21 +0000 Subject: [PATCH 022/439] fix more tests --- tests/queries/0_stateless/00955_test_final_mark_use.sh | 2 +- .../0_stateless/02245_weird_partitions_pruning.sql | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/00955_test_final_mark_use.sh b/tests/queries/0_stateless/00955_test_final_mark_use.sh index 2c3219fbffd..b1bccd2d6e6 100755 --- a/tests/queries/0_stateless/00955_test_final_mark_use.sh +++ b/tests/queries/0_stateless/00955_test_final_mark_use.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-random-merge-tree-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02245_weird_partitions_pruning.sql b/tests/queries/0_stateless/02245_weird_partitions_pruning.sql index 6273a9f3d59..8b0208da6aa 100644 --- a/tests/queries/0_stateless/02245_weird_partitions_pruning.sql +++ b/tests/queries/0_stateless/02245_weird_partitions_pruning.sql @@ -1,9 +1,11 @@ --- We use a hack - partition by ignore(d1). In some cases there are two columns +-- Tags: no-random-merge-tree-settings + +-- We use a hack - partition by ignore(d1). In some cases there are two columns -- not fully correlated (<1) (date_begin - date_end or datetime - datetime_in_TZ_with_DST) -- If we partition by these columns instead of one it will be twice more partitions. --- Partition by (.., ignore(d1)) allows to partition by the first column but build +-- Partition by (.., ignore(d1)) allows to partition by the first column but build -- min_max indexes for both column, so partition pruning works for both columns. --- It's very similar to min_max skip index but gives bigger performance boost, +-- It's very similar to min_max skip index but gives bigger performance boost, -- because partition pruning happens on very early query stage. From 499565a604fe7cab5a05eb826c1f6441ee6e997d Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 20 Dec 2022 19:32:59 +0100 Subject: [PATCH 023/439] Fix tests --- .../integration/test_storage_rabbitmq/test.py | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index ddb7a185c2f..cc86f63c2e3 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -49,6 +49,7 @@ def rabbitmq_check_result(result, check=False, ref_file="test_rabbitmq_json.refe def wait_rabbitmq_to_start(rabbitmq_docker_id, timeout=180): + logging.getLogger("pika").propagate = False start = time.time() while time.time() - start < timeout: try: @@ -609,6 +610,7 @@ def test_rabbitmq_big_message(rabbitmq_cluster): def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): NUM_CONSUMERS = 10 NUM_QUEUES = 10 + logging.getLogger("pika").propagate = False instance.query( """ @@ -618,7 +620,8 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): rabbitmq_exchange_name = 'test_sharding', rabbitmq_num_queues = 10, rabbitmq_num_consumers = 10, - rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size = 100, + rabbitmq_flush_interval_ms=10000, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64, channel_id String) @@ -671,8 +674,10 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): while True: result1 = instance.query("SELECT count() FROM test.view") time.sleep(1) - if int(result1) == messages_num * threads_num: + expected = messages_num * threads_num + if int(result1) == expected: break + print(f"Result {result1} / {expected}") result2 = instance.query("SELECT count(DISTINCT channel_id) FROM test.view") @@ -688,6 +693,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): def test_rabbitmq_mv_combo(rabbitmq_cluster): NUM_MV = 5 NUM_CONSUMERS = 4 + logging.getLogger("pika").propagate = False instance.query( """ @@ -696,6 +702,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'combo', rabbitmq_queue_base = 'combo', + rabbitmq_max_block_size = 100, rabbitmq_flush_interval_ms=1000, rabbitmq_num_consumers = 2, rabbitmq_num_queues = 5, @@ -761,8 +768,10 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): result += int( instance.query("SELECT count() FROM test.combo_{0}".format(mv_id)) ) - if int(result) == messages_num * threads_num * NUM_MV: + expected = messages_num * threads_num * NUM_MV + if int(result) == expected: break + print(f"Result: {result} / {expected}") time.sleep(1) for thread in threads: @@ -1005,8 +1014,8 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 5, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size = 100, rabbitmq_num_queues = 10, - rabbitmq_max_block_size = 10000, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; @@ -1017,6 +1026,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'over', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size = 100, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view_overload (key UInt64, value UInt64) @@ -1059,8 +1069,10 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): while True: result = instance.query("SELECT count() FROM test.view_overload") time.sleep(1) - if int(result) == messages_num * threads_num: + expected = messages_num * threads_num + if int(result) == expected: break + print(f"Result: {result} / {expected}") instance.query( """ @@ -1849,6 +1861,7 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster): rabbitmq_num_queues = 2, rabbitmq_num_consumers = 2, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_queue_base = 'many_consumers', rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -1939,6 +1952,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size = 100, rabbitmq_exchange_name = 'producer_reconnect', rabbitmq_format = 'JSONEachRow', rabbitmq_num_consumers = 2, @@ -2008,6 +2022,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): + logging.getLogger("pika").propagate = False instance.query( """ CREATE TABLE test.consumer_reconnect (key UInt64, value UInt64) @@ -2015,7 +2030,8 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'consumer_reconnect', rabbitmq_num_consumers = 10, - rabbitmq_flush_interval_ms=1000, + rabbitmq_flush_interval_ms = 100, + rabbitmq_max_block_size = 100, rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -2070,10 +2086,11 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): # revive_rabbitmq() while True: - result = instance.query("SELECT count(DISTINCT key) FROM test.view") - time.sleep(1) + result = instance.query("SELECT count(DISTINCT key) FROM test.view").strip() if int(result) == messages_num: break + print(f"Result: {result} / {messages_num}") + time.sleep(1) instance.query( """ @@ -2088,6 +2105,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): + logging.getLogger("pika").propagate = False instance.query( """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) @@ -2195,6 +2213,7 @@ def test_rabbitmq_no_connection_at_startup_2(rabbitmq_cluster): rabbitmq_format = 'JSONEachRow', rabbitmq_num_consumers = '5', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree @@ -2423,12 +2442,11 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): time.sleep(5) - result = instance.query( - "SELECT count() FROM test.rabbitmq_settings", ignore_error=True - ) - while int(result) != 10: - time.sleep(0.5) + while True: result = instance.query("SELECT count() FROM test.view", ignore_error=True) + if int(result) == 10: + break + time.sleep(0.5) instance.query("DROP TABLE test.rabbitmq_settings") @@ -2696,7 +2714,8 @@ def test_rabbitmq_drop_mv(rabbitmq_cluster): rabbitmq_check_result(result, True) - instance.query("DROP VIEW test.consumer") + instance.query("DROP VIEW test.consumer NO DELAY") + time.sleep(10) for i in range(50, 60): channel.basic_publish( exchange="mv", routing_key="", body=json.dumps({"key": i, "value": i}) From 9711355cc934f7f1609eb11bd00889b4cebd6fc5 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 21 Dec 2022 08:43:55 +0000 Subject: [PATCH 024/439] Fix tests --- src/Coordination/Changelog.cpp | 233 +++++++++++------- src/Coordination/Changelog.h | 6 +- src/Coordination/tests/gtest_coordination.cpp | 5 +- src/IO/ZstdDeflatingAppendableWriteBuffer.cpp | 16 +- src/IO/ZstdDeflatingAppendableWriteBuffer.h | 4 +- src/IO/ZstdInflatingReadBuffer.cpp | 2 +- 6 files changed, 158 insertions(+), 108 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 793b540bbb2..51dc32bb37e 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -36,20 +36,27 @@ std::string formatChangelogPath( return path; } -ChangelogFileDescription getChangelogFileDescription(const std::filesystem::path & path) +ChangelogFileDescriptionPtr getChangelogFileDescription(const std::filesystem::path & path) { - std::string filename = path.stem(); + // we can have .bin.zstd so we cannot use std::filesystem stem and extension + std::string filename_with_extension = path.filename(); + std::string_view filename_with_extension_view = filename_with_extension; + + auto first_dot = filename_with_extension.find('.'); + if (first_dot == std::string::npos) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid changelog file {}", path.generic_string()); + Strings filename_parts; - boost::split(filename_parts, filename, boost::is_any_of("_")); + boost::split(filename_parts, filename_with_extension_view.substr(0, first_dot), boost::is_any_of("_")); if (filename_parts.size() < 3) throw Exception(ErrorCodes::CORRUPTED_DATA, "Invalid changelog {}", path.generic_string()); - ChangelogFileDescription result; - result.prefix = filename_parts[0]; - result.from_log_index = parse(filename_parts[1]); - result.to_log_index = parse(filename_parts[2]); - result.extension = path.extension(); - result.path = path.generic_string(); + auto result = std::make_shared(); + result->prefix = filename_parts[0]; + result->from_log_index = parse(filename_parts[1]); + result->to_log_index = parse(filename_parts[2]); + result->extension = std::string(filename_with_extension.substr(first_dot + 1)); + result->path = path.generic_string(); return result; } @@ -72,7 +79,7 @@ class ChangelogWriter { public: ChangelogWriter( - std::map & existing_changelogs_, + std::map & existing_changelogs_, bool compress_logs_, bool force_fsync_, const std::filesystem::path & changelogs_dir_, @@ -96,15 +103,36 @@ public: } } - void setFile(ChangelogFileDescription & file_description, WriteMode mode) + void setFile(ChangelogFileDescriptionPtr file_description, WriteMode mode) { - file_buf = std::make_unique( - file_description.path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); - start_index = file_description.from_log_index; - last_index = start_index; - current_file_description = &file_description; + if (mode == WriteMode::Append && file_description->expectedEntriesCountInLog() != rotate_interval) + LOG_TRACE(&Poco::Logger::get("Changelog"), "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, file_description->expectedEntriesCountInLog()); - if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(file_description.path)) + if (file_buf) + { + finalizeCurrentFile(/* final */ false); + + assert(current_file_description); + if (last_index != current_file_description->to_log_index) + { + auto new_path = formatChangelogPath( + changelogs_dir, + current_file_description->prefix, + current_file_description->from_log_index, + last_index, + current_file_description->extension); + std::filesystem::rename(current_file_description->path, new_path); + current_file_description->path = std::move(new_path); + } + } + + file_buf = std::make_unique( + file_description->path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); + start_index = file_description->from_log_index; + last_index = start_index; + current_file_description = std::move(file_description); + + if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) { ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); file_buf->sync(); @@ -118,7 +146,7 @@ public: bool appendRecord(ChangelogRecord && record) { - const bool log_is_complete = record.header.index - getStartIndex() == rotate_interval; + const bool log_is_complete = record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog(); if (log_is_complete) rotate(record.header.index); @@ -146,7 +174,7 @@ public: if (compressed_buffer) { - /// Flush compressed data to WriteBufferFromFile working_buffer + /// Flush compressed data to WriteBufferMemory working_buffer compressed_buffer->next(); } @@ -156,7 +184,7 @@ public: cur_memory_buf.finalize(); // if the file is too big, rotate - if (total_bytes_available != 0 && written + memory.size() > total_bytes_available) + if (max_log_file_size != 0 && written + memory.size() > total_bytes_available) { rotate(record.header.index); @@ -185,47 +213,24 @@ public: void rotate(uint64_t new_start_log_index) { - if (file_buf) - { - if (compress_logs && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) - { - ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); - file_buf->sync(); - written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); - } - - flush(); - - ftruncate(file_buf->getFD(), written); - - auto new_path = formatChangelogPath( - changelogs_dir, - current_file_description->prefix, - current_file_description->from_log_index, - last_index, - current_file_description->extension); - std::filesystem::rename(current_file_description->path, new_path); - current_file_description->path = std::move(new_path); - } - /// Start new one - ChangelogFileDescription new_description; - new_description.prefix = DEFAULT_PREFIX; - new_description.from_log_index = new_start_log_index; - new_description.to_log_index = new_start_log_index + rotate_interval - 1; - new_description.extension = "bin"; + auto new_description = std::make_shared(); + new_description->prefix = DEFAULT_PREFIX; + new_description->from_log_index = new_start_log_index; + new_description->to_log_index = new_start_log_index + rotate_interval - 1; + new_description->extension = "bin"; if (compress_logs) - new_description.extension += "." + toContentEncodingName(CompressionMethod::Zstd); + new_description->extension += "." + toContentEncodingName(CompressionMethod::Zstd); - new_description.path = formatChangelogPath( + new_description->path = formatChangelogPath( changelogs_dir, - new_description.prefix, + new_description->prefix, new_start_log_index, new_start_log_index + rotate_interval - 1, - new_description.extension); + new_description->extension); - LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description.path); + LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description->path); auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); setFile(it->second, WriteMode::Rewrite); @@ -234,10 +239,46 @@ public: void finalize() { if (file_buf) - ftruncate(file_buf->getFD(), written); + finalizeCurrentFile(/* final */ true); } private: + + void finalizeCurrentFile(bool final) + { + assert(file_buf); + // reset compression buffer + if (compress_logs) + { + compressed_buffer->finalize(); + WriteBufferMemory & cur_memory_buf + = dynamic_cast(*compressed_buffer->getNestedBuffer()); + + cur_memory_buf.finalize(); + + file_buf->write(memory.raw_data(), memory.size()); + flush(); + + assert(max_log_file_size == 0 || memory.size() < total_bytes_available - written); + written += memory.size(); + + if (final) + compressed_buffer.reset(); + else + compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); + } + else + { + flush(); + } + + + if (max_log_file_size != 0) + ftruncate(file_buf->getFD(), written); + + file_buf.reset(); + } + WriteBuffer & getBuffer() { if (compressed_buffer) @@ -250,6 +291,7 @@ private: if (max_log_file_size == 0) { total_bytes_available = 0; + written = 0; prealloc_done = true; return; } @@ -277,14 +319,14 @@ private: prealloc_done = true; } - std::map & existing_changelogs; + std::map & existing_changelogs; using MemoryBuffer = PODArray; using WriteBufferMemory = WriteBufferFromVector; MemoryBuffer memory; std::unique_ptr memory_buf; - ChangelogFileDescription * current_file_description{nullptr}; + ChangelogFileDescriptionPtr current_file_description{nullptr}; std::unique_ptr file_buf; size_t written{0}; size_t total_bytes_available{0}; @@ -459,7 +501,7 @@ Changelog::Changelog( continue; auto file_description = getChangelogFileDescription(p.path()); - existing_changelogs[file_description.from_log_index] = file_description; + existing_changelogs[file_description->from_log_index] = std::move(file_description); } if (existing_changelogs.empty()) @@ -490,9 +532,12 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin else start_to_read_from = 1; + SCOPE_EXIT({ initialized = true; }); + /// Got through changelog files in order of start_index - for (const auto & [changelog_start_index, changelog_description] : existing_changelogs) + for (const auto & [changelog_start_index, changelog_description_ptr] : existing_changelogs) { + const auto & changelog_description = *changelog_description_ptr; /// [from_log_index.>=.......start_to_read_from.....<=.to_log_index] if (changelog_description.to_log_index >= start_to_read_from) { @@ -602,12 +647,12 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin assert(existing_changelogs.find(last_log_read_result->log_start_index)->first == existing_changelogs.rbegin()->first); /// Continue to write into incomplete existing log if it doesn't finished with error - auto & description = existing_changelogs[last_log_read_result->log_start_index]; + const auto & description = existing_changelogs[last_log_read_result->log_start_index]; if (last_log_read_result->last_read_index == 0 || last_log_read_result->error) /// If it's broken log then remove it { - LOG_INFO(log, "Removing chagelog {} because it's empty or read finished with error", description.path); - std::filesystem::remove(description.path); + LOG_INFO(log, "Removing chagelog {} because it's empty or read finished with error", description->path); + std::filesystem::remove(description->path); existing_changelogs.erase(last_log_read_result->log_start_index); std::erase_if(logs, [last_log_read_result](const auto & item) { return item.first >= last_log_read_result->log_start_index; }); } @@ -620,32 +665,30 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin /// Start new log if we don't initialize writer from previous log. All logs can be "complete". if (!current_writer->isFileSet()) current_writer->rotate(max_log_id + 1); - - initialized = true; } -void Changelog::initWriter(ChangelogFileDescription & description) +void Changelog::initWriter(ChangelogFileDescriptionPtr description) { - LOG_TRACE(log, "Continue to write into {}", description.path); - current_writer->setFile(description, WriteMode::Append); + LOG_TRACE(log, "Continue to write into {}", description->path); + current_writer->setFile(std::move(description), WriteMode::Append); } namespace { - std::string getCurrentTimestampFolder() - { - const auto timestamp = LocalDateTime{std::time(nullptr)}; - return fmt::format( - "{:02}{:02}{:02}T{:02}{:02}{:02}", - timestamp.year(), - timestamp.month(), - timestamp.day(), - timestamp.hour(), - timestamp.minute(), - timestamp.second()); - } +std::string getCurrentTimestampFolder() +{ + const auto timestamp = LocalDateTime{std::time(nullptr)}; + return fmt::format( + "{:02}{:02}{:02}T{:02}{:02}{:02}", + timestamp.year(), + timestamp.month(), + timestamp.day(), + timestamp.hour(), + timestamp.minute(), + timestamp.second()); +} } @@ -661,8 +704,8 @@ void Changelog::removeExistingLogs(ChangelogIter begin, ChangelogIter end) std::filesystem::create_directories(timestamp_folder); } - LOG_WARNING(log, "Removing changelog {}", itr->second.path); - const std::filesystem::path path = itr->second.path; + LOG_WARNING(log, "Removing changelog {}", itr->second->path); + const std::filesystem::path & path = itr->second->path; const auto new_path = timestamp_folder / path.filename(); std::filesystem::rename(path, new_path); itr = existing_changelogs.erase(itr); @@ -788,21 +831,20 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) { auto index_changelog = existing_changelogs.lower_bound(index); - ChangelogFileDescription * description; + ChangelogFileDescriptionPtr description{nullptr}; if (index_changelog->first == index) /// exactly this file starts from index - description = &index_changelog->second; + description = index_changelog->second; else - description = &std::prev(index_changelog)->second; + description = std::prev(index_changelog)->second; - current_writer->flush(); - current_writer->setFile(*description, WriteMode::Append); + current_writer->setFile(std::move(description), WriteMode::Append); /// Remove all subsequent files if overwritten something in previous one auto to_remove_itr = existing_changelogs.upper_bound(index); for (auto itr = to_remove_itr; itr != existing_changelogs.end();) { - std::filesystem::remove(itr->second.path); + std::filesystem::remove(itr->second->path); itr = existing_changelogs.erase(itr); } } @@ -833,29 +875,29 @@ void Changelog::compact(uint64_t up_to_log_index) bool need_rotate = false; for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { + const auto & changelog_description = *itr->second; /// Remove all completely outdated changelog files - if (remove_all_logs || itr->second.to_log_index <= up_to_log_index) + if (remove_all_logs || changelog_description.to_log_index <= up_to_log_index) { - if (current_writer && itr->second.from_log_index == current_writer->getStartIndex()) + if (current_writer && changelog_description.from_log_index == current_writer->getStartIndex()) { LOG_INFO( log, "Trying to remove log {} which is current active log for write. Possibly this node recovers from snapshot", - itr->second.path); + changelog_description.path); need_rotate = true; - current_writer.reset(); } - LOG_INFO(log, "Removing changelog {} because of compaction", itr->second.path); + LOG_INFO(log, "Removing changelog {} because of compaction", changelog_description.path); /// If failed to push to queue for background removing, then we will remove it now - if (!log_files_to_delete_queue.tryPush(itr->second.path, 1)) + if (!log_files_to_delete_queue.tryPush(changelog_description.path, 1)) { std::error_code ec; - std::filesystem::remove(itr->second.path, ec); + std::filesystem::remove(changelog_description.path, ec); if (ec) - LOG_WARNING(log, "Failed to remove changelog {} in compaction, error message: {}", itr->second.path, ec.message()); + LOG_WARNING(log, "Failed to remove changelog {} in compaction, error message: {}", changelog_description.path, ec.message()); else - LOG_INFO(log, "Removed changelog {} because of compaction", itr->second.path); + LOG_INFO(log, "Removed changelog {} because of compaction", changelog_description.path); } itr = existing_changelogs.erase(itr); @@ -1010,6 +1052,7 @@ void Changelog::shutdown() write_thread.join(); current_writer->finalize(); + current_writer.reset(); } Changelog::~Changelog() diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 99f936ba75a..34b2dcfc92f 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -64,6 +64,8 @@ struct ChangelogFileDescription uint64_t expectedEntriesCountInLog() const { return to_log_index - from_log_index + 1; } }; +using ChangelogFileDescriptionPtr = std::shared_ptr; + class ChangelogWriter; /// Simplest changelog with files rotation. @@ -140,7 +142,7 @@ private: static ChangelogRecord buildRecord(uint64_t index, const LogEntryPtr & log_entry); /// Currently existing changelogs - std::map existing_changelogs; + std::map existing_changelogs; using ChangelogIter = decltype(existing_changelogs)::iterator; void removeExistingLogs(ChangelogIter begin, ChangelogIter end); @@ -151,7 +153,7 @@ private: /// Remove all logs from disk void removeAllLogs(); /// Init writer for existing log with some entries already written - void initWriter(ChangelogFileDescription & description); + void initWriter(ChangelogFileDescriptionPtr description); /// Clean useless log files in a background thread void cleanLogThread(); diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 628fe408d01..4e2a6a184d2 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -235,6 +235,7 @@ TEST_P(CoordinationTest, ChangelogTestSimple) { auto params = GetParam(); ChangelogDirTest test("./logs"); + DB::KeeperLogStore changelog("./logs", 5, true, params.enable_compression); changelog.init(1, 0); auto entry = getLogEntry("hello world", 77); @@ -405,6 +406,7 @@ TEST_P(CoordinationTest, ChangelogTestAppendAfterRead) EXPECT_EQ(changelog_reader.size(), 11); waitDurableLogs(changelog_reader); + EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension)); EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension)); EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension)); @@ -874,7 +876,7 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2) EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + params.extension)); DB::WriteBufferFromFile plain_buf("./logs/changelog_1_20.bin" + params.extension, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY); - plain_buf.truncate(140); + plain_buf.truncate(30); DB::KeeperLogStore changelog_reader("./logs", 20, true, params.enable_compression); changelog_reader.init(1, 0); @@ -2161,6 +2163,7 @@ TEST_P(CoordinationTest, TestDurableState) const auto reload_state_manager = [&] { state_manager.emplace(1, "localhost", 9181, "./logs", "./state"); + state_manager->loadLogStore(1, 0); }; reload_state_manager(); diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp index ea2bb8f0c5a..bcde03b3a68 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp @@ -39,12 +39,6 @@ void ZstdDeflatingAppendableWriteBuffer::nextImpl() input.size = offset(); input.pos = 0; - //if (first_write && append_to_existing_file && isNeedToAddEmptyBlock()) - //{ - // addEmptyBlock(); - // first_write = false; - //} - try { bool ended = false; @@ -134,6 +128,16 @@ void ZstdDeflatingAppendableWriteBuffer::finalizeBefore() throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "Zstd stream encoder end failed: error: '{}' zstd version: {}", ZSTD_getErrorName(remaining), ZSTD_VERSION_STRING); remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); + + out->position() = out->buffer().begin() + output.pos; + + if (!out->hasPendingData()) + { + out->next(); + output.dst = reinterpret_cast(out->buffer().begin()); + output.size = out->buffer().size(); + output.pos = out->offset(); + } } out->position() = out->buffer().begin() + output.pos; } diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.h b/src/IO/ZstdDeflatingAppendableWriteBuffer.h index 4819efbc1ab..84242c0e605 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.h +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.h @@ -69,12 +69,10 @@ private: std::unique_ptr out; - //bool append_to_existing_file = false; ZSTD_CCtx * cctx; ZSTD_inBuffer input; ZSTD_outBuffer output; - /// Flipped on the first nextImpl call - bool first_write = true; + bool first_write; }; } diff --git a/src/IO/ZstdInflatingReadBuffer.cpp b/src/IO/ZstdInflatingReadBuffer.cpp index 0d026cdab9a..f70da2ac364 100644 --- a/src/IO/ZstdInflatingReadBuffer.cpp +++ b/src/IO/ZstdInflatingReadBuffer.cpp @@ -58,7 +58,7 @@ bool ZstdInflatingReadBuffer::nextImpl() size_t ret = ZSTD_decompressStream(dctx, &output, &input); if (ZSTD_isError(ret)) throw Exception( - ErrorCodes::ZSTD_DECODER_FAILED, "Zstd stream encoding failed: error '{}'; zstd version: {}", ZSTD_getErrorName(ret), ZSTD_VERSION_STRING); + ErrorCodes::ZSTD_DECODER_FAILED, "Zstd stream decoding failed: error '{}'; zstd version: {}", ZSTD_getErrorName(ret), ZSTD_VERSION_STRING); /// Check that something has changed after decompress (input or output position) assert(in->eof() || output.pos > 0 || in->position() < in->buffer().begin() + input.pos); From da14883715ae179a9370460fd5c6b18a1e9315fc Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Wed, 21 Dec 2022 22:17:09 +0800 Subject: [PATCH 025/439] fix spark bar OOM --- .../AggregateFunctionSparkbar.h | 23 +++++++++++++------ .../02016_aggregation_spark_bar.sql | 6 +++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.h b/src/AggregateFunctions/AggregateFunctionSparkbar.h index f0fbdd2f2e4..177cc352f7c 100644 --- a/src/AggregateFunctions/AggregateFunctionSparkbar.h +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -103,6 +105,7 @@ private: X min_x; X max_x; bool specified_min_max_x; + using Chars = PaddedPODArray; template String getBar(const T value) const @@ -136,9 +139,9 @@ private: * the actual y value of the first position + the actual second position y*0.1, and the remaining y*0.9 is reserved for the next bucket. * The next bucket will use the last y*0.9 + the actual third position y*0.2, and the remaining y*0.8 will be reserved for the next bucket. And so on. */ - String render(const AggregateFunctionSparkbarData & data) const + Chars render(const AggregateFunctionSparkbarData & data) const { - String value; + Chars value; if (data.points.empty() || !width) return value; @@ -167,13 +170,17 @@ private: { auto it = data.points.find(static_cast(min_x_local + i)); bool found = it != data.points.end(); - value += getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); + auto bar = getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); + value.insert(bar.begin(), bar.end()); } } else { for (size_t i = 0; i <= diff_x; ++i) - value += getBar(data.points.has(min_x_local + static_cast(i)) ? 1 : 0); + { + auto bar = getBar(data.points.has(min_x_local + static_cast(i)) ? 1 : 0); + value.insert(bar.begin(), bar.end()); + } } } else @@ -242,11 +249,13 @@ private: auto get_bars = [&] (const std::optional & point_y) { - value += getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); + auto bar = getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); + value.insert(bar.begin(), bar.end()); }; auto get_bars_for_constant = [&] (const std::optional & point_y) { - value += getBar(point_y ? 1 : 0); + auto bar = getBar(point_y ? 1 : 0); + value.insert(bar.begin(), bar.end()); }; if (diff_y != 0.0) @@ -319,7 +328,7 @@ public: { auto & to_column = assert_cast(to); const auto & data = this->data(place); - const String & value = render(data); + const auto & value = render(data); to_column.insertData(value.data(), value.size()); } }; diff --git a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql index 8b5b62305ec..490fd7c01d8 100644 --- a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql +++ b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql @@ -35,3 +35,9 @@ SELECT sparkbar(5,toDate('2020-01-01'),toDate('2020-01-10'))(event_date,cnt) FRO DROP TABLE IF EXISTS spark_bar_test; WITH number DIV 50 AS k, number % 50 AS value SELECT k, sparkbar(50, 0, 99)(number, value) FROM numbers(100) GROUP BY k ORDER BY k; + +-- OOM guard +DROP TABLE IF EXISTS spark_bar_oom; +CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple(); +INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163); +SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000 -- { serverError 241 } From 64ade259cfbf4f2f5fd3de361e50e5259e70a766 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 22 Dec 2022 10:20:55 +0000 Subject: [PATCH 026/439] Fix tests --- src/Coordination/Changelog.cpp | 87 ++++++++++++--------- src/IO/ZstdDeflatingAppendableWriteBuffer.h | 2 +- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 51dc32bb37e..2cdd7b2384e 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -105,37 +105,47 @@ public: void setFile(ChangelogFileDescriptionPtr file_description, WriteMode mode) { - if (mode == WriteMode::Append && file_description->expectedEntriesCountInLog() != rotate_interval) - LOG_TRACE(&Poco::Logger::get("Changelog"), "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, file_description->expectedEntriesCountInLog()); - - if (file_buf) + try { - finalizeCurrentFile(/* final */ false); + if (mode == WriteMode::Append && file_description->expectedEntriesCountInLog() != rotate_interval) + LOG_TRACE( + &Poco::Logger::get("Changelog"), + "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", + rotate_interval, + file_description->expectedEntriesCountInLog()); - assert(current_file_description); - if (last_index != current_file_description->to_log_index) + if (file_buf) { - auto new_path = formatChangelogPath( - changelogs_dir, - current_file_description->prefix, - current_file_description->from_log_index, - last_index, - current_file_description->extension); - std::filesystem::rename(current_file_description->path, new_path); - current_file_description->path = std::move(new_path); + finalizeCurrentFile(/* final */ false); + + assert(current_file_description); + if (last_index && *last_index != current_file_description->to_log_index) + { + auto new_path = formatChangelogPath( + changelogs_dir, + current_file_description->prefix, + current_file_description->from_log_index, + *last_index, + current_file_description->extension); + std::filesystem::rename(current_file_description->path, new_path); + current_file_description->path = std::move(new_path); + } } - } - file_buf = std::make_unique( - file_description->path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); - start_index = file_description->from_log_index; - last_index = start_index; - current_file_description = std::move(file_description); + file_buf = std::make_unique( + file_description->path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); + start_index = file_description->from_log_index; + last_index.reset(); + current_file_description = std::move(file_description); - if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) - { - ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); - file_buf->sync(); + if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) + { + ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); + flush(); + } + } catch (...) { + tryLogCurrentException(&Poco::Logger::get("Changelog")); + throw; } prealloc_done = false; @@ -259,7 +269,7 @@ private: file_buf->write(memory.raw_data(), memory.size()); flush(); - assert(max_log_file_size == 0 || memory.size() < total_bytes_available - written); + assert(max_log_file_size == 0 || !prealloc_done || memory.size() < total_bytes_available - written); written += memory.size(); if (final) @@ -296,18 +306,25 @@ private: return; } - int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, max_log_file_size); - if (res == ENOSPC) + bool fallocate_ok = false; +#ifdef OS_LINUX { - LOG_FATAL(&Poco::Logger::get("Changelog"), "Failed to allocate enough space on disk for logs"); - return; - } + int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, max_log_file_size); + if (res == ENOSPC) + { + LOG_FATAL(&Poco::Logger::get("Changelog"), "Failed to allocate enough space on disk for logs"); + return; + } - bool fallocate_ok = res == 0; + fallocate_ok = res == 0; + } +#endif struct stat buf; - res = fstat(file_buf->getFD(), &buf); - assert(res == 0); + { + int res = fstat(file_buf->getFD(), &buf); + assert(res == 0); + } written = buf.st_size; total_bytes_available = fallocate_ok ? buf.st_blocks * 512 : max_log_file_size; @@ -346,7 +363,7 @@ private: const std::filesystem::path changelogs_dir; uint64_t start_index; - uint64_t last_index; + std::optional last_index; }; struct ChangelogReadResult diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.h b/src/IO/ZstdDeflatingAppendableWriteBuffer.h index 84242c0e605..1d9cc127803 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.h +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.h @@ -72,7 +72,7 @@ private: ZSTD_CCtx * cctx; ZSTD_inBuffer input; ZSTD_outBuffer output; - bool first_write; + bool first_write{true}; }; } From ab458e68cdc1f37d502d29392c2bcd52a4fc497b Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 22 Dec 2022 12:24:20 +0100 Subject: [PATCH 027/439] Fix tests --- src/Storages/RabbitMQ/RabbitMQSource.cpp | 3 +-- .../integration/test_storage_rabbitmq/test.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index b997e74abfc..8ec1f4c556b 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -173,12 +173,11 @@ Chunk RabbitMQSource::generateImpl() } total_rows += new_rows; + buffer->allowNext(); } if (total_rows >= max_block_size || buffer->isConsumerStopped() || isTimeLimitExceeded()) break; - - buffer->allowNext(); } LOG_TEST( diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index cc86f63c2e3..ea0fd99af6c 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -172,6 +172,7 @@ def test_rabbitmq_json_without_delimiter(rabbitmq_cluster): SETTINGS rabbitmq_host_port = '{}:5672', rabbitmq_commit_on_select = 1, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_name = 'json', rabbitmq_format = 'JSONEachRow' """.format( @@ -225,6 +226,7 @@ def test_rabbitmq_csv_with_delimiter(rabbitmq_cluster): rabbitmq_commit_on_select = 1, rabbitmq_format = 'CSV', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_row_delimiter = '\\n'; """ ) @@ -267,6 +269,7 @@ def test_rabbitmq_tsv_with_delimiter(rabbitmq_cluster): rabbitmq_format = 'TSV', rabbitmq_commit_on_select = 1, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_queue_base = 'tsv', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) @@ -309,6 +312,7 @@ def test_rabbitmq_macros(rabbitmq_cluster): SETTINGS rabbitmq_host_port = '{rabbitmq_host}:{rabbitmq_port}', rabbitmq_commit_on_select = 1, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_name = '{rabbitmq_exchange_name}', rabbitmq_format = '{rabbitmq_format}' """ @@ -349,6 +353,7 @@ def test_rabbitmq_materialized_view(rabbitmq_cluster): rabbitmq_exchange_name = 'mv', rabbitmq_format = 'JSONEachRow', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree() @@ -406,6 +411,7 @@ def test_rabbitmq_materialized_view_with_subquery(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mvsq', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64) @@ -450,6 +456,7 @@ def test_rabbitmq_many_materialized_views(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'mmv', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view1 (key UInt64, value UInt64) @@ -508,6 +515,7 @@ def test_rabbitmq_protobuf(rabbitmq_cluster): rabbitmq_exchange_name = 'pb', rabbitmq_format = 'Protobuf', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_schema = 'rabbitmq.proto:KeyValueProto'; CREATE TABLE test.view (key UInt64, value UInt64) ENGINE = MergeTree() @@ -583,6 +591,7 @@ def test_rabbitmq_big_message(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'big', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow'; CREATE TABLE test.view (key UInt64, value String) ENGINE = MergeTree @@ -800,6 +809,7 @@ def test_rabbitmq_insert(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'insert', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert1', rabbitmq_format = 'TSV', @@ -858,6 +868,7 @@ def test_rabbitmq_insert_headers_exchange(rabbitmq_cluster): rabbitmq_exchange_name = 'insert_headers', rabbitmq_exchange_type = 'headers', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_routing_key_list = 'test=insert,topic=headers', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; @@ -925,6 +936,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert2', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.rabbitmq_consume (key UInt64, value UInt64) @@ -934,6 +946,7 @@ def test_rabbitmq_many_inserts(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'insert2', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; """ @@ -1115,6 +1128,7 @@ def test_rabbitmq_direct_exchange(rabbitmq_cluster): rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_name = 'direct_exchange_testing', rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'direct_{0}', @@ -1207,6 +1221,7 @@ def test_rabbitmq_fanout_exchange(rabbitmq_cluster): rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_routing_key_list = 'key_{0}', rabbitmq_exchange_name = 'fanout_exchange_testing', rabbitmq_exchange_type = 'fanout', @@ -1294,6 +1309,7 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_name = 'topic_exchange_testing', rabbitmq_exchange_type = 'topic', rabbitmq_routing_key_list = '*.{0}', @@ -1318,6 +1334,7 @@ def test_rabbitmq_topic_exchange(rabbitmq_cluster): rabbitmq_num_consumers = 2, rabbitmq_num_queues = 2, rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_exchange_name = 'topic_exchange_testing', rabbitmq_exchange_type = 'topic', rabbitmq_routing_key_list = '*.logs', @@ -1518,6 +1535,7 @@ def test_rabbitmq_multiple_bindings(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'key1,key2,key3,key4,key5', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.bindings_mv TO test.destination AS @@ -1608,6 +1626,7 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_exchange_name = 'headers_exchange_testing', rabbitmq_exchange_type = 'headers', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_routing_key_list = 'x-match=all,format=logs,type=report,year=2020', rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -1633,6 +1652,7 @@ def test_rabbitmq_headers_exchange(rabbitmq_cluster): rabbitmq_routing_key_list = 'x-match=all,format=logs,type=report,year=2019', rabbitmq_format = 'JSONEachRow', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_row_delimiter = '\\n'; CREATE MATERIALIZED VIEW test.headers_exchange_{0}_mv TO test.destination AS SELECT key, value FROM test.headers_exchange_{0}; @@ -1706,6 +1726,7 @@ def test_rabbitmq_virtual_columns(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'virtuals', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow'; CREATE MATERIALIZED VIEW test.view Engine=Log AS SELECT value, key, _exchange_name, _channel_id, _delivery_tag, _redelivered FROM test.rabbitmq_virtuals; @@ -1775,6 +1796,7 @@ def test_rabbitmq_virtual_columns_with_materialized_view(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'virtuals_mv', rabbitmq_flush_interval_ms=1000, + rabbitmq_max_block_size=100, rabbitmq_format = 'JSONEachRow'; CREATE TABLE test.view (key UInt64, value UInt64, exchange_name String, channel_id String, delivery_tag UInt64, redelivered UInt8) ENGINE = MergeTree() From 8f1145f6e40794568637be15a41762fda5096cab Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 23 Dec 2022 11:47:51 +0100 Subject: [PATCH 028/439] Fix tests --- tests/integration/test_storage_rabbitmq/test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index ea0fd99af6c..5511196fe1d 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -627,7 +627,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'test_sharding', - rabbitmq_num_queues = 10, + rabbitmq_num_queues = 5, rabbitmq_num_consumers = 10, rabbitmq_max_block_size = 100, rabbitmq_flush_interval_ms=10000, @@ -671,7 +671,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): connection.close() threads = [] - threads_num = 20 + threads_num = 10 for _ in range(threads_num): threads.append(threading.Thread(target=produce)) @@ -696,7 +696,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): assert ( int(result1) == messages_num * threads_num ), "ClickHouse lost some messages: {}".format(result) - assert int(result2) == 10 + assert int(result2) == 5 def test_rabbitmq_mv_combo(rabbitmq_cluster): @@ -1025,10 +1025,10 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_name = 'over', rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', - rabbitmq_num_consumers = 5, + rabbitmq_num_consumers = 3, rabbitmq_flush_interval_ms=1000, rabbitmq_max_block_size = 100, - rabbitmq_num_queues = 10, + rabbitmq_num_queues = 2, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; From 71633bd3419d4c37a70b3c15413e15f44da903ea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Dec 2022 05:14:50 +0300 Subject: [PATCH 029/439] Update AggregateFunctionSparkbar.h --- src/AggregateFunctions/AggregateFunctionSparkbar.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.h b/src/AggregateFunctions/AggregateFunctionSparkbar.h index 177cc352f7c..9b04ddd0b79 100644 --- a/src/AggregateFunctions/AggregateFunctionSparkbar.h +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -105,7 +104,7 @@ private: X min_x; X max_x; bool specified_min_max_x; - using Chars = PaddedPODArray; + using Chars = PODArray; template String getBar(const T value) const From d65f23b2f6091fd8ea64ac0be5cbaf6ee4cb1097 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Tue, 27 Dec 2022 19:24:59 +0800 Subject: [PATCH 030/439] update frame in-place --- .../AggregateFunctionSparkbar.h | 43 ++++++------------- .../02016_aggregation_spark_bar.sql | 9 ++-- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.h b/src/AggregateFunctions/AggregateFunctionSparkbar.h index 5db2104ead8..6324454f85c 100644 --- a/src/AggregateFunctions/AggregateFunctionSparkbar.h +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.h @@ -107,24 +107,11 @@ private: using Chars = PODArray; template - String getBar(const T value) const + void updateFrame(Chars & frame, const T value) const { - if (isNaN(value) || value > 8 || value < 1) - return " "; - - // ▁▂▃▄▅▆▇█ - switch (static_cast(value)) - { - case 1: return "▁"; - case 2: return "▂"; - case 3: return "▃"; - case 4: return "▄"; - case 5: return "▅"; - case 6: return "▆"; - case 7: return "▇"; - case 8: return "█"; - } - return " "; + static const String bars[9] = {" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"}; + const auto & bar = (isNaN(value) || value > 8 || value < 1) ? bars[0] : bars[static_cast(value)]; + frame.insert(bar.begin(), bar.end()); } /** @@ -169,17 +156,13 @@ private: { auto it = data.points.find(static_cast(min_x_local + i)); bool found = it != data.points.end(); - auto bar = getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); - value.insert(bar.begin(), bar.end()); + updateFrame(value, found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); } } else { for (size_t i = 0; i <= diff_x; ++i) - { - auto bar = getBar(data.points.has(min_x_local + static_cast(i)) ? 1 : 0); - value.insert(bar.begin(), bar.end()); - } + updateFrame(value, data.points.has(min_x_local + static_cast(i)) ? 1 : 0); } } else @@ -246,21 +229,19 @@ private: Float64 diff_y = max_y.value() - min_y.value(); - auto get_bars = [&] (const std::optional & point_y) + auto update_frame = [&] (const std::optional & point_y) { - auto bar = getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); - value.insert(bar.begin(), bar.end()); + updateFrame(value, point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); }; - auto get_bars_for_constant = [&] (const std::optional & point_y) + auto update_frame_for_constant = [&] (const std::optional & point_y) { - auto bar = getBar(point_y ? 1 : 0); - value.insert(bar.begin(), bar.end()); + updateFrame(value, point_y ? 1 : 0); }; if (diff_y != 0.0) - std::for_each(new_points.begin(), new_points.end(), get_bars); + std::for_each(new_points.begin(), new_points.end(), update_frame); else - std::for_each(new_points.begin(), new_points.end(), get_bars_for_constant); + std::for_each(new_points.begin(), new_points.end(), update_frame_for_constant); } return value; } diff --git a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql index 490fd7c01d8..ec4f8825d1c 100644 --- a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql +++ b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql @@ -37,7 +37,8 @@ DROP TABLE IF EXISTS spark_bar_test; WITH number DIV 50 AS k, number % 50 AS value SELECT k, sparkbar(50, 0, 99)(number, value) FROM numbers(100) GROUP BY k ORDER BY k; -- OOM guard -DROP TABLE IF EXISTS spark_bar_oom; -CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple(); -INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163); -SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000 -- { serverError 241 } +-- DROP TABLE IF EXISTS spark_bar_oom; +-- CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple(); +-- INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163); +-- SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000; -- { serverError 241 } +-- DROP TABLE IF EXISTS spark_bar_oom; From 9c8b5636dfffca0524a239c341e6a67b879ba662 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Tue, 27 Dec 2022 21:22:36 +0800 Subject: [PATCH 031/439] fix test --- .../0_stateless/02016_aggregation_spark_bar.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql index ec4f8825d1c..5237f832d25 100644 --- a/tests/queries/0_stateless/02016_aggregation_spark_bar.sql +++ b/tests/queries/0_stateless/02016_aggregation_spark_bar.sql @@ -37,8 +37,8 @@ DROP TABLE IF EXISTS spark_bar_test; WITH number DIV 50 AS k, number % 50 AS value SELECT k, sparkbar(50, 0, 99)(number, value) FROM numbers(100) GROUP BY k ORDER BY k; -- OOM guard --- DROP TABLE IF EXISTS spark_bar_oom; --- CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple(); --- INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163); --- SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000; -- { serverError 241 } --- DROP TABLE IF EXISTS spark_bar_oom; +DROP TABLE IF EXISTS spark_bar_oom; +CREATE TABLE spark_bar_oom (x UInt64, y UInt8) Engine=MergeTree ORDER BY tuple(); +INSERT INTO spark_bar_oom VALUES (18446744073709551615,255),(0,0),(0,0),(4036797895307271799,163); +SELECT sparkbar(9)(x,y) FROM spark_bar_oom SETTINGS max_memory_usage = 100000000; -- { serverError 241 } +DROP TABLE IF EXISTS spark_bar_oom; From c559ffe5b923f3921018690d82b642a1e52b36df Mon Sep 17 00:00:00 2001 From: fhbai Date: Wed, 28 Dec 2022 23:32:37 +0800 Subject: [PATCH 032/439] add corrMatrix,covarSampMatrix,covarPopMatrix --- .../AggregateFunctionStatisticsSimple.cpp | 16 ++ .../AggregateFunctionStatisticsSimple.h | 141 ++++++++++++++++++ ...5_aggregate_functions_statistics.reference | 24 +++ .../02515_aggregate_functions_statistics.sql | 56 +++++++ 4 files changed, 237 insertions(+) create mode 100644 tests/queries/0_stateless/02515_aggregate_functions_statistics.reference create mode 100644 tests/queries/0_stateless/02515_aggregate_functions_statistics.sql diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.cpp b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.cpp index 4af21f9542c..0d623f0a0cf 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.cpp +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.cpp @@ -51,6 +51,18 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary( return res; } +template +AggregateFunctionPtr createAggregateFunctionStatisticsArbitrary( + const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) +{ + assertNoParameters(name, parameters); + for (const auto & argument_type : argument_types) + if (!isNumber(argument_type)) + throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(argument_types); +} + } void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory & factory) @@ -75,6 +87,10 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory & facto factory.registerAlias("STDDEV_POP", "stddevPop", AggregateFunctionFactory::CaseInsensitive); factory.registerAlias("COVAR_SAMP", "covarSamp", AggregateFunctionFactory::CaseInsensitive); factory.registerAlias("COVAR_POP", "covarPop", AggregateFunctionFactory::CaseInsensitive); + + factory.registerFunction("covarSampMatrix", createAggregateFunctionStatisticsArbitrary); + factory.registerFunction("covarPopMatrix", createAggregateFunctionStatisticsArbitrary); + factory.registerFunction("corrMatrix", createAggregateFunctionStatisticsArbitrary); } } diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 9ef62363a75..98e15db92ef 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -12,8 +12,13 @@ #include #include +#include #include #include +#include +#include "Columns/ColumnsNumber.h" +#include +#include /** This is simple, not numerically stable @@ -277,4 +282,140 @@ template using AggregateFunctionCovarPopSimple = Aggr template using AggregateFunctionCovarSampSimple = AggregateFunctionVarianceSimple>; template using AggregateFunctionCorrSimple = AggregateFunctionVarianceSimple>; + +template +struct StatFuncArbitraryArgData +{ + using DataType = std::conditional_t<_kind == StatisticsFunctionKind::corr, CorrMoments, CovarMoments>; + + StatFuncArbitraryArgData() = default; + + explicit StatFuncArbitraryArgData(const size_t _num_args) + : num_args(_num_args) + { + data_matrix.resize_fill(num_args * (num_args + 1) / 2, DataType()); + } + + void add(const IColumn ** column, const size_t row_num) + { + for (size_t i = 0; i < num_args; ++i) + for (size_t j = 0; j <= i; ++j) + data_matrix[i * (i + 1) / 2 + j].add(column[i]->getFloat64(row_num), column[j]->getFloat64(row_num)); + } + + void merge(const StatFuncArbitraryArgData & other) + { + for (size_t i = 0; i < num_args; ++i) + for (size_t j = 0; j <= i; ++j) + data_matrix[i * (i + 1) / 2 + j].merge(other.data_matrix[i * (i + 1) / 2 + j]); + } + + void serialize(WriteBuffer & buf) const + { + for (size_t i = 0; i < num_args; ++i) + for (size_t j = 0; j <= i; ++j) + data_matrix[i * (i + 1) / 2 + j].write(buf); + } + + void deserialize(ReadBuffer & buf) + { + for (size_t i = 0; i < num_args; ++i) + for (size_t j = 0; j <= i; ++j) + data_matrix[i * (i + 1) / 2 + j].read(buf); + } + + void insertResultInto(IColumn & to) const + { + auto & data_to = assert_cast(assert_cast(assert_cast(to).getData()).getData()).getData(); + auto & root_offsets_to = assert_cast(to).getOffsets(); + auto & nested_offsets_to = assert_cast(assert_cast(to).getData()).getOffsets(); + for (size_t i = 0; i < num_args; ++i) + { + for (size_t j = 0; j < num_args; ++j) + { + auto & data = i < j ? data_matrix[j * (j + 1) / 2 + i] : data_matrix[i * (i + 1) / 2 + j]; + if constexpr (kind == StatisticsFunctionKind::covarPop) + data_to.push_back(data.getPopulation()); + if constexpr (kind == StatisticsFunctionKind::covarSamp) + data_to.push_back(data.getSample()); + if constexpr (kind == StatisticsFunctionKind::corr) + data_to.push_back(data.get()); + } + nested_offsets_to.push_back(nested_offsets_to.back() + num_args); + } + root_offsets_to.push_back(root_offsets_to.back() + num_args); + } + + static constexpr StatisticsFunctionKind kind = _kind; + PaddedPODArray data_matrix; + size_t num_args; +}; + +template +class AggregateFunctionVarianceSimpleMatrix final + : public IAggregateFunctionDataHelper> +{ +public: + + explicit AggregateFunctionVarianceSimpleMatrix(const DataTypes & argument_types_) + : IAggregateFunctionDataHelper>(argument_types_, {}, createResultType()) + {} + + AggregateFunctionVarianceSimpleMatrix(const IDataType &, const DataTypes & argument_types_) + : IAggregateFunctionDataHelper>(argument_types_, {}, createResultType()) + {} + + String getName() const override + { + if constexpr (StatFuncData::kind == StatisticsFunctionKind::covarPop) + return "covarPopMatrix"; + if constexpr (StatFuncData::kind == StatisticsFunctionKind::covarSamp) + return "covarSampMatrix"; + if constexpr (StatFuncData::kind == StatisticsFunctionKind::corr) + return "corrMatrix"; + UNREACHABLE(); + } + + void create(AggregateDataPtr __restrict place) const override + { + new (place) StatFuncData(this->argument_types.size()); + } + + static DataTypePtr createResultType() + { + return std::make_shared(std::make_shared(std::make_shared())); + } + + bool allocatesMemoryInArena() const override { return false; } + + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override + { + this->data(place).add(columns, row_num); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional /* version */) const override + { + this->data(place).serialize(buf); + } + + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override + { + this->data(place).insertResultInto(to); + } +}; + +using AggregateFunctionCovarPopSimpleMatrix = AggregateFunctionVarianceSimpleMatrix>; +using AggregateFunctionCovarSampSimpleMatrix = AggregateFunctionVarianceSimpleMatrix>; +using AggregateFunctionCorrSimpleMatrix = AggregateFunctionVarianceSimpleMatrix>; + } diff --git a/tests/queries/0_stateless/02515_aggregate_functions_statistics.reference b/tests/queries/0_stateless/02515_aggregate_functions_statistics.reference new file mode 100644 index 00000000000..5320cb8c6c5 --- /dev/null +++ b/tests/queries/0_stateless/02515_aggregate_functions_statistics.reference @@ -0,0 +1,24 @@ +[[nan]] +[[nan]] +[[nan,nan],[nan,nan]] +[[nan,nan],[nan,nan]] +[[nan,nan,nan],[nan,nan,nan],[nan,nan,nan]] +[[nan,nan,nan],[nan,nan,nan],[nan,nan,nan]] +[[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan]] +[[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan]] +[[nan]] +[[nan]] +[[nan,nan],[nan,nan]] +[[nan,nan],[nan,nan]] +[[nan,nan,nan],[nan,nan,nan],[nan,nan,nan]] +[[nan,nan,nan],[nan,nan,nan],[nan,nan,nan]] +[[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan]] +[[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan]] +[[nan]] +[[0]] +[[nan,nan],[nan,nan]] +[[0,0],[0,0]] +[[nan,nan,nan],[nan,nan,nan],[nan,nan,nan]] +[[0,0,0],[0,0,0],[0,0,0]] +[[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan],[nan,nan,nan,nan]] +[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]] diff --git a/tests/queries/0_stateless/02515_aggregate_functions_statistics.sql b/tests/queries/0_stateless/02515_aggregate_functions_statistics.sql new file mode 100644 index 00000000000..51850406366 --- /dev/null +++ b/tests/queries/0_stateless/02515_aggregate_functions_statistics.sql @@ -0,0 +1,56 @@ +DROP TABLE IF EXISTS fh; + +CREATE TABLE fh(a_value UInt32, b_value Float64, c_value Float64, d_value Float64) ENGINE = Memory; + +INSERT INTO fh(a_value, b_value, c_value, d_value) VALUES (1, 5.6,-4.4, 2.6),(2, -9.6, 3, 3.3),(3, -1.3,-4, 1.2),(4, 5.3,9.7,2.3),(5, 4.4,0.037,1.222),(6, -8.6,-7.8,2.1233),(7, 5.1,9.3,8.1222),(8, 7.9,-3.6,9.837),(9, -8.2,0.62,8.43555),(10, -3,7.3,6.762); + +SELECT corrMatrix(a_value) FROM (select a_value from fh limit 0); + +SELECT corrMatrix(a_value) FROM (select a_value from fh limit 1); + +SELECT corrMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 0); + +SELECT corrMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 1); + +SELECT corrMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 0); + +SELECT corrMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 1); + +SELECT corrMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 0); + +SELECT corrMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 1); + + +SELECT covarSampMatrix(a_value) FROM (select a_value from fh limit 0); + +SELECT covarSampMatrix(a_value) FROM (select a_value from fh limit 1); + +SELECT covarSampMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 0); + +SELECT covarSampMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 1); + +SELECT covarSampMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 0); + +SELECT covarSampMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 1); + +SELECT covarSampMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 0); + +SELECT covarSampMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 1); + + +SELECT covarPopMatrix(a_value) FROM (select a_value from fh limit 0); + +SELECT covarPopMatrix(a_value) FROM (select a_value from fh limit 1); + +SELECT covarPopMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 0); + +SELECT covarPopMatrix(a_value, b_value) FROM (select a_value, b_value from fh limit 1); + +SELECT covarPopMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 0); + +SELECT covarPopMatrix(a_value, b_value, c_value) FROM (select a_value, b_value, c_value from fh limit 1); + +SELECT covarPopMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 0); + +SELECT covarPopMatrix(a_value, b_value, c_value, d_value) FROM (select a_value, b_value, c_value, d_value from fh limit 1); + From 8366ac5232d20a94f6b5231049c61280478dc9da Mon Sep 17 00:00:00 2001 From: FFFFFFFHHHHHHH <916677625@qq.com> Date: Thu, 29 Dec 2022 11:44:24 +0800 Subject: [PATCH 033/439] fix style --- src/AggregateFunctions/AggregateFunctionStatisticsSimple.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 98e15db92ef..1b51be41bad 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -299,7 +299,7 @@ struct StatFuncArbitraryArgData void add(const IColumn ** column, const size_t row_num) { for (size_t i = 0; i < num_args; ++i) - for (size_t j = 0; j <= i; ++j) + for (size_t j = 0; j <= i; ++j) data_matrix[i * (i + 1) / 2 + j].add(column[i]->getFloat64(row_num), column[j]->getFloat64(row_num)); } From 55548a6678918dfaec66b3cf36f21d2f3a0a2b5b Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 22 Dec 2022 10:31:07 +0000 Subject: [PATCH 034/439] style fix --- src/Coordination/Changelog.cpp | 150 ++++++++++-------- src/Coordination/tests/gtest_coordination.cpp | 51 ++++++ src/IO/WriteBufferFromVector.h | 11 +- 3 files changed, 142 insertions(+), 70 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 2cdd7b2384e..bb1f09b7b23 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -114,18 +114,18 @@ public: rotate_interval, file_description->expectedEntriesCountInLog()); - if (file_buf) + if (file_buf && prealloc_done) { - finalizeCurrentFile(/* final */ false); + finalizeCurrentFile(/*final*/ false); assert(current_file_description); - if (last_index && *last_index != current_file_description->to_log_index) + if (last_index_written && *last_index_written != current_file_description->to_log_index) { auto new_path = formatChangelogPath( changelogs_dir, current_file_description->prefix, current_file_description->from_log_index, - *last_index, + *last_index_written, current_file_description->extension); std::filesystem::rename(current_file_description->path, new_path); current_file_description->path = std::move(new_path); @@ -134,8 +134,7 @@ public: file_buf = std::make_unique( file_description->path, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)); - start_index = file_description->from_log_index; - last_index.reset(); + last_index_written.reset(); current_file_description = std::move(file_description); if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) @@ -143,12 +142,14 @@ public: ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); flush(); } - } catch (...) { + + prealloc_done = false; + } + catch (...) + { tryLogCurrentException(&Poco::Logger::get("Changelog")); throw; } - - prealloc_done = false; } bool isFileSet() const { return file_buf != nullptr; } @@ -166,49 +167,62 @@ public: tryPreallocateForFile(); if (!prealloc_done) - { return false; - } } - writeIntBinary(computeRecordChecksum(record), getBuffer()); - - writeIntBinary(record.header.version, getBuffer()); - writeIntBinary(record.header.index, getBuffer()); - writeIntBinary(record.header.term, getBuffer()); - writeIntBinary(record.header.value_type, getBuffer()); - writeIntBinary(record.header.blob_size, getBuffer()); - - if (record.header.blob_size != 0) - getBuffer().write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); - - if (compressed_buffer) + const auto write_record = [&] { - /// Flush compressed data to WriteBufferMemory working_buffer - compressed_buffer->next(); - } + writeIntBinary(computeRecordChecksum(record), getBuffer()); - WriteBufferMemory & cur_memory_buf - = compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; + writeIntBinary(record.header.version, getBuffer()); + writeIntBinary(record.header.index, getBuffer()); + writeIntBinary(record.header.term, getBuffer()); + writeIntBinary(record.header.value_type, getBuffer()); + writeIntBinary(record.header.blob_size, getBuffer()); - cur_memory_buf.finalize(); + if (record.header.blob_size != 0) + getBuffer().write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); + + if (compressed_buffer) + { + /// Flush compressed data to WriteBufferMemory working_buffer + compressed_buffer->next(); + } + }; + + write_record(); + + const auto get_memory_buffer = [&]() -> WriteBufferMemory & + { + return compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; + }; // if the file is too big, rotate - if (max_log_file_size != 0 && written + memory.size() > total_bytes_available) + if (max_log_file_size != 0 && total_bytes_written + get_memory_buffer().valuesWritten() > total_bytes_available) { + LOG_TRACE(&Poco::Logger::get("Changelog"), "Log file reached maximum allowed size ({} bytes), creating new log file", max_log_file_size); rotate(record.header.index); tryPreallocateForFile(); if (!prealloc_done) return false; + + // if we use compression we need to write again to the buffer because it's a new one + if (compress_logs) + write_record(); } - file_buf->write(memory.raw_data(), memory.size()); + // after possible rotation, we don't check if we have enough space + // because writing at least 1 log is requirement - we don't want empty log files - written += memory.size(); + auto & cur_memory_buf = get_memory_buffer(); + + file_buf->write(memory.raw_data(), cur_memory_buf.valuesWritten()); + + total_bytes_written += cur_memory_buf.valuesWritten(); cur_memory_buf.restart(); - last_index = record.header.index; + last_index_written = record.header.index; return true; } @@ -219,7 +233,11 @@ public: file_buf->sync(); } - uint64_t getStartIndex() const { return start_index; } + uint64_t getStartIndex() const + { + assert(current_file_description); + return current_file_description->from_log_index; + } void rotate(uint64_t new_start_log_index) { @@ -248,43 +266,42 @@ public: void finalize() { - if (file_buf) - finalizeCurrentFile(/* final */ true); + if (file_buf && prealloc_done) + finalizeCurrentFile(/*final*/ true); } private: void finalizeCurrentFile(bool final) { - assert(file_buf); + assert(file_buf && prealloc_done); + // reset compression buffer if (compress_logs) { - compressed_buffer->finalize(); + if (ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) + { + ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); + assert(max_log_file_size == 0 || memory.size() < total_bytes_available - total_bytes_written); + total_bytes_written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + } + WriteBufferMemory & cur_memory_buf = dynamic_cast(*compressed_buffer->getNestedBuffer()); - cur_memory_buf.finalize(); - - file_buf->write(memory.raw_data(), memory.size()); - flush(); - - assert(max_log_file_size == 0 || !prealloc_done || memory.size() < total_bytes_available - written); - written += memory.size(); + // so finalize can be done + cur_memory_buf.restart(); if (final) compressed_buffer.reset(); else compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); } - else - { - flush(); - } + flush(); if (max_log_file_size != 0) - ftruncate(file_buf->getFD(), written); + ftruncate(file_buf->getFD(), total_bytes_written); file_buf.reset(); } @@ -301,7 +318,7 @@ private: if (max_log_file_size == 0) { total_bytes_available = 0; - written = 0; + total_bytes_written = 0; prealloc_done = true; return; } @@ -322,11 +339,11 @@ private: struct stat buf; { - int res = fstat(file_buf->getFD(), &buf); + [[maybe_unused]] int res = fstat(file_buf->getFD(), &buf); assert(res == 0); } - written = buf.st_size; + total_bytes_written = buf.st_size; total_bytes_available = fallocate_ok ? buf.st_blocks * 512 : max_log_file_size; // always leave space for last termination block @@ -341,29 +358,27 @@ private: using MemoryBuffer = PODArray; using WriteBufferMemory = WriteBufferFromVector; MemoryBuffer memory; + // we write everything to memory buffer to see how many bytes are required + // when we have that information we can try to write to the disk std::unique_ptr memory_buf; ChangelogFileDescriptionPtr current_file_description{nullptr}; std::unique_ptr file_buf; - size_t written{0}; + std::optional last_index_written; + size_t total_bytes_written{0}; size_t total_bytes_available{0}; std::unique_ptr compressed_buffer; bool prealloc_done{false}; - bool compress_logs; - - bool force_fsync; - - uint64_t rotate_interval; - - uint64_t max_log_file_size; + // Changelog configuration + const bool compress_logs; + const bool force_fsync; + const uint64_t rotate_interval; + const uint64_t max_log_file_size; const std::filesystem::path changelogs_dir; - - uint64_t start_index; - std::optional last_index; }; struct ChangelogReadResult @@ -1068,8 +1083,11 @@ void Changelog::shutdown() if (write_thread.joinable()) write_thread.join(); - current_writer->finalize(); - current_writer.reset(); + if (current_writer) + { + current_writer->finalize(); + current_writer.reset(); + } } Changelog::~Changelog() diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 4e2a6a184d2..88d758c6867 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -2271,6 +2271,57 @@ TEST_P(CoordinationTest, TestSystemNodeModify) assert_create("/keeper1/test", Error::ZOK); } +TEST_P(CoordinationTest, ChangelogTestMaxLogSize) +{ + auto params = GetParam(); + ChangelogDirTest test("./logs"); + + uint64_t last_entry_index{0}; + size_t i{0}; + { + SCOPED_TRACE("Small rotation interval, big size limit"); + DB::KeeperLogStore changelog("./logs", 20, true, params.enable_compression, 50 * 1024 * 1024); + changelog.init(1, 0); + + for (; i < 100; ++i) + { + auto entry = getLogEntry(std::to_string(i) + "_hello_world", (i + 44) * 10); + last_entry_index = changelog.append(entry); + } + changelog.end_of_append_batch(0, 0); + + waitDurableLogs(changelog); + + ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10); + } + { + SCOPED_TRACE("Large rotation interval, small size limit"); + DB::KeeperLogStore changelog("./logs", 100'000, true, params.enable_compression, 4000); + changelog.init(1, 0); + + ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10); + + for (; i < 500; ++i) + { + auto entry = getLogEntry(std::to_string(i) + "_hello_world", (i + 44) * 10); + last_entry_index = changelog.append(entry); + } + changelog.end_of_append_batch(0, 0); + + waitDurableLogs(changelog); + + ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10); + } + { + SCOPED_TRACE("Final verify all logs"); + DB::KeeperLogStore changelog("./logs", 100'000, true, params.enable_compression, 4000); + changelog.init(1, 0); + ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10); + } + +} + + INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, CoordinationTest, ::testing::ValuesIn(std::initializer_list{ diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index 521acb6c8d6..da9947ee2f9 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -58,6 +58,12 @@ public: finalized = false; } + size_t valuesWritten() { + return ((position() - reinterpret_cast(vector.data())) /// NOLINT + + sizeof(typename VectorType::value_type) - 1) /// Align up. + / sizeof(typename VectorType::value_type); + } + ~WriteBufferFromVector() override { finalize(); @@ -66,10 +72,7 @@ public: private: void finalizeImpl() override { - vector.resize( - ((position() - reinterpret_cast(vector.data())) /// NOLINT - + sizeof(typename VectorType::value_type) - 1) /// Align up. - / sizeof(typename VectorType::value_type)); + vector.resize(valuesWritten()); /// Prevent further writes. set(nullptr, 0); From b845f880fb1aacf15c593b3a6757d819cb6d48c0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 2 Jan 2023 10:42:15 +0000 Subject: [PATCH 035/439] Polish --- src/Coordination/Changelog.cpp | 86 ++++++++++++++++++++++------------ src/Coordination/Changelog.h | 2 + 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index bb1f09b7b23..b3929d362ef 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -75,6 +75,14 @@ Checksum computeRecordChecksum(const ChangelogRecord & record) } +/// Appendable log writer +/// Writes are done in 2 phases: +/// - write everything in-memory +/// - if enough space on disk, write everything to file +/// New file on disk will be created when: +/// - we have already "rotation_interval" amount of logs in a single file +/// - maximum log file size is reached +/// At least 1 log should be contained in each log class ChangelogWriter { public: @@ -92,6 +100,7 @@ public: , rotate_interval(rotate_interval_) , max_log_file_size(max_log_file_size_) , changelogs_dir(changelogs_dir_) + , log(&Poco::Logger::get("Changelog")) { if (compress_logs) { @@ -109,16 +118,19 @@ public: { if (mode == WriteMode::Append && file_description->expectedEntriesCountInLog() != rotate_interval) LOG_TRACE( - &Poco::Logger::get("Changelog"), + log, "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, file_description->expectedEntriesCountInLog()); + // we have a file we need to finalize first if (file_buf && prealloc_done) { finalizeCurrentFile(/*final*/ false); assert(current_file_description); + // if we wrote at least 1 log in the log file we can rename the file to reflect correctly the + // contained logs if (last_index_written && *last_index_written != current_file_description->to_log_index) { auto new_path = formatChangelogPath( @@ -137,6 +149,7 @@ public: last_index_written.reset(); current_file_description = std::move(file_description); + // try fixing ZstdStream for the file if we are appending to some old file if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) { ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); @@ -147,7 +160,7 @@ public: } catch (...) { - tryLogCurrentException(&Poco::Logger::get("Changelog")); + tryLogCurrentException(log); throw; } } @@ -170,11 +183,12 @@ public: return false; } - const auto write_record = [&] + const auto write_record = [&] { writeIntBinary(computeRecordChecksum(record), getBuffer()); writeIntBinary(record.header.version, getBuffer()); + writeIntBinary(record.header.index, getBuffer()); writeIntBinary(record.header.term, getBuffer()); writeIntBinary(record.header.value_type, getBuffer()); @@ -190,6 +204,7 @@ public: } }; + // write record to an in-memory block write_record(); const auto get_memory_buffer = [&]() -> WriteBufferMemory & @@ -198,16 +213,17 @@ public: }; // if the file is too big, rotate - if (max_log_file_size != 0 && total_bytes_written + get_memory_buffer().valuesWritten() > total_bytes_available) + if (last_index_written.has_value() && max_log_file_size != 0 && total_bytes_written + get_memory_buffer().valuesWritten() > total_bytes_available) { - LOG_TRACE(&Poco::Logger::get("Changelog"), "Log file reached maximum allowed size ({} bytes), creating new log file", max_log_file_size); + LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", max_log_file_size); rotate(record.header.index); + // we need to preallocate space on disk for the new file tryPreallocateForFile(); if (!prealloc_done) return false; - // if we use compression we need to write again to the buffer because it's a new one + // if we use compression we need to write again to the buffer because it's a new one if (compress_logs) write_record(); } @@ -217,12 +233,14 @@ public: auto & cur_memory_buf = get_memory_buffer(); + // write the in-memory block to file file_buf->write(memory.raw_data(), cur_memory_buf.valuesWritten()); total_bytes_written += cur_memory_buf.valuesWritten(); cur_memory_buf.restart(); last_index_written = record.header.index; + return true; } @@ -233,8 +251,8 @@ public: file_buf->sync(); } - uint64_t getStartIndex() const - { + uint64_t getStartIndex() const + { assert(current_file_description); return current_file_description->from_log_index; } @@ -258,7 +276,7 @@ public: new_start_log_index + rotate_interval - 1, new_description->extension); - LOG_TRACE(&Poco::Logger::get("Changelog"), "Starting new changelog {}", new_description->path); + LOG_TRACE(log, "Starting new changelog {}", new_description->path); auto [it, inserted] = existing_changelogs.insert(std::make_pair(new_start_log_index, std::move(new_description))); setFile(it->second, WriteMode::Rewrite); @@ -276,32 +294,39 @@ private: { assert(file_buf && prealloc_done); - // reset compression buffer - if (compress_logs) + // compact can delete the file and we don't need to do anything + if (std::filesystem::exists(current_file_description->path)) { - if (ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) + // finish the stream on disk if needed (finalize will write to in-memory block) + if (compress_logs) { - ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); - assert(max_log_file_size == 0 || memory.size() < total_bytes_available - total_bytes_written); - total_bytes_written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + if (ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) + { + ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); + assert(max_log_file_size == 0 || memory.size() < total_bytes_available - total_bytes_written); + total_bytes_written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + } + + WriteBufferMemory & cur_memory_buf + = dynamic_cast(*compressed_buffer->getNestedBuffer()); + + // so finalize can be done + cur_memory_buf.restart(); + compressed_buffer.reset(); } - WriteBufferMemory & cur_memory_buf - = dynamic_cast(*compressed_buffer->getNestedBuffer()); + flush(); - // so finalize can be done - cur_memory_buf.restart(); - - if (final) - compressed_buffer.reset(); - else - compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); + if (max_log_file_size != 0) + ftruncate(file_buf->getFD(), total_bytes_written); + } + else + { + LOG_WARNING(log, "Log {} is already deleted", file_buf->getFileName()); } - flush(); - - if (max_log_file_size != 0) - ftruncate(file_buf->getFD(), total_bytes_written); + if (compress_logs && !final) + compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); file_buf.reset(); } @@ -329,7 +354,7 @@ private: int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, max_log_file_size); if (res == ENOSPC) { - LOG_FATAL(&Poco::Logger::get("Changelog"), "Failed to allocate enough space on disk for logs"); + LOG_FATAL(log, "Failed to allocate enough space on disk for logs"); return; } @@ -344,6 +369,7 @@ private: } total_bytes_written = buf.st_size; + // if fallocate passed, we know exact allocated bytes on disk which can be larger than the max_log_file_size total_bytes_available = fallocate_ok ? buf.st_blocks * 512 : max_log_file_size; // always leave space for last termination block @@ -379,6 +405,8 @@ private: const uint64_t max_log_file_size; const std::filesystem::path changelogs_dir; + + Poco::Logger * const log; }; struct ChangelogReadResult diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 34b2dcfc92f..35ce010f084 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -82,6 +82,8 @@ public: bool compress_logs_ = true, uint64_t max_log_file_size = 0); + Changelog(Changelog &&) = delete; + /// Read changelog from files on changelogs_dir_ skipping all entries before from_log_index /// Truncate broken entries, remove files after broken entries. void readChangelogAndInitWriter(uint64_t last_commited_log_index, uint64_t logs_to_keep); From 8533f6d299a7be9e8ef2b1a69b8a0fa2ebd6af37 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 2 Jan 2023 10:48:35 +0000 Subject: [PATCH 036/439] Style fix --- src/IO/WriteBufferFromVector.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index da9947ee2f9..e6f899f59f0 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -58,7 +58,8 @@ public: finalized = false; } - size_t valuesWritten() { + size_t valuesWritten() + { return ((position() - reinterpret_cast(vector.data())) /// NOLINT + sizeof(typename VectorType::value_type) - 1) /// Align up. / sizeof(typename VectorType::value_type); From 2e9364061437a721d4ae1bde8dbf1fcecedc4e79 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 2 Jan 2023 13:37:40 +0000 Subject: [PATCH 037/439] Fix ASAN error --- src/Coordination/Changelog.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index b3929d362ef..e7fda79bd44 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -306,13 +306,6 @@ private: assert(max_log_file_size == 0 || memory.size() < total_bytes_available - total_bytes_written); total_bytes_written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); } - - WriteBufferMemory & cur_memory_buf - = dynamic_cast(*compressed_buffer->getNestedBuffer()); - - // so finalize can be done - cur_memory_buf.restart(); - compressed_buffer.reset(); } flush(); @@ -325,8 +318,18 @@ private: LOG_WARNING(log, "Log {} is already deleted", file_buf->getFileName()); } - if (compress_logs && !final) - compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); + if (compress_logs) + { + WriteBufferMemory & cur_memory_buf + = dynamic_cast(*compressed_buffer->getNestedBuffer()); + + // so finalize can be done + cur_memory_buf.restart(); + compressed_buffer.reset(); + + if (!final) + compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); + } file_buf.reset(); } From f32fab5025ffc5c62bfea47ef97a03a83bd59828 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 2 Jan 2023 13:48:13 +0000 Subject: [PATCH 038/439] Check if file exists before rename --- src/Coordination/Changelog.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index e7fda79bd44..e922f42a77f 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -131,7 +131,9 @@ public: assert(current_file_description); // if we wrote at least 1 log in the log file we can rename the file to reflect correctly the // contained logs - if (last_index_written && *last_index_written != current_file_description->to_log_index) + // file can be deleted from disk earlier by compaction + if (std::filesystem::exists(current_file_description->path) && last_index_written + && *last_index_written != current_file_description->to_log_index) { auto new_path = formatChangelogPath( changelogs_dir, @@ -297,6 +299,8 @@ private: // compact can delete the file and we don't need to do anything if (std::filesystem::exists(current_file_description->path)) { + assert(current_file_description); + // finish the stream on disk if needed (finalize will write to in-memory block) if (compress_logs) { From f2653c016a285d4e355ec237b6a16b4675f8c49f Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 2 Jan 2023 14:36:58 +0000 Subject: [PATCH 039/439] Manually set deleted flag --- src/Coordination/Changelog.cpp | 12 +++++++----- src/Coordination/Changelog.h | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index e922f42a77f..a743d61bbf9 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -132,7 +132,7 @@ public: // if we wrote at least 1 log in the log file we can rename the file to reflect correctly the // contained logs // file can be deleted from disk earlier by compaction - if (std::filesystem::exists(current_file_description->path) && last_index_written + if (!current_file_description->deleted && last_index_written && *last_index_written != current_file_description->to_log_index) { auto new_path = formatChangelogPath( @@ -296,11 +296,10 @@ private: { assert(file_buf && prealloc_done); + assert(current_file_description); // compact can delete the file and we don't need to do anything - if (std::filesystem::exists(current_file_description->path)) + if (!current_file_description->deleted) { - assert(current_file_description); - // finish the stream on disk if needed (finalize will write to in-memory block) if (compress_logs) { @@ -942,7 +941,7 @@ void Changelog::compact(uint64_t up_to_log_index) bool need_rotate = false; for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { - const auto & changelog_description = *itr->second; + auto & changelog_description = *itr->second; /// Remove all completely outdated changelog files if (remove_all_logs || changelog_description.to_log_index <= up_to_log_index) { @@ -956,6 +955,7 @@ void Changelog::compact(uint64_t up_to_log_index) } LOG_INFO(log, "Removing changelog {} because of compaction", changelog_description.path); + /// If failed to push to queue for background removing, then we will remove it now if (!log_files_to_delete_queue.tryPush(changelog_description.path, 1)) { @@ -967,6 +967,8 @@ void Changelog::compact(uint64_t up_to_log_index) LOG_INFO(log, "Removed changelog {} because of compaction", changelog_description.path); } + changelog_description.deleted = true; + itr = existing_changelogs.erase(itr); } else /// Files are ordered, so all subsequent should exist diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 35ce010f084..34938b05c83 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -60,6 +60,8 @@ struct ChangelogFileDescription std::string path; + bool deleted = false; + /// How many entries should be stored in this log uint64_t expectedEntriesCountInLog() const { return to_log_index - from_log_index + 1; } }; From ae2bd711d92b54515234bd99fe4b38b75372e545 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 4 Jan 2023 21:47:10 +0100 Subject: [PATCH 040/439] Catch exceptions in BackgroundSchedulePool I saw the case when one of users of BackgroundSchedulePool (DirectoryMonitor for async INSERT into Distributed table) throws (in a very unlikely scenaior, which wich #44922 may help), and this case the task is leaved in an undefined state, and instead of fix this I put assert to catch such code instead. Signed-off-by: Azat Khuzhin --- src/Core/BackgroundSchedulePool.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index c3b2af40980..165d8902e85 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -95,7 +95,15 @@ void BackgroundSchedulePoolTaskInfo::execute() executing = true; } - function(); + try + { + function(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + chassert(false && "Tasks in BackgroundSchedulePool cannot throw"); + } UInt64 milliseconds = watch.elapsedMilliseconds(); /// If the task is executed longer than specified time, it will be logged. From 245bdd8e5e953254c74d772a17ae529aad2d446f Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Thu, 5 Jan 2023 14:47:23 +0800 Subject: [PATCH 041/439] update column inplace --- .../AggregateFunctionSparkbar.h | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.h b/src/AggregateFunctions/AggregateFunctionSparkbar.h index 6324454f85c..8f1d63a027b 100644 --- a/src/AggregateFunctions/AggregateFunctionSparkbar.h +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -104,14 +105,14 @@ private: X min_x; X max_x; bool specified_min_max_x; - using Chars = PODArray; template - void updateFrame(Chars & frame, const T value) const + int updateFrame(ColumnString::Chars & frame, const T value) const { static const String bars[9] = {" ", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"}; const auto & bar = (isNaN(value) || value > 8 || value < 1) ? bars[0] : bars[static_cast(value)]; frame.insert(bar.begin(), bar.end()); + return static_cast(bar.size()); } /** @@ -125,11 +126,18 @@ private: * the actual y value of the first position + the actual second position y*0.1, and the remaining y*0.9 is reserved for the next bucket. * The next bucket will use the last y*0.9 + the actual third position y*0.2, and the remaining y*0.8 will be reserved for the next bucket. And so on. */ - Chars render(const AggregateFunctionSparkbarData & data) const + void render(ColumnString & to_column, const AggregateFunctionSparkbarData & data) const { - Chars value; + size_t sz = 0; + auto & values = to_column.getChars(); + auto & offsets = to_column.getOffsets(); + auto update_column = [&]() { + values.push_back('\0'); + offsets.push_back(offsets.empty() ? sz + 1 : offsets.back() + sz + 1); + }; + if (data.points.empty() || !width) - return value; + return update_column(); size_t diff_x; X min_x_local; @@ -156,13 +164,13 @@ private: { auto it = data.points.find(static_cast(min_x_local + i)); bool found = it != data.points.end(); - updateFrame(value, found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); + sz += updateFrame(values, found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0); } } else { for (size_t i = 0; i <= diff_x; ++i) - updateFrame(value, data.points.has(min_x_local + static_cast(i)) ? 1 : 0); + sz += updateFrame(values, data.points.has(min_x_local + static_cast(i)) ? 1 : 0); } } else @@ -225,17 +233,17 @@ private: } if (!min_y || !max_y) // No value is set - return {}; + return update_column(); Float64 diff_y = max_y.value() - min_y.value(); auto update_frame = [&] (const std::optional & point_y) { - updateFrame(value, point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); + sz += updateFrame(values, point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0); }; auto update_frame_for_constant = [&] (const std::optional & point_y) { - updateFrame(value, point_y ? 1 : 0); + sz += updateFrame(values, point_y ? 1 : 0); }; if (diff_y != 0.0) @@ -243,7 +251,7 @@ private: else std::for_each(new_points.begin(), new_points.end(), update_frame_for_constant); } - return value; + update_column(); } @@ -303,8 +311,7 @@ public: { auto & to_column = assert_cast(to); const auto & data = this->data(place); - const auto & value = render(data); - to_column.insertData(value.data(), value.size()); + render(to_column, data); } }; From d02227306c1789a48dfca75bfefb5e5d83f90df4 Mon Sep 17 00:00:00 2001 From: Duc Canh Le Date: Thu, 5 Jan 2023 16:11:05 +0800 Subject: [PATCH 042/439] fix style check --- src/AggregateFunctions/AggregateFunctionSparkbar.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.h b/src/AggregateFunctions/AggregateFunctionSparkbar.h index 8f1d63a027b..acde00ab88e 100644 --- a/src/AggregateFunctions/AggregateFunctionSparkbar.h +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.h @@ -131,7 +131,8 @@ private: size_t sz = 0; auto & values = to_column.getChars(); auto & offsets = to_column.getOffsets(); - auto update_column = [&]() { + auto update_column = [&] () + { values.push_back('\0'); offsets.push_back(offsets.empty() ? sz + 1 : offsets.back() + sz + 1); }; From dd6a2c2d3759a663cf9fdbc1d1b5e477a4fca8d1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 6 Jan 2023 16:53:24 +0100 Subject: [PATCH 043/439] Fix style check --- .../integration/test_storage_rabbitmq/test.py | 236 +++++++++--------- 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 299b59ad374..6ec89ccaa9b 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -3296,8 +3296,8 @@ def test_block_based_formats_2(rabbitmq_cluster): def test_rabbitmq_flush_by_block_size(rabbitmq_cluster): - instance.query( - """ + instance.query( + """ DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; @@ -3317,146 +3317,146 @@ def test_rabbitmq_flush_by_block_size(rabbitmq_cluster): CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.rabbitmq; """ - ) + ) - cancel = threading.Event() + cancel = threading.Event() - def produce(): - credentials = pika.PlainCredentials("root", "clickhouse") - parameters = pika.ConnectionParameters( - rabbitmq_cluster.rabbitmq_ip, - rabbitmq_cluster.rabbitmq_port, - "/", - credentials, - ) - connection = pika.BlockingConnection(parameters) + def produce(): + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, + rabbitmq_cluster.rabbitmq_port, + "/", + credentials, + ) + connection = pika.BlockingConnection(parameters) - while not cancel.is_set(): - try: - channel = connection.channel() - channel.basic_publish( - exchange="flush_by_block", - routing_key="", - body=json.dumps({"key": 0, "value": 0}), - ) - except e: - print(f"Got error: {str(e)}") + while not cancel.is_set(): + try: + channel = connection.channel() + channel.basic_publish( + exchange="flush_by_block", + routing_key="", + body=json.dumps({"key": 0, "value": 0}), + ) + except e: + print(f"Got error: {str(e)}") - produce_thread = threading.Thread(target=produce) - produce_thread.start() + produce_thread = threading.Thread(target=produce) + produce_thread.start() - while 0 == int( - instance.query( - "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" - ) - ): - time.sleep(0.5) + while 0 == int( + instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" + ) + ): + time.sleep(0.5) - cancel.set() - produce_thread.join() + cancel.set() + produce_thread.join() - # more flushes can happens during test, we need to check only result of first flush (part named all_1_1_0). - result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") - # logging.debug(result) + # more flushes can happens during test, we need to check only result of first flush (part named all_1_1_0). + result = instance.query("SELECT count() FROM test.view WHERE _part='all_1_1_0'") + # logging.debug(result) - instance.query( - """ + instance.query( + """ DROP TABLE test.consumer; DROP TABLE test.view; DROP TABLE test.rabbitmq; """ - ) + ) - # 100 = first poll should return 100 messages (and rows) - # not waiting for stream_flush_interval_ms - assert ( - int(result) == 100 - ), "Messages from rabbitmq should be flushed when block of size rabbitmq_max_block_size is formed!" + # 100 = first poll should return 100 messages (and rows) + # not waiting for stream_flush_interval_ms + assert ( + int(result) == 100 + ), "Messages from rabbitmq should be flushed when block of size rabbitmq_max_block_size is formed!" - def test_rabbitmq_flush_by_time(rabbitmq_cluster): - instance.query( - """ - DROP TABLE IF EXISTS test.view; - DROP TABLE IF EXISTS test.consumer; +def test_rabbitmq_flush_by_time(rabbitmq_cluster): + instance.query( + """ + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; - CREATE TABLE test.rabbitmq (key UInt64, value UInt64) - ENGINE = RabbitMQ - SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', - rabbitmq_exchange_name = 'flush_by_time', - rabbitmq_queue_base = 'flush_by_time', - rabbitmq_max_block_size = 100, - rabbitmq_flush_interval_ms = 5000, - rabbitmq_format = 'JSONEachRow'; + CREATE TABLE test.rabbitmq (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'flush_by_time', + rabbitmq_queue_base = 'flush_by_time', + rabbitmq_max_block_size = 100, + rabbitmq_flush_interval_ms = 5000, + rabbitmq_format = 'JSONEachRow'; - CREATE TABLE test.view (key UInt64, value UInt64, ts DateTime64(3) MATERIALIZED now64(3)) - ENGINE = MergeTree() - ORDER BY key; - """ - ) + CREATE TABLE test.view (key UInt64, value UInt64, ts DateTime64(3) MATERIALIZED now64(3)) + ENGINE = MergeTree() + ORDER BY key; + """ + ) - cancel = threading.Event() + cancel = threading.Event() - def produce(): - credentials = pika.PlainCredentials("root", "clickhouse") - parameters = pika.ConnectionParameters( - rabbitmq_cluster.rabbitmq_ip, - rabbitmq_cluster.rabbitmq_port, - "/", - credentials, - ) - connection = pika.BlockingConnection(parameters) + def produce(): + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + rabbitmq_cluster.rabbitmq_ip, + rabbitmq_cluster.rabbitmq_port, + "/", + credentials, + ) + connection = pika.BlockingConnection(parameters) - while not cancel.is_set(): - try: - channel = connection.channel() - channel.basic_publish( - exchange="flush_by_time", - routing_key="", - body=json.dumps({"key": 0, "value": 0}), - ) - print("Produced a message") - time.sleep(0.8) - except e: - print(f"Got error: {str(e)}") + while not cancel.is_set(): + try: + channel = connection.channel() + channel.basic_publish( + exchange="flush_by_time", + routing_key="", + body=json.dumps({"key": 0, "value": 0}), + ) + print("Produced a message") + time.sleep(0.8) + except e: + print(f"Got error: {str(e)}") - produce_thread = threading.Thread(target=produce) - produce_thread.start() + produce_thread = threading.Thread(target=produce) + produce_thread.start() - instance.query( - """ - CREATE MATERIALIZED VIEW test.consumer TO test.view AS - SELECT * FROM test.rabbitmq; - """ - ) + instance.query( + """ + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.rabbitmq; + """ + ) - while True: - time.sleep(0.2) - count = instance.query( - "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view'" - ) - print(f"kssenii total count: {count}") - count = int( - instance.query( - "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" - ) - ) - print(f"kssenii count: {count}") - if count > 0: - break + while True: + time.sleep(0.2) + count = instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view'" + ) + print(f"kssenii total count: {count}") + count = int( + instance.query( + "SELECT count() FROM system.parts WHERE database = 'test' AND table = 'view' AND name = 'all_1_1_0'" + ) + ) + print(f"kssenii count: {count}") + if count > 0: + break - time.sleep(12) - result = instance.query("SELECT uniqExact(ts) FROM test.view") + time.sleep(12) + result = instance.query("SELECT uniqExact(ts) FROM test.view") - cancel.set() - produce_thread.join() + cancel.set() + produce_thread.join() - instance.query( - """ - DROP TABLE test.consumer; - DROP TABLE test.view; - DROP TABLE test.rabbitmq; - """ - ) + instance.query( + """ + DROP TABLE test.consumer; + DROP TABLE test.view; + DROP TABLE test.rabbitmq; + """ + ) - assert int(result) == 3 + assert int(result) == 3 From 0651048cbc7ce6bc8c061dd3557674491de4bafc Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 12 Jan 2023 13:42:55 +0000 Subject: [PATCH 044/439] fix more tests --- .../queries/0_stateless/01200_mutations_memory_consumption.sql | 2 +- .../01903_correct_block_size_prediction_with_default.sql | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index ff4918c0810..bca2286aa22 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,4 +1,4 @@ --- Tags: no-debug, no-parallel, long, no-s3-storage +-- Tags: no-debug, no-parallel, long, no-s3-storage, no-random-merge-tree-settings DROP TABLE IF EXISTS table_with_single_pk; diff --git a/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql index 7aa1b0112a6..2eec08635eb 100644 --- a/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql +++ b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + CREATE TABLE test_extract(str String, arr Array(Array(String)) ALIAS extractAllGroupsHorizontal(str, '\\W(\\w+)=("[^"]*?"|[^",}]*)')) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY tuple(); INSERT INTO test_extract (str) WITH range(8) as range_arr, arrayMap(x-> concat(toString(x),'Id'), range_arr) as key, arrayMap(x -> rand() % 8, range_arr) as val, arrayStringConcat(arrayMap((x,y) -> concat(x,'=',toString(y)), key, val),',') as str SELECT str FROM numbers(500000); From f6131101bb5d3597c42cd84f0489514f15489e29 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 17 Jan 2023 15:51:49 +0100 Subject: [PATCH 045/439] Fix no shared id during drop for the fourth time --- src/Storages/StorageReplicatedMergeTree.cpp | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7c1894b3652..f6267b845e2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7892,20 +7892,7 @@ String StorageReplicatedMergeTree::getTableSharedID() const /// Can happen if table was partially initialized before drop by DatabaseCatalog if (table_shared_id == UUIDHelpers::Nil) - { - if (has_metadata_in_zookeeper.has_value()) - { - if (*has_metadata_in_zookeeper) - createTableSharedID(); - else - throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is already dropped", getStorageID().getNameForLogs()); - } - else - { - throw Exception(ErrorCodes::NO_ZOOKEEPER, "No connection to ZooKeeper, cannot get shared table ID for table {}. " - "It will be resolve automatically when connection will be established", getStorageID().getNameForLogs()); - } - } + createTableSharedID(); return toString(table_shared_id); } @@ -7951,6 +7938,10 @@ void StorageReplicatedMergeTree::createTableSharedID() const id = zookeeper->get(zookeeper_table_id_path); LOG_DEBUG(log, "Shared ID on path {} concurrently created, will set ID {}", zookeeper_table_id_path, id); } + else if (code == Coordination::Error::ZNONODE) /// table completely dropped, we can choose any id we want + { + LOG_DEBUG(log, "Table was completely drop, we can use anything as ID (will use {})", id); + } else if (code != Coordination::Error::ZOK) { throw zkutil::KeeperException(code, zookeeper_table_id_path); From 69925647ebeebfc7401683e7438181d1139533e2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 17 Jan 2023 15:59:55 +0100 Subject: [PATCH 046/439] Fix style --- src/Storages/StorageReplicatedMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f6267b845e2..3fd04c511e5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -142,7 +142,6 @@ namespace ErrorCodes extern const int ABORTED; extern const int REPLICA_IS_NOT_IN_QUORUM; extern const int TABLE_IS_READ_ONLY; - extern const int TABLE_IS_DROPPED; extern const int NOT_FOUND_NODE; extern const int NO_ACTIVE_REPLICAS; extern const int NOT_A_LEADER; From 1c120ee7517c43b37c19966852f7d3096a0a4fda Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 18 Jan 2023 15:13:47 +0800 Subject: [PATCH 047/439] add fastpath for vectorConstant when needle is empty --- src/Functions/PositionImpl.h | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 76f10373a58..9b774edfa29 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -113,7 +113,7 @@ struct PositionCaseSensitiveUTF8 static const char * advancePos(const char * pos, const char * end, size_t n) { - for (auto it = pos; it != end; ++it) + for (const auto *it = pos; it != end; ++it) { if (!UTF8::isContinuationOctet(static_cast(*it))) { @@ -128,7 +128,7 @@ struct PositionCaseSensitiveUTF8 static size_t countChars(const char * begin, const char * end) { size_t res = 0; - for (auto it = begin; it != end; ++it) + for (const auto *it = begin; it != end; ++it) if (!UTF8::isContinuationOctet(static_cast(*it))) ++res; return res; @@ -198,6 +198,32 @@ struct PositionImpl const UInt8 * const end = haystack_data.data() + haystack_data.size(); const UInt8 * pos = begin; + /// Fastpath when needle is empty + if (needle.empty()) + { + if (start_pos == nullptr) + { + for (auto & x : res) + x = 1; + } + else + { + ColumnString::Offset prev_offset = 0; + for (size_t i = 0; i < haystack_offsets.size(); ++i) + { + size_t haystack_size = Impl::countChars( + reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1)); + UInt64 start = start_pos->getUInt(i); + start = std::max(UInt64(1), start); + res[i] = start <= haystack_size + 1 ? 1 : 0; + + pos = begin + haystack_offsets[i]; + prev_offset = haystack_offsets[i]; + } + } + return; + } + /// Current index in the array of strings. size_t i = 0; @@ -249,7 +275,7 @@ struct PositionImpl { auto start = std::max(start_pos, UInt64(1)); - if (needle.size() == 0) + if (needle.empty()) { size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size()); res = start <= haystack_size + 1 ? start : 0; From 8d63dc2608da51dd235bce0f6a97385d6d09e9df Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Wed, 18 Jan 2023 20:13:26 +0800 Subject: [PATCH 048/439] fix tests --- src/Functions/PositionImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 9b774edfa29..0b85518bad6 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -215,7 +215,7 @@ struct PositionImpl reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1)); UInt64 start = start_pos->getUInt(i); start = std::max(UInt64(1), start); - res[i] = start <= haystack_size + 1 ? 1 : 0; + res[i] = start <= haystack_size + 1 ? start : 0; pos = begin + haystack_offsets[i]; prev_offset = haystack_offsets[i]; From 16a2585d5557b9947035649f712a5621524b869a Mon Sep 17 00:00:00 2001 From: Bharat Nallan Chakravarthy Date: Wed, 18 Jan 2023 16:36:25 -0800 Subject: [PATCH 049/439] fixes to odbc connection pooling --- programs/odbc-bridge/IdentifierQuoteHandler.cpp | 11 ++++++++--- programs/odbc-bridge/MainHandler.cpp | 4 +++- programs/odbc-bridge/SchemaAllowedHandler.cpp | 12 +++++++++--- src/BridgeHelper/XDBCBridgeHelper.h | 8 +++++++- src/Dictionaries/XDBCDictionarySource.cpp | 6 +++++- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index 8157e3b6159..f622995bf15 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -61,13 +61,18 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ return; } + bool use_connection_pooling = params.getParsed("use_connection_pooling", true); + try { std::string connection_string = params.get("connection_string"); - auto connection = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), - getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + nanodbc::ConnectionHolderPtr connection; + if (use_connection_pooling) + connection = ODBCPooledConnectionFactory::instance().get( + validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + else + connection = std::make_shared(validateODBCConnectionString(connection_string)); auto identifier = getIdentifierQuote(std::move(connection)); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 0875cc2e9d9..9130b3e0f47 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -102,7 +102,9 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse std::string format = params.get("format", "RowBinary"); std::string connection_string = params.get("connection_string"); + bool use_connection_pooling = params.getParsed("use_connection_pooling", true); LOG_TRACE(log, "Connection string: '{}'", connection_string); + LOG_TRACE(log, "Use pooling: {}", use_connection_pooling); UInt64 max_block_size = DEFAULT_BLOCK_SIZE; if (params.has("max_block_size")) @@ -134,7 +136,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse try { nanodbc::ConnectionHolderPtr connection_handler; - if (getContext()->getSettingsRef().odbc_bridge_use_connection_pooling) + if (use_connection_pooling) connection_handler = ODBCPooledConnectionFactory::instance().get( validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); else diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index 4d20c8bc3b7..020359f51fd 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -70,13 +70,19 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer return; } + bool use_connection_pooling = params.getParsed("use_connection_pooling", true); + try { std::string connection_string = params.get("connection_string"); - auto connection = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), - getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + nanodbc::ConnectionHolderPtr connection; + + if (use_connection_pooling) + connection = ODBCPooledConnectionFactory::instance().get( + validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + else + connection = std::make_shared(validateODBCConnectionString(connection_string)); bool result = isSchemaAllowed(std::move(connection)); diff --git a/src/BridgeHelper/XDBCBridgeHelper.h b/src/BridgeHelper/XDBCBridgeHelper.h index 139c1ab9726..204edcef82c 100644 --- a/src/BridgeHelper/XDBCBridgeHelper.h +++ b/src/BridgeHelper/XDBCBridgeHelper.h @@ -63,10 +63,12 @@ public: XDBCBridgeHelper( ContextPtr context_, Poco::Timespan http_timeout_, - const std::string & connection_string_) + const std::string & connection_string_, + bool use_connection_pooling_=true) : IXDBCBridgeHelper(context_->getGlobalContext()) , log(&Poco::Logger::get(BridgeHelperMixin::getName() + "BridgeHelper")) , connection_string(connection_string_) + , use_connection_pooling(use_connection_pooling_) , http_timeout(http_timeout_) , config(context_->getGlobalContext()->getConfigRef()) { @@ -132,6 +134,7 @@ protected: uri.setHost(bridge_host); uri.setPort(bridge_port); uri.setScheme("http"); + uri.addQueryParameter("use_connection_pooling", toString(use_connection_pooling)); return uri; } @@ -146,6 +149,7 @@ private: Poco::Logger * log; std::string connection_string; + bool use_connection_pooling; Poco::Timespan http_timeout; std::string bridge_host; size_t bridge_port; @@ -189,6 +193,7 @@ protected: uri.setPath(SCHEMA_ALLOWED_HANDLER); uri.addQueryParameter("version", std::to_string(XDBC_BRIDGE_PROTOCOL_VERSION)); uri.addQueryParameter("connection_string", getConnectionString()); + uri.addQueryParameter("use_connection_pooling", toString(use_connection_pooling)); ReadWriteBufferFromHTTP buf(uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts::getHTTPTimeouts(getContext()), credentials); @@ -210,6 +215,7 @@ protected: uri.setPath(IDENTIFIER_QUOTE_HANDLER); uri.addQueryParameter("version", std::to_string(XDBC_BRIDGE_PROTOCOL_VERSION)); uri.addQueryParameter("connection_string", getConnectionString()); + uri.addQueryParameter("use_connection_pooling", toString(use_connection_pooling)); ReadWriteBufferFromHTTP buf(uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts::getHTTPTimeouts(getContext()), credentials); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index dec4feb5ced..7ff8a8ece15 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -233,7 +233,11 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) bool /* check_config */) -> DictionarySourcePtr { #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( - global_context, global_context->getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); + global_context, + global_context->getSettings().http_receive_timeout, + config.getString(config_prefix + ".odbc.connection_string"), + config.getBool(config_prefix + ".settings.odbc_bridge_use_connection_pooling", + global_context->getSettingsRef().odbc_bridge_use_connection_pooling)); std::string settings_config_prefix = config_prefix + ".odbc"; From f80246644ccd146ad2cecba985bf2a8e45a72959 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:04:09 +0100 Subject: [PATCH 050/439] Update test.py --- tests/integration/test_storage_rabbitmq/test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index be066818c94..bc7e633dfb3 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1,10 +1,5 @@ import pytest -# FIXME This test is too flaky -# https://github.com/ClickHouse/ClickHouse/issues/45160 - -pytestmark = pytest.mark.skip - import json import os.path as p import random From 1452d2588ce0cdc984dd6ec9161f34eb24475e25 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:06:43 +0100 Subject: [PATCH 051/439] Update test.py --- tests/integration/test_storage_rabbitmq/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index bc7e633dfb3..b4e7c98e282 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -631,7 +631,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): rabbitmq_num_queues = 5, rabbitmq_num_consumers = 10, rabbitmq_max_block_size = 100, - rabbitmq_flush_interval_ms=10000, + rabbitmq_flush_interval_ms=500, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view (key UInt64, value UInt64, channel_id String) @@ -1974,7 +1974,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_1(rabbitmq_cluster): CREATE TABLE test.consume (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', - rabbitmq_flush_interval_ms=1000, + rabbitmq_flush_interval_ms=500, rabbitmq_max_block_size = 100, rabbitmq_exchange_name = 'producer_reconnect', rabbitmq_format = 'JSONEachRow', From 2fb49503dacee99091bb3804a71006394440e125 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 19 Jan 2023 14:02:06 +0800 Subject: [PATCH 052/439] Make addAttribute exception safe --- src/Common/OpenTelemetryTraceContext.cpp | 71 ++++++++++-------------- src/Common/OpenTelemetryTraceContext.h | 19 +++---- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 515060803d6..365725c1a98 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -14,76 +14,65 @@ namespace OpenTelemetry thread_local TracingContextOnThread current_thread_trace_context; -void Span::addAttribute(std::string_view name, UInt64 value) +bool Span::addAttribute(std::string_view name, UInt64 value) noexcept { if (!this->isTraceEnabled() || name.empty()) - return; + return false; - this->attributes.push_back(Tuple{name, toString(value)}); + return addAttribute(name, toString(value)); } -void Span::addAttributeIfNotZero(std::string_view name, UInt64 value) +bool Span::addAttributeIfNotZero(std::string_view name, UInt64 value) noexcept { - if (value != 0) - addAttribute(name, value); + if (!this->isTraceEnabled() || name.empty() || value == 0) + return false; + + return addAttribute(name, toString(value)); } -void Span::addAttribute(std::string_view name, std::string_view value) +bool Span::addAttribute(std::string_view name, std::string_view value) noexcept { if (!this->isTraceEnabled() || name.empty()) - return; + return false; - this->attributes.push_back(Tuple{name, value}); + try + { + this->attributes.push_back(Tuple{name, value}); + } + catch (...) + { + return false; + } + return true; } -void Span::addAttributeIfNotEmpty(std::string_view name, std::string_view value) +bool Span::addAttributeIfNotEmpty(std::string_view name, std::string_view value) noexcept { - if (!this->isTraceEnabled() || name.empty() || value.empty()) - return; - - this->attributes.push_back(Tuple{name, value}); + return value.empty() ? false : addAttribute(name, value); } -void Span::addAttribute(std::string_view name, std::function value_supplier) +bool Span::addAttribute(std::string_view name, std::function value_supplier) noexcept { if (!this->isTraceEnabled() || !value_supplier) - return; + return false; - String value = value_supplier(); - if (value.empty()) - return; - - this->attributes.push_back(Tuple{name, value}); + return addAttributeIfNotEmpty(name, value_supplier()); } -void Span::addAttribute(const Exception & e) noexcept +bool Span::addAttribute(const Exception & e) noexcept { if (!this->isTraceEnabled()) - return; + return false; - try - { - this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)}); - } - catch (...) - { - /// Ignore exceptions - } + return addAttribute("clickhouse.exception", getExceptionMessage(e, false)); } -void Span::addAttribute(std::exception_ptr e) noexcept +bool Span::addAttribute(std::exception_ptr e) noexcept { if (!this->isTraceEnabled() || e == nullptr) - return; + return false; - try - { - this->attributes.push_back(Tuple{"clickhouse.exception", getExceptionMessage(e, false)}); - } - catch (...) - { - /// Ignore exceptions - } + return addAttribute("clickhouse.exception", getExceptionMessage(e, false)); } SpanHolder::SpanHolder(std::string_view _operation_name) diff --git a/src/Common/OpenTelemetryTraceContext.h b/src/Common/OpenTelemetryTraceContext.h index 03bac2891fc..930262f2c92 100644 --- a/src/Common/OpenTelemetryTraceContext.h +++ b/src/Common/OpenTelemetryTraceContext.h @@ -23,16 +23,15 @@ struct Span UInt64 finish_time_us = 0; Map attributes; - void addAttribute(std::string_view name, UInt64 value); - void addAttributeIfNotZero(std::string_view name, UInt64 value); - void addAttribute(std::string_view name, std::string_view value); - void addAttributeIfNotEmpty(std::string_view name, std::string_view value); - void addAttribute(std::string_view name, std::function value_supplier); - - /// Following two methods are declared as noexcept to make sure they're exception safe - /// This is because they're usually called in exception handler - void addAttribute(const Exception & e) noexcept; - void addAttribute(std::exception_ptr e) noexcept; + /// Following methods are declared as noexcept to make sure they're exception safe + /// This is because sometimes they will be called in exception handlers/dtor + bool addAttribute(std::string_view name, UInt64 value) noexcept; + bool addAttributeIfNotZero(std::string_view name, UInt64 value) noexcept; + bool addAttribute(std::string_view name, std::string_view value) noexcept; + bool addAttributeIfNotEmpty(std::string_view name, std::string_view value) noexcept; + bool addAttribute(std::string_view name, std::function value_supplier) noexcept; + bool addAttribute(const Exception & e) noexcept; + bool addAttribute(std::exception_ptr e) noexcept; bool isTraceEnabled() const { From 75191fd86853f7b656d6c868bd0c1ed52c398438 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 20 Jan 2023 14:51:44 +0800 Subject: [PATCH 053/439] Catch exception raised by function object --- src/Common/OpenTelemetryTraceContext.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 365725c1a98..65e66b1da96 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -56,7 +56,15 @@ bool Span::addAttribute(std::string_view name, std::function value_sup if (!this->isTraceEnabled() || !value_supplier) return false; - return addAttributeIfNotEmpty(name, value_supplier()); + try + { + return addAttributeIfNotEmpty(name, value_supplier()); + } + catch (...) + { + /// Ignore exception raised by value_supplier + return false; + } } bool Span::addAttribute(const Exception & e) noexcept From fb8452aa4909058962f2ddc055fe3e029fee44f3 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 13 Jan 2023 11:26:41 +0800 Subject: [PATCH 054/439] Add last_exception_time to replication_queue --- src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp | 1 + src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h | 1 + src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 1 + src/Storages/System/StorageSystemReplicationQueue.cpp | 4 +++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 36a5957bf1d..de416c3388a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -91,6 +91,7 @@ bool ReplicatedMergeMutateTaskBase::executeStep() auto & log_entry = selected_entry->log_entry; log_entry->exception = saved_exception; + log_entry->last_exception_time = time(nullptr); if (log_entry->type == ReplicatedMergeTreeLogEntryData::MUTATE_PART) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 84115cb2b8a..476d4797eef 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -157,6 +157,7 @@ struct ReplicatedMergeTreeLogEntryData /// Access under queue_mutex, see ReplicatedMergeTreeQueue. size_t num_tries = 0; /// The number of attempts to perform the action (since the server started, including the running one). std::exception_ptr exception; /// The last exception, in the case of an unsuccessful attempt to perform the action. + time_t last_exception_time = 0; /// The time at which the last exception occurred. time_t last_attempt_time = 0; /// The time at which the last attempt was attempted to complete the action. size_t num_postponed = 0; /// The number of times the action was postponed. String postpone_reason; /// The reason why the action was postponed, if it was postponed. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index bd75d76109a..628b3048cfc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1691,6 +1691,7 @@ bool ReplicatedMergeTreeQueue::processEntry( { std::lock_guard lock(state_mutex); entry->exception = saved_exception; + entry->last_exception_time = time(nullptr); return false; } diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 8acd192eac4..fd9f874052d 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -38,6 +38,7 @@ NamesAndTypesList StorageSystemReplicationQueue::getNamesAndTypes() { "is_currently_executing", std::make_shared() }, { "num_tries", std::make_shared() }, { "last_exception", std::make_shared() }, + { "last_exception_time", std::make_shared() }, { "last_attempt_time", std::make_shared() }, { "num_postponed", std::make_shared() }, { "postpone_reason", std::make_shared() }, @@ -141,7 +142,8 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte res_columns[col_num++]->insert(entry.detach); res_columns[col_num++]->insert(entry.currently_executing); res_columns[col_num++]->insert(entry.num_tries); - res_columns[col_num++]->insert(entry.exception ? getExceptionMessage(entry.exception, false) : ""); + res_columns[col_num++]->insert(entry.exception ? getExceptionMessage(entry.exception, true) : ""); + res_columns[col_num++]->insert(UInt64(entry.last_exception_time)); res_columns[col_num++]->insert(UInt64(entry.last_attempt_time)); res_columns[col_num++]->insert(entry.num_postponed); res_columns[col_num++]->insert(entry.postpone_reason); From 1fe9bde59409db16bc2da8f720877986899d9111 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 20 Jan 2023 11:04:36 +0800 Subject: [PATCH 055/439] Set replica name to zero copy exclusive lock --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 7 ++++--- src/Storages/MergeTree/ZeroCopyLock.cpp | 4 ++-- src/Storages/MergeTree/ZeroCopyLock.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.h | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index bd75d76109a..09b9365cf71 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1363,10 +1363,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!disk->supportZeroCopyReplication()) only_s3_storage = false; - if (!disks.empty() && only_s3_storage && storage.checkZeroCopyLockExists(entry.new_part_name, disks[0])) + String replica_to_execute_merge; + if (!disks.empty() && only_s3_storage && storage.checkZeroCopyLockExists(entry.new_part_name, disks[0], replica_to_execute_merge)) { - constexpr auto fmt_string = "Not executing merge/mutation for the part {}, waiting other replica to execute it and will fetch after."; - out_postpone_reason = fmt::format(fmt_string, entry.new_part_name); + constexpr auto fmt_string = "Not executing merge/mutation for the part {}, waiting for {} to execute it and will fetch after."; + out_postpone_reason = fmt::format(fmt_string, entry.new_part_name, replica_to_execute_merge;); return false; } } diff --git a/src/Storages/MergeTree/ZeroCopyLock.cpp b/src/Storages/MergeTree/ZeroCopyLock.cpp index dbb12d0d610..53dfe0c769f 100644 --- a/src/Storages/MergeTree/ZeroCopyLock.cpp +++ b/src/Storages/MergeTree/ZeroCopyLock.cpp @@ -2,8 +2,8 @@ namespace DB { - ZeroCopyLock::ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path) - : lock(zkutil::createSimpleZooKeeperLock(zookeeper, lock_path, "part_exclusive_lock", "")) + ZeroCopyLock::ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message) + : lock(zkutil::createSimpleZooKeeperLock(zookeeper, lock_path, "part_exclusive_lock", lock_message)) { } } diff --git a/src/Storages/MergeTree/ZeroCopyLock.h b/src/Storages/MergeTree/ZeroCopyLock.h index 96709fb01c9..4e73b27804c 100644 --- a/src/Storages/MergeTree/ZeroCopyLock.h +++ b/src/Storages/MergeTree/ZeroCopyLock.h @@ -12,7 +12,7 @@ namespace DB /// because due to bad abstraction we use it in MergeTreeData. struct ZeroCopyLock { - ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path); + ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message); /// Actual lock std::unique_ptr lock; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d4fd329ef90..320e3754b83 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8453,14 +8453,14 @@ Strings StorageReplicatedMergeTree::getZeroCopyPartPath( return res; } -bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk) +bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica) { auto path = getZeroCopyPartPath(part_name, disk); if (path) { /// FIXME auto lock_path = fs::path(*path) / "part_exclusive_lock"; - if (getZooKeeper()->exists(lock_path)) + if (getZooKeeper()->tryGet(lock_path, lock_replica)) { return true; } @@ -8493,7 +8493,7 @@ std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusi zookeeper->createIfNotExists(zc_zookeeper_path, ""); /// Create actual lock - ZeroCopyLock lock(zookeeper, zc_zookeeper_path); + ZeroCopyLock lock(zookeeper, zc_zookeeper_path, replica_name); if (lock.lock->tryLock()) return lock; else diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 71f143aa8a4..842ff584a8a 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -858,7 +858,7 @@ private: // Create table id if needed void createTableSharedID() const; - bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk); + bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); std::optional getZeroCopyPartPath(const String & part_name, const DiskPtr & disk); From be8ceeb4bb857028c8bba7076e65dc33dbdd5fce Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 13 Jan 2023 18:42:52 +0800 Subject: [PATCH 056/439] Save stack trace in part_log --- src/Common/Exception.cpp | 6 ++++++ src/Common/Exception.h | 2 ++ src/Storages/MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergePlainMergeTreeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MutateFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp | 2 +- .../MergeTree/ReplicatedMergeMutateTaskBase.cpp | 4 ++-- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 11 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 35231354651..b83ddf09c7f 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -546,6 +546,12 @@ ExecutionStatus ExecutionStatus::fromCurrentException(const std::string & start_ return ExecutionStatus(getCurrentExceptionCode(), msg); } +ExecutionStatus ExecutionStatus::fromCurrentExceptionWithStackTrace(const std::string & start_of_message) +{ + String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(true, true); + return ExecutionStatus(getCurrentExceptionCode(), msg); +} + ExecutionStatus ExecutionStatus::fromText(const std::string & data) { ExecutionStatus status; diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 0f459a887f1..58da20359ad 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -204,6 +204,8 @@ struct ExecutionStatus : code(return_code), message(exception_message) {} static ExecutionStatus fromCurrentException(const std::string & start_of_message = ""); + + static ExecutionStatus fromCurrentExceptionWithStackTrace(const std::string & start_of_message = ""); static ExecutionStatus fromText(const std::string & data); diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 97226825629..ca3e25ef077 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -333,7 +333,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite " We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(part), "merge-not-byte-identical"); diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index cc5e87956a1..c38fefd9d32 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -52,7 +52,7 @@ bool MergePlainMergeTreeTask::executeStep() } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b1384dc799f..e1e1312cd6c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7480,7 +7480,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); if (cloned_part) cloned_part->remove(); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index b432841d5b0..65b3f8b15a8 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -209,7 +209,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit LOG_ERROR(log, "{}. Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(new_part), "mutate-not-byte-identical"); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index b1714076a46..28452aaf856 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -101,7 +101,7 @@ bool MutatePlainMergeTreeTask::executeStep() String exception_message = getCurrentExceptionMessage(false); LOG_ERROR(&Poco::Logger::get("MutatePlainMergeTreeTask"), "{}", exception_message); storage.updateMutationEntriesErrors(future_part, false, exception_message); - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); tryLogCurrentException(__PRETTY_FUNCTION__); return false; } diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 36a5957bf1d..09fda943600 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -188,7 +188,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentException()); + part_log_writer(ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } @@ -204,7 +204,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentException()); + part_log_writer(ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 14ae620176a..9ebed2467ad 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -520,7 +520,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF } catch (...) { - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } } @@ -588,7 +588,7 @@ void ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData: } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c8dc53f5036..ec17c765e05 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1784,7 +1784,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } } @@ -1864,7 +1864,7 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d4fd329ef90..04fbdad22d1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2336,7 +2336,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (...) { - PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); for (const auto & res_part : res_parts) unlockSharedData(*res_part); @@ -4185,7 +4185,7 @@ bool StorageReplicatedMergeTree::fetchPart( catch (...) { if (!to_detached) - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } @@ -4296,7 +4296,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException()); + write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); throw; } @@ -7029,7 +7029,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); for (const auto & dst_part : dst_parts) unlockSharedData(*dst_part); @@ -7255,7 +7255,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); for (const auto & dst_part : dst_parts) dest_table_storage->unlockSharedData(*dst_part); From 38799eb85238003b15f6a0e397a3e0650f0aa9de Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 20 Jan 2023 16:35:28 +0800 Subject: [PATCH 057/439] Merge two methods as one --- src/Storages/MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergePlainMergeTreeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MutateFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp | 2 +- .../MergeTree/ReplicatedMergeMutateTaskBase.cpp | 4 ++-- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index ca3e25ef077..16efde3e4cc 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -333,7 +333,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite " We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(part), "merge-not-byte-identical"); diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index c38fefd9d32..9f24839f1e1 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -52,7 +52,7 @@ bool MergePlainMergeTreeTask::executeStep() } catch (...) { - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); throw; } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e1e1312cd6c..6184669b223 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7480,7 +7480,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge } catch (...) { - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); if (cloned_part) cloned_part->remove(); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 65b3f8b15a8..8b5c6bf013d 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -209,7 +209,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit LOG_ERROR(log, "{}. Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(new_part), "mutate-not-byte-identical"); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 28452aaf856..e801c0a8879 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -101,7 +101,7 @@ bool MutatePlainMergeTreeTask::executeStep() String exception_message = getCurrentExceptionMessage(false); LOG_ERROR(&Poco::Logger::get("MutatePlainMergeTreeTask"), "{}", exception_message); storage.updateMutationEntriesErrors(future_part, false, exception_message); - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); tryLogCurrentException(__PRETTY_FUNCTION__); return false; } diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 09fda943600..7225a7c41d3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -188,7 +188,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + part_log_writer(ExecutionStatus::fromCurrentException("", true)); throw; } @@ -204,7 +204,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + part_log_writer(ExecutionStatus::fromCurrentException("", true)); throw; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 9ebed2467ad..c4a8f837ec9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -520,7 +520,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF } catch (...) { - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException("", true)); throw; } } @@ -588,7 +588,7 @@ void ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData: } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); throw; } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index ec17c765e05..5536e8af793 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1784,7 +1784,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); throw; } } @@ -1864,7 +1864,7 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); throw; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 04fbdad22d1..c0d0780a550 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2336,7 +2336,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (...) { - PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); for (const auto & res_part : res_parts) unlockSharedData(*res_part); @@ -4185,7 +4185,7 @@ bool StorageReplicatedMergeTree::fetchPart( catch (...) { if (!to_detached) - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); throw; } @@ -4296,7 +4296,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( } catch (...) { - write_part_log(ExecutionStatus::fromCurrentExceptionWithStackTrace()); + write_part_log(ExecutionStatus::fromCurrentException("", true)); throw; } @@ -7029,7 +7029,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); for (const auto & dst_part : dst_parts) unlockSharedData(*dst_part); @@ -7255,7 +7255,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentExceptionWithStackTrace()); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); for (const auto & dst_part : dst_parts) dest_table_storage->unlockSharedData(*dst_part); From f867c4c2061858bc2a3e58855acaa90b93e753dc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 20 Jan 2023 16:43:10 +0800 Subject: [PATCH 058/439] Add an overriden fromCurrentException to simply the code --- src/Common/Exception.cpp | 9 ++++----- src/Common/Exception.h | 5 ++--- src/Storages/MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergePlainMergeTreeTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MutateFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp | 2 +- .../MergeTree/ReplicatedMergeMutateTaskBase.cpp | 4 ++-- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 4 ++-- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 11 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index b83ddf09c7f..10fc3dcbfae 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -540,15 +540,14 @@ bool ExecutionStatus::tryDeserializeText(const std::string & data) return true; } -ExecutionStatus ExecutionStatus::fromCurrentException(const std::string & start_of_message) +ExecutionStatus ExecutionStatus::fromCurrentException(bool with_stacktrace) { - String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(false, true); - return ExecutionStatus(getCurrentExceptionCode(), msg); + return ExecutionStatus(getCurrentExceptionCode(), getCurrentExceptionMessage(with_stacktrace, true)); } -ExecutionStatus ExecutionStatus::fromCurrentExceptionWithStackTrace(const std::string & start_of_message) +ExecutionStatus ExecutionStatus::fromCurrentException(const std::string & start_of_message, bool with_stacktrace) { - String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(true, true); + String msg = (start_of_message.empty() ? "" : (start_of_message + ": ")) + getCurrentExceptionMessage(with_stacktrace, true); return ExecutionStatus(getCurrentExceptionCode(), msg); } diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 58da20359ad..96668c68cd3 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -203,9 +203,8 @@ struct ExecutionStatus explicit ExecutionStatus(int return_code, const std::string & exception_message = "") : code(return_code), message(exception_message) {} - static ExecutionStatus fromCurrentException(const std::string & start_of_message = ""); - - static ExecutionStatus fromCurrentExceptionWithStackTrace(const std::string & start_of_message = ""); + static ExecutionStatus fromCurrentException(bool with_stacktrace = false); + static ExecutionStatus fromCurrentException(const std::string & start_of_message, bool with_stacktrace = false); static ExecutionStatus fromText(const std::string & data); diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 16efde3e4cc..b3934098d0d 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -333,7 +333,7 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite " We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(part), "merge-not-byte-identical"); diff --git a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp index 9f24839f1e1..e59f7fda44c 100644 --- a/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MergePlainMergeTreeTask.cpp @@ -52,7 +52,7 @@ bool MergePlainMergeTreeTask::executeStep() } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); throw; } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6184669b223..5addb24b0ca 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7480,7 +7480,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); if (cloned_part) cloned_part->remove(); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 8b5c6bf013d..7bf3a95876b 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -209,7 +209,7 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit LOG_ERROR(log, "{}. Data after mutation is not byte-identical to data on another replicas. " "We will download merged part from replica to force byte-identical result.", getCurrentExceptionMessage(false)); - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); if (storage.getSettings()->detach_not_byte_identical_parts) storage.forcefullyMovePartToDetachedAndRemoveFromMemory(std::move(new_part), "mutate-not-byte-identical"); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index e801c0a8879..1ab1e2a82ec 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -101,7 +101,7 @@ bool MutatePlainMergeTreeTask::executeStep() String exception_message = getCurrentExceptionMessage(false); LOG_ERROR(&Poco::Logger::get("MutatePlainMergeTreeTask"), "{}", exception_message); storage.updateMutationEntriesErrors(future_part, false, exception_message); - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); tryLogCurrentException(__PRETTY_FUNCTION__); return false; } diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index 7225a7c41d3..bbe83814497 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -188,7 +188,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentException("", true)); + part_log_writer(ExecutionStatus::fromCurrentException(true)); throw; } @@ -204,7 +204,7 @@ bool ReplicatedMergeMutateTaskBase::executeImpl() catch (...) { if (part_log_writer) - part_log_writer(ExecutionStatus::fromCurrentException("", true)); + part_log_writer(ExecutionStatus::fromCurrentException(true)); throw; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index c4a8f837ec9..03b2dadde5c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -520,7 +520,7 @@ void ReplicatedMergeTreeSinkImpl::finishDelayedChunk(const ZooKeeperWithF } catch (...) { - PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException(true)); throw; } } @@ -588,7 +588,7 @@ void ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData: } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); throw; } } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 5536e8af793..20b0c16edb7 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1784,7 +1784,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); throw; } } @@ -1864,7 +1864,7 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); throw; } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index c0d0780a550..82ff0a0bd83 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2336,7 +2336,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) } catch (...) { - PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), res_parts, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); for (const auto & res_part : res_parts) unlockSharedData(*res_part); @@ -4185,7 +4185,7 @@ bool StorageReplicatedMergeTree::fetchPart( catch (...) { if (!to_detached) - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); throw; } @@ -4296,7 +4296,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::fetchExistsPart( } catch (...) { - write_part_log(ExecutionStatus::fromCurrentException("", true)); + write_part_log(ExecutionStatus::fromCurrentException(true)); throw; } @@ -7029,7 +7029,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); for (const auto & dst_part : dst_parts) unlockSharedData(*dst_part); @@ -7255,7 +7255,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta } catch (...) { - PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException("", true)); + PartLog::addNewParts(getContext(), dst_parts, watch.elapsed(), ExecutionStatus::fromCurrentException(true)); for (const auto & dst_part : dst_parts) dest_table_storage->unlockSharedData(*dst_part); From 01048c6568fbeeab00fb52fd2e209460cc66c59b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 20 Jan 2023 17:05:28 +0800 Subject: [PATCH 059/439] change as request --- src/Functions/PositionImpl.h | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 0b85518bad6..6e968c16351 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -205,21 +205,37 @@ struct PositionImpl { for (auto & x : res) x = 1; + return; } - else + + ColumnString::Offset prev_offset = 0; + auto rows = haystack_offsets.size(); + const ColumnConst * start_pos_const = typeid_cast(&*start_pos); + if (start_pos_const) { - ColumnString::Offset prev_offset = 0; - for (size_t i = 0; i < haystack_offsets.size(); ++i) + UInt64 start = std::max(start_pos_const->getUInt(0), UInt64(1)); + for (size_t i = 0; i < rows; ++i) { size_t haystack_size = Impl::countChars( reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1)); - UInt64 start = start_pos->getUInt(i); - start = std::max(UInt64(1), start); res[i] = start <= haystack_size + 1 ? start : 0; pos = begin + haystack_offsets[i]; prev_offset = haystack_offsets[i]; } + return; + } + + for (size_t i = 0; i < rows; ++i) + { + size_t haystack_size = Impl::countChars( + reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1)); + UInt64 start = start_pos->getUInt(i); + start = std::max(UInt64(1), start); + res[i] = start <= haystack_size + 1 ? start : 0; + + pos = begin + haystack_offsets[i]; + prev_offset = haystack_offsets[i]; } return; } From 838acb22b70633f9e65d64bd9d6a14287b08d8e6 Mon Sep 17 00:00:00 2001 From: Aleksandr Musorin Date: Sun, 23 Oct 2022 21:50:30 +0200 Subject: [PATCH 060/439] added num_processed_files and processed_files_size --- src/Backups/BackupImpl.cpp | 30 ++++ src/Backups/BackupImpl.h | 12 +- src/Backups/BackupsWorker.cpp | 19 ++- src/Backups/BackupsWorker.h | 10 +- src/Backups/IBackup.h | 6 + src/Storages/System/StorageSystemBackups.cpp | 6 + .../test_backup_restore_new/test.py | 128 +++++++++++++++++- 7 files changed, 201 insertions(+), 10 deletions(-) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index ec35b8ed07a..32e0d74b90d 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -271,6 +271,18 @@ size_t BackupImpl::getNumFiles() const return num_files; } +size_t BackupImpl::getNumProcessedFiles() const +{ + std::lock_guard lock{mutex}; + return num_processed_files; +} + +UInt64 BackupImpl::getProcessedFilesSize() const +{ + std::lock_guard lock{mutex}; + return processed_files_size; +} + UInt64 BackupImpl::getUncompressedSize() const { std::lock_guard lock{mutex}; @@ -355,6 +367,7 @@ void BackupImpl::writeBackupMetadata() out->finalize(); increaseUncompressedSize(str.size()); + increaseProcessedSize(str.size()); } @@ -380,6 +393,7 @@ void BackupImpl::readBackupMetadata() String str; readStringUntilEOF(str, *in); increaseUncompressedSize(str.size()); + increaseProcessedSize(str.size()); Poco::XML::DOMParser dom_parser; Poco::AutoPtr config = dom_parser.parseMemory(str.data(), str.size()); const Poco::XML::Node * config_root = getRootNode(config); @@ -598,6 +612,8 @@ BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) c if (open_mode != OpenMode::READ) throw Exception("Backup is not opened for reading", ErrorCodes::LOGICAL_ERROR); + increaseProcessedSize(size_and_checksum.first); + if (!size_and_checksum.first) { /// Entry's data is empty. @@ -761,6 +777,8 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry) .base_checksum = 0, }; + increaseProcessedSize(info); + /// Empty file, nothing to backup if (info.size == 0 && deduplicate_files) { @@ -971,6 +989,18 @@ void BackupImpl::increaseUncompressedSize(const FileInfo & info) increaseUncompressedSize(info.size - info.base_size); } +void BackupImpl::increaseProcessedSize(UInt64 file_size) const +{ + processed_files_size += file_size; + ++num_processed_files; +} + +void BackupImpl::increaseProcessedSize(const FileInfo & info) +{ + if ((info.size > info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name))) + increaseProcessedSize(info.size - info.base_size); +} + void BackupImpl::setCompressedSize() { if (use_archives) diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 9fc881bf680..45c471aa825 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -59,6 +59,8 @@ public: time_t getTimestamp() const override { return timestamp; } UUID getUUID() const override { return *uuid; } size_t getNumFiles() const override; + size_t getNumProcessedFiles() const override; + UInt64 getProcessedFilesSize() const override; UInt64 getUncompressedSize() const override; UInt64 getCompressedSize() const override; Strings listFiles(const String & directory, bool recursive) const override; @@ -101,10 +103,16 @@ private: std::shared_ptr getArchiveReader(const String & suffix) const; std::shared_ptr getArchiveWriter(const String & suffix); - /// Increases `uncompressed_size` by a specific value and `num_files` by 1. + /// Increases `uncompressed_size` by a specific value, + /// also increases `num_files` by 1. void increaseUncompressedSize(UInt64 file_size); void increaseUncompressedSize(const FileInfo & info); + /// Increases `num_processed_files` by a specific value, + /// also increases `num_processed_files` by 1. + void increaseProcessedSize(UInt64 file_size) const; + void increaseProcessedSize(const FileInfo & info); + /// Calculates and sets `compressed_size`. void setCompressedSize(); @@ -121,6 +129,8 @@ private: std::optional uuid; time_t timestamp = 0; size_t num_files = 0; + mutable size_t num_processed_files = 0; + mutable UInt64 processed_files_size = 0; UInt64 uncompressed_size = 0; UInt64 compressed_size = 0; int version; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 53bebaf06d7..2c012464ca6 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -341,16 +341,20 @@ void BackupsWorker::doBackup( } size_t num_files = 0; + size_t num_processed_files = 0; UInt64 uncompressed_size = 0; UInt64 compressed_size = 0; + UInt64 processed_files_size = 0; /// Finalize backup (write its metadata). if (!backup_settings.internal) { backup->finalizeWriting(); num_files = backup->getNumFiles(); + num_processed_files = backup->getNumProcessedFiles(); uncompressed_size = backup->getUncompressedSize(); compressed_size = backup->getCompressedSize(); + processed_files_size = backup->getProcessedFilesSize(); } /// Close the backup. @@ -358,7 +362,7 @@ void BackupsWorker::doBackup( LOG_INFO(log, "{} {} was created successfully", (backup_settings.internal ? "Internal backup" : "Backup"), backup_name_for_logging); setStatus(backup_id, BackupStatus::BACKUP_CREATED); - setNumFilesAndSize(backup_id, num_files, uncompressed_size, compressed_size); + setNumFilesAndSize(backup_id, num_files, num_processed_files, processed_files_size, uncompressed_size, compressed_size); } catch (...) { @@ -501,8 +505,6 @@ void BackupsWorker::doRestore( backup_open_params.password = restore_settings.password; BackupPtr backup = BackupFactory::instance().createBackup(backup_open_params); - setNumFilesAndSize(restore_id, backup->getNumFiles(), backup->getUncompressedSize(), backup->getCompressedSize()); - String current_database = context->getCurrentDatabase(); /// Checks access rights if this is ON CLUSTER query. @@ -583,6 +585,13 @@ void BackupsWorker::doRestore( LOG_INFO(log, "Restored from {} {} successfully", (restore_settings.internal ? "internal backup" : "backup"), backup_name_for_logging); setStatus(restore_id, BackupStatus::RESTORED); + setNumFilesAndSize( + restore_id, + backup->getNumFiles(), + backup->getNumProcessedFiles(), + backup->getProcessedFilesSize(), + backup->getUncompressedSize(), + backup->getCompressedSize()); } catch (...) { @@ -663,7 +672,7 @@ void BackupsWorker::setStatus(const String & id, BackupStatus status, bool throw } -void BackupsWorker::setNumFilesAndSize(const String & id, size_t num_files, UInt64 uncompressed_size, UInt64 compressed_size) +void BackupsWorker::setNumFilesAndSize(const String & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size) { std::lock_guard lock{infos_mutex}; auto it = infos.find(id); @@ -672,6 +681,8 @@ void BackupsWorker::setNumFilesAndSize(const String & id, size_t num_files, UInt auto & info = it->second; info.num_files = num_files; + info.num_processed_files = num_processed_files; + info.processed_files_size = processed_files_size; info.uncompressed_size = uncompressed_size; info.compressed_size = compressed_size; } diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index b6d9729833e..19d2e4d0d1d 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -56,6 +56,14 @@ public: /// Number of files in the backup (including backup's metadata; only unique files are counted). size_t num_files = 0; + /// Number of processed files during backup or restore process + /// For restore it includes files from base backups + size_t num_processed_files = 0; + + /// Size of processed files during backup or restore + /// For restore in includes sizes from base backups + UInt64 processed_files_size = 0; + /// Size of all files in the backup (including backup's metadata; only unique files are counted). UInt64 uncompressed_size = 0; @@ -102,7 +110,7 @@ private: void addInfo(const OperationID & id, const String & name, bool internal, BackupStatus status); void setStatus(const OperationID & id, BackupStatus status, bool throw_if_error = true); void setStatusSafe(const String & id, BackupStatus status) { setStatus(id, status, false); } - void setNumFilesAndSize(const OperationID & id, size_t num_files, UInt64 uncompressed_size, UInt64 compressed_size); + void setNumFilesAndSize(const OperationID & id, size_t num_files, size_t num_processed_files, UInt64 processed_files_size, UInt64 uncompressed_size, UInt64 compressed_size); std::vector getAllActiveBackupInfos() const; std::vector getAllActiveRestoreInfos() const; diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h index 43763c5bfde..13c21fb0287 100644 --- a/src/Backups/IBackup.h +++ b/src/Backups/IBackup.h @@ -40,6 +40,12 @@ public: /// Returns the number of unique files in the backup. virtual size_t getNumFiles() const = 0; + /// Returns the number of files were processed for backup or restore + virtual size_t getNumProcessedFiles() const = 0; + + // Returns the total size of processed files for backup or restore + virtual UInt64 getProcessedFilesSize() const = 0; + /// Returns the total size of unique files in the backup. virtual UInt64 getUncompressedSize() const = 0; diff --git a/src/Storages/System/StorageSystemBackups.cpp b/src/Storages/System/StorageSystemBackups.cpp index 52a26fe0cd6..268cc9d0963 100644 --- a/src/Storages/System/StorageSystemBackups.cpp +++ b/src/Storages/System/StorageSystemBackups.cpp @@ -19,6 +19,8 @@ NamesAndTypesList StorageSystemBackups::getNamesAndTypes() {"name", std::make_shared()}, {"status", std::make_shared(getBackupStatusEnumValues())}, {"num_files", std::make_shared()}, + {"num_processed_files", std::make_shared()}, + {"processed_files_size", std::make_shared()}, {"uncompressed_size", std::make_shared()}, {"compressed_size", std::make_shared()}, {"error", std::make_shared()}, @@ -36,6 +38,8 @@ void StorageSystemBackups::fillData(MutableColumns & res_columns, ContextPtr con auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_status = assert_cast(*res_columns[column_index++]); auto & column_num_files = assert_cast(*res_columns[column_index++]); + auto & column_num_processed_files = assert_cast(*res_columns[column_index++]); + auto & column_processed_files_size = assert_cast(*res_columns[column_index++]); auto & column_uncompressed_size = assert_cast(*res_columns[column_index++]); auto & column_compressed_size = assert_cast(*res_columns[column_index++]); auto & column_error = assert_cast(*res_columns[column_index++]); @@ -48,6 +52,8 @@ void StorageSystemBackups::fillData(MutableColumns & res_columns, ContextPtr con column_name.insertData(info.name.data(), info.name.size()); column_status.insertValue(static_cast(info.status)); column_num_files.insertValue(info.num_files); + column_num_processed_files.insertValue(info.num_processed_files); + column_processed_files_size.insertValue(info.processed_files_size); column_uncompressed_size.insertValue(info.uncompressed_size); column_compressed_size.insertValue(info.compressed_size); column_error.insertData(info.error_message.data(), info.error_message.size()); diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 90e8acc702d..b1fab60f33b 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -192,6 +192,101 @@ def test_incremental_backup(): assert instance.query("SELECT count(), sum(x) FROM test.table2") == "102\t5081\n" +def test_increment_backup_without_changes(): + backup_name = new_backup_name() + incremental_backup_name = new_backup_name() + create_and_fill_table(n=1) + + system_backup_qry = "SELECT status, num_files, num_processed_files, processed_files_size, uncompressed_size, compressed_size, error FROM system.backups WHERE id='{id_backup}'" + + assert instance.query("SELECT count(), sum(x) FROM test.table") == TSV([["1", "0"]]) + + # prepare first backup without base_backup + (id_backup, status) = instance.query( + f"BACKUP TABLE test.table TO {backup_name}" + ).split("\t") + + ( + backup_status, + num_files, + num_processed_files, + processed_files_size, + uncompressed_size, + compressed_size, + error, + ) = ( + instance.query(system_backup_qry.format(id_backup=id_backup)) + .strip("\n") + .split("\t") + ) + + assert backup_status == "BACKUP_CREATED" + assert num_files == "11" + assert int(uncompressed_size) > 0 + assert int(compressed_size) > 0 + assert error == "" + + # create second backup without changes based on the first one + (id_backup_wo_changes, status_backup_wo_changes) = instance.query( + f"BACKUP TABLE test.table TO {incremental_backup_name} SETTINGS base_backup = {backup_name}" + ).split("\t") + + ( + backup_status_wo_changes, + num_files_backup_wo_changes, + num_processed_files_backup_wo_changes, + processed_files_size_backup_wo_changes, + uncompressed_size_backup_wo_changes, + compressed_size_backup_wo_changes, + error_snd, + ) = ( + instance.query(system_backup_qry.format(id_backup=id_backup_wo_changes)) + .strip("\n") + .split("\t") + ) + + assert backup_status_wo_changes == "BACKUP_CREATED" + assert num_files_backup_wo_changes == "1" + assert num_processed_files_backup_wo_changes == "11" + assert int(processed_files_size_backup_wo_changes) > 0 + assert int(uncompressed_size_backup_wo_changes) > 0 + assert int(compressed_size_backup_wo_changes) > 0 + assert error_snd == "" + + # restore the second backup + # we expect to see all files in the meta info of the restore and a sum of uncompressed and compressed sizes + (id_restore, status_restore) = instance.query( + f"RESTORE TABLE test.table AS test.table2 FROM {incremental_backup_name}" + ).split("\t") + + assert instance.query("SELECT count(), sum(x) FROM test.table2") == TSV( + [["1", "0"]] + ) + + ( + restore_status, + restore_num_files, + restore_num_processed_files, + restore_processed_files_size, + restore_uncompressed_size, + restore_compressed_size, + restore_error, + ) = ( + instance.query(system_backup_qry.format(id_backup=id_restore)) + .strip("\n") + .split("\t") + ) + + assert restore_status == "RESTORED" + assert int(restore_num_files) == 1 + assert int(restore_num_processed_files) == int( + num_processed_files_backup_wo_changes + ) + assert int(restore_uncompressed_size) > 0 + assert int(restore_compressed_size) > 0 + assert restore_error == "" + + def test_incremental_backup_overflow(): backup_name = new_backup_name() incremental_backup_name = new_backup_name() @@ -1089,9 +1184,18 @@ def test_system_backups(): id = instance.query(f"BACKUP TABLE test.table TO {backup_name}").split("\t")[0] - [name, status, num_files, uncompressed_size, compressed_size, error] = ( + [ + name, + status, + num_files, + num_processed_files, + processed_files_size, + uncompressed_size, + compressed_size, + error, + ] = ( instance.query( - f"SELECT name, status, num_files, uncompressed_size, compressed_size, error FROM system.backups WHERE id='{id}'" + f"SELECT name, status, num_files, num_processed_files, processed_files_size, uncompressed_size, compressed_size, error FROM system.backups WHERE id='{id}'" ) .strip("\n") .split("\t") @@ -1101,9 +1205,13 @@ def test_system_backups(): num_files = int(num_files) compressed_size = int(compressed_size) uncompressed_size = int(uncompressed_size) + num_processed_files = int(num_processed_files) + processed_files_size = int(processed_files_size) assert name == escaped_backup_name assert status == "BACKUP_CREATED" assert num_files > 1 + assert num_processed_files > 1 + assert processed_files_size > 1 assert uncompressed_size > 1 assert compressed_size == uncompressed_size assert error == "" @@ -1115,9 +1223,17 @@ def test_system_backups(): ) escaped_backup_name = backup_name.replace("'", "\\'") - [status, num_files, uncompressed_size, compressed_size, error] = ( + [ + status, + num_files, + num_processed_files, + processed_files_size, + uncompressed_size, + compressed_size, + error, + ] = ( instance.query( - f"SELECT status, num_files, uncompressed_size, compressed_size, error FROM system.backups WHERE name='{escaped_backup_name}'" + f"SELECT status, num_files, num_processed_files, processed_files_size, uncompressed_size, compressed_size, error FROM system.backups WHERE name='{escaped_backup_name}'" ) .strip("\n") .split("\t") @@ -1126,10 +1242,14 @@ def test_system_backups(): num_files = int(num_files) compressed_size = int(compressed_size) uncompressed_size = int(uncompressed_size) + num_processed_files = int(num_processed_files) + processed_files_size = int(processed_files_size) assert status == "BACKUP_FAILED" assert num_files == 0 assert uncompressed_size == 0 assert compressed_size == 0 + assert num_processed_files == 0 + assert processed_files_size == 0 assert expected_error in error From a2b342bcbe9807351e0c1cbd34397726aada7ea2 Mon Sep 17 00:00:00 2001 From: Diego Nieto Date: Fri, 20 Jan 2023 12:32:50 +0100 Subject: [PATCH 061/439] Update skipping-index.md ADD and DROP are not such lightweight command they generate mutations to deal with the changes and they will take time to complete depending on how much data the table has. --- docs/en/sql-reference/statements/alter/skipping-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/skipping-index.md b/docs/en/sql-reference/statements/alter/skipping-index.md index 037e4bc38c5..6b68afc0e6f 100644 --- a/docs/en/sql-reference/statements/alter/skipping-index.md +++ b/docs/en/sql-reference/statements/alter/skipping-index.md @@ -16,7 +16,7 @@ The following operations are available: - `ALTER TABLE [db.]table_name [ON CLUSTER cluster] MATERIALIZE INDEX name [IN PARTITION partition_name]` - Rebuilds the secondary index `name` for the specified `partition_name`. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations). If `IN PARTITION` part is omitted then it rebuilds the index for the whole table data. -The first two commands are lightweight in a sense that they only change metadata or remove files. +The first two commands are lightweight in a sense that they only change metadata or remove files but they also do create a mutation that will add/drop a column for the index for each part. Also, they are replicated, syncing indices metadata via ZooKeeper. From e6167d6b3632a8cfbe5e3a1f0010da0d30093d88 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 20 Jan 2023 14:30:53 +0000 Subject: [PATCH 062/439] Deprecate Gorilla compression of non-float columns Reasons: 1. The original Gorilla paper proposed a compression schema for pairs of time stamps and double-precision FP values. ClickHouse's Gorilla codec only implements compression of the latter and it does not impose any data type restrictions. - Data types != Float* or (U)Int* (e.g. Decimal, Point etc.) are definitely not supposed to be used with Gorilla. - (U)Int* types are debatable. The paper only considers integers-stored-as-FP-values, a practical use case for which Gorilla works well. Standalone integers are not considered which makes them at least suspicious. 2. Achieve consistency with FPC, another specialized floating-point timeseries codec, which rejects non-float data. 3. On practical datasets, ZSTD is often "good enough" (**) so it should be okay to disincentive non-ZSTD codecs a little bit. If needed, Delta and DoubleDelta codecs are viable alternative for slowly changing (time-series-like) integer sequences. Since on-prem and hosted users may still have Gorilla-compressed non-float data, this combination is only deprecated for now. No warning or error will be emitted. Users are encouraged to migrate Gorilla-compressed non-float data to an alternative codec. It is planned to treat Gorilla-compressed non-float columns as "suspicious" six months after this commit (i.e. in v23.6). Even then, it will still be possible to set "allow_suspicious_codecs = true" and read and write Gorilla-compressed non-float data. (*) Sec. 4.1.2, "Gorilla restricts the value element in its tuple to a double floating point type.", https://doi.org/10.14778/2824032.2824078 (**) https://clickhouse.com/blog/optimize-clickhouse-codecs-compression-schema --- .../sql-reference/statements/create/table.md | 4 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionCodecFPC.cpp | 1 + src/Compression/CompressionCodecGorilla.cpp | 7 +- src/Compression/CompressionFactory.h | 6 +- .../CompressionFactoryAdditions.cpp | 43 ++++++---- src/Compression/ICompressionCodec.h | 2 + .../tests/gtest_compressionCodec.cpp | 83 +++++-------------- src/Core/Settings.h | 4 + src/Interpreters/InterpreterCreateQuery.cpp | 3 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 10 ++- src/Storages/ColumnsDescription.cpp | 2 +- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- tests/performance/codecs_int_insert.xml | 1 - tests/performance/codecs_int_select.xml | 1 - .../0_stateless/01272_suspicious_codecs.sql | 18 ++-- .../02533_gorilla_on_nonfloat.reference | 0 .../0_stateless/02533_gorilla_on_nonfloat.sql | 34 ++++++++ 20 files changed, 122 insertions(+), 105 deletions(-) create mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference create mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 68fb968c609..fdd479b5da4 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -293,7 +293,9 @@ These codecs are designed to make compression more effective by using specific f #### Gorilla -`Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see Compressing Values in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +`Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). + +It is currently possible to apply Gorilla compression to columns of non-floating-point type but this practice is strongly discouraged and will be removed in future. #### FPC diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 6a8acace238..e7840feeeab 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -541,7 +541,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_gorilla_codec_for_non_float_data); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp index 48eba210b60..0468adc4610 100644 --- a/src/Compression/CompressionCodecFPC.cpp +++ b/src/Compression/CompressionCodecFPC.cpp @@ -39,6 +39,7 @@ protected: bool isCompression() const override { return true; } bool isGenericCompression() const override { return false; } + bool isFloatingPointTimeSeries() const override { return true; } private: static constexpr UInt32 HEADER_SIZE = 2; diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 88b8c2bc3bb..93e6fba16f9 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -123,6 +123,7 @@ protected: bool isCompression() const override { return true; } bool isGenericCompression() const override { return false; } + bool isFloatingPointTimeSeries() const override { return true; } private: UInt8 data_bytes_size; @@ -444,14 +445,14 @@ void CompressionCodecGorilla::doDecompressData(const char * source, UInt32 sourc void registerCodecGorilla(CompressionCodecFactory & factory) { UInt8 method_code = static_cast(CompressionMethodByte::Gorilla); - factory.registerCompressionCodecWithType("Gorilla", method_code, - [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr + auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { if (arguments) throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS); UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0; return std::make_shared(data_bytes_size); - }); + }; + factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder); } } diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index a4451f9ed2e..a85676d5964 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -40,10 +40,12 @@ public: CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const; + /// + /// Note: enable_gorilla_coded_for_non_float_data is a transitory parameter and can be removed after v23.6 (being implicitly false then), see comments in Core/Settings.h. + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_coded_for_non_float_data) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_coded_for_non_float_data) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 3e215076871..d909a3dc70f 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -32,7 +32,7 @@ namespace ErrorCodes void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const { if (family_name.empty()) throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -41,33 +41,34 @@ void CompressionCodecFactory::validateCodec( { auto literal = std::make_shared(static_cast(*level)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs); + {}, sanity_check, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs); + {}, sanity_check, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); } } ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const + const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const { if (const auto * func = ast->as()) { ASTPtr codecs_descriptions = std::make_shared(); - bool is_compression = false; - bool has_none = false; + bool with_compression_codec = false; + bool with_none_codec = false; + bool with_floating_point_timeseries_codec = false; std::optional generic_compression_codec_pos; - std::set encryption_codecs; + std::set encryption_codecs_pos; bool can_substitute_codec_arguments = true; for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i) { - const auto & inner_codec_ast = func->arguments->children[i]; + const ASTPtr & inner_codec_ast = func->arguments->children[i]; String codec_family_name; ASTPtr codec_arguments; if (const auto * family_name = inner_codec_ast->as()) @@ -136,21 +137,22 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); } - is_compression |= result_codec->isCompression(); - has_none |= result_codec->isNone(); + with_compression_codec |= result_codec->isCompression(); + with_none_codec |= result_codec->isNone(); + with_floating_point_timeseries_codec |= result_codec->isFloatingPointTimeSeries(); if (!generic_compression_codec_pos && result_codec->isGenericCompression()) generic_compression_codec_pos = i; if (result_codec->isEncryption()) - encryption_codecs.insert(i); + encryption_codecs_pos.insert(i); } String codec_description = queryToString(codecs_descriptions); if (sanity_check) { - if (codecs_descriptions->children.size() > 1 && has_none) + if (codecs_descriptions->children.size() > 1 && with_none_codec) throw Exception( "It does not make sense to have codec NONE along with other compression codecs: " + codec_description + ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", @@ -159,7 +161,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( /// Allow to explicitly specify single NONE codec if user don't want any compression. /// But applying other transformations solely without compression (e.g. Delta) does not make sense. /// It's okay to apply encryption codecs solely without anything else. - if (!is_compression && !has_none && encryption_codecs.size() != codecs_descriptions->children.size()) + if (!with_compression_codec && !with_none_codec && encryption_codecs_pos.size() != codecs_descriptions->children.size()) throw Exception( "Compression codec " + codec_description + " does not compress anything." @@ -171,17 +173,26 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( /// It does not make sense to apply any non-encryption codecs /// after encryption one. - if (!encryption_codecs.empty() && - *encryption_codecs.begin() != codecs_descriptions->children.size() - encryption_codecs.size()) + if (!encryption_codecs_pos.empty() && + *encryption_codecs_pos.begin() != codecs_descriptions->children.size() - encryption_codecs_pos.size()) throw Exception("The combination of compression codecs " + codec_description + " is meaningless," " because it does not make sense to apply any non-post-processing codecs after" " post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'" " to skip this check).", ErrorCodes::BAD_ARGUMENTS); + if (column_type) + if (with_floating_point_timeseries_codec && + !WhichDataType(*column_type).isFloat() && + !enable_gorilla_codec_for_non_float_data) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The combination of compression codecs {} is meaningless," + " because it does not make sense to apply a floating-point time series codec to non-floating-point columns" + " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", codec_description); + /// It does not make sense to apply any transformations after generic compression algorithm /// So, generic compression can be only one and only at the end. if (generic_compression_codec_pos && - *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs.size()) + *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - encryption_codecs_pos.size()) throw Exception("The combination of compression codecs " + codec_description + " is meaningless," " because it does not make sense to apply any transformations after generic compression algorithm." " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index f40404a84f3..05507d99863 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -113,6 +113,8 @@ public: /// If it does nothing. virtual bool isNone() const { return false; } + virtual bool isFloatingPointTimeSeries() const { return false; } + protected: /// This is used for fuzz testing friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size); diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index b4c29bf9ce6..a843144c7c4 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -529,6 +529,13 @@ public: TEST_P(CodecTest, TranscodingWithDataType) { + /// Gorilla can only be applied to floating point columns + bool codec_is_gorilla = std::get<0>(GetParam()).codec_statement.find("Gorilla") != std::string::npos; + WhichDataType which(std::get<1>(GetParam()).data_type.get()); + bool data_is_float = which.isFloat(); + if (codec_is_gorilla && !data_is_float) + GTEST_SKIP() << "Skipping Gorilla-compressed integer column"; + const auto codec = makeCodec(CODEC_WITH_DATA_TYPE); testTranscoding(*codec); } @@ -1204,68 +1211,20 @@ auto DDperformanceTestSequence() } // prime numbers in ascending order with some random repetitions hit all the cases of Gorilla. -auto PrimesWithMultiplierGenerator = [](int multiplier = 1) -{ - return [multiplier](auto i) - { - static const int vals[] = { - 2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43, - 47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103, - 107, 107, 109, 113, 113, 127, 127, 127 - }; - static const size_t count = sizeof(vals)/sizeof(vals[0]); - - return static_cast(vals[i % count]) * multiplier; - }; -}; - -template -auto GCompatibilityTestSequence() -{ - // Also multiply result by some factor to test large values on types that can hold those. - return generateSeq(G(PrimesWithMultiplierGenerator(intExp10(sizeof(ValueType)))), 0, 42); -} - -INSTANTIATE_TEST_SUITE_P(Gorilla, - CodecTestCompatibility, - ::testing::Combine( - ::testing::Values(Codec("Gorilla")), - ::testing::ValuesIn(std::initializer_list>{ - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x35\x00\x00\x00\x2a\x00\x00\x00\x01\x00\x2a\x00\x00\x00\x14\xe1\xdd\x25\xe5\x7b\x29\x86\xee\x2a\x16\x5a\xc5\x0b\x23\x75\x1b\x3c\xb1\x97\x8b\x5f\xcb\x43\xd9\xc5\x48\xab\x23\xaf\x62\x93\x71\x4a\x73\x0f\xc6\x0a") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x52\x00\x00\x00\x54\x00\x00\x00\x02\x00\x2a\x00\x00\x00\xc8\x00\xdc\xfe\x66\xdb\x1f\x4e\xa7\xde\xdc\xd5\xec\x6e\xf7\x37\x3a\x23\xe7\x63\xf5\x6a\x8e\x99\x37\x34\xf9\xf8\x2e\x76\x35\x2d\x51\xbb\x3b\xc3\x6d\x13\xbf\x86\x53\x9e\x25\xe4\xaf\xaf\x63\xd5\x6a\x6e\x76\x35\x3a\x27\xd3\x0f\x91\xae\x6b\x33\x57\x6e\x64\xcc\x55\x81\xe4") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x65\x00\x00\x00\xa8\x00\x00\x00\x04\x00\x2a\x00\x00\x00\x20\x4e\x00\x00\xe4\x57\x63\xc0\xbb\x67\xbc\xce\x91\x97\x99\x15\x9e\xe3\x36\x3f\x89\x5f\x8e\xf2\xec\x8e\xd3\xbf\x75\x43\x58\xc4\x7e\xcf\x93\x43\x38\xc6\x91\x36\x1f\xe7\xb6\x11\x6f\x02\x73\x46\xef\xe0\xec\x50\xfb\x79\xcb\x9c\x14\xfa\x13\xea\x8d\x66\x43\x48\xa0\xde\x3a\xcf\xff\x26\xe0\x5f\x93\xde\x5e\x7f\x6e\x36\x5e\xe6\xb4\x66\x5d\xb0\x0e\xc4") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10") - }, - { - GCompatibilityTestSequence(), - BIN_STR("\x95\x91\x00\x00\x00\x50\x01\x00\x00\x08\x00\x2a\x00\x00\x00\x00\xc2\xeb\x0b\x00\x00\x00\x00\xe3\x2b\xa0\xa6\x19\x85\x98\xdc\x45\x74\x74\x43\xc2\x57\x41\x4c\x6e\x42\x79\xd9\x8f\x88\xa5\x05\xf3\xf1\x94\xa3\x62\x1e\x02\xdf\x05\x10\xf1\x15\x97\x35\x2a\x50\x71\x0f\x09\x6c\x89\xf7\x65\x1d\x11\xb7\xcc\x7d\x0b\x70\xc1\x86\x88\x48\x47\x87\xb6\x32\x26\xa7\x86\x87\x88\xd3\x93\x3d\xfc\x28\x68\x85\x05\x0b\x13\xc6\x5f\xd4\x70\xe1\x5e\x76\xf1\x9f\xf3\x33\x2a\x14\x14\x5e\x40\xc1\x5c\x28\x3f\xec\x43\x03\x05\x11\x91\xe8\xeb\x8e\x0a\x0e\x27\x21\x55\xcb\x39\xbc\x6a\xff\x11\x5d\x81\xa0\xa6\x10") - }, - }) - ) -); +// auto PrimesWithMultiplierGenerator = [](int multiplier = 1) +// { +// return [multiplier](auto i) +// { +// static const int vals[] = { +// 2, 3, 5, 7, 11, 11, 13, 17, 19, 23, 29, 29, 31, 37, 41, 43, +// 47, 47, 53, 59, 61, 61, 67, 71, 73, 79, 83, 89, 89, 97, 101, 103, +// 107, 107, 109, 113, 113, 127, 127, 127 +// }; +// static const size_t count = sizeof(vals)/sizeof(vals[0]); +// +// return static_cast(vals[i % count]) * multiplier; +// }; +// }; // These 'tests' try to measure performance of encoding and decoding and hence only make sense to be run locally, // also they require pretty big data to run against and generating this data slows down startup of unit test process. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1ade4ba2868..35172883c96 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -627,6 +627,10 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ \ + /* Transitory setting needed for ca. six months. After v23.6, the setting can be moved into the obsolete section below and assumed as false. */ \ + /* Gorilla is a float-codec which happend to be enabled for non-float datatypes. Users (self-hosted or on-premise) might have non-float Gorilla-compressed data so allow some time for migration. */ \ + M(Bool, enable_gorilla_codec_for_non_float_data, true, "Enable Gorilla compression of columns with non-float data type", 0) \ + \ M(UInt64, use_structure_from_insertion_table_in_table_functions, 2, "Use structure from insertion table instead of schema inference from data. Possible values: 0 - disabled, 1 - enabled, 2 - auto", 0) \ \ M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index bea88885d20..28b3d91c094 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -556,6 +556,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; + bool enable_gorilla_codec_for_non_float_data = context_->getSettingsRef().enable_gorilla_codec_for_non_float_data; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -617,7 +618,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS}; column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 5b3d18c66f6..97bec765e77 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1692,7 +1692,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_gorilla_codec_for_non_float_data); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 1d4df05c723..49d95c09d74 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -389,7 +389,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.comment = *comment; if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true, true); column.ttl = ttl; @@ -430,7 +430,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) else { if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true, true); if (comment) column.comment = *comment; @@ -1080,7 +1080,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const ErrorCodes::ILLEGAL_COLUMN}; if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, + !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -1104,7 +1105,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const ErrorCodes::NOT_IMPLEMENTED}; if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, + !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); auto column_default = all_columns.getDefault(column_name); if (column_default) { diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 0fdb21e064f..22641573c63 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -130,7 +130,7 @@ void ColumnDescription::readText(ReadBuffer & buf) comment = col_ast->comment->as().value.get(); if (col_ast->codec) - codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true); + codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true, true); if (col_ast->ttl) ttl = col_ast->ttl; diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 8cee3e9ee91..cb38a2e2df5 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -701,7 +701,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_gorilla_codec_for_non_float_data); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 2971d977099..4f6df4ab95a 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -291,7 +291,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); } } diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index caefaba3725..618e20160f8 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -13,7 +13,6 @@ Delta T64 DoubleDelta - Gorilla diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml index 7d47cd300d8..62c1ee16e7b 100644 --- a/tests/performance/codecs_int_select.xml +++ b/tests/performance/codecs_int_select.xml @@ -13,7 +13,6 @@ Delta T64 DoubleDelta - Gorilla diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql index 5baa30e3cf4..7776e511725 100644 --- a/tests/queries/0_stateless/01272_suspicious_codecs.sql +++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql @@ -11,7 +11,7 @@ CREATE TABLE codecs c Float32 CODEC(Gorilla), d UInt8 CODEC(Delta, LZ4), e Float64 CODEC(Gorilla, ZSTD), - f UInt32 CODEC(Delta, Delta, Gorilla), + f UInt32 CODEC(Delta, Delta, T64), g DateTime CODEC(DoubleDelta), h DateTime64 CODEC(DoubleDelta, LZ4), i String CODEC(NONE) @@ -21,14 +21,14 @@ DROP TABLE codecs; -- test what should not work -CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } +CREATE TABLE codecs (a UInt8 CODEC(NONE, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(NONE, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(LZ4, NONE)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(LZ4, LZ4)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } -- test that sanity check is not performed in ATTACH query diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql new file mode 100644 index 00000000000..d64c04cdbf9 --- /dev/null +++ b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql @@ -0,0 +1,34 @@ +-- Welcome visitor from the future! If it is >= July 2023 and your intention is to adjust the test because "enable_gorilla_codec_for_non_float_data" +-- is now obsolete, then please also extend 01272_suspicious_codecs.sql with new tests cases for Gorilla on non-float data. + +DROP TABLE IF EXISTS test; + +-- current default behavior is to enable non-float Gorilla compressed data + +CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; + +CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; + +CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; + +-- this can be changed (and it is planned to be changed by default in future) with a setting +SET enable_gorilla_codec_for_non_float_data = false; + +CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } + +-- even with above setting, it will still be possible to create non-float Gorilla-compressed data using allow_suspicious_codecs +SET allow_suspicious_codecs = true; + +CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; + +CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; + +CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; +DROP TABLE IF EXISTS test; From 206eb4d4461e4aca9410397bdb6e69d53ef48c87 Mon Sep 17 00:00:00 2001 From: Aleksandr Date: Sat, 21 Jan 2023 15:07:25 +0100 Subject: [PATCH 063/439] removed unused if statement for increaseProcessedSize Co-authored-by: Vitaly Baranov --- src/Backups/BackupImpl.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 32e0d74b90d..3e8c9b36886 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -997,8 +997,7 @@ void BackupImpl::increaseProcessedSize(UInt64 file_size) const void BackupImpl::increaseProcessedSize(const FileInfo & info) { - if ((info.size > info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name))) - increaseProcessedSize(info.size - info.base_size); + increaseProcessedSize(info.size); } void BackupImpl::setCompressedSize() From 430203de79f8da45d180b3f7cb4190e86d5388e6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 21 Jan 2023 19:46:18 +0000 Subject: [PATCH 064/439] Fix typo --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 35172883c96..72a494ce38a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -628,7 +628,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ \ /* Transitory setting needed for ca. six months. After v23.6, the setting can be moved into the obsolete section below and assumed as false. */ \ - /* Gorilla is a float-codec which happend to be enabled for non-float datatypes. Users (self-hosted or on-premise) might have non-float Gorilla-compressed data so allow some time for migration. */ \ + /* Gorilla is a float-codec which happened to be enabled for non-float datatypes. Users (self-hosted or on-premise) might have non-float Gorilla-compressed data so allow some time for migration. */ \ M(Bool, enable_gorilla_codec_for_non_float_data, true, "Enable Gorilla compression of columns with non-float data type", 0) \ \ M(UInt64, use_structure_from_insertion_table_in_table_functions, 2, "Use structure from insertion table instead of schema inference from data. Possible values: 0 - disabled, 1 - enabled, 2 - auto", 0) \ From 4ece499f19f0ccba8ded324d4905fa4e9fc148ce Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 22 Jan 2023 12:26:03 +0000 Subject: [PATCH 065/439] Fix build --- src/Compression/CompressionFactory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index a85676d5964..2afe0b1594c 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -41,11 +41,11 @@ public: /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) /// - /// Note: enable_gorilla_coded_for_non_float_data is a transitory parameter and can be removed after v23.6 (being implicitly false then), see comments in Core/Settings.h. - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_coded_for_non_float_data) const; + /// Note: enable_gorilla_codec_for_non_float_data is a transitory parameter and can be removed after v23.6 (being implicitly false then), see comments in Core/Settings.h. + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_coded_for_non_float_data) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should From 39b384fa3138086e038e8db94925e9b1af2fa900 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 23 Jan 2023 15:38:41 +0000 Subject: [PATCH 066/439] fix tests --- .../0_stateless/01605_adaptive_granularity_block_borders.sql | 2 ++ .../queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql index 030e0c5574c..85ad7636201 100644 --- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql +++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql @@ -1,3 +1,5 @@ +-- Tags: no-random-merge-tree-settings + SET use_uncompressed_cache = 0; DROP TABLE IF EXISTS adaptive_table; diff --git a/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh b/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh index d3cbc6ec861..1223d7957b5 100755 --- a/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh +++ b/tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-random-merge-tree-settings # shellcheck disable=SC2154 From 82f194fbc679fa6d53534a59f660d34c28003716 Mon Sep 17 00:00:00 2001 From: AVMusorin Date: Mon, 23 Jan 2023 17:15:50 +0100 Subject: [PATCH 067/439] added mutex for increaseProcessedSize --- src/Backups/BackupImpl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 4159ff8bc96..9c50d0ed1ee 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -991,6 +991,7 @@ void BackupImpl::increaseUncompressedSize(const FileInfo & info) void BackupImpl::increaseProcessedSize(UInt64 file_size) const { + std::lock_guard lock{mutex}; processed_files_size += file_size; ++num_processed_files; } From 3f6594f4c6da2380692c27ffd4bd902aa3ac8be5 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 23 Jan 2023 14:16:14 +0100 Subject: [PATCH 068/439] forbid old ctor of Exception --- programs/benchmark/Benchmark.cpp | 2 +- programs/client/Client.cpp | 4 ++-- programs/format/Format.cpp | 5 ++--- programs/keeper/Keeper.cpp | 4 ++-- programs/server/Server.cpp | 4 ++-- src/Access/Common/AuthenticationData.cpp | 4 +--- src/Access/ContextAccess.cpp | 4 ++-- src/Access/LDAPClient.cpp | 4 ++-- src/Access/SettingsConstraints.cpp | 2 +- .../AggregateFunctionMaxIntersections.h | 6 ++--- .../AggregateFunctionSequenceNextNode.cpp | 9 ++++---- .../ReservoirSamplerDeterministic.h | 2 +- src/Backups/BackupIO_S3.cpp | 6 ++--- src/Client/ClientBase.cpp | 6 ++--- src/Client/LocalConnection.cpp | 4 ++-- src/Columns/ColumnDecimal.cpp | 2 +- src/Columns/ColumnNullable.cpp | 2 +- src/Common/Allocator.h | 5 +++-- src/Common/Exception.h | 12 ++++++++-- src/Common/PoolWithFailoverBase.h | 11 ++++------ src/Common/SensitiveDataMasker.cpp | 8 +++---- src/Common/Throttler.cpp | 2 +- src/Common/assert_cast.h | 2 +- src/Common/typeid_cast.h | 6 ++--- src/Compression/CompressedReadBufferBase.cpp | 6 ++--- src/Compression/CompressionCodecEncrypted.cpp | 8 +++---- src/Core/Block.cpp | 2 +- src/Core/MySQL/MySQLClient.cpp | 6 ++--- src/Core/MySQL/MySQLReplication.cpp | 14 ++++++------ src/Core/SettingsFields.h | 4 ++-- src/Daemon/BaseDaemon.cpp | 2 +- src/DataTypes/DataTypeTuple.cpp | 4 ++-- src/DataTypes/getMostSubtype.cpp | 2 +- src/DataTypes/registerDataTypeDateTime.cpp | 6 ++--- src/Databases/DatabaseDictionary.cpp | 9 ++++---- src/Databases/DatabaseFactory.cpp | 6 ++--- src/Databases/DatabaseOnDisk.cpp | 2 +- .../MySQL/MaterializedMySQLSyncThread.cpp | 2 +- .../Embedded/RegionsHierarchy.cpp | 6 ++--- src/Dictionaries/Embedded/RegionsNames.cpp | 6 ++--- src/Disks/DiskLocal.cpp | 5 ++--- src/Formats/ProtobufReader.cpp | 2 +- src/Functions/FunctionsAES.cpp | 2 +- src/Functions/FunctionsAES.h | 9 +++----- src/Functions/FunctionsConversion.h | 21 +++++++++--------- src/Functions/FunctionsLogical.cpp | 9 ++++---- src/Functions/GregorianDate.h | 22 ++++++------------- src/Functions/PolygonUtils.h | 4 ++-- src/Functions/array/arrayScalarProduct.h | 7 ++---- src/IO/HadoopSnappyReadBuffer.cpp | 4 ++-- src/IO/ReadHelpers.cpp | 4 ++-- src/IO/S3/PocoHTTPClient.cpp | 3 +-- src/IO/ZlibDeflatingWriteBuffer.cpp | 8 +++---- src/Interpreters/Context.cpp | 10 ++++----- src/Interpreters/DatabaseCatalog.cpp | 4 ++-- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- .../NormalizeSelectWithUnionQueryVisitor.cpp | 5 ++--- src/Interpreters/QueryAliasesVisitor.cpp | 2 +- .../SelectIntersectExceptQueryVisitor.cpp | 10 ++++----- src/Interpreters/TreeRewriter.cpp | 2 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 14 +++++------- src/Parsers/ASTColumnsMatcher.cpp | 10 ++++----- src/Parsers/ASTColumnsTransformers.cpp | 5 ++--- src/Parsers/parseQuery.cpp | 2 +- .../Formats/Impl/AvroRowInputFormat.cpp | 9 ++++---- .../Formats/Impl/AvroRowOutputFormat.cpp | 7 +++--- .../Formats/Impl/CHColumnToArrowColumn.cpp | 9 +++----- .../Formats/Impl/CSVRowInputFormat.cpp | 5 ++--- .../Formats/Impl/ORCBlockInputFormat.cpp | 4 ++-- .../Formats/Impl/ParquetBlockInputFormat.cpp | 2 +- src/Processors/Merges/Algorithms/Graphite.cpp | 17 ++++++-------- src/Server/GRPCServer.cpp | 4 +--- src/Server/MySQLHandlerFactory.cpp | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 3 +-- src/Storages/ColumnsDescription.cpp | 4 ++-- src/Storages/ColumnsDescription.h | 2 +- src/Storages/Hive/HiveFile.cpp | 2 +- src/Storages/Kafka/StorageKafka.cpp | 14 ++++++------ .../MeiliSearch/StorageMeiliSearch.cpp | 9 ++++---- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 2 +- src/Storages/ProjectionsDescription.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/WindowView/StorageWindowView.cpp | 7 +++--- src/TableFunctions/TableFunctionRemote.cpp | 8 +++---- src/TableFunctions/TableFunctionS3Cluster.cpp | 2 +- 88 files changed, 226 insertions(+), 265 deletions(-) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 26099b352a3..dae3aea2d2e 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -277,7 +277,7 @@ private: } if (queries.empty()) - throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED); + throw Exception(ErrorCodes::EMPTY_DATA_PASSED, "Empty list of queries."); } else { diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 74e18b68b1f..1ce29555f2f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -719,7 +719,7 @@ bool Client::processWithFuzzing(const String & full_query) // uniformity. // Surprisingly, this is a client exception, because we get the // server exception w/o throwing (see onReceiveException()). - client_exception = std::make_unique(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); + client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); have_error = true; } @@ -854,7 +854,7 @@ bool Client::processWithFuzzing(const String & full_query) } catch (...) { - client_exception = std::make_unique(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); + client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); have_error = true; } diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 5eed58b15d0..8bab24b5e37 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -165,9 +165,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv) /// should throw exception early and make exception message more readable. if (const auto * insert_query = res->as(); insert_query && insert_query->data) { - throw Exception( - "Can't format ASTInsertQuery with data, since data will be lost", - DB::ErrorCodes::INVALID_FORMAT_INSERT_QUERY_WITH_DATA); + throw Exception(DB::ErrorCodes::INVALID_FORMAT_INSERT_QUERY_WITH_DATA, + "Can't format ASTInsertQuery with data, since data will be lost"); } if (!quiet) { diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index f16dbc91344..3a0d3d3a6ca 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -196,7 +196,7 @@ void Keeper::createServer(const std::string & listen_host, const char * port_nam } else { - throw Exception{message, ErrorCodes::NETWORK_ERROR}; + throw Exception::createDeprecated(message, ErrorCodes::NETWORK_ERROR); } } } @@ -375,7 +375,7 @@ try if (effective_user_id == 0) { message += " Run under 'sudo -u " + data_owner + "'."; - throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); + throw Exception::createDeprecated(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); } else { diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index d905cb7e03e..4b0ab4b69ee 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -416,7 +416,7 @@ void Server::createServer( } else { - throw Exception{message, ErrorCodes::NETWORK_ERROR}; + throw Exception::createDeprecated(message, ErrorCodes::NETWORK_ERROR); } } } @@ -946,7 +946,7 @@ try if (effective_user_id == 0) { message += " Run under 'sudo -u " + data_owner + "'."; - throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); + throw Exception::createDeprecated(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); } else { diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index c10297312f2..064e6c78be0 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -79,9 +79,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_vi ::DB::encodeSHA256(text, hash.data()); return hash; #else - throw DB::Exception( - "SHA256 passwords support is disabled, because ClickHouse was built without SSL library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, " "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); #endif } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 2700216112c..ed9f131860c 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -490,7 +490,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), (grant_option ? " WITH GRANT OPTION" : "")); if constexpr (throw_if_denied) - throw Exception(getUserName() + ": " + error_msg, error_code); + throw Exception::createDeprecated(getUserName() + ": " + error_msg, error_code); return false; }; @@ -683,7 +683,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const { UNUSED(this); if constexpr (throw_if_denied) - throw Exception(getUserName() + ": " + msg, error_code); + throw Exception::createDeprecated(getUserName() + ": " + msg, error_code); }; if (is_full_access) diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index e933bd49408..9606656f732 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -205,7 +205,7 @@ void LDAPClient::handleError(int result_code, String text) } } - throw Exception(text, ErrorCodes::LDAP_ERROR); + throw Exception::createDeprecated(text, ErrorCodes::LDAP_ERROR); } } @@ -569,7 +569,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) message += matched_msg; } - throw Exception(message, ErrorCodes::LDAP_ERROR); + throw Exception::createDeprecated(message, ErrorCodes::LDAP_ERROR); } break; diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 4f69a12f175..ded33005fc0 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -266,7 +266,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n if (!explain.empty()) { if (reaction == THROW_ON_VIOLATION) - throw Exception(explain, code); + throw Exception::createDeprecated(explain, code); else return false; } diff --git a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h index e46d5da03f9..9737e2e8257 100644 --- a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h +++ b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h @@ -66,13 +66,13 @@ public: , kind(kind_) { if (!isNativeNumber(arguments[0])) - throw Exception{getName() + ": first argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: first argument must be represented by integer", getName()); if (!isNativeNumber(arguments[1])) - throw Exception{getName() + ": second argument must be represented by integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: second argument must be represented by integer", getName()); if (!arguments[0]->equals(*arguments[1])) - throw Exception{getName() + ": arguments must have the same type", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "{}: arguments must have the same type", getName()); } String getName() const override diff --git a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp index 85bc2d4be30..7bb19b13ca0 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp @@ -88,9 +88,9 @@ createAggregateFunctionSequenceNode(const std::string & name, const DataTypes & name, toString(min_required_args + 1)); if (argument_types.size() > max_events_size + min_required_args) - throw Exception(fmt::format( + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function '{}' requires at most {} (timestamp, value_column, ...{} events) arguments.", - name, max_events_size + min_required_args, max_events_size), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + name, max_events_size + min_required_args, max_events_size); if (const auto * cond_arg = argument_types[2].get(); cond_arg && !isUInt8(cond_arg)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of third argument of aggregate function {}, " @@ -100,9 +100,8 @@ createAggregateFunctionSequenceNode(const std::string & name, const DataTypes & { const auto * cond_arg = argument_types[i].get(); if (!isUInt8(cond_arg)) - throw Exception(fmt::format( - "Illegal type '{}' of {} argument of aggregate function '{}', must be UInt8", cond_arg->getName(), i + 1, name), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type '{}' of {} argument of aggregate function '{}', must be UInt8", cond_arg->getName(), i + 1, name); } if (WhichDataType(argument_types[1].get()).idx != TypeIndex::String) diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 39c962242f9..17e4ce0e494 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -235,7 +235,7 @@ private: if (skip_degree_ == skip_degree) return; if (skip_degree_ > detail::MAX_SKIP_DEGREE) - throw DB::Exception{"skip_degree exceeds maximum value", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED}; + throw DB::Exception(DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, "skip_degree exceeds maximum value"); skip_degree = skip_degree_; if (skip_degree == detail::MAX_SKIP_DEGREE) skip_mask = static_cast(-1); diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index db90a3504c5..ef1adecfd8f 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -79,7 +79,7 @@ namespace request.SetMaxKeys(1); auto outcome = client.ListObjects(request); if (!outcome.IsSuccess()) - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); return outcome.GetResult().GetContents(); } @@ -233,7 +233,7 @@ void BackupWriterS3::removeFile(const String & file_name) request.SetKey(fs::path(s3_uri.key) / file_name); auto outcome = client->DeleteObject(request); if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType())) - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } void BackupWriterS3::removeFiles(const Strings & file_names) @@ -291,7 +291,7 @@ void BackupWriterS3::removeFilesBatch(const Strings & file_names) auto outcome = client->DeleteObjects(request); if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType())) - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } } diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index a40d6272ff7..0caedf67513 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -451,7 +451,7 @@ void ClientBase::onData(Block & block, ASTPtr parsed_query) catch (const Exception &) { /// Catch client errors like NO_ROW_DELIMITER - throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); + throw LocalFormatError(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); } /// Received data block is immediately displayed to the user. @@ -629,7 +629,7 @@ try } catch (...) { - throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); + throw LocalFormatError(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); } @@ -1897,7 +1897,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) { // Surprisingly, this is a client error. A server error would // have been reported without throwing (see onReceiveSeverException()). - client_exception = std::make_unique(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode()); + client_exception = std::make_unique(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode()); have_error = true; } diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index d5e4aa4fc40..d86a097b910 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -187,7 +187,7 @@ void LocalConnection::sendQuery( catch (...) { state->io.onException(); - state->exception = std::make_unique("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION); + state->exception = std::make_unique(ErrorCodes::UNKNOWN_EXCEPTION, "Unknown exception"); } } @@ -291,7 +291,7 @@ bool LocalConnection::poll(size_t) catch (...) { state->io.onException(); - state->exception = std::make_unique("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION); + state->exception = std::make_unique(ErrorCodes::UNKNOWN_EXCEPTION, "Unknown exception"); } } diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index e06593c5f45..d9fed8c87e5 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -83,7 +83,7 @@ template UInt64 ColumnDecimal::get64([[maybe_unused]] size_t n) const { if constexpr (sizeof(T) > sizeof(UInt64)) - throw Exception(String("Method get64 is not supported for ") + getFamilyName(), ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method get64 is not supported for {}", getFamilyName()); else return static_cast(data[n]); } diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index df2537dcbb5..9398c66bef0 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -35,7 +35,7 @@ ColumnNullable::ColumnNullable(MutableColumnPtr && nested_column_, MutableColumn nested_column = getNestedColumn().convertToFullColumnIfConst(); if (!getNestedColumn().canBeInsideNullable()) - throw Exception{getNestedColumn().getName() + " cannot be inside Nullable column", ErrorCodes::ILLEGAL_COLUMN}; + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "{} cannot be inside Nullable column", getNestedColumn().getName()); if (isColumnConst(*null_map)) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnNullable cannot have constant null map"); diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index c348eaea006..3739ce6dfdf 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -207,8 +207,9 @@ private: if (size >= MMAP_THRESHOLD) { if (alignment > mmap_min_alignment) - throw DB::Exception(fmt::format("Too large alignment {}: more than page size when allocating {}.", - ReadableSize(alignment), ReadableSize(size)), DB::ErrorCodes::BAD_ARGUMENTS); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "Too large alignment {}: more than page size when allocating {}.", + ReadableSize(alignment), ReadableSize(size)); buf = mmap(getMmapHint(), size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 514e030cf40..357841a7745 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -48,6 +48,12 @@ public: Exception() = default; + Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.message), code) + { + message_format_string = msg.format_string; + } + +protected: // used to remove the sensitive information from exceptions if query_masking_rules is configured struct MessageMasked { @@ -62,9 +68,11 @@ public: // delegating constructor to mask sensitive information from the message Exception(const std::string & msg, int code, bool remote_ = false): Exception(MessageMasked(msg), code, remote_) {} Exception(std::string && msg, int code, bool remote_ = false): Exception(MessageMasked(std::move(msg)), code, remote_) {} - Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.message), code) + +public: + static Exception createDeprecated(const std::string & msg, int code, bool remote_ = false) { - message_format_string = msg.format_string; + return Exception(msg, code, remote_); } template>> diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index b02235b6c58..fabfd4517d6 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -211,9 +211,8 @@ PoolWithFailoverBase::get(size_t max_ignored_errors, bool fallback_ max_ignored_errors, fallback_to_stale_replicas, try_get_entry, get_priority); if (results.empty() || results[0].entry.isNull()) - throw DB::Exception( - "PoolWithFailoverBase::getMany() returned less than min_entries entries.", - DB::ErrorCodes::LOGICAL_ERROR); + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "PoolWithFailoverBase::getMany() returned less than min_entries entries."); return results[0].entry; } @@ -320,10 +319,8 @@ PoolWithFailoverBase::getMany( try_results.resize(up_to_date_count); } else - throw DB::Exception( - "Could not find enough connections to up-to-date replicas. Got: " + std::to_string(up_to_date_count) - + ", needed: " + std::to_string(min_entries), - DB::ErrorCodes::ALL_REPLICAS_ARE_STALE); + throw DB::Exception(DB::ErrorCodes::ALL_REPLICAS_ARE_STALE, + "Could not find enough connections to up-to-date replicas. Got: {}, needed: {}", up_to_date_count, max_entries); return try_results; } diff --git a/src/Common/SensitiveDataMasker.cpp b/src/Common/SensitiveDataMasker.cpp index 124f0a16b46..5336b48524f 100644 --- a/src/Common/SensitiveDataMasker.cpp +++ b/src/Common/SensitiveDataMasker.cpp @@ -62,10 +62,10 @@ public: , replacement(replacement_string) { if (!regexp.ok()) - throw DB::Exception( - "SensitiveDataMasker: cannot compile re2: " + regexp_string_ + ", error: " + regexp.error() - + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", - DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, + "SensitiveDataMasker: cannot compile re2: {}, error: {}. " + "Look at https://github.com/google/re2/wiki/Syntax for reference.", + regexp_string_, regexp.error()); } uint64_t apply(std::string & data) const diff --git a/src/Common/Throttler.cpp b/src/Common/Throttler.cpp index d1d56e90209..4f99c24cc8d 100644 --- a/src/Common/Throttler.cpp +++ b/src/Common/Throttler.cpp @@ -58,7 +58,7 @@ UInt64 Throttler::add(size_t amount) } if (limit && count_value > limit) - throw Exception(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED); + throw Exception::createDeprecated(limit_exceeded_exception_message + std::string(" Maximum: ") + toString(limit), ErrorCodes::LIMIT_EXCEEDED); /// Wait unless there is positive amount of tokens - throttling Int64 sleep_time = 0; diff --git a/src/Common/assert_cast.h b/src/Common/assert_cast.h index ec2abc7b548..604cfaed6e2 100644 --- a/src/Common/assert_cast.h +++ b/src/Common/assert_cast.h @@ -41,7 +41,7 @@ To assert_cast(From && from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); + throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Bad cast from type {} to {}", diff --git a/src/Common/typeid_cast.h b/src/Common/typeid_cast.h index fe5e5d5c5a1..1568d380938 100644 --- a/src/Common/typeid_cast.h +++ b/src/Common/typeid_cast.h @@ -37,7 +37,7 @@ To typeid_cast(From & from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); + throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Bad cast from type {} to {}", @@ -58,7 +58,7 @@ To typeid_cast(From * from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); + throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } } @@ -93,6 +93,6 @@ To typeid_cast(const std::shared_ptr & from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); + throw DB::Exception::createDeprecated(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 206fd957d41..ab856cc9801 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -86,7 +86,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " << message_hardware_failure; - throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); } flip_bit(tmp_data, bit_pos); /// Restore @@ -101,10 +101,10 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in checksum. " << message_hardware_failure; - throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); } - throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); } static void readHeaderAndGetCodecAndSize( diff --git a/src/Compression/CompressionCodecEncrypted.cpp b/src/Compression/CompressionCodecEncrypted.cpp index a4734757106..17bac8234a5 100644 --- a/src/Compression/CompressionCodecEncrypted.cpp +++ b/src/Compression/CompressionCodecEncrypted.cpp @@ -141,7 +141,7 @@ size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, Encryption reinterpret_cast(key.data()), key.size(), tag_size, nullptr); if (!ok_init) - throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR); + throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR); /// encrypt data using context and given nonce. size_t out_len; @@ -152,7 +152,7 @@ size_t encrypt(std::string_view plaintext, char * ciphertext_and_tag, Encryption reinterpret_cast(plaintext.data()), plaintext.size(), nullptr, 0); if (!ok_open) - throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR); + throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR); return out_len; } @@ -171,7 +171,7 @@ size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod m reinterpret_cast(key.data()), key.size(), tag_size, nullptr); if (!ok_init) - throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR); + throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR); /// decrypt data using given nonce size_t out_len; @@ -182,7 +182,7 @@ size_t decrypt(std::string_view ciphertext, char * plaintext, EncryptionMethod m reinterpret_cast(ciphertext.data()), ciphertext.size(), nullptr, 0); if (!ok_open) - throw Exception(lastErrorString(), ErrorCodes::OPENSSL_ERROR); + throw Exception::createDeprecated(lastErrorString(), ErrorCodes::OPENSSL_ERROR); return out_len; } diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index d7b6e8ae19f..93f25d6d0e2 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -33,7 +33,7 @@ template static ReturnType onError(const std::string & message [[maybe_unused]], int code [[maybe_unused]]) { if constexpr (std::is_same_v) - throw Exception(message, code); + throw Exception::createDeprecated(message, code); else return false; } diff --git a/src/Core/MySQL/MySQLClient.cpp b/src/Core/MySQL/MySQLClient.cpp index 903d8a5804b..0a859e7259f 100644 --- a/src/Core/MySQL/MySQLClient.cpp +++ b/src/Core/MySQL/MySQLClient.cpp @@ -95,7 +95,7 @@ void MySQLClient::handshake() packet_endpoint->resetSequenceId(); if (packet_response.getType() == PACKET_ERR) - throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); else if (packet_response.getType() == PACKET_AUTH_SWITCH) throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Access denied for user {}", user); } @@ -110,7 +110,7 @@ void MySQLClient::writeCommand(char command, String query) switch (packet_response.getType()) { case PACKET_ERR: - throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); case PACKET_OK: break; default: @@ -128,7 +128,7 @@ void MySQLClient::registerSlaveOnMaster(UInt32 slave_id) packet_endpoint->receivePacket(packet_response); packet_endpoint->resetSequenceId(); if (packet_response.getType() == PACKET_ERR) - throw Exception(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + throw Exception::createDeprecated(packet_response.err.error_message, ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); } void MySQLClient::ping() diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 57625fdb239..f8cceb3f2b0 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -111,7 +111,7 @@ namespace MySQLReplication else if (query.starts_with("XA")) { if (query.starts_with("XA ROLLBACK")) - throw ReplicationError("ParseQueryEvent: Unsupported query event:" + query, ErrorCodes::LOGICAL_ERROR); + throw ReplicationError(ErrorCodes::LOGICAL_ERROR, "ParseQueryEvent: Unsupported query event: {}", query); typ = QUERY_EVENT_XA; if (!query.starts_with("XA COMMIT")) transaction_complete = false; @@ -247,7 +247,7 @@ namespace MySQLReplication break; } default: - throw ReplicationError("ParseMetaData: Unhandled data type:" + std::to_string(typ), ErrorCodes::UNKNOWN_EXCEPTION); + throw ReplicationError(ErrorCodes::UNKNOWN_EXCEPTION, "ParseMetaData: Unhandled data type: {}", std::to_string(typ)); } } } @@ -770,8 +770,8 @@ namespace MySQLReplication break; } default: - throw ReplicationError( - "ParseRow: Unhandled MySQL field type:" + std::to_string(field_type), ErrorCodes::UNKNOWN_EXCEPTION); + throw ReplicationError(ErrorCodes::UNKNOWN_EXCEPTION, + "ParseRow: Unhandled MySQL field type: {}" + std::to_string(field_type)); } } null_index++; @@ -873,7 +873,7 @@ namespace MySQLReplication break; } default: - throw ReplicationError("Position update with unsupported event", ErrorCodes::LOGICAL_ERROR); + throw ReplicationError(ErrorCodes::LOGICAL_ERROR, "Position update with unsupported event"); } } @@ -901,11 +901,11 @@ namespace MySQLReplication switch (header) { case PACKET_EOF: - throw ReplicationError("Master maybe lost", ErrorCodes::CANNOT_READ_ALL_DATA); + throw ReplicationError(ErrorCodes::CANNOT_READ_ALL_DATA, "Master maybe lost"); case PACKET_ERR: ERRPacket err; err.readPayloadWithUnpacked(payload); - throw ReplicationError(err.error_message, ErrorCodes::UNKNOWN_EXCEPTION); + throw ReplicationError::createDeprecated(err.error_message, ErrorCodes::UNKNOWN_EXCEPTION); } // skip the generic response packets header flag. payload.ignore(1); diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index f01ac37d3cc..4f339380f0e 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -402,7 +402,7 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) auto it = map.find(value); \ if (it != map.end()) \ return it->second; \ - throw Exception( \ + throw Exception::createDeprecated( \ "Unexpected value of " #NEW_NAME ":" + std::to_string(std::underlying_type::type(value)), \ ERROR_CODE_FOR_UNEXPECTED_NAME); \ } \ @@ -428,7 +428,7 @@ void SettingFieldEnum::readBinary(ReadBuffer & in) msg += "'" + String{name} + "'"; \ } \ msg += "]"; \ - throw Exception(msg, ERROR_CODE_FOR_UNEXPECTED_NAME); \ + throw Exception::createDeprecated(msg, ERROR_CODE_FOR_UNEXPECTED_NAME); \ } // Mostly like SettingFieldEnum, but can have multiple enum values (or none) set at once. diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index b98ae8fb23c..922c1087815 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -933,7 +933,7 @@ void BaseDaemon::handleSignal(int signal_id) onInterruptSignals(signal_id); } else - throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context + throw DB::Exception::createDeprecated(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context } void BaseDaemon::onInterruptSignals(int signal_id) diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index 80994906eed..768f87fe3d4 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -53,10 +53,10 @@ static std::optional checkTupleNames(const Strings & names) for (const auto & name : names) { if (name.empty()) - return Exception("Names of tuple elements cannot be empty", ErrorCodes::BAD_ARGUMENTS); + return Exception(ErrorCodes::BAD_ARGUMENTS, "Names of tuple elements cannot be empty"); if (!names_set.insert(name).second) - return Exception("Names of tuple elements must be unique", ErrorCodes::DUPLICATE_COLUMN); + return Exception(ErrorCodes::DUPLICATE_COLUMN, "Names of tuple elements must be unique"); } return {}; diff --git a/src/DataTypes/getMostSubtype.cpp b/src/DataTypes/getMostSubtype.cpp index 636bb5b431b..e5d529205d2 100644 --- a/src/DataTypes/getMostSubtype.cpp +++ b/src/DataTypes/getMostSubtype.cpp @@ -49,7 +49,7 @@ DataTypePtr getMostSubtype(const DataTypes & types, bool throw_if_result_is_noth auto get_nothing_or_throw = [throw_if_result_is_nothing, & types](const std::string & reason) { if (throw_if_result_is_nothing) - throw Exception(getExceptionMessagePrefix(types) + reason, ErrorCodes::NO_COMMON_TYPE); + throw Exception::createDeprecated(getExceptionMessagePrefix(types) + reason, ErrorCodes::NO_COMMON_TYPE); return std::make_shared(); }; diff --git a/src/DataTypes/registerDataTypeDateTime.cpp b/src/DataTypes/registerDataTypeDateTime.cpp index 902ef9c38b1..5a3e10656b4 100644 --- a/src/DataTypes/registerDataTypeDateTime.cpp +++ b/src/DataTypes/registerDataTypeDateTime.cpp @@ -47,10 +47,10 @@ getArgument(const ASTPtr & arguments, size_t argument_index, const char * argume else { if (argument && argument->value.getType() != field_type) - throw Exception(getExceptionMessage(fmt::format(" has wrong type: {}", argument->value.getTypeName()), + throw Exception::createDeprecated(getExceptionMessage(fmt::format(" has wrong type: {}", argument->value.getTypeName()), argument_index, argument_name, context_data_type_name, field_type), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else - throw Exception(getExceptionMessage(" is missing", argument_index, argument_name, context_data_type_name, field_type), + throw Exception::createDeprecated(getExceptionMessage(" is missing", argument_index, argument_name, context_data_type_name, field_type), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } } @@ -67,7 +67,7 @@ static DataTypePtr create(const ASTPtr & arguments) const auto timezone = getArgument(arguments, !!scale, "timezone", "DateTime"); if (!scale && !timezone) - throw Exception(getExceptionMessage(" has wrong type: ", 0, "scale", "DateTime", Field::Types::Which::UInt64), + throw Exception::createDeprecated(getExceptionMessage(" has wrong type: ", 0, "scale", "DateTime", Field::Types::Which::UInt64), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); /// If scale is defined, the data type is DateTime when scale = 0 otherwise the data type is DateTime64 diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index 43524487912..04529ff5293 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -41,10 +41,9 @@ namespace } catch (Exception & e) { - throw Exception( - fmt::format("Error while loading dictionary '{}.{}': {}", - database_name, load_result.name, e.displayText()), - e.code()); + throw Exception(e.code(), + "Error while loading dictionary '{}.{}': {}", + database_name, load_result.name, e.displayText()); } } } @@ -118,7 +117,7 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co /* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth); if (!ast && throw_on_error) - throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); + throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR); return ast; } diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 31f2aeafb63..51378b66b08 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -252,13 +252,11 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { auto print_create_ast = create.clone(); print_create_ast->as()->attach = false; - throw Exception( - fmt::format( + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "The MaterializedMySQL database engine no longer supports Ordinary databases. To re-create the database, delete " "the old one by executing \"rm -rf {}{{,.sql}}\", then re-create the database with the following query: {}", metadata_path, - queryToString(print_create_ast)), - ErrorCodes::NOT_IMPLEMENTED); + queryToString(print_create_ast)); } return std::make_shared( diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 6d1b759a47e..c36ed269bf7 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -672,7 +672,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( "in file " + metadata_file_path, /* allow_multi_statements = */ false, 0, settings.max_parser_depth); if (!ast && throw_on_error) - throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); + throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR); else if (!ast) return nullptr; diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index f367e968f11..aaf6b00dc4f 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -147,7 +147,7 @@ static void checkMySQLVariables(const mysqlxx::Pool::Entry & connection, const S first = false; } - throw Exception(error_message.str(), ErrorCodes::ILLEGAL_MYSQL_VARIABLE); + throw Exception::createDeprecated(error_message.str(), ErrorCodes::ILLEGAL_MYSQL_VARIABLE); } } diff --git a/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/src/Dictionaries/Embedded/RegionsHierarchy.cpp index f06dcef0171..c266bf7efb8 100644 --- a/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ b/src/Dictionaries/Embedded/RegionsHierarchy.cpp @@ -54,9 +54,9 @@ void RegionsHierarchy::reload() if (region_entry.id > max_region_id) { if (region_entry.id > max_size) - throw DB::Exception( - "Region id is too large: " + DB::toString(region_entry.id) + ", should be not more than " + DB::toString(max_size), - DB::ErrorCodes::INCORRECT_DATA); + throw DB::Exception(DB::ErrorCodes::INCORRECT_DATA, + "Region id is too large: {}, should be not more than {}", + DB::toString(region_entry.id), DB::toString(max_size)); max_region_id = region_entry.id; diff --git a/src/Dictionaries/Embedded/RegionsNames.cpp b/src/Dictionaries/Embedded/RegionsNames.cpp index 58fafc0d666..93ca9e6dbc9 100644 --- a/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/src/Dictionaries/Embedded/RegionsNames.cpp @@ -84,9 +84,9 @@ void RegionsNames::reload() max_region_id = name_entry.id; if (name_entry.id > max_size) - throw DB::Exception( - "Region id is too large: " + DB::toString(name_entry.id) + ", should be not more than " + DB::toString(max_size), - DB::ErrorCodes::INCORRECT_DATA); + throw DB::Exception(DB::ErrorCodes::INCORRECT_DATA, + "Region id is too large: {}, should be not more than {}", + DB::toString(name_entry.id), DB::toString(max_size)); } while (name_entry.id >= new_names_refs.size()) diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index e750f765eb1..a3b7e413014 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -61,9 +61,8 @@ static void loadDiskLocalConfig(const String & name, if (name == "default") { if (!path.empty()) - throw Exception( - "\"default\" disk path should be provided in not it ", - ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, + "\"default\" disk path should be provided in not it "); path = context->getPath(); } else diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp index c92b0b1cc71..159a1d33c49 100644 --- a/src/Formats/ProtobufReader.cpp +++ b/src/Formats/ProtobufReader.cpp @@ -429,7 +429,7 @@ void ProtobufReader::ignoreGroup() [[noreturn]] void ProtobufReader::throwUnknownFormat() const { - throw Exception( + throw Exception::createDeprecated( std::string("Protobuf messages are corrupted or don't match the provided schema.") + (root_message_has_length_delimiter ? " Please note that Protobuf stream is length-delimited: every message is prefixed by its length in varint." diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 87fa794955a..2911d115e35 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -22,7 +22,7 @@ namespace OpenSSLDetails void onError(std::string error_message) { error_message += ". OpenSSL error code: " + std::to_string(ERR_get_error()); - throw DB::Exception(error_message, DB::ErrorCodes::OPENSSL_ERROR); + throw DB::Exception::createDeprecated(error_message, DB::ErrorCodes::OPENSSL_ERROR); } StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, StringRef key, std::array & folded_key) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index c017239dd6f..f0edf6237eb 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -60,8 +60,7 @@ struct KeyHolder inline StringRef setKey(size_t cipher_key_size, StringRef key) const { if (key.size != cipher_key_size) - throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), - DB::ErrorCodes::BAD_ARGUMENTS); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size); return key; } @@ -73,8 +72,7 @@ struct KeyHolder inline StringRef setKey(size_t cipher_key_size, StringRef key) { if (key.size < cipher_key_size) - throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), - DB::ErrorCodes::BAD_ARGUMENTS); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size); // MySQL does something fancy with the keys that are too long, // ruining compatibility with OpenSSL and not improving security. @@ -129,8 +127,7 @@ inline void validateIV(StringRef iv_value, const size_t cipher_iv_size) // In MySQL mode we don't care if IV is longer than expected, only if shorter. if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size) || (mode == CipherMode::OpenSSLCompatibility && iv_value.size != 0 && iv_value.size != cipher_iv_size)) - throw DB::Exception(fmt::format("Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size), - DB::ErrorCodes::BAD_ARGUMENTS); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size); } } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 9dcc60b06ac..d1c0d80d346 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -3181,8 +3181,8 @@ private: const auto * from_type = checkAndGetDataType(from_type_untyped.get()); if (!from_type) - throw Exception{"CAST AS Tuple can only be performed between tuple types or from String.\nLeft type: " - + from_type_untyped->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Tuple can only be performed between tuple types or from String.\n" + "Left type: {}, right type: {}", from_type_untyped->getName(), to_type->getName()); const auto & from_element_types = from_type->getElements(); const auto & to_element_types = to_type->getElements(); @@ -3223,8 +3223,9 @@ private: else { if (from_element_types.size() != to_element_types.size()) - throw Exception{"CAST AS Tuple can only be performed between tuple types with the same number of elements or from String.\n" - "Left type: " + from_type->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Tuple can only be performed between tuple types " + "with the same number of elements or from String.\nLeft type: {}, right type: {}", + from_type->getName(), to_type->getName()); element_wrappers = getElementWrappers(from_element_types, to_element_types); to_reverse_index.reserve(to_element_types.size()); @@ -3337,8 +3338,8 @@ private: if (const auto * from_tuple = checkAndGetDataType(from_type_untyped.get())) { if (from_tuple->getElements().size() != 2) - throw Exception{"CAST AS Map from tuple requeires 2 elements.\n" - "Left type: " + from_tuple->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Map from tuple requeires 2 elements. " + "Left type: {}, right type: {}", from_tuple->getName(), to_type->getName()); DataTypes from_kv_types; const auto & to_kv_types = to_type->getKeyValueTypes(); @@ -3359,8 +3360,8 @@ private: { const auto * nested_tuple = typeid_cast(from_array->getNestedType().get()); if (!nested_tuple || nested_tuple->getElements().size() != 2) - throw Exception{"CAST AS Map from array requeires nested tuple of 2 elements.\n" - "Left type: " + from_array->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + throw Exception(ErrorCodes::TYPE_MISMATCH, "CAST AS Map from array requeires nested tuple of 2 elements. " + "Left type: {}, right type: {}", from_array->getName(), to_type->getName()); return createArrayToMapWrrapper(nested_tuple->getElements(), to_type->getKeyValueTypes()); } @@ -3370,8 +3371,8 @@ private: } else { - throw Exception{"Unsupported types to CAST AS Map\n" - "Left type: " + from_type_untyped->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; + throw Exception(ErrorCodes::TYPE_MISMATCH, "Unsupported types to CAST AS Map. " + "Left type: {}, right type: {}", from_type_untyped->getName(), to_type->getName()); } } diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index dd8469a2234..d01fdc99076 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -265,9 +265,8 @@ struct TernaryValueBuilderImpl<> { [[noreturn]] static void build(const IColumn * x, UInt8 * /* nullable_ternary_column_data */) { - throw Exception( - std::string("Unknown numeric column of type: ") + demangle(typeid(*x).name()), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unknown numeric column of type: {}", demangle(typeid(*x).name())); } }; @@ -440,13 +439,13 @@ struct TypedExecutorInvoker template static void apply(const ColumnVector &, const IColumn & y, Result &) { - throw Exception(std::string("Unknown numeric column y of type: ") + demangle(typeid(y).name()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown numeric column y of type: {}", demangle(typeid(y).name())); } template static void apply(const IColumn & x, const IColumn &, Result &) { - throw Exception(std::string("Unknown numeric column x of type: ") + demangle(typeid(x).name()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown numeric column x of type: {}", demangle(typeid(x).name())); } }; diff --git a/src/Functions/GregorianDate.h b/src/Functions/GregorianDate.h index 4a9846bf01c..63bc443fa31 100644 --- a/src/Functions/GregorianDate.h +++ b/src/Functions/GregorianDate.h @@ -278,9 +278,8 @@ namespace DB { if (year_ < 0 || year_ > 9999) { - throw Exception( - "Impossible to stringify: year too big or small: " + DB::toString(year_), - ErrorCodes::CANNOT_FORMAT_DATETIME); + throw Exception(ErrorCodes::CANNOT_FORMAT_DATETIME, + "Impossible to stringify: year too big or small: {}", DB::toString(year_)); } else { @@ -372,9 +371,7 @@ namespace DB , day_of_month_(day_of_month) { if (month < 1 || month > 12) - throw Exception( - "Invalid month: " + DB::toString(month), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid month: {}", DB::toString(month)); /* We can't validate day_of_month here, because we don't know if * it's a leap year. */ } @@ -382,10 +379,8 @@ namespace DB inline MonthDay::MonthDay(bool is_leap_year, uint16_t day_of_year) { if (day_of_year < 1 || day_of_year > (is_leap_year ? 366 : 365)) - throw Exception( - std::string("Invalid day of year: ") + - (is_leap_year ? "leap, " : "non-leap, ") + DB::toString(day_of_year), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid day of year: {}{}", + (is_leap_year ? "leap, " : "non-leap, "), DB::toString(day_of_year)); month_ = 1; uint16_t d = day_of_year; @@ -404,11 +399,8 @@ namespace DB { if (day_of_month_ < 1 || day_of_month_ > gd::monthLength(is_leap_year, month_)) { - throw Exception( - std::string("Invalid day of month: ") + - (is_leap_year ? "leap, " : "non-leap, ") + DB::toString(month_) + - "-" + DB::toString(day_of_month_), - ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid day of month: {}{}-{}", + (is_leap_year ? "leap, " : "non-leap, "), DB::toString(month_), DB::toString(day_of_month_)); } const auto k = month_ <= 2 ? 0 : is_leap_year ? -1 :-2; return (367 * month_ - 362) / 12 + k + day_of_month_; diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index 63d9a351b6a..d1504ad2c2e 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -600,13 +600,13 @@ struct CallPointInPolygon<> template static ColumnPtr call(const ColumnVector &, const IColumn & y, PointInPolygonImpl &&) { - throw Exception(std::string("Unknown numeric column type: ") + demangle(typeid(y).name()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown numeric column type: {}", demangle(typeid(y).name())); } template static ColumnPtr call(const IColumn & x, const IColumn &, PointInPolygonImpl &&) { - throw Exception(std::string("Unknown numeric column type: ") + demangle(typeid(x).name()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown numeric column type: {}", demangle(typeid(x).name())); } }; diff --git a/src/Functions/array/arrayScalarProduct.h b/src/Functions/array/arrayScalarProduct.h index 010b525426b..94ce1bc533c 100644 --- a/src/Functions/array/arrayScalarProduct.h +++ b/src/Functions/array/arrayScalarProduct.h @@ -143,11 +143,8 @@ public: || (res = executeNumber(arguments)) || (res = executeNumber(arguments)) || (res = executeNumber(arguments)))) - throw Exception - { - "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN - }; + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName()); return res; } diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index 48a24151e75..6ba31997b37 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -196,7 +196,7 @@ bool HadoopSnappyReadBuffer::nextImpl() if (decoder->result == Status::NEEDS_MORE_INPUT && (!in_available || in->eof())) { - throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); + throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "hadoop snappy decode error: {}", statusToString(decoder->result)); } out_capacity = internal_buffer.size(); @@ -221,7 +221,7 @@ bool HadoopSnappyReadBuffer::nextImpl() } else if (decoder->result == Status::INVALID_INPUT || decoder->result == Status::BUFFER_TOO_SMALL) { - throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); + throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "hadoop snappy decode error: {}", statusToString(decoder->result)); } return true; } diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index d2648d2a802..1f38b81e948 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -381,7 +381,7 @@ static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) auto error = [](const char * message [[maybe_unused]], int code [[maybe_unused]]) { if constexpr (throw_exception) - throw Exception(message, code); + throw Exception::createDeprecated(message, code); return ReturnType(false); }; @@ -1341,7 +1341,7 @@ Exception readException(ReadBuffer & buf, const String & additional_message, boo if (!stack_trace.empty()) out << " Stack trace:\n\n" << stack_trace; - return Exception(out.str(), code, remote_exception); + return Exception::createDeprecated(out.str(), code, remote_exception); } void readAndThrowException(ReadBuffer & buf, const String & additional_message) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index ba485c84386..b0ab1c52409 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -501,8 +501,7 @@ void PocoHTTPClient::makeRequestInternal( return; } - throw Exception(String("Too many redirects while trying to access ") + request.GetUri().GetURIString(), - ErrorCodes::TOO_MANY_REDIRECTS); + throw Exception(ErrorCodes::TOO_MANY_REDIRECTS, "Too many redirects while trying to access {}", request.GetUri().GetURIString()); } catch (...) { diff --git a/src/IO/ZlibDeflatingWriteBuffer.cpp b/src/IO/ZlibDeflatingWriteBuffer.cpp index 798e98b51ce..82442ea9699 100644 --- a/src/IO/ZlibDeflatingWriteBuffer.cpp +++ b/src/IO/ZlibDeflatingWriteBuffer.cpp @@ -63,7 +63,7 @@ void ZlibDeflatingWriteBuffer::nextImpl() out->position() = out->buffer().end() - zstr.avail_out; if (rc != Z_OK) - throw Exception(std::string("deflate failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED); + throw Exception(ErrorCodes::ZLIB_DEFLATE_FAILED, "deflate failed: {}", zError(rc)); } while (zstr.avail_in > 0 || zstr.avail_out == 0); } @@ -102,7 +102,7 @@ void ZlibDeflatingWriteBuffer::finalizeBefore() out->position() = out->buffer().end() - zstr.avail_out; if (rc != Z_OK) - throw Exception(std::string("deflate failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED); + throw Exception(ErrorCodes::ZLIB_DEFLATE_FAILED, "deflate failed: {}", zError(rc)); } while (zstr.avail_out == 0); @@ -121,7 +121,7 @@ void ZlibDeflatingWriteBuffer::finalizeBefore() } if (rc != Z_OK) - throw Exception(std::string("deflate finalizeImpl() failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED); + throw Exception(ErrorCodes::ZLIB_DEFLATE_FAILED, "deflate finalizeImpl() failed: {}", zError(rc)); } } @@ -131,7 +131,7 @@ void ZlibDeflatingWriteBuffer::finalizeAfter() { int rc = deflateEnd(&zstr); if (rc != Z_OK) - throw Exception(std::string("deflateEnd failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED); + throw Exception(ErrorCodes::ZLIB_DEFLATE_FAILED, "deflateEnd failed: {}", zError(rc)); } catch (...) { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6be0678a01a..6872d9c0dce 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3437,7 +3437,7 @@ StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace w if (!storage_id) { if (exception) - exception->emplace("Both table name and UUID are empty", ErrorCodes::UNKNOWN_TABLE); + exception->emplace(ErrorCodes::UNKNOWN_TABLE, "Both table name and UUID are empty"); return storage_id; } @@ -3454,8 +3454,8 @@ StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace w if (in_specified_database) return storage_id; /// NOTE There is no guarantees that table actually exists in database. if (exception) - exception->emplace("External and temporary tables have no database, but " + - storage_id.database_name + " is specified", ErrorCodes::UNKNOWN_TABLE); + exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "External and temporary tables have no database, but {} is specified", + storage_id.database_name)); return StorageID::createEmpty(); } @@ -3498,7 +3498,7 @@ StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace w if (current_database.empty()) { if (exception) - exception->emplace("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + exception->emplace(ErrorCodes::UNKNOWN_DATABASE, "Default database is not selected"); return StorageID::createEmpty(); } storage_id.database_name = current_database; @@ -3507,7 +3507,7 @@ StorageID Context::resolveStorageIDImpl(StorageID storage_id, StorageNamespace w } if (exception) - exception->emplace("Cannot resolve database name for table " + storage_id.getNameForLogs(), ErrorCodes::UNKNOWN_TABLE); + exception->emplace(Exception(ErrorCodes::UNKNOWN_TABLE, "Cannot resolve database name for table {}", storage_id.getNameForLogs())); return StorageID::createEmpty(); } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 6abc95b6738..d51b6916799 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -298,7 +298,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( /// If table_id has no UUID, then the name of database was specified by user and table_id was not resolved through context. /// Do not allow access to TEMPORARY_DATABASE because it contains all temporary tables of all contexts and users. if (exception) - exception->emplace(fmt::format("Direct access to `{}` database is not allowed", TEMPORARY_DATABASE), ErrorCodes::DATABASE_ACCESS_DENIED); + exception->emplace(Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "Direct access to `{}` database is not allowed", TEMPORARY_DATABASE)); return {}; } @@ -309,7 +309,7 @@ DatabaseAndTable DatabaseCatalog::getTableImpl( if (databases.end() == it) { if (exception) - exception->emplace(fmt::format("Database {} doesn't exist", backQuoteIfNeed(table_id.getDatabaseName())), ErrorCodes::UNKNOWN_DATABASE); + exception->emplace(Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} doesn't exist", backQuoteIfNeed(table_id.getDatabaseName()))); return {}; } database = it->second; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 1d0b299295f..c20aa66313f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -121,7 +121,7 @@ void executeCommandsAndThrowIfError(Callables && ... commands) { auto status = getOverallExecutionStatusOfCommands(std::forward(commands)...); if (status.code != 0) - throw Exception(status.message, status.code); + throw Exception::createDeprecated(status.message, status.code); } diff --git a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp index b3c2063c6f6..cbdd656fb8c 100644 --- a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp +++ b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.cpp @@ -65,9 +65,8 @@ void NormalizeSelectWithUnionQueryMatcher::visit(ASTSelectWithUnionQuery & ast, else if (data.union_default_mode == SetOperationMode::DISTINCT) union_modes[i] = SelectUnionMode::UNION_DISTINCT; else - throw Exception( - "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + throw Exception(DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT, + "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty"); } if (union_modes[i] == SelectUnionMode::UNION_ALL) diff --git a/src/Interpreters/QueryAliasesVisitor.cpp b/src/Interpreters/QueryAliasesVisitor.cpp index 1b6f37ac67a..18fb4b75365 100644 --- a/src/Interpreters/QueryAliasesVisitor.cpp +++ b/src/Interpreters/QueryAliasesVisitor.cpp @@ -127,7 +127,7 @@ void QueryAliasesMatcher::visitOther(const ASTPtr & ast, Data & data) if (!alias.empty()) { if (aliases.contains(alias) && ast->getTreeHash() != aliases[alias]->getTreeHash()) - throw Exception(wrongAliasMessage(ast, aliases[alias], alias), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS); + throw Exception::createDeprecated(wrongAliasMessage(ast, aliases[alias], alias), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS); aliases[alias] = ast; } diff --git a/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp b/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp index 756a8a48e25..c72961493f0 100644 --- a/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp +++ b/src/Interpreters/SelectIntersectExceptQueryVisitor.cpp @@ -59,9 +59,8 @@ void SelectIntersectExceptQueryMatcher::visit(ASTSelectWithUnionQuery & ast, Dat else if (data.except_default_mode == SetOperationMode::DISTINCT) mode = SelectUnionMode::EXCEPT_DISTINCT; else - throw Exception( - "Expected ALL or DISTINCT in EXCEPT query, because setting (except_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + throw Exception(DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT, + "Expected ALL or DISTINCT in EXCEPT query, because setting (except_default_mode) is empty"); } else if (mode == SelectUnionMode::INTERSECT_DEFAULT) { @@ -70,9 +69,8 @@ void SelectIntersectExceptQueryMatcher::visit(ASTSelectWithUnionQuery & ast, Dat else if (data.intersect_default_mode == SetOperationMode::DISTINCT) mode = SelectUnionMode::INTERSECT_DISTINCT; else - throw Exception( - "Expected ALL or DISTINCT in INTERSECT query, because setting (intersect_default_mode) is empty", - DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); + throw Exception(DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT, + "Expected ALL or DISTINCT in INTERSECT query, because setting (intersect_default_mode) is empty"); } switch (mode) diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 72ce13fee4e..71ad3fdb7a2 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1276,7 +1276,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select ss << " '" << name << "'"; } - throw Exception(ss.str(), ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception::createDeprecated(ss.str(), ErrorCodes::UNKNOWN_IDENTIFIER); } required_source_columns.swap(source_columns); diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index bd10af953ff..342852f16c7 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -398,9 +398,8 @@ Chunk DDLQueryStatusSource::generate() if (throw_on_timeout) { if (!first_exception) - first_exception = std::make_unique( - fmt::format(msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts), - ErrorCodes::TIMEOUT_EXCEEDED); + first_exception = std::make_unique(ErrorCodes::TIMEOUT_EXCEEDED, + fmt::format(msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts)); /// For Replicated database print a list of unfinished hosts as well. Will return empty block on next iteration. if (is_replicated_database) @@ -423,12 +422,11 @@ Chunk DDLQueryStatusSource::generate() /// Paradoxically, this exception will be throw even in case of "never_throw" mode. if (!first_exception) - first_exception = std::make_unique( + first_exception = std::make_unique(ErrorCodes::UNFINISHED, fmt::format( "Cannot provide query execution status. The query's node {} has been deleted by the cleaner" " since it was finished (or its lifetime is expired)", - node_path), - ErrorCodes::UNFINISHED); + node_path)); return {}; } @@ -464,8 +462,8 @@ Chunk DDLQueryStatusSource::generate() throw Exception(ErrorCodes::LOGICAL_ERROR, "There was an error on {}: {} (probably it's a bug)", host_id, status.message); auto [host, port] = parseHostAndPort(host_id); - first_exception = std::make_unique( - fmt::format("There was an error on [{}:{}]: {}", host, port, status.message), status.code); + first_exception = std::make_unique(status.code, + fmt::format("There was an error on [{}:{}]: {}", host, port, status.message)); } ++num_hosts_finished; diff --git a/src/Parsers/ASTColumnsMatcher.cpp b/src/Parsers/ASTColumnsMatcher.cpp index d301394cc54..940030577d6 100644 --- a/src/Parsers/ASTColumnsMatcher.cpp +++ b/src/Parsers/ASTColumnsMatcher.cpp @@ -69,9 +69,8 @@ void ASTColumnsRegexpMatcher::setPattern(String pattern) original_pattern = std::move(pattern); column_matcher = std::make_shared(original_pattern, RE2::Quiet); if (!column_matcher->ok()) - throw DB::Exception( - "COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), - DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); } const String & ASTColumnsRegexpMatcher::getPattern() const @@ -177,9 +176,8 @@ void ASTQualifiedColumnsRegexpMatcher::setPattern(String pattern, bool set_match column_matcher = std::make_shared(original_pattern, RE2::Quiet); if (!column_matcher->ok()) - throw DB::Exception( - "COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), - DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, + "COLUMNS pattern {} cannot be compiled: {}", original_pattern, column_matcher->error()); } void ASTQualifiedColumnsRegexpMatcher::setMatcher(std::shared_ptr matcher) diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 3bed3428d38..769503fb7fe 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -273,9 +273,8 @@ void ASTColumnsExceptTransformer::setPattern(String pattern) original_pattern = std::move(pattern); column_matcher = std::make_shared(original_pattern, RE2::Quiet); if (!column_matcher->ok()) - throw DB::Exception( - "COLUMNS pattern " + original_pattern + " cannot be compiled: " + column_matcher->error(), - DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "COLUMNS pattern {} cannot be compiled: {}", + original_pattern, column_matcher->error()); } const std::shared_ptr & ASTColumnsExceptTransformer::getMatcher() const diff --git a/src/Parsers/parseQuery.cpp b/src/Parsers/parseQuery.cpp index da8450ac301..8d794409f78 100644 --- a/src/Parsers/parseQuery.cpp +++ b/src/Parsers/parseQuery.cpp @@ -359,7 +359,7 @@ ASTPtr parseQueryAndMovePosition( if (res) return res; - throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); + throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR); } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index fa437b16ba7..935ea13a0c8 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -468,10 +468,9 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node }; } - throw Exception( - "Type " + target_type->getName() + " is not compatible with Avro " + avro::toString(root_node->type()) + ":\n" - + nodeToJson(root_node), - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Type {} is not compatible with Avro {}:\n{}", + target_type->getName(), avro::toString(root_node->type()), nodeToJson(root_node)); } AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) @@ -842,7 +841,7 @@ private: } catch (const avro::Exception & e) { - throw Exception(e.what(), ErrorCodes::INCORRECT_DATA); + throw Exception::createDeprecated(e.what(), ErrorCodes::INCORRECT_DATA); } } catch (Exception & e) diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 3f29469c883..7f71cc6e947 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -49,10 +49,9 @@ public: : string_to_string_regexp(settings_.avro.string_column_pattern) { if (!string_to_string_regexp.ok()) - throw DB::Exception( - "Avro: cannot compile re2: " + settings_.avro.string_column_pattern + ", error: " + string_to_string_regexp.error() - + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", - DB::ErrorCodes::CANNOT_COMPILE_REGEXP); + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " + "Look at https://github.com/google/re2/wiki/Syntax for reference.", + settings_.avro.string_column_pattern, string_to_string_regexp.error()); } bool isStringAsString(const String & column_name) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index f529c6e9363..8ef57ad4917 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -85,7 +85,7 @@ namespace DB static void checkStatus(const arrow::Status & status, const String & column_name, const String & format_name) { if (!status.ok()) - throw Exception{fmt::format("Error with a {} column \"{}\": {}.", format_name, column_name, status.ToString()), ErrorCodes::UNKNOWN_EXCEPTION}; + throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, "Error with a {} column \"{}\": {}.", format_name, column_name, status.ToString()); } template @@ -589,11 +589,8 @@ namespace DB #undef DISPATCH else { - throw Exception - { - fmt::format("Internal type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type_name, column_name, format_name), - ErrorCodes::UNKNOWN_TYPE - }; + throw Exception(ErrorCodes::UNKNOWN_TYPE, + "Internal type '{}' of a column '{}' is not supported for conversion into {} data format.", column_type_name, column_name, format_name); } } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index b95feb33469..ecaf9d79ebb 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -78,10 +78,9 @@ static void skipEndOfLine(ReadBuffer & in) if (!in.eof() && *in.position() == '\n') ++in.position(); else - throw Exception( + throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse CSV format: found \\r (CR) not followed by \\n (LF)." - " Line must end by \\n (LF) or \\r\\n (CR LF) or \\n\\r.", - ErrorCodes::INCORRECT_DATA); + " Line must end by \\n (LF) or \\r\\n (CR LF) or \\n\\r."); } else if (!in.eof()) throw Exception(ErrorCodes::INCORRECT_DATA, "Expected end of line"); diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp index 5855b6e1ce0..2e45d817506 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp @@ -104,12 +104,12 @@ static void getFileReaderAndSchema( auto result = arrow::adapters::orc::ORCFileReader::Open(arrow_file, arrow::default_memory_pool()); if (!result.ok()) - throw Exception(result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); + throw Exception::createDeprecated(result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); file_reader = std::move(result).ValueOrDie(); auto read_schema_result = file_reader->ReadSchema(); if (!read_schema_result.ok()) - throw Exception(read_schema_result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); + throw Exception::createDeprecated(read_schema_result.status().ToString(), ErrorCodes::BAD_ARGUMENTS); schema = std::move(read_schema_result).ValueOrDie(); } diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index abe262d17bd..22a605dd5be 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -30,7 +30,7 @@ namespace ErrorCodes do \ { \ if (::arrow::Status _s = (status); !_s.ok()) \ - throw Exception(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ + throw Exception::createDeprecated(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ } while (false) ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_, const FormatSettings & format_settings_) diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 6b4e3217bbe..418bf5e3f13 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -166,7 +166,7 @@ static bool compareRetentions(const Retention & a, const Retention & b) String error_msg = "age and precision should only grow up: " + std::to_string(a.age) + ":" + std::to_string(a.precision) + " vs " + std::to_string(b.age) + ":" + std::to_string(b.precision); - throw Exception( + throw Exception::createDeprecated( error_msg, DB::ErrorCodes::BAD_ARGUMENTS); } @@ -389,15 +389,13 @@ static const Pattern & appendGraphitePattern( } if (!pattern.function && pattern.retentions.empty()) - throw Exception( - "At least one of an aggregate function or retention rules is mandatory for rollup patterns in GraphiteMergeTree", - DB::ErrorCodes::NO_ELEMENTS_IN_CONFIG); + throw Exception(DB::ErrorCodes::NO_ELEMENTS_IN_CONFIG, + "At least one of an aggregate function or retention rules is mandatory for rollup patterns in GraphiteMergeTree"); if (default_rule && pattern.rule_type != RuleTypeAll) { - throw Exception( - "Default must have rule_type all for rollup patterns in GraphiteMergeTree", - DB::ErrorCodes::BAD_ARGUMENTS); + throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "Default must have rule_type all for rollup patterns in GraphiteMergeTree"); } if (!pattern.function) @@ -415,9 +413,8 @@ static const Pattern & appendGraphitePattern( if (pattern.type & pattern.TypeAggregation) /// TypeAggregation or TypeAll if (pattern.function->allocatesMemoryInArena()) - throw Exception( - "Aggregate function " + pattern.function->getName() + " isn't supported " - "in GraphiteMergeTree", DB::ErrorCodes::NOT_IMPLEMENTED); + throw Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "Aggregate function {} isn't supported in GraphiteMergeTree", pattern.function->getName()); /// retention should be in descending order of age. if (pattern.type & pattern.TypeRetention) /// TypeRetention or TypeAll diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 106f488f83a..2a17589759c 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -130,9 +130,7 @@ namespace } return grpc::SslServerCredentials(options); #else - throw DB::Exception( - "Can't use SSL in grpc, because ClickHouse was built without SSL library", - DB::ErrorCodes::SUPPORT_IS_DISABLED); + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, " "Can't use SSL in grpc, because ClickHouse was built without SSL library"); #endif } return grpc::InsecureServerCredentials(); diff --git a/src/Server/MySQLHandlerFactory.cpp b/src/Server/MySQLHandlerFactory.cpp index 3b5f28477ad..deadb10f9a9 100644 --- a/src/Server/MySQLHandlerFactory.cpp +++ b/src/Server/MySQLHandlerFactory.cpp @@ -92,7 +92,7 @@ void MySQLHandlerFactory::readRSAKeys() FILE * fp = fopen(private_key_file.data(), "r"); if (fp == nullptr) - throw Exception ("Cannot open private key file " + private_key_file + ".", ErrorCodes::CANNOT_OPEN_FILE); + throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Cannot open private key file {}.", private_key_file); SCOPE_EXIT( if (0 != fclose(fp)) throwFromErrno("Cannot close file with the certificate in MySQLHandlerFactory", ErrorCodes::CANNOT_CLOSE_FILE); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 073bde0198f..42b235bf40f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -515,7 +515,7 @@ void TCPHandler::runImpl() catch (...) { state.io.onException(); - exception = std::make_unique("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION); + exception = std::make_unique(ErrorCodes::UNKNOWN_EXCEPTION, "Unknown exception"); } try diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 319451bd39a..911ff7b97b9 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1090,8 +1090,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { String exception_message = fmt::format("Wrong column. Cannot find column {} to modify", backQuote(column_name)); all_columns.appendHintsMessage(exception_message, column_name); - throw Exception{exception_message, - ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK}; + throw Exception::createDeprecated(exception_message, ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); } else continue; diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index b882fee6cfa..ebbf81f1faa 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -234,7 +234,7 @@ void ColumnsDescription::remove(const String & column_name) { String exception_message = fmt::format("There is no column {} in table", column_name); appendHintsMessage(exception_message, column_name); - throw Exception(exception_message, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + throw Exception::createDeprecated(exception_message, ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } for (auto list_it = range.first; list_it != range.second;) @@ -251,7 +251,7 @@ void ColumnsDescription::rename(const String & column_from, const String & colum { String exception_message = fmt::format("Cannot find column {} in ColumnsDescription", column_from); appendHintsMessage(exception_message, column_from); - throw Exception(exception_message, ErrorCodes::LOGICAL_ERROR); + throw Exception::createDeprecated(exception_message, ErrorCodes::LOGICAL_ERROR); } columns.get<1>().modify_key(it, [&column_to] (String & old_name) diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index e40187fe782..32c52bdcb9e 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -161,7 +161,7 @@ public: { String exception_message = fmt::format("Cannot find column {} in ColumnsDescription", column_name); appendHintsMessage(exception_message, column_name); - throw Exception(exception_message, ErrorCodes::LOGICAL_ERROR); + throw Exception::createDeprecated(exception_message, ErrorCodes::LOGICAL_ERROR); } removeSubcolumns(it->name); diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index af727b744f3..629c8689263 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -32,7 +32,7 @@ namespace ErrorCodes do \ { \ if (const ::arrow::Status & _s = (status); !_s.ok()) \ - throw Exception(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ + throw Exception::createDeprecated(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \ } while (false) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index bc0ef9c2c0a..50fb7dffa34 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -772,10 +772,10 @@ void registerStorageKafka(StorageFactory & factory) if (args_count < (ARG_NUM) && (ARG_NUM) <= 4 && \ !kafka_settings->PAR_NAME.changed) \ { \ - throw Exception( \ - "Required parameter '" #PAR_NAME "' " \ + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,\ + "Required parameter '{}' " \ "for storage Kafka not specified", \ - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); \ + #PAR_NAME); \ } \ if (args_count >= (ARG_NUM)) \ { \ @@ -783,11 +783,11 @@ void registerStorageKafka(StorageFactory & factory) if (has_settings && \ kafka_settings->PAR_NAME.changed) \ { \ - throw Exception( \ - "The argument №" #ARG_NUM " of storage Kafka " \ - "and the parameter '" #PAR_NAME "' " \ + throw Exception(ErrorCodes::BAD_ARGUMENTS, \ + "The argument №{} of storage Kafka " \ + "and the parameter '{}' " \ "in SETTINGS cannot be specified at the same time", \ - ErrorCodes::BAD_ARGUMENTS); \ + #ARG_NUM, #PAR_NAME); \ } \ /* move engine args to settings */ \ else \ diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp index fc91f58c837..56dad2a0d13 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp @@ -139,8 +139,8 @@ MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args, if (url.empty() || index.empty()) { - throw Exception( - "Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")"); } return MeiliSearchConfiguration(url, index, key); @@ -149,9 +149,8 @@ MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args, { if (engine_args.size() < 2 || 3 < engine_args.size()) { - throw Exception( - "Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")"); } for (auto & engine_arg : engine_args) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index bb507263be0..8c67f0f1625 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3503,7 +3503,7 @@ bool MergeTreeData::renameTempPartAndReplaceImpl( if (part->isEmpty() || (hierarchy.intersected_parts.size() == 1 && hierarchy.intersected_parts.back()->isEmpty())) { message += fmt::format(" One of them is empty part. That is a race between drop operation under transaction and a merge/mutation."); - throw Exception(message, ErrorCodes::SERIALIZATION_ERROR); + throw Exception::createDeprecated(message, ErrorCodes::SERIALIZATION_ERROR); } if (hierarchy.intersected_parts.size() > 1) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index f43d7705d00..0409cadc1e9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -303,7 +303,7 @@ void ReplicatedMergeTreeCleanupThread::markLostReplicas(const std::unordered_map { auto multi_responses = futures[i].get(); if (multi_responses.responses[0]->error == Coordination::Error::ZBADVERSION) - throw Exception(candidate_lost_replicas[i] + " became active when we marked lost replicas.", DB::ErrorCodes::REPLICA_STATUS_CHANGED); + throw Exception(DB::ErrorCodes::REPLICA_STATUS_CHANGED, "{} became active when we marked lost replicas.", candidate_lost_replicas[i]); zkutil::KeeperMultiException::check(multi_responses.error, requests[i], multi_responses.responses); } } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e3827170709..e58a320d897 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -273,7 +273,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) msg += getMergeTreeVerboseHelp(is_extended_storage_def); - throw Exception(msg, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(msg, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } if (is_extended_storage_def) diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index 3d93894eaff..e568fba0495 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -341,7 +341,7 @@ const ProjectionDescription & ProjectionsDescription::get(const String & project { String exception_message = fmt::format("There is no projection {} in table", projection_name); appendHintsMessage(exception_message, projection_name); - throw Exception(exception_message, ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE); + throw Exception::createDeprecated(exception_message, ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE); } return *(it->second); @@ -386,7 +386,7 @@ void ProjectionsDescription::remove(const String & projection_name, bool if_exis String exception_message = fmt::format("There is no projection {} in table", projection_name); appendHintsMessage(exception_message, projection_name); - throw Exception(exception_message, ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE); + throw Exception::createDeprecated(exception_message, ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE); } projections.erase(it->second); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 2d88f151571..0bf79581961 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4798,7 +4798,7 @@ bool StorageReplicatedMergeTree::optimize( PreformattedMessage message = fmt_string.format(std::forward(args)...); LOG_DEBUG(log, message); if (query_context->getSettingsRef().optimize_throw_if_noop) - throw Exception(message, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception::createDeprecated(message, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; }; diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 652063c780f..54a8ba1afaa 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -256,7 +256,7 @@ namespace const auto * arg = ast->as(); if (!arg || !startsWith(arg->name, "toInterval") || !IntervalKind::tryParseString(Poco::toLower(arg->name.substr(10)), kind)) - throw Exception(err_msg, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception::createDeprecated(err_msg, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); const auto * interval_unit = arg->children.front()->children.front()->as(); if (!interval_unit @@ -373,10 +373,9 @@ static void extractDependentTable(ContextPtr context, ASTPtr & query, String & s extractDependentTable(context, inner_select_query, select_database_name, select_table_name); } else - throw Exception( + throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Logical error while creating StorageWindowView." - " Could not retrieve table name from select query.", - DB::ErrorCodes::LOGICAL_ERROR); + " Could not retrieve table name from select query."); } UInt32 StorageWindowView::getCleanupBound() diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 7fd6ea10ddc..383f317fd5c 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -40,7 +40,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr String cluster_description; if (args_func.size() != 1) - throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); ASTs & args = args_func.at(0)->children; @@ -119,7 +119,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr /// clusterAllReplicas() - same as cluster() if (args.size() < 2 || args.size() > max_args) - throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); size_t arg_num = 0; auto get_string_literal = [](const IAST & node, String & res) @@ -168,7 +168,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr { if (arg_num >= args.size()) { - throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } else { @@ -220,7 +220,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr } if (arg_num < args.size()) - throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } if (!cluster_name.empty()) diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 5c3e60c125e..3c0d8d0ea10 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -60,7 +60,7 @@ void TableFunctionS3Cluster::parseArguments(const ASTPtr & ast_function, Context " - cluster, url, access_key_id, secret_access_key, format, structure, compression_method"; auto message = PreformattedMessage{fmt::format(fmt_string, getName()), fmt_string}; if (args.size() < 2 || args.size() > 7) - throw Exception(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception::createDeprecated(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// This arguments are always the first configuration.cluster_name = checkAndGetLiteralArgument(args[0], "cluster_name"); From bb4c8e169f59ec8dbb8f2995fa347554cc4ec918 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 23 Jan 2023 23:16:16 +0100 Subject: [PATCH 069/439] check number of parameters in format string --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 4 +- src/Common/Exception.h | 5 +- src/Common/LoggingFormatStringHelpers.h | 61 +++++++++++++++++++ .../NamedCollections/NamedCollectionUtils.cpp | 2 +- src/Common/TLDListsHolder.cpp | 2 +- src/Common/logger_useful.h | 1 + src/DataTypes/DataTypeObject.cpp | 2 +- src/Databases/MySQL/DatabaseMySQL.cpp | 4 +- .../PostgreSQL/DatabasePostgreSQL.cpp | 4 +- src/Functions/FunctionsAES.h | 2 +- src/Functions/FunctionsExternalDictionaries.h | 4 +- src/IO/Lz4DeflatingWriteBuffer.cpp | 3 +- src/IO/S3/copyDataToS3.cpp | 4 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- src/Interpreters/ProcessList.cpp | 2 +- .../Impl/BSONEachRowRowInputFormat.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- .../PartMetadataManagerWithCache.cpp | 2 +- .../PostgreSQLReplicationHandler.cpp | 5 +- src/Storages/StorageDictionary.cpp | 2 +- src/Storages/StorageS3Settings.cpp | 9 +-- 23 files changed, 95 insertions(+), 33 deletions(-) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 3351d08bd8a..fee040d973d 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1943,7 +1943,7 @@ void QueryAnalyzer::validateTableExpressionModifiers(const QueryTreeNodePtr & ta if (!table_node && !table_function_node && !query_node && !union_node) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Unexpected table expression. Expected table, table function, query or union node. Actual {}", + "Unexpected table expression. Expected table, table function, query or union node. Table node: {}, scope node: {}", table_expression_node->formatASTForErrorMessage(), scope.scope_node->formatASTForErrorMessage()); @@ -5726,7 +5726,7 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, case QueryTreeNodeType::IDENTIFIER: { throw Exception(ErrorCodes::LOGICAL_ERROR, - "Identifiers in FROM section must be already resolved. In scope {}", + "Identifiers in FROM section must be already resolved. Node {}, scope {}", join_tree_node->formatASTForErrorMessage(), scope.scope_node->formatASTForErrorMessage()); } diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 357841a7745..8177d92b59b 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -23,7 +23,10 @@ struct FormatStringHelperImpl std::string_view message_format_string; fmt::format_string fmt_str; template - consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward(str)) {} + consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward(str)) + { + formatStringCheckArgsNumImpl(message_format_string, sizeof...(Args)); + } template FormatStringHelperImpl(fmt::basic_runtime && str) : message_format_string(), fmt_str(std::forward>(str)) {} diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h index 8d1a1a8ec67..18cab0c84fe 100644 --- a/src/Common/LoggingFormatStringHelpers.h +++ b/src/Common/LoggingFormatStringHelpers.h @@ -53,3 +53,64 @@ template constexpr auto firstArg(T && x, Ts &&...) /// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor template constexpr auto firstArg(fmt::basic_runtime && data, Ts &&...) { return data.str.data(); } +consteval ssize_t formatStringCountArgsNum(const char * const str, size_t len) +{ + /// It does not count named args, but we don't use them + size_t cnt = 0; + size_t i = 0; + while (i + 1 < len) + { + if (str[i] == '{' && str[i + 1] == '}') + { + i += 2; + cnt += 1; + } + else if (str[i] == '{') + { + /// Ignore checks for complex formatting like "{:.3f}" + return -1; + } + else + { + i += 1; + } + } + return cnt; +} + +[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error) +{ + throw std::runtime_error(error); +} + +/// fmt::format checks that there are enough arguments, but ignores extra arguments (e.g. fmt::format("{}", 1, 2) compiles) +/// This function will fail to compile if the number of "{}" substitutions does not exactly match +consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs) +{ + if (str.empty()) + return; + ssize_t cnt = formatStringCountArgsNum(str.data(), str.size()); + if (0 <= cnt && cnt != nargs) + functionThatFailsCompilationOfConstevalFunctions("unexpected number of arguments in a format string"); +} + +template +struct CheckArgsNumHelperImpl +{ + //std::enable_if_t, PreformattedMessage>> + template + consteval CheckArgsNumHelperImpl(T && str) + { + formatStringCheckArgsNumImpl(tryGetStaticFormatString(str), sizeof...(Args)); + } + + /// No checks for fmt::runtime and PreformattedMessage + template CheckArgsNumHelperImpl(fmt::basic_runtime &&) {} + template<> CheckArgsNumHelperImpl(PreformattedMessage &) {} + template<> CheckArgsNumHelperImpl(const PreformattedMessage &) {} + template<> CheckArgsNumHelperImpl(PreformattedMessage &&) {} + +}; + +template using CheckArgsNumHelper = CheckArgsNumHelperImpl...>; +template void formatStringCheckArgsNum(CheckArgsNumHelper, Args &&...) {} diff --git a/src/Common/NamedCollections/NamedCollectionUtils.cpp b/src/Common/NamedCollections/NamedCollectionUtils.cpp index d38a889554b..8beaa38210e 100644 --- a/src/Common/NamedCollections/NamedCollectionUtils.cpp +++ b/src/Common/NamedCollections/NamedCollectionUtils.cpp @@ -187,7 +187,7 @@ public: { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Value with key `{}` is used twice in the SET query", + "Value with key `{}` is used twice in the SET query (collection name: {})", name, query.collection_name); } } diff --git a/src/Common/TLDListsHolder.cpp b/src/Common/TLDListsHolder.cpp index 75e57d9b9d4..c493a88010b 100644 --- a/src/Common/TLDListsHolder.cpp +++ b/src/Common/TLDListsHolder.cpp @@ -100,7 +100,7 @@ size_t TLDListsHolder::parseAndAddTldList(const std::string & name, const std::s tld_list_tmp.emplace(line, TLDType::TLD_REGULAR); } if (!in.eof()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not all list had been read", name); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not all list had been read: {}", name); TLDList tld_list(tld_list_tmp.size()); for (const auto & [host, type] : tld_list_tmp) diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h index 52c32ff1715..170ca1c3b10 100644 --- a/src/Common/logger_useful.h +++ b/src/Common/logger_useful.h @@ -57,6 +57,7 @@ namespace if (_is_clients_log || _logger->is((PRIORITY))) \ { \ std::string formatted_message = numArgs(__VA_ARGS__) > 1 ? fmt::format(__VA_ARGS__) : firstArg(__VA_ARGS__); \ + formatStringCheckArgsNum(__VA_ARGS__); \ if (auto _channel = _logger->getChannel()) \ { \ std::string file_function; \ diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp index 659f69b6c68..720436d0e0d 100644 --- a/src/DataTypes/DataTypeObject.cpp +++ b/src/DataTypes/DataTypeObject.cpp @@ -57,7 +57,7 @@ static DataTypePtr create(const ASTPtr & arguments) { if (func->name != "Nullable" || func->arguments->children.size() != 1) throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, - "Expected 'Nullable()' as parameter for type Object", func->name); + "Expected 'Nullable()' as parameter for type Object (function: {})", func->name); schema_argument = func->arguments->children[0]; is_nullable = true; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 6c1c8b35f43..85944319999 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -136,7 +136,7 @@ ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, Context if (local_tables_cache.find(table_name) == local_tables_cache.end()) { if (throw_on_error) - throw Exception(ErrorCodes::UNKNOWN_TABLE, "MySQL table {} doesn't exist.", database_name_in_mysql, table_name); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "MySQL table {}.{} doesn't exist.", database_name_in_mysql, table_name); return nullptr; } @@ -180,7 +180,7 @@ time_t DatabaseMySQL::getObjectMetadataModificationTime(const String & table_nam fetchTablesIntoLocalCache(getContext()); if (local_tables_cache.find(table_name) == local_tables_cache.end()) - throw Exception(ErrorCodes::UNKNOWN_TABLE, "MySQL table {} doesn't exist.", database_name_in_mysql, table_name); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "MySQL table {}.{} doesn't exist.", database_name_in_mysql, table_name); return time_t(local_tables_cache[table_name].first); } diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 4465dbb3d9d..f2d57968baa 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -214,12 +214,12 @@ void DatabasePostgreSQL::attachTable(ContextPtr /* context_ */, const String & t if (!checkPostgresTable(table_name)) throw Exception(ErrorCodes::UNKNOWN_TABLE, - "Cannot attach PostgreSQL table {} because it does not exist in PostgreSQL", + "Cannot attach PostgreSQL table {} because it does not exist in PostgreSQL (database: {})", getTableNameForLogs(table_name), database_name); if (!detached_or_dropped.contains(table_name)) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, - "Cannot attach PostgreSQL table {} because it already exists", + "Cannot attach PostgreSQL table {} because it already exists (database: {})", getTableNameForLogs(table_name), database_name); if (cache_tables) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index f0edf6237eb..c748be5b9b8 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -597,7 +597,7 @@ private: if (input_value.size < tag_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Encrypted data is too short: only {} bytes, " "should contain at least {} bytes of a tag.", - input_value.size, block_size, tag_size); + input_value.size, tag_size); input_value.size -= tag_size; } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 5c54fc5e27d..48ecc13da39 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -182,7 +182,7 @@ public: if (!isString(arguments[0])) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of first argument of function, expected a string", + "Illegal type {} of first argument of function {}, expected a string", arguments[0]->getName(), getName()); @@ -416,7 +416,7 @@ public: if (!(range_col_type->isValueRepresentedByInteger() && range_col_type->getSizeOfValueInMemory() <= sizeof(Int64))) throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal type {} of fourth argument of function must be convertible to Int64.", + "Illegal type {} of fourth argument of function {} must be convertible to Int64.", range_col_type->getName(), getName()); diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index c2eb175d5a9..c3a1b8282c3 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -37,8 +37,7 @@ Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer( throw Exception( ErrorCodes::LZ4_ENCODER_FAILED, "creation of LZ4 compression context failed. LZ4F version: {}", - LZ4F_VERSION, - ErrorCodes::LZ4_ENCODER_FAILED); + LZ4F_VERSION); } Lz4DeflatingWriteBuffer::~Lz4DeflatingWriteBuffer() diff --git a/src/IO/S3/copyDataToS3.cpp b/src/IO/S3/copyDataToS3.cpp index f7018c51359..804e67803a9 100644 --- a/src/IO/S3/copyDataToS3.cpp +++ b/src/IO/S3/copyDataToS3.cpp @@ -131,7 +131,7 @@ namespace LOG_TRACE(log, "Completing multipart upload. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", dest_bucket, dest_key, multipart_upload_id, part_tags.size()); if (part_tags.empty()) - throw Exception("Failed to complete multipart upload. No parts have uploaded", ErrorCodes::S3_ERROR); + throw Exception(ErrorCodes::S3_ERROR, "Failed to complete multipart upload. No parts have uploaded"); Aws::S3::Model::CompleteMultipartUploadRequest request; request.SetBucket(dest_bucket); @@ -680,7 +680,7 @@ namespace if (!outcome.IsSuccess()) { abortMultipartUpload(); - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + throw Exception::createDeprecated(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } return outcome.GetResult().GetCopyPartResult().GetETag(); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6872d9c0dce..08d3d368159 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2046,7 +2046,7 @@ void Context::setQueryResultCache(size_t max_size_in_bytes, size_t max_entries, auto lock = getLock(); if (shared->query_result_cache) - throw Exception("Query result cache has been already created.", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Query result cache has been already created."); shared->query_result_cache = std::make_shared(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_records); } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index c20aa66313f..5fa5e20be18 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -288,7 +288,7 @@ BlockIO InterpreterSystemQuery::execute() copyData(res->out, out); copyData(res->err, out); if (!out.str().empty()) - LOG_DEBUG(log, "The command returned output: {}", command, out.str()); + LOG_DEBUG(log, "The command {} returned output: {}", command, out.str()); res->wait(); break; } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index aab8cff71ad..9ee6cf1afe8 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -637,7 +637,7 @@ void ProcessList::decreaseQueryKindAmount(const IAST::QueryKind & query_kind) if (found == query_kind_amounts.end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong query kind amount: decrease before increase on '{}'", query_kind); else if (found->second == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong query kind amount: decrease to negative on '{}'", query_kind, found->second); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong query kind amount: decrease to negative on '{}', {}", query_kind, found->second); else found->second -= 1; } diff --git a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp index 349b063fabe..c4ffce2bc65 100644 --- a/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp @@ -458,7 +458,7 @@ bool BSONEachRowRowInputFormat::readField(IColumn & column, const DataTypePtr & if (!format_settings.null_as_default) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert BSON Null value into non-nullable column with type {}", - getBSONTypeName(bson_type), data_type->getName()); + data_type->getName()); column.insertDefault(); return false; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8c67f0f1625..50e311efaa1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1776,7 +1776,7 @@ catch (...) { LOG_ERROR(log, "Loading of outdated parts failed. " "Will terminate to avoid undefined behaviour due to inconsistent set of parts. " - "Exception: ", getCurrentExceptionMessage(true)); + "Exception: {}", getCurrentExceptionMessage(true)); std::terminate(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index a31b88365c2..a80f7d97326 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -207,7 +207,7 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const if (!checksums.files.contains(bin_file_name)) throw Exception( ErrorCodes::NO_FILE_IN_DATA_PART, - "No {} file checksum for column {} in part ", + "No {} file checksum for column {} in part {}", bin_file_name, name_type.name, getDataPartStorage().getFullPath()); }); } diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp index 90fd25bc4e7..b6260d5edb6 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp @@ -177,7 +177,7 @@ void PartMetadataManagerWithCache::assertAllDeleted(bool include_projection) con { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Data part {} with type {} with meta file {} with projection name still in cache", + "Data part {} with type {} with meta file {} with projection name {} still in cache", part->name, part->getType().toString(), file_path, diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index 2f290c93591..f450604fded 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -619,7 +619,8 @@ void PostgreSQLReplicationHandler::removeTableFromPublication(pqxx::nontransacti catch (const pqxx::undefined_table &) { /// Removing table from replication must succeed even if table does not exist in PostgreSQL. - LOG_WARNING(log, "Did not remove table {} from publication, because table does not exist in PostgreSQL", doubleQuoteWithSchema(table_name), publication_name); + LOG_WARNING(log, "Did not remove table {} from publication, because table does not exist in PostgreSQL (publication: {})", + doubleQuoteWithSchema(table_name), publication_name); } } @@ -756,7 +757,7 @@ std::set PostgreSQLReplicationHandler::fetchRequiredTables() } LOG_ERROR(log, - "Publication {} already exists, but specified tables list differs from publication tables list in tables: {}. ", + "Publication {} already exists, but specified tables list differs from publication tables list in tables: {}. " "Will use tables list from setting. " "To avoid redundant work, you can try ALTER PUBLICATION query to remove redundant tables. " "Or you can you ALTER SETTING. " diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index 0eb3eb6b94e..5d1f08771be 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -153,7 +153,7 @@ void StorageDictionary::checkTableCanBeDropped() const dictionary_name); if (location == Location::DictionaryDatabase) throw Exception(ErrorCodes::CANNOT_DETACH_DICTIONARY_AS_TABLE, - "Cannot drop/detach table from a database with DICTIONARY engine, use DROP DICTIONARY or DETACH DICTIONARY query instead", + "Cannot drop/detach table '{}' from a database with DICTIONARY engine, use DROP DICTIONARY or DETACH DICTIONARY query instead", dictionary_name); } diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index fc505f67966..ee704b3e750 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -116,20 +116,17 @@ void S3Settings::RequestSettings::PartUploadSettings::validate() if (!upload_part_size_multiply_factor) throw Exception( ErrorCodes::INVALID_SETTING_VALUE, - "Setting upload_part_size_multiply_factor cannot be zero", - upload_part_size_multiply_factor); + "Setting upload_part_size_multiply_factor cannot be zero"); if (!upload_part_size_multiply_parts_count_threshold) throw Exception( ErrorCodes::INVALID_SETTING_VALUE, - "Setting upload_part_size_multiply_parts_count_threshold cannot be zero", - upload_part_size_multiply_parts_count_threshold); + "Setting upload_part_size_multiply_parts_count_threshold cannot be zero"); if (!max_part_number) throw Exception( ErrorCodes::INVALID_SETTING_VALUE, - "Setting max_part_number cannot be zero", - max_part_number); + "Setting max_part_number cannot be zero"); static constexpr size_t max_part_number_limit = 10000; if (max_part_number > max_part_number_limit) From 414693feb22a637c0c09f956afadde374780f162 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 00:46:03 +0100 Subject: [PATCH 070/439] fixes --- src/Access/KerberosInit.cpp | 24 +++++++++---------- src/Common/LoggingFormatStringHelpers.cpp | 6 +++++ src/Common/LoggingFormatStringHelpers.h | 6 +---- src/Functions/array/mapPopulateSeries.cpp | 2 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/h3EdgeAngle.cpp | 2 +- src/Functions/h3EdgeLengthKm.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3GetPentagonIndexes.cpp | 2 +- src/Functions/h3HexAreaKm2.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- src/Functions/h3NumHexagons.cpp | 2 +- src/Functions/randDistribution.cpp | 2 +- src/Functions/throwIf.cpp | 11 ++++++--- src/Interpreters/TreeRewriter.cpp | 3 ++- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../TableFunctionHDFSCluster.cpp | 6 ++--- 17 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 src/Common/LoggingFormatStringHelpers.cpp diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 0855b338ca9..58e4a46f2aa 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -81,7 +81,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co { ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving cache: {}", fmtError(ret)); LOG_TRACE(log,"Resolved cache"); } else @@ -89,7 +89,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Resolve the default cache and get its type and default principal (if it is initialized). ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while getting default cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while getting default cache: {}", fmtError(ret)); LOG_TRACE(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) @@ -99,7 +99,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Use the specified principal name. ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when parsing principal name {}", principal + fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when parsing principal name ({}): {}", principal, fmtError(ret)); // Cache related commands if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) @@ -107,7 +107,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Use an existing cache for the client principal if we can. ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); if (ret && ret != KRB5_CC_NOTFOUND) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while searching for cache for {}", principal + fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while searching for cache for ({}): {}", principal, fmtError(ret)); if (0 == ret) { LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); @@ -118,7 +118,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Create a new cache to avoid overwriting the initialized default cache. ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while generating new cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while generating new cache: {}", fmtError(ret)); LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } @@ -134,24 +134,24 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when unparsing name{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when unparsing name: {}", fmtError(ret)); LOG_TRACE(log,"Using principal: {}", k5.name); // Allocate a new initial credential options structure. ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in options allocation{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in options allocation: {}", fmtError(ret)); // Resolve keytab ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving keytab {}{}", keytab_file, fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in resolving keytab ({}): {}", keytab_file, fmtError(ret)); LOG_TRACE(log,"Using keytab: {}", keytab_file); // Set an output credential cache in initial credential options. ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in setting output credential cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in setting output credential cache: {}", fmtError(ret)); // Action: init or renew LOG_TRACE(log,"Trying to renew credentials"); @@ -165,7 +165,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in getting initial credentials{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error in getting initial credentials: {}", fmtError(ret)); else LOG_TRACE(log,"Got initial credentials"); } @@ -175,7 +175,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when initializing cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error when initializing cache: {}", fmtError(ret)); LOG_TRACE(log,"Initialized cache"); // Store credentials in a credential cache. ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); @@ -189,7 +189,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co // Make a credential cache the primary cache for its collection. ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while switching to new cache{}", fmtError(ret)); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while switching to new cache: {}", fmtError(ret)); } LOG_TRACE(log,"Authenticated to Kerberos v5"); diff --git a/src/Common/LoggingFormatStringHelpers.cpp b/src/Common/LoggingFormatStringHelpers.cpp new file mode 100644 index 00000000000..9d1053335b9 --- /dev/null +++ b/src/Common/LoggingFormatStringHelpers.cpp @@ -0,0 +1,6 @@ +#include + +[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error) +{ + throw std::runtime_error(error); +} diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h index 18cab0c84fe..e00f2463044 100644 --- a/src/Common/LoggingFormatStringHelpers.h +++ b/src/Common/LoggingFormatStringHelpers.h @@ -78,10 +78,7 @@ consteval ssize_t formatStringCountArgsNum(const char * const str, size_t len) return cnt; } -[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error) -{ - throw std::runtime_error(error); -} +[[noreturn]] void functionThatFailsCompilationOfConstevalFunctions(const char * error); /// fmt::format checks that there are enough arguments, but ignores extra arguments (e.g. fmt::format("{}", 1, 2) compiles) /// This function will fail to compile if the number of "{}" substitutions does not exactly match @@ -97,7 +94,6 @@ consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs) template struct CheckArgsNumHelperImpl { - //std::enable_if_t, PreformattedMessage>> template consteval CheckArgsNumHelperImpl(T && str) { diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index 8814ce26189..45732d8957c 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -80,7 +80,7 @@ private: if (!(max_key_data_type.isInt() || max_key_data_type.isUInt())) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Function {} max key should be of signed or unsigned integer type. Actual type {}.", + "Function {} max key should be of signed or unsigned integer type. Actual type {}, max type {}.", getName(), key_type->getName(), max_key_type->getName()); diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 91c0e5b2361..52008f2e165 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -118,7 +118,7 @@ public: if (res > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(res), getName(), MAX_H3_RES); diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index bd6d5d2b47f..3cf8f653024 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -77,7 +77,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index 821b699e8bb..dbcdcbf6791 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -78,7 +78,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 5544f8555ff..3e5659077d1 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -83,7 +83,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); Float64 res = getHexagonEdgeLengthAvgM(resolution); diff --git a/src/Functions/h3GetPentagonIndexes.cpp b/src/Functions/h3GetPentagonIndexes.cpp index 098a577f05c..15b5f2ab68b 100644 --- a/src/Functions/h3GetPentagonIndexes.cpp +++ b/src/Functions/h3GetPentagonIndexes.cpp @@ -83,7 +83,7 @@ public: if (data[row] > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(data[row]), getName(), MAX_H3_RES); diff --git a/src/Functions/h3HexAreaKm2.cpp b/src/Functions/h3HexAreaKm2.cpp index b6c9434077f..e127096ac33 100644 --- a/src/Functions/h3HexAreaKm2.cpp +++ b/src/Functions/h3HexAreaKm2.cpp @@ -78,7 +78,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index 07b276fe155..c38211b49f8 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -78,7 +78,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 5414d42b49c..677ea3cd170 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -77,7 +77,7 @@ public: if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, - "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", toString(resolution), getName(), MAX_H3_RES); Int64 res = getNumCells(resolution); dst_data[row] = res; diff --git a/src/Functions/randDistribution.cpp b/src/Functions/randDistribution.cpp index ef4fd34e802..f92a2c7c034 100644 --- a/src/Functions/randDistribution.cpp +++ b/src/Functions/randDistribution.cpp @@ -217,7 +217,7 @@ private: const IColumn * col = arguments[parameter_number].column.get(); if (!isColumnConst(*col)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Parameter number {} of function must be constant.", parameter_number, getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Parameter number {} of function {} must be constant.", parameter_number, getName()); auto parameter = applyVisitor(FieldVisitorConvertToNumber(), assert_cast(*col).getField()); diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 631a1ef2188..0a05621d99d 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -132,9 +132,14 @@ private: const auto & in_data = in->getData(); if (!memoryIsZero(in_data.data(), 0, in_data.size() * sizeof(in_data[0]))) { - throw Exception( - error_code.value_or(ErrorCodes::FUNCTION_THROW_IF_VALUE_IS_NON_ZERO), - message.value_or("Value passed to '" + getName() + "' function is non-zero")); + if (message.has_value()) + throw Exception( + error_code.value_or(ErrorCodes::FUNCTION_THROW_IF_VALUE_IS_NON_ZERO), + *message); + else + throw Exception( + error_code.value_or(ErrorCodes::FUNCTION_THROW_IF_VALUE_IS_NON_ZERO), + "Value passed to '{}' function is non-zero", getName()); } size_t result_size = in_untyped->size(); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 71ad3fdb7a2..1810f3e7c2a 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1220,6 +1220,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select if (!unknown_required_source_columns.empty()) { + constexpr auto format_string = "Missing columns: {} while processing query: '{}', required columns:{}{}"; WriteBufferFromOwnString ss; ss << "Missing columns:"; for (const auto & name : unknown_required_source_columns) @@ -1276,7 +1277,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select ss << " '" << name << "'"; } - throw Exception::createDeprecated(ss.str(), ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception::createDeprecated(PreformattedMessage{ss.str(), format_string}, ErrorCodes::UNKNOWN_IDENTIFIER); } required_source_columns.swap(source_columns); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0bf79581961..666b7534379 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4798,7 +4798,7 @@ bool StorageReplicatedMergeTree::optimize( PreformattedMessage message = fmt_string.format(std::forward(args)...); LOG_DEBUG(log, message); if (query_context->getSettingsRef().optimize_throw_if_noop) - throw Exception::createDeprecated(message, ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); + throw Exception(std::move(message), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; }; diff --git a/src/TableFunctions/TableFunctionHDFSCluster.cpp b/src/TableFunctions/TableFunctionHDFSCluster.cpp index e2cb2168b5d..87a071929d5 100644 --- a/src/TableFunctions/TableFunctionHDFSCluster.cpp +++ b/src/TableFunctions/TableFunctionHDFSCluster.cpp @@ -48,9 +48,9 @@ void TableFunctionHDFSCluster::parseArguments(const ASTPtr & ast_function, Conte if (args.size() < 2 || args.size() > 5) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n" - " - cluster, uri\n", - " - cluster, uri, format\n", - " - cluster, uri, format, structure\n", + " - cluster, uri\n" + " - cluster, uri, format\n" + " - cluster, uri, format, structure\n" " - cluster, uri, format, structure, compression_method", getName()); From c6910f39b9102fa95fcee66178c55a700679f25c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 01:11:58 +0100 Subject: [PATCH 071/439] fix --- src/Common/LoggingFormatStringHelpers.h | 6 +++--- src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h index e00f2463044..fd9eb43840e 100644 --- a/src/Common/LoggingFormatStringHelpers.h +++ b/src/Common/LoggingFormatStringHelpers.h @@ -5,11 +5,11 @@ /// Saves a format string for already formatted message struct PreformattedMessage { - String message; + std::string message; std::string_view format_string; - operator const String & () const { return message; } - operator String () && { return std::move(message); } + operator const std::string & () const { return message; } + operator std::string () && { return std::move(message); } operator fmt::format_string<> () const { UNREACHABLE(); } }; diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 7f71cc6e947..8483a91df62 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -49,7 +49,7 @@ public: : string_to_string_regexp(settings_.avro.string_column_pattern) { if (!string_to_string_regexp.ok()) - throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " + throw DB::Exception(DB::ErrorCodes::CANNOT_COMPILE_REGEXP, "Avro: cannot compile re2: {}, error: {}. " "Look at https://github.com/google/re2/wiki/Syntax for reference.", settings_.avro.string_column_pattern, string_to_string_regexp.error()); } From 6ddd76c5a0f526a9bb9f9b598505862fd9bd670c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 01:30:26 +0100 Subject: [PATCH 072/439] fix --- src/Server/TCPHandler.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 42b235bf40f..a48a3bb1ed6 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1475,9 +1475,8 @@ void TCPHandler::receiveQuery() session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address); } #else - auto exception = Exception( - "Inter-server secret support is disabled, because ClickHouse was built without SSL library", - ErrorCodes::AUTHENTICATION_FAILED); + auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, + "Inter-server secret support is disabled, because ClickHouse was built without SSL library"); session->onAuthenticationFailure(/* user_name */ std::nullopt, socket().peerAddress(), exception); throw exception; /// NOLINT #endif From 9d13ca4a164253efd99b0590038a0ef0bf5cfe2a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 01:49:16 +0100 Subject: [PATCH 073/439] fix --- src/Access/Common/AuthenticationData.cpp | 2 +- src/Server/GRPCServer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/Common/AuthenticationData.cpp b/src/Access/Common/AuthenticationData.cpp index 064e6c78be0..0a22eeb92b3 100644 --- a/src/Access/Common/AuthenticationData.cpp +++ b/src/Access/Common/AuthenticationData.cpp @@ -79,7 +79,7 @@ AuthenticationData::Digest AuthenticationData::Util::encodeSHA256(std::string_vi ::DB::encodeSHA256(text, hash.data()); return hash; #else - throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, " "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, "SHA256 passwords support is disabled, because ClickHouse was built without SSL library"); #endif } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 2a17589759c..595f5a8c2b7 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -130,7 +130,7 @@ namespace } return grpc::SslServerCredentials(options); #else - throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, " "Can't use SSL in grpc, because ClickHouse was built without SSL library"); + throw DB::Exception(DB::ErrorCodes::SUPPORT_IS_DISABLED, "Can't use SSL in grpc, because ClickHouse was built without SSL library"); #endif } return grpc::InsecureServerCredentials(); From 574cab5d7edc5c772dd36ca0ab4df07d48da3ace Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 24 Jan 2023 10:35:23 +0000 Subject: [PATCH 074/439] Remove transitory parameter --- .../sql-reference/statements/create/table.md | 4 +-- src/Client/Connection.cpp | 2 +- src/Compression/CompressionFactory.h | 6 ++-- .../CompressionFactoryAdditions.cpp | 12 +++---- src/Compression/ICompressionCodec.h | 5 +-- src/Core/Settings.h | 4 --- src/Interpreters/InterpreterCreateQuery.cpp | 3 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 8 ++--- src/Storages/ColumnsDescription.cpp | 2 +- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- .../01272_suspicious_codecs.reference | 6 ++++ .../0_stateless/01272_suspicious_codecs.sql | 27 +++++++++++++++ .../02533_gorilla_on_nonfloat.reference | 0 .../0_stateless/02533_gorilla_on_nonfloat.sql | 34 ------------------- 16 files changed, 54 insertions(+), 65 deletions(-) delete mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference delete mode 100644 tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 978e4ef923f..a2d363da042 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -293,9 +293,7 @@ These codecs are designed to make compression more effective by using specific f #### Gorilla -`Gorilla` — Calculates XOR between current and previous value and writes it in compact binary form. Efficient when storing a series of floating point values that change slowly, because the best compression rate is achieved when neighboring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). - -It is currently possible to apply Gorilla compression to columns of non-floating-point type but this practice is strongly discouraged and will be removed in future. +`Gorilla` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). #### FPC diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 5f6adc4dd0c..701e26e4f67 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -541,7 +541,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_gorilla_codec_for_non_float_data); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index 2afe0b1594c..a4451f9ed2e 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -40,12 +40,10 @@ public: CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - /// - /// Note: enable_gorilla_codec_for_non_float_data is a transitory parameter and can be removed after v23.6 (being implicitly false then), see comments in Core/Settings.h. - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index b5bd0a47597..3e81e2dd903 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -32,7 +32,7 @@ namespace ErrorCodes void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const { if (family_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty"); @@ -41,19 +41,19 @@ void CompressionCodecFactory::validateCodec( { auto literal = std::make_shared(static_cast(*level)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); + {}, sanity_check, allow_experimental_codecs); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); + {}, sanity_check, allow_experimental_codecs); } } ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_gorilla_codec_for_non_float_data) const + const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const { if (const auto * func = ast->as()) { @@ -177,9 +177,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( "to skip this check).", codec_description); if (column_type) - if (with_floating_point_timeseries_codec && - !WhichDataType(*column_type).isFloat() && - !enable_gorilla_codec_for_non_float_data) + if (with_floating_point_timeseries_codec && !WhichDataType(*column_type).isFloat()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The combination of compression codecs {} is meaningless," " because it does not make sense to apply a floating-point time series codec to non-floating-point columns" diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index 05507d99863..d5c70fceb8c 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -106,6 +106,9 @@ public: /// If it is a post-processing codec such as encryption. Usually it does not make sense to apply non-post-processing codecs after this. virtual bool isEncryption() const { return false; } + /// If it is a specialized codec for floating-point time series. Applying it to non-floating point data is suspicious. + virtual bool isFloatingPointTimeSeries() const { return false; } + /// It is a codec available only for evaluation purposes and not meant to be used in production. /// It will not be allowed to use unless the user will turn off the safety switch. virtual bool isExperimental() const { return false; } @@ -113,8 +116,6 @@ public: /// If it does nothing. virtual bool isNone() const { return false; } - virtual bool isFloatingPointTimeSeries() const { return false; } - protected: /// This is used for fuzz testing friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a73c8277ddb..842c00be794 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -624,10 +624,6 @@ class IColumn; \ M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ \ - /* Transitory setting needed for ca. six months. After v23.6, the setting can be moved into the obsolete section below and assumed as false. */ \ - /* Gorilla is a float-codec which happened to be enabled for non-float datatypes. Users (self-hosted or on-premise) might have non-float Gorilla-compressed data so allow some time for migration. */ \ - M(Bool, enable_gorilla_codec_for_non_float_data, true, "Enable Gorilla compression of columns with non-float data type", 0) \ - \ M(UInt64, use_structure_from_insertion_table_in_table_functions, 2, "Use structure from insertion table instead of schema inference from data. Possible values: 0 - disabled, 1 - enabled, 2 - auto", 0) \ \ M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 38a4736f0c7..1f2b81bcc08 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -560,7 +560,6 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; - bool enable_gorilla_codec_for_non_float_data = context_->getSettingsRef().enable_gorilla_codec_for_non_float_data; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -621,7 +620,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_gorilla_codec_for_non_float_data); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index a98edea6f8d..073bde0198f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1692,7 +1692,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_gorilla_codec_for_non_float_data); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index ceac45c0fe5..ecf0333f860 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -388,7 +388,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.comment = *comment; if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true); column.ttl = ttl; @@ -429,7 +429,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) else { if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true); if (comment) column.comment = *comment; @@ -1079,7 +1079,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (command.codec) CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, - !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); + !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -1104,7 +1104,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (command.codec) CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, - !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); + !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); auto column_default = all_columns.getDefault(column_name); if (column_default) { diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index c878a027ad0..b882fee6cfa 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -130,7 +130,7 @@ void ColumnDescription::readText(ReadBuffer & buf) comment = col_ast->comment->as().value.get(); if (col_ast->codec) - codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true, true); + codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true); if (col_ast->ttl) ttl = col_ast->ttl; diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 7e65db68d6c..3cf1ef2678f 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -701,7 +701,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_gorilla_codec_for_non_float_data); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e1ff319d6de..e1a80800630 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_gorilla_codec_for_non_float_data); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); } } diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.reference b/tests/queries/0_stateless/01272_suspicious_codecs.reference index 559b6df2693..aa93c3192d0 100644 --- a/tests/queries/0_stateless/01272_suspicious_codecs.reference +++ b/tests/queries/0_stateless/01272_suspicious_codecs.reference @@ -6,6 +6,9 @@ CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs9\n(\n `a` UInt8 CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs10\n(\n `a` FixedString(2) CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs11\n(\n `a` Decimal(15, 5) CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs1\n(\n `a` UInt8 CODEC(NONE, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs2\n(\n `a` UInt8 CODEC(NONE, LZ4)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs3\n(\n `a` UInt8 CODEC(LZ4, NONE)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 @@ -14,3 +17,6 @@ CREATE TABLE default.codecs5\n(\n `a` UInt8 CODEC(LZ4, ZSTD(1))\n)\nENGINE = CREATE TABLE default.codecs6\n(\n `a` UInt8 CODEC(Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs7\n(\n `a` UInt8 CODEC(Delta(1), Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.codecs8\n(\n `a` UInt8 CODEC(LZ4, Delta(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs9\n(\n `a` UInt8 CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs10\n(\n `a` FixedString(2) CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.codecs11\n(\n `a` Decimal(15, 5) CODEC(Gorilla)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01272_suspicious_codecs.sql b/tests/queries/0_stateless/01272_suspicious_codecs.sql index 7776e511725..3210a280944 100644 --- a/tests/queries/0_stateless/01272_suspicious_codecs.sql +++ b/tests/queries/0_stateless/01272_suspicious_codecs.sql @@ -29,6 +29,9 @@ CREATE TABLE codecs (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tuple CREATE TABLE codecs (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } CREATE TABLE codecs (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } CREATE TABLE codecs (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a UInt8 CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a FixedString(2) CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE codecs (a Decimal(15,5) CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } -- test that sanity check is not performed in ATTACH query @@ -40,6 +43,9 @@ DROP TABLE IF EXISTS codecs5; DROP TABLE IF EXISTS codecs6; DROP TABLE IF EXISTS codecs7; DROP TABLE IF EXISTS codecs8; +DROP TABLE IF EXISTS codecs9; +DROP TABLE IF EXISTS codecs10; +DROP TABLE IF EXISTS codecs11; SET allow_suspicious_codecs = 1; @@ -51,6 +57,9 @@ CREATE TABLE codecs5 (a UInt8 CODEC(LZ4, ZSTD)) ENGINE = MergeTree ORDER BY tupl CREATE TABLE codecs6 (a UInt8 CODEC(Delta)) ENGINE = MergeTree ORDER BY tuple(); CREATE TABLE codecs7 (a UInt8 CODEC(Delta, Delta)) ENGINE = MergeTree ORDER BY tuple(); CREATE TABLE codecs8 (a UInt8 CODEC(LZ4, Delta)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE codecs9 (a UInt8 CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE codecs10 (a FixedString(2) CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE codecs11 (a Decimal(15,5) CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); SET allow_suspicious_codecs = 0; @@ -62,6 +71,9 @@ SHOW CREATE TABLE codecs5; SHOW CREATE TABLE codecs6; SHOW CREATE TABLE codecs7; SHOW CREATE TABLE codecs8; +SHOW CREATE TABLE codecs9; +SHOW CREATE TABLE codecs10; +SHOW CREATE TABLE codecs11; DETACH TABLE codecs1; DETACH TABLE codecs2; @@ -71,6 +83,9 @@ DETACH TABLE codecs5; DETACH TABLE codecs6; DETACH TABLE codecs7; DETACH TABLE codecs8; +DETACH TABLE codecs9; +DETACH TABLE codecs10; +DETACH TABLE codecs11; ATTACH TABLE codecs1; ATTACH TABLE codecs2; @@ -80,6 +95,9 @@ ATTACH TABLE codecs5; ATTACH TABLE codecs6; ATTACH TABLE codecs7; ATTACH TABLE codecs8; +ATTACH TABLE codecs9; +ATTACH TABLE codecs10; +ATTACH TABLE codecs11; SHOW CREATE TABLE codecs1; SHOW CREATE TABLE codecs2; @@ -89,6 +107,9 @@ SHOW CREATE TABLE codecs5; SHOW CREATE TABLE codecs6; SHOW CREATE TABLE codecs7; SHOW CREATE TABLE codecs8; +SHOW CREATE TABLE codecs9; +SHOW CREATE TABLE codecs10; +SHOW CREATE TABLE codecs11; SELECT * FROM codecs1; SELECT * FROM codecs2; @@ -98,6 +119,9 @@ SELECT * FROM codecs5; SELECT * FROM codecs6; SELECT * FROM codecs7; SELECT * FROM codecs8; +SELECT * FROM codecs9; +SELECT * FROM codecs10; +SELECT * FROM codecs11; DROP TABLE codecs1; DROP TABLE codecs2; @@ -107,3 +131,6 @@ DROP TABLE codecs5; DROP TABLE codecs6; DROP TABLE codecs7; DROP TABLE codecs8; +DROP TABLE codecs9; +DROP TABLE codecs10; +DROP TABLE codecs11; diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql b/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql deleted file mode 100644 index d64c04cdbf9..00000000000 --- a/tests/queries/0_stateless/02533_gorilla_on_nonfloat.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Welcome visitor from the future! If it is >= July 2023 and your intention is to adjust the test because "enable_gorilla_codec_for_non_float_data" --- is now obsolete, then please also extend 01272_suspicious_codecs.sql with new tests cases for Gorilla on non-float data. - -DROP TABLE IF EXISTS test; - --- current default behavior is to enable non-float Gorilla compressed data - -CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; - -CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; - -CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; - --- this can be changed (and it is planned to be changed by default in future) with a setting -SET enable_gorilla_codec_for_non_float_data = false; - -CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } -CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } -CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } - --- even with above setting, it will still be possible to create non-float Gorilla-compressed data using allow_suspicious_codecs -SET allow_suspicious_codecs = true; - -CREATE TABLE test (id UInt64, val Decimal(15,5) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; - -CREATE TABLE test (id UInt64, val FixedString(2) CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; - -CREATE TABLE test (id UInt64, val UInt64 CODEC (Gorilla)) ENGINE = MergeTree() ORDER BY id; -DROP TABLE IF EXISTS test; From 41b94b4954648899264569a685ec34564d52c7b9 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 24 Jan 2023 13:38:21 +0000 Subject: [PATCH 075/439] Enable query_plan_remove_redundant_sorting optimization by default --- src/Core/Settings.h | 2 +- tests/performance/sort.xml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 193072184e8..285a7ca62de 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -585,7 +585,7 @@ class IColumn; M(Bool, query_plan_optimize_primary_key, true, "Analyze primary key using query plan (instead of AST)", 0) \ M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimisation", 0) \ M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimisation", 0) \ - M(Bool, query_plan_remove_redundant_sorting, false, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \ + M(Bool, query_plan_remove_redundant_sorting, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \ M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \ \ M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ diff --git a/tests/performance/sort.xml b/tests/performance/sort.xml index 39de5fc04a0..4f126047dfb 100644 --- a/tests/performance/sort.xml +++ b/tests/performance/sort.xml @@ -1,4 +1,8 @@ + + 0 + + CREATE TABLE rand_unlimited_10m_8 (key UInt8) Engine = Memory CREATE TABLE rand_1k_10m_16 (key UInt16) Engine = Memory From 31e16c4b4d4f5e5e79a2aaa1ad24e20d3367f643 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 15:29:19 +0100 Subject: [PATCH 076/439] fix --- programs/library-bridge/CatBoostLibraryHandler.cpp | 10 ++++++---- src/Interpreters/HashJoin.cpp | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/programs/library-bridge/CatBoostLibraryHandler.cpp b/programs/library-bridge/CatBoostLibraryHandler.cpp index f08b4ca49ee..46cebf1186a 100644 --- a/programs/library-bridge/CatBoostLibraryHandler.cpp +++ b/programs/library-bridge/CatBoostLibraryHandler.cpp @@ -243,7 +243,6 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( const ColumnRawPtrs & columns, bool cat_features_are_strings) const { - std::string error_msg = "Error occurred while applying CatBoost model: "; size_t column_size = columns.front()->size(); auto result = ColumnFloat64::create(column_size * tree_count); @@ -265,7 +264,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( result_buf, column_size * tree_count)) { - throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL, + "Error occurred while applying CatBoost model: {}", api.GetErrorString()); } return result; } @@ -288,7 +288,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( cat_features_buf, cat_features_count, result_buf, column_size * tree_count)) { - throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL, + "Error occurred while applying CatBoost model: {}", api.GetErrorString()); } } else @@ -304,7 +305,8 @@ ColumnFloat64::MutablePtr CatBoostLibraryHandler::evalImpl( cat_features_buf, cat_features_count, result_buf, column_size * tree_count)) { - throw Exception(error_msg + api.GetErrorString(), ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL); + throw Exception(ErrorCodes::CANNOT_APPLY_CATBOOST_MODEL, + "Error occurred while applying CatBoost model: {}", api.GetErrorString()); } } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 22d02c4dbdf..fba985da41c 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -707,7 +707,7 @@ Block HashJoin::prepareRightBlock(const Block & block) const bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) { if (!data) - throw Exception("Join data was released", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Join data was released"); /// RowRef::SizeT is uint32_t (not size_t) for hash table Cell memory efficiency. /// It's possible to split bigger blocks and insert them by parts here. But it would be a dead code. From 159f49266e647acb25fa34d9fae66d0d56cb4e34 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 24 Jan 2023 15:45:27 +0000 Subject: [PATCH 077/439] Don't infer Dates from 8 digit numbers --- src/Formats/SchemaInferenceUtils.cpp | 12 ++++++++++++ .../02536_date_from_number_inference_fix.reference | 2 ++ .../02536_date_from_number_inference_fix.sql | 3 +++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/02536_date_from_number_inference_fix.reference create mode 100644 tests/queries/0_stateless/02536_date_from_number_inference_fix.sql diff --git a/src/Formats/SchemaInferenceUtils.cpp b/src/Formats/SchemaInferenceUtils.cpp index 6d0853f6169..07fee20c08f 100644 --- a/src/Formats/SchemaInferenceUtils.cpp +++ b/src/Formats/SchemaInferenceUtils.cpp @@ -444,7 +444,19 @@ namespace bool tryInferDate(std::string_view field) { + if (field.empty()) + return false; + ReadBufferFromString buf(field); + Float64 tmp_float; + /// Check if it's just a number, and if so, don't try to infer Date from it, + /// because we can interpret this number as a Date (for example 20000101 will be 2000-01-01) + /// and it will lead to inferring Date instead of simple Int64/Float64 in some cases. + if (tryReadFloatText(tmp_float, buf) && buf.eof()) + return false; + + buf.seek(0, SEEK_SET); /// Return position to the beginning + DayNum tmp; return tryReadDateText(tmp, buf) && buf.eof(); } diff --git a/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference b/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference new file mode 100644 index 00000000000..7ba1111651a --- /dev/null +++ b/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference @@ -0,0 +1,2 @@ +x Nullable(Int64) +20000101 diff --git a/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql b/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql new file mode 100644 index 00000000000..75c62c66731 --- /dev/null +++ b/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql @@ -0,0 +1,3 @@ +desc format(JSONEachRow, '{"x" : "20000101"}'); +select * from format(JSONEachRow, '{"x" : "20000101"}'); + From 7eeb2a0bc725ad05ff1e766dbc77055396f20e2d Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 24 Jan 2023 15:46:32 +0000 Subject: [PATCH 078/439] Change comment --- src/Formats/SchemaInferenceUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Formats/SchemaInferenceUtils.cpp b/src/Formats/SchemaInferenceUtils.cpp index 07fee20c08f..b20a15a4613 100644 --- a/src/Formats/SchemaInferenceUtils.cpp +++ b/src/Formats/SchemaInferenceUtils.cpp @@ -451,7 +451,7 @@ namespace Float64 tmp_float; /// Check if it's just a number, and if so, don't try to infer Date from it, /// because we can interpret this number as a Date (for example 20000101 will be 2000-01-01) - /// and it will lead to inferring Date instead of simple Int64/Float64 in some cases. + /// and it will lead to inferring Date instead of simple Int64/UInt64 in some cases. if (tryReadFloatText(tmp_float, buf) && buf.eof()) return false; From d237ae36f820b44201fb4553906d5cf2310eb620 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 24 Jan 2023 15:47:32 +0000 Subject: [PATCH 079/439] Better test --- .../0_stateless/02536_date_from_number_inference_fix.reference | 1 + .../queries/0_stateless/02536_date_from_number_inference_fix.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference b/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference index 7ba1111651a..3fb7eba4357 100644 --- a/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference +++ b/tests/queries/0_stateless/02536_date_from_number_inference_fix.reference @@ -1,2 +1,3 @@ x Nullable(Int64) 20000101 +19000101 diff --git a/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql b/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql index 75c62c66731..912057265e7 100644 --- a/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql +++ b/tests/queries/0_stateless/02536_date_from_number_inference_fix.sql @@ -1,3 +1,4 @@ desc format(JSONEachRow, '{"x" : "20000101"}'); select * from format(JSONEachRow, '{"x" : "20000101"}'); +select * from format(JSONEachRow, '{"x" : "19000101"}'); From 6621834e378ad3228eec2be4dff41e3942830344 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 17:10:54 +0100 Subject: [PATCH 080/439] fix --- src/IO/PeekableReadBuffer.cpp | 2 +- src/Interpreters/TreeRewriter.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/PeekableReadBuffer.cpp b/src/IO/PeekableReadBuffer.cpp index d9d5398708f..ce9c20e7a53 100644 --- a/src/IO/PeekableReadBuffer.cpp +++ b/src/IO/PeekableReadBuffer.cpp @@ -363,7 +363,7 @@ size_t PeekableReadBuffer::offsetFromCheckpointInOwnMemory() const size_t PeekableReadBuffer::offsetFromCheckpoint() const { if (!checkpoint) - throw DB::Exception("There is no checkpoint", ErrorCodes::LOGICAL_ERROR); + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "There is no checkpoint"); if (checkpointInOwnMemory() == currentlyReadFromOwnMemory()) { diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 1810f3e7c2a..cc70d255c01 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1277,7 +1277,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select ss << " '" << name << "'"; } - throw Exception::createDeprecated(PreformattedMessage{ss.str(), format_string}, ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception(PreformattedMessage{ss.str(), format_string}, ErrorCodes::UNKNOWN_IDENTIFIER); } required_source_columns.swap(source_columns); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 607af3c3a42..b3d0189f093 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7355,7 +7355,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( /// canMergeSinglePart is overlapping with dropPart, let's try to use the same code. String out_reason; if (!merge_pred.canMergeSinglePart(part, &out_reason)) - throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Part is busy, reason: " + out_reason); + throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Part is busy, reason: {}", out_reason); } { From ccfea134f7841bc005cb82755938bc22476e0aec Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 19:14:27 +0100 Subject: [PATCH 081/439] add a test for log stats --- ...nd_exception_messages_formatting.reference | 13 +++++ ..._log_and_exception_messages_formatting.sql | 51 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference create mode 100644 tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference new file mode 100644 index 00000000000..75dd0426f4a --- /dev/null +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference @@ -0,0 +1,13 @@ +10 1 +20 1 +30 1 +40 1 +50 1 +60 1 +70 1 +80 1 +90 1 +100 1 +110 1 +120 1 +130 1 diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql new file mode 100644 index 00000000000..45862bb1717 --- /dev/null +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -0,0 +1,51 @@ +-- Tags: no-parallel +-- no-parallel because we want to run this test when most of the other tests already passed + +-- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log + +drop table if exists logs; +create view logs as select * from system.text_log where now() - toIntervalMinute(120) < event_time; +system flush logs; + + +-- Check that we don't have too many messages formatted with fmt::runtime or strings concatenation. +-- 0.001 threshold should be always enough, the value was about 0.00025 +select 10, (sum(length(message_format_string) = 0) / count()) < 0.001 from logs; + +-- Check the same for exceptions. The value was 0.03 +select 20, (sum(length(message_format_string) = 0) / count()) < 0.05 from logs where message like '%DB::Exception%'; + +-- Check that we don't have too many short meaningless message patterns. +select 30, countDistinct(message_format_string) < 10 from logs where length(message_format_string) < 10; + +-- Same as above. Feel free to update the threshold or remove this query if really necessary +select 40, countDistinct(message_format_string) < 35 from logs where length(message_format_string) < 16; + +-- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary +select 50, countDistinct(message_format_string) < 90 from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%'; + + +-- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold +select 60, (select count() from logs group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.30; + +-- Same as above, but excluding Test level (actually finds top 1 Trace message) +select 70, (select count() from logs where level!='Test' group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.16; + +-- Same as above for Debug +select 80, (select count() from logs where level <= 'Debug' group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.08; + +-- Same as above for Info +select 90, (select count() from logs where level <= 'Information' group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.04; + +-- Same as above for Warning +select 100, (select count() from logs where level = 'Warning' group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.0001; + +-- Same as above for Error (it's funny that we have 100 time less warnings than errors) +select 110, (select count() from logs where level = 'Warning' group by message_format_string order by count() desc limit 1) / (select count() from logs) < 0.001; + +-- Avoid too noisy messages: limit the number of messages with high frequency +select 120, count() < 3 from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.10); +select 130, count() < 10 from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.05); + + +drop table logs; From 76d0eb0a94fd7209d0361c442f8561e19ab5caf1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 24 Jan 2023 19:33:12 +0100 Subject: [PATCH 082/439] Raw changelog --- CHANGELOG.md | 2392 ++++++++++++-------------------------------------- 1 file changed, 551 insertions(+), 1841 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddc10c1eb2f..38862b72b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,1857 +1,567 @@ ### Table of Contents -**[ClickHouse release v22.12, 2022-12-15](#2212)**
-**[ClickHouse release v22.11, 2022-11-17](#2211)**
-**[ClickHouse release v22.10, 2022-10-25](#2210)**
-**[ClickHouse release v22.9, 2022-09-22](#229)**
-**[ClickHouse release v22.8-lts, 2022-08-18](#228)**
-**[ClickHouse release v22.7, 2022-07-21](#227)**
-**[ClickHouse release v22.6, 2022-06-16](#226)**
-**[ClickHouse release v22.5, 2022-05-19](#225)**
-**[ClickHouse release v22.4, 2022-04-20](#224)**
-**[ClickHouse release v22.3-lts, 2022-03-17](#223)**
-**[ClickHouse release v22.2, 2022-02-17](#222)**
-**[ClickHouse release v22.1, 2022-01-18](#221)**
-**[Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021/)**
+**[ClickHouse release v23.1, 2023-01-25](#231)**
+**[Changelog for 2022](https://clickhouse.com/docs/en/whats-new/changelog/2022/)**
-# 2022 Changelog +# 2023 Changelog -### ClickHouse release 22.12, 2022-12-15 +### ClickHouse release 23.1, 2023-01-25 + +# 2023 Changelog + +### ClickHouse release master (2f1092e6d24) FIXME as compared to v22.12.1.1752-stable (688e488e930) #### Backward Incompatible Change -* Add `GROUP BY ALL` syntax: [#37631](https://github.com/ClickHouse/ClickHouse/issues/37631). [#42265](https://github.com/ClickHouse/ClickHouse/pull/42265) ([刘陶峰](https://github.com/taofengliu)). If you have a column or an alias named `all` and doing `GROUP BY all` without the intention to group by all the columns, the query will have a different semantic. To keep the old semantic, put `all` into backticks or double quotes `"all"` to make it an identifier instead of a keyword. - -#### Upgrade Notes -* Fixed backward incompatibility in (de)serialization of states of `min`, `max`, `any*`, `argMin`, `argMax` aggregate functions with `String` argument. The incompatibility affects 22.9, 22.10 and 22.11 branches (fixed since 22.9.6, 22.10.4 and 22.11.2 correspondingly). Some minor releases of 22.3, 22.7 and 22.8 branches are also affected: 22.3.13...22.3.14 (fixed since 22.3.15), 22.8.6...22.8.9 (fixed since 22.8.10), 22.7.6 and newer (will not be fixed in 22.7, we recommend upgrading from 22.7.* to 22.8.10 or newer). This release note does not concern users that have never used affected versions. Incompatible versions append an extra `'\0'` to strings when reading states of the aggregate functions mentioned above. For example, if an older version saved state of `anyState('foobar')` to `state_column` then the incompatible version will print `'foobar\0'` on `anyMerge(state_column)`. Also incompatible versions write states of the aggregate functions without trailing `'\0'`. Newer versions (that have the fix) can correctly read data written by all versions including incompatible versions, except one corner case. If an incompatible version saved a state with a string that actually ends with null character, then newer version will trim trailing `'\0'` when reading state of affected aggregate function. For example, if an incompatible version saved state of `anyState('abrac\0dabra\0')` to `state_column` then newer versions will print `'abrac\0dabra'` on `anyMerge(state_column)`. The issue also affects distributed queries when an incompatible version works in a cluster together with older or newer versions. [#43038](https://github.com/ClickHouse/ClickHouse/pull/43038) ([Alexander Tokmakov](https://github.com/tavplubix), [Raúl Marín](https://github.com/Algunenano)). Note: all the official ClickHouse builds already include the patches. This is not necessarily true for unofficial third-party builds that should be avoided. +* Remove query `SYSTEM RESTART DISK`. [#44647](https://github.com/ClickHouse/ClickHouse/pull/44647) ([alesapin](https://github.com/alesapin)). +* Disallow Gorilla compression on columns of non-Float32 or non-Float64 type. [#45252](https://github.com/ClickHouse/ClickHouse/pull/45252) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove PREALLOCATE for HASHED/SPARSE_HASHED dictionaries. [#45388](https://github.com/ClickHouse/ClickHouse/pull/45388) ([Azat Khuzhin](https://github.com/azat)). +* Parallel quorum inserts might work incorrectly with `*MergeTree` tables created with deprecated syntax. Therefore, parallel quorum inserts support is completely disabled for such tables. It does not affect tables created with a new syntax. [#45430](https://github.com/ClickHouse/ClickHouse/pull/45430) ([Alexander Tokmakov](https://github.com/tavplubix)). #### New Feature -* Add `BSONEachRow` input/output format. In this format, ClickHouse formats/parses each row as a separate BSON document and each column is formatted/parsed as a single BSON field with the column name as the key. [#42033](https://github.com/ClickHouse/ClickHouse/pull/42033) ([mark-polokhov](https://github.com/mark-polokhov)). -* Add `grace_hash` JOIN algorithm, it can be enabled with `SET join_algorithm = 'grace_hash'`. [#38191](https://github.com/ClickHouse/ClickHouse/pull/38191) ([BigRedEye](https://github.com/BigRedEye), [Vladimir C](https://github.com/vdimir)). -* Allow configuring password complexity rules and checks for creating and changing users. [#43719](https://github.com/ClickHouse/ClickHouse/pull/43719) ([Nikolay Degterinsky](https://github.com/evillique)). -* Mask sensitive information in logs; mask secret parts in the output of queries `SHOW CREATE TABLE` and `SELECT FROM system.tables`. Also resolves [#41418](https://github.com/ClickHouse/ClickHouse/issues/41418). [#43227](https://github.com/ClickHouse/ClickHouse/pull/43227) ([Vitaly Baranov](https://github.com/vitlibar)). -* Add `GROUP BY ALL` syntax: [#37631](https://github.com/ClickHouse/ClickHouse/issues/37631). [#42265](https://github.com/ClickHouse/ClickHouse/pull/42265) ([刘陶峰](https://github.com/taofengliu)). -* Add `FROM table SELECT column` syntax. [#41095](https://github.com/ClickHouse/ClickHouse/pull/41095) ([Nikolay Degterinsky](https://github.com/evillique)). -* Added function `concatWithSeparator` and `concat_ws` as an alias for Spark SQL compatibility. A function `concatWithSeparatorAssumeInjective` added as a variant to enable GROUP BY optimization, similarly to `concatAssumeInjective`. [#43749](https://github.com/ClickHouse/ClickHouse/pull/43749) ([李扬](https://github.com/taiyang-li)). -* Added `multiplyDecimal` and `divideDecimal` functions for decimal operations with fixed precision. [#42438](https://github.com/ClickHouse/ClickHouse/pull/42438) ([Andrey Zvonov](https://github.com/zvonand)). -* Added `system.moves` table with list of currently moving parts. [#42660](https://github.com/ClickHouse/ClickHouse/pull/42660) ([Sergei Trifonov](https://github.com/serxa)). -* Add support for embedded Prometheus endpoint for ClickHouse Keeper. [#43087](https://github.com/ClickHouse/ClickHouse/pull/43087) ([Antonio Andelic](https://github.com/antonio2368)). -* Support numeric literals with `_` as the separator, for example, `1_000_000`. [#43925](https://github.com/ClickHouse/ClickHouse/pull/43925) ([jh0x](https://github.com/jh0x)). -* Added possibility to use an array as a second parameter for `cutURLParameter` function. It will cut multiple parameters. Close [#6827](https://github.com/ClickHouse/ClickHouse/issues/6827). [#43788](https://github.com/ClickHouse/ClickHouse/pull/43788) ([Roman Vasin](https://github.com/rvasin)). -* Add a column with the expression of the index in the `system.data_skipping_indices` table. [#43308](https://github.com/ClickHouse/ClickHouse/pull/43308) ([Guillaume Tassery](https://github.com/YiuRULE)). -* Add column `engine_full` to system table `databases` so that users can access the entire engine definition of a database via system tables. [#43468](https://github.com/ClickHouse/ClickHouse/pull/43468) ([凌涛](https://github.com/lingtaolf)). -* New hash function [xxh3](https://github.com/Cyan4973/xxHash) added. Also, the performance of `xxHash32` and `xxHash64` are improved on ARM thanks to a library update. [#43411](https://github.com/ClickHouse/ClickHouse/pull/43411) ([Nikita Taranov](https://github.com/nickitat)). -* Added support to define constraints for merge tree settings. For example you can forbid overriding the `storage_policy` by users. [#43903](https://github.com/ClickHouse/ClickHouse/pull/43903) ([Sergei Trifonov](https://github.com/serxa)). -* Add a new setting `input_format_json_read_objects_as_strings` that allows the parsing of nested JSON objects into Strings in all JSON input formats. This setting is disabled by default. [#44052](https://github.com/ClickHouse/ClickHouse/pull/44052) ([Kruglov Pavel](https://github.com/Avogar)). - -#### Experimental Feature -* Support deduplication for asynchronous inserts. Before this change, async inserts did not support deduplication, because multiple small inserts coexisted in one inserted batch. Closes [#38075](https://github.com/ClickHouse/ClickHouse/issues/38075). [#43304](https://github.com/ClickHouse/ClickHouse/pull/43304) ([Han Fei](https://github.com/hanfei1991)). -* Add support for cosine distance for the experimental Annoy (vector similarity search) index. [#42778](https://github.com/ClickHouse/ClickHouse/pull/42778) ([Filatenkov Artur](https://github.com/FArthur-cmd)). -* Add `CREATE / ALTER / DROP NAMED COLLECTION` queries. [#43252](https://github.com/ClickHouse/ClickHouse/pull/43252) ([Kseniia Sumarokova](https://github.com/kssenii)). This feature is under development and the queries are not effective as of version 22.12. This changelog entry is added only to avoid confusion. Restrict default access to named collections to the user defined in config. This requires that `show_named_collections = 1` is set to be able to see them. [#43325](https://github.com/ClickHouse/ClickHouse/pull/43325) ([Kseniia Sumarokova](https://github.com/kssenii)). The `system.named_collections` table is introduced [#43147](https://github.com/ClickHouse/ClickHouse/pull/43147) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `quantileInterpolatedWeighted`/`quantilesInterpolatedWeighted` functions. [#38252](https://github.com/ClickHouse/ClickHouse/pull/38252) ([Bharat Nallan](https://github.com/bharatnc)). +* Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)). +* Add column `ptr` to `system.trace_log` for `trace_type = 'MemorySample'`. This column contains an address of allocation. Added function `flameGraph` which can build flamegraph containing allocated and not released memory. Reworking of [#38391](https://github.com/ClickHouse/ClickHouse/issues/38391). [#38953](https://github.com/ClickHouse/ClickHouse/pull/38953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Dictionary source for extracting keys by traversing regular expressions tree. [#40878](https://github.com/ClickHouse/ClickHouse/pull/40878) ([Vage Ogannisian](https://github.com/nooblose)). +* Added parametrized view functionality, now it's possible to specify query parameters for View table engine. resolves [#40907](https://github.com/ClickHouse/ClickHouse/issues/40907). [#41687](https://github.com/ClickHouse/ClickHouse/pull/41687) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* added extendable and configurable scheduling subsystem for IO requests (not yet integrated with IO code itself). [#41840](https://github.com/ClickHouse/ClickHouse/pull/41840) ([Sergei Trifonov](https://github.com/serxa)). +* Added `SYSTEM DROP DATABASE REPLICA` that removes metadata of dead replica of `Replicated` database. Resolves [#41794](https://github.com/ClickHouse/ClickHouse/issues/41794). [#42807](https://github.com/ClickHouse/ClickHouse/pull/42807) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Array join support map type, like function explode in spark. [#43239](https://github.com/ClickHouse/ClickHouse/pull/43239) ([李扬](https://github.com/taiyang-li)). +* Support SQL standard binary and hex string literals. [#43785](https://github.com/ClickHouse/ClickHouse/pull/43785) ([Mo Xuan](https://github.com/mo-avatar)). +* Add experimental query result cache. [#43797](https://github.com/ClickHouse/ClickHouse/pull/43797) ([Robert Schulze](https://github.com/rschu1ze)). +* format datetime in joda datetime style. Refer to https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html. [#43818](https://github.com/ClickHouse/ClickHouse/pull/43818) ([李扬](https://github.com/taiyang-li)). +* to merge [#40878](https://github.com/ClickHouse/ClickHouse/issues/40878) , supporting regexp dictionary. [#43858](https://github.com/ClickHouse/ClickHouse/pull/43858) ([Han Fei](https://github.com/hanfei1991)). +* Implemented a fractional second formatter (`%f`) for formatDateTime. [#44060](https://github.com/ClickHouse/ClickHouse/pull/44060) ([ltrk2](https://github.com/ltrk2)). +* Added age function to calculate difference between two dates or dates with time values expressed as number of full units. Closes [#41115](https://github.com/ClickHouse/ClickHouse/issues/41115). [#44421](https://github.com/ClickHouse/ClickHouse/pull/44421) ([Robert Schulze](https://github.com/rschu1ze)). +* Implemented a fractional second formatter (%f) for formatDateTime. This is slightly modified PR [#44060](https://github.com/ClickHouse/ClickHouse/issues/44060) by @ltrk2. [#44497](https://github.com/ClickHouse/ClickHouse/pull/44497) ([Alexander Gololobov](https://github.com/davenger)). +* Add null source for dictionaries. Closes [#44240](https://github.com/ClickHouse/ClickHouse/issues/44240). [#44502](https://github.com/ClickHouse/ClickHouse/pull/44502) ([mayamika](https://github.com/mayamika)). +* We can use `s3_storage_class` to set different tier. Such as ``` s3 xxx xxx xxx STANDARD/INTELLIGENT_TIERING ``` Closes [#44443](https://github.com/ClickHouse/ClickHouse/issues/44443). [#44707](https://github.com/ClickHouse/ClickHouse/pull/44707) ([chen](https://github.com/xiedeyantu)). +* Insert default values in case of missing elements in JSON object while parsing named tuple. Add setting `input_format_json_defaults_for_missing_elements_in_named_tuple` that controls this behaviour. Closes [#45142](https://github.com/ClickHouse/ClickHouse/issues/45142)#issuecomment-1380153217. [#45231](https://github.com/ClickHouse/ClickHouse/pull/45231) ([Kruglov Pavel](https://github.com/Avogar)). #### Performance Improvement -* Add settings `max_streams_for_merge_tree_reading` and `allow_asynchronous_read_from_io_pool_for_merge_tree`. Setting `max_streams_for_merge_tree_reading` limits the number of reading streams for MergeTree tables. Setting `allow_asynchronous_read_from_io_pool_for_merge_tree` enables a background I/O pool to read from `MergeTree` tables. This may increase performance for I/O bound queries if used together with `max_streams_to_max_threads_ratio` or `max_streams_for_merge_tree_reading`. [#43260](https://github.com/ClickHouse/ClickHouse/pull/43260) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). This improves performance up to 100 times in case of high latency storage, low number of CPU and high number of data parts. -* Settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem/merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem` did not respect adaptive granularity. Fat rows did not decrease the number of read rows (as it was done for `merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read`, which could lead to high memory usage when using remote filesystems. [#43965](https://github.com/ClickHouse/ClickHouse/pull/43965) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Optimized the number of list requests to ZooKeeper or ClickHouse Keeper when selecting a part to merge. Previously it could produce thousands of requests in some cases. Fixes [#43647](https://github.com/ClickHouse/ClickHouse/issues/43647). [#43675](https://github.com/ClickHouse/ClickHouse/pull/43675) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Optimization is getting skipped now if `max_size_to_preallocate_for_aggregation` has too small a value. The default value of this setting increased to `10^8`. [#43945](https://github.com/ClickHouse/ClickHouse/pull/43945) ([Nikita Taranov](https://github.com/nickitat)). -* Speed-up server shutdown by avoiding cleaning up of old data parts. Because it is unnecessary after https://github.com/ClickHouse/ClickHouse/pull/41145. [#43760](https://github.com/ClickHouse/ClickHouse/pull/43760) ([Sema Checherinda](https://github.com/CheSema)). -* Merging on initiator now uses the same memory bound approach as merging of local aggregation results if `enable_memory_bound_merging_of_aggregation_results` is set. [#40879](https://github.com/ClickHouse/ClickHouse/pull/40879) ([Nikita Taranov](https://github.com/nickitat)). -* Keeper improvement: try syncing logs to disk in parallel with replication. [#43450](https://github.com/ClickHouse/ClickHouse/pull/43450) ([Antonio Andelic](https://github.com/antonio2368)). -* Keeper improvement: requests are batched more often. The batching can be controlled with the new setting `max_requests_quick_batch_size`. [#43686](https://github.com/ClickHouse/ClickHouse/pull/43686) ([Antonio Andelic](https://github.com/antonio2368)). +* Added sharding support in HashedDictionary to allow parallel load (almost linear scaling based on number of shards). [#40003](https://github.com/ClickHouse/ClickHouse/pull/40003) ([Azat Khuzhin](https://github.com/azat)). +* Do not load inactive parts at startup of `MergeTree` tables. [#42181](https://github.com/ClickHouse/ClickHouse/pull/42181) ([Anton Popov](https://github.com/CurtizJ)). +* - Speed up query parsing. [#42284](https://github.com/ClickHouse/ClickHouse/pull/42284) ([Raúl Marín](https://github.com/Algunenano)). +* Always replace OR chain `expr = x1 OR ... OR expr = xN` to `expr IN (x1, ..., xN)` in case if `expr` is a `LowCardinality` column. Setting `optimize_min_equality_disjunction_chain_length` is ignored in this case. [#42889](https://github.com/ClickHouse/ClickHouse/pull/42889) ([Guo Wangyang](https://github.com/guowangy)). +* > Original changelog In the original implementation, the memory of ThreadGroupStatus:: finished_threads_counters_memory is released by moving it to a temporary std::vector, which soon expired and gets destructed. This method is viable, however not straightforward enough. To enhance the code readability, this commit releases the memory in the vector by firstly resizing it to 0 and then shrinking the capacity accordingly. [#43586](https://github.com/ClickHouse/ClickHouse/pull/43586) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* As a follow-up of [#42214](https://github.com/ClickHouse/ClickHouse/issues/42214), this PR tries to optimize the column-wise ternary logic evaluation by achieving auto-vectorization. In the performance test of this [microbenchmark](https://github.com/ZhiguoZh/ClickHouse/blob/20221123-ternary-logic-opt-example/src/Functions/examples/associative_applier_perf.cpp), we've observed a peak **performance gain** of **21x** on the ICX device (Intel Xeon Platinum 8380 CPU). [#43669](https://github.com/ClickHouse/ClickHouse/pull/43669) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Improved latency of reading from storage `S3` and table function `s3` with large number of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)). +* - Avoid acquiring read locks in system.tables if possible. [#43840](https://github.com/ClickHouse/ClickHouse/pull/43840) ([Raúl Marín](https://github.com/Algunenano)). +* The performance experiments of SSB (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could effectively decrease the lock contention for ThreadPoolImpl::mutex by **75%**, increasing the CPU utilization and improving the overall performance by **2.4%**. [#44308](https://github.com/ClickHouse/ClickHouse/pull/44308) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Now optimisation is applied only if the cached HT size is sufficiently large (thresholds were determined empirically and hardcoded). [#44455](https://github.com/ClickHouse/ClickHouse/pull/44455) ([Nikita Taranov](https://github.com/nickitat)). +* ... The whole struct field will be loaded at current, even though we just want to read one field of the struct. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)). +* Small performance improvement for asynchronous reading from remote fs. [#44868](https://github.com/ClickHouse/ClickHouse/pull/44868) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add fast path for: - col like '%%' - col like '%' - col not like '%' - col not like '%' - match(col, '.*'). [#45244](https://github.com/ClickHouse/ClickHouse/pull/45244) ([李扬](https://github.com/taiyang-li)). +* todo. [#45289](https://github.com/ClickHouse/ClickHouse/pull/45289) ([Nikita Taranov](https://github.com/nickitat)). #### Improvement -* Implement referential dependencies and use them to create tables in the correct order while restoring from a backup. [#43834](https://github.com/ClickHouse/ClickHouse/pull/43834) ([Vitaly Baranov](https://github.com/vitlibar)). -* Substitute UDFs in `CREATE` query to avoid failures during loading at startup. Additionally, UDFs can now be used as `DEFAULT` expressions for columns. [#43539](https://github.com/ClickHouse/ClickHouse/pull/43539) ([Antonio Andelic](https://github.com/antonio2368)). -* Change how the following queries delete parts: TRUNCATE TABLE, ALTER TABLE DROP PART, ALTER TABLE DROP PARTITION. Now, these queries make empty parts which cover the old parts. This makes the TRUNCATE query work without a followedexclusive lock which means concurrent reads aren't locked. Also achieved durability in all those queries. If the request succeeds, then no resurrected parts appear later. Note that atomicity is achieved only with transaction scope. [#41145](https://github.com/ClickHouse/ClickHouse/pull/41145) ([Sema Checherinda](https://github.com/CheSema)). -* `SET param_x` query no longer requires manual string serialization for the value of the parameter. For example, query `SET param_a = '[\'a\', \'b\']'` can now be written like `SET param_a = ['a', 'b']`. [#41874](https://github.com/ClickHouse/ClickHouse/pull/41874) ([Nikolay Degterinsky](https://github.com/evillique)). -* Show read rows in the progress indication while reading from STDIN from client. Closes [#43423](https://github.com/ClickHouse/ClickHouse/issues/43423). [#43442](https://github.com/ClickHouse/ClickHouse/pull/43442) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Show progress bar while reading from s3 table function / engine. [#43454](https://github.com/ClickHouse/ClickHouse/pull/43454) ([Kseniia Sumarokova](https://github.com/kssenii)). -* `filesystemAvailable` and related functions support one optional argument with disk name, and change `filesystemFree` to `filesystemUnreserved`. Closes [#35076](https://github.com/ClickHouse/ClickHouse/issues/35076). [#42064](https://github.com/ClickHouse/ClickHouse/pull/42064) ([flynn](https://github.com/ucasfl)). -* Integration with LDAP: increased the default value of search_limit to 256, and added LDAP server config option to change that to an arbitrary value. Closes: [#42276](https://github.com/ClickHouse/ClickHouse/issues/42276). [#42461](https://github.com/ClickHouse/ClickHouse/pull/42461) ([Vasily Nemkov](https://github.com/Enmk)). -* Allow the removal of sensitive information (see the `query_masking_rules` in the configuration file) from the exception messages as well. Resolves [#41418](https://github.com/ClickHouse/ClickHouse/issues/41418). [#42940](https://github.com/ClickHouse/ClickHouse/pull/42940) ([filimonov](https://github.com/filimonov)). -* Support queries like `SHOW FULL TABLES ...` for MySQL compatibility. [#43910](https://github.com/ClickHouse/ClickHouse/pull/43910) ([Filatenkov Artur](https://github.com/FArthur-cmd)). -* Keeper improvement: Add 4lw command `rqld` which can manually assign a node as leader. [#43026](https://github.com/ClickHouse/ClickHouse/pull/43026) ([JackyWoo](https://github.com/JackyWoo)). -* Apply connection timeout settings for Distributed async INSERT from the query. [#43156](https://github.com/ClickHouse/ClickHouse/pull/43156) ([Azat Khuzhin](https://github.com/azat)). -* The `unhex` function now supports `FixedString` arguments. [issue42369](https://github.com/ClickHouse/ClickHouse/issues/42369). [#43207](https://github.com/ClickHouse/ClickHouse/pull/43207) ([DR](https://github.com/freedomDR)). -* Priority is given to deleting completely expired parts according to the TTL rules, see [#42869](https://github.com/ClickHouse/ClickHouse/issues/42869). [#43222](https://github.com/ClickHouse/ClickHouse/pull/43222) ([zhongyuankai](https://github.com/zhongyuankai)). -* More precise and reactive CPU load indication in clickhouse-client. [#43307](https://github.com/ClickHouse/ClickHouse/pull/43307) ([Sergei Trifonov](https://github.com/serxa)). -* Support reading of subcolumns of nested types from storage `S3` and table function `s3` with formats `Parquet`, `Arrow` and `ORC`. [#43329](https://github.com/ClickHouse/ClickHouse/pull/43329) ([chen](https://github.com/xiedeyantu)). -* Add `table_uuid` column to the `system.parts` table. [#43404](https://github.com/ClickHouse/ClickHouse/pull/43404) ([Azat Khuzhin](https://github.com/azat)). -* Added client option to display the number of locally processed rows in non-interactive mode (`--print-num-processed-rows`). [#43407](https://github.com/ClickHouse/ClickHouse/pull/43407) ([jh0x](https://github.com/jh0x)). -* Implement `aggregation-in-order` optimization on top of a query plan. It is enabled by default (but works only together with `optimize_aggregation_in_order`, which is disabled by default). Set `query_plan_aggregation_in_order = 0` to use the previous AST-based version. [#43592](https://github.com/ClickHouse/ClickHouse/pull/43592) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Allow to collect profile events with `trace_type = 'ProfileEvent'` to `system.trace_log` on each increment with current stack, profile event name and value of the increment. It can be enabled by the setting `trace_profile_events` and used to investigate performance of queries. [#43639](https://github.com/ClickHouse/ClickHouse/pull/43639) ([Anton Popov](https://github.com/CurtizJ)). -* Add a new setting `input_format_max_binary_string_size` to limit string size in RowBinary format. [#43842](https://github.com/ClickHouse/ClickHouse/pull/43842) ([Kruglov Pavel](https://github.com/Avogar)). -* When ClickHouse requests a remote HTTP server, and it returns an error, the numeric HTTP code was not displayed correctly in the exception message. Closes [#43919](https://github.com/ClickHouse/ClickHouse/issues/43919). [#43920](https://github.com/ClickHouse/ClickHouse/pull/43920) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Correctly report errors in queries even when multiple JOINs optimization is taking place. [#43583](https://github.com/ClickHouse/ClickHouse/pull/43583) ([Salvatore](https://github.com/tbsal)). +* Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted complitely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `optimize_or_like_chain` in the new infrastructure. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42797](https://github.com/ClickHouse/ClickHouse/pull/42797) ([Dmitry Novik](https://github.com/novikd)). +* Improve the Asterisk and ColumnMatcher parsers. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42884](https://github.com/ClickHouse/ClickHouse/pull/42884) ([Nikolay Degterinsky](https://github.com/evillique)). +* Implement `optimize_redundant_functions_in_order_by` on top of QueryTree. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42970](https://github.com/ClickHouse/ClickHouse/pull/42970) ([Dmitry Novik](https://github.com/novikd)). +* Support `optimize_group_by_function_keys` in the new analyzer architecture. Also, add support for optimizing GROUPING SETS keys. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43261](https://github.com/ClickHouse/ClickHouse/pull/43261) ([Dmitry Novik](https://github.com/novikd)). +* Improve reading CSV field in CustomSeparated/Template format. Closes [#42352](https://github.com/ClickHouse/ClickHouse/issues/42352) Closes [#39620](https://github.com/ClickHouse/ClickHouse/issues/39620). [#43332](https://github.com/ClickHouse/ClickHouse/pull/43332) ([Kruglov Pavel](https://github.com/Avogar)). +* Support reading/writing `Nested` tables as `List` of `Struct` in CapnProto format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)). +* - Unify query elapsed time measurements. [#43455](https://github.com/ClickHouse/ClickHouse/pull/43455) ([Raúl Marín](https://github.com/Algunenano)). +* Support scalar subqueries cache Implementation: * Added a map with hash of the node (without alias) and the evaluated value to Context. Testing: * Added a test-case with new analyser in 02174_cte_scalar_cache.sql. [#43640](https://github.com/ClickHouse/ClickHouse/pull/43640) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Improve automatic usage of structure from insertion table in table functions file/hdfs/s3 when virtual columns present in select query, it fixes possible error `Block structure mismatch` or `number of columns mismatch`. [#43695](https://github.com/ClickHouse/ClickHouse/pull/43695) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for signed arguments in range(). Fixes [#43333](https://github.com/ClickHouse/ClickHouse/issues/43333). [#43733](https://github.com/ClickHouse/ClickHouse/pull/43733) ([sanyu](https://github.com/wineternity)). +* Remove redundant sorting, for example, sorting related ORDER BY clauses in subqueries. Implemented on top of query plan. It does similar optimization as `optimize_duplicate_order_by_and_distinct` regarding `ORDER BY` clauses, but more generic, since it's applied to any redundant sorting steps (not only caused by ORDER BY clause) and applied to subqueries of any depth. Related to [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43905](https://github.com/ClickHouse/ClickHouse/pull/43905) ([Igor Nikonov](https://github.com/devcrafter)). +* Added mmap support for StorageFile, which should improve the performance of clickhouse-local. [#43927](https://github.com/ClickHouse/ClickHouse/pull/43927) ([pufit](https://github.com/pufit)). +* Add ability to disable deduplication for BACKUP (for backups wiithout deduplication ATTACH can be used instead of full RESTORE), example `BACKUP foo TO S3(...) SETTINGS deduplicate_files=0` (default `deduplicate_files=1`). [#43947](https://github.com/ClickHouse/ClickHouse/pull/43947) ([Azat Khuzhin](https://github.com/azat)). +* Make `system.replicas` table do parallel fetches of replicas statuses. Closes [#43918](https://github.com/ClickHouse/ClickHouse/issues/43918). [#43998](https://github.com/ClickHouse/ClickHouse/pull/43998) ([Nikolay Degterinsky](https://github.com/evillique)). +* Refactor and improve schema inference for text formats. Add new setting `schema_inference_make_columns_nullable` that controls making result types `Nullable` (enabled by default);. [#44019](https://github.com/ClickHouse/ClickHouse/pull/44019) ([Kruglov Pavel](https://github.com/Avogar)). +* Better support for PROXYv1. [#44135](https://github.com/ClickHouse/ClickHouse/pull/44135) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Add information about the latest part check by cleanup thread into `system.parts` table. [#44244](https://github.com/ClickHouse/ClickHouse/pull/44244) ([Dmitry Novik](https://github.com/novikd)). +* Disable functions in readonly for inserts. [#44290](https://github.com/ClickHouse/ClickHouse/pull/44290) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Add a setting `simultaneous_parts_removal_limit` to allow to limit the number of parts being processed by one iteration of CleanupThread. [#44461](https://github.com/ClickHouse/ClickHouse/pull/44461) ([Dmitry Novik](https://github.com/novikd)). +* If user only need virtual columns, we don't need to initialize ReadBufferFromS3. May be helpful to [#44246](https://github.com/ClickHouse/ClickHouse/issues/44246). [#44493](https://github.com/ClickHouse/ClickHouse/pull/44493) ([chen](https://github.com/xiedeyantu)). +* Prevent duplicate column names hints. Closes [#44130](https://github.com/ClickHouse/ClickHouse/issues/44130). [#44519](https://github.com/ClickHouse/ClickHouse/pull/44519) ([Joanna Hulboj](https://github.com/jh0x)). +* Allow macro substitution in endpoint of disks resolve [#40951](https://github.com/ClickHouse/ClickHouse/issues/40951). [#44533](https://github.com/ClickHouse/ClickHouse/pull/44533) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve schema inference when `input_format_json_read_object_as_string` is enabled. [#44546](https://github.com/ClickHouse/ClickHouse/pull/44546) ([Kruglov Pavel](https://github.com/Avogar)). +* Add user-level setting `database_replicated_allow_replicated_engine_arguments` which allow to ban creation of `ReplicatedMergeTree` tables with arguments in `DatabaseReplicated`. [#44566](https://github.com/ClickHouse/ClickHouse/pull/44566) ([alesapin](https://github.com/alesapin)). +* Prevent users from mistakenly specifying zero (invalid) value for `index_granularity`. This closes [#44536](https://github.com/ClickHouse/ClickHouse/issues/44536). [#44578](https://github.com/ClickHouse/ClickHouse/pull/44578) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added possibility to set path to service keytab file in `keytab` parameter in `kerberos` section of config.xml. [#44594](https://github.com/ClickHouse/ClickHouse/pull/44594) ([Roman Vasin](https://github.com/rvasin)). +* Use already written part of the query for fuzzy search (pass to skim). [#44600](https://github.com/ClickHouse/ClickHouse/pull/44600) ([Azat Khuzhin](https://github.com/azat)). +* Enable input_format_json_read_objects_as_strings by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)). +* When users do duplicate async inserts, we should dedup inside the memory before we query keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)). +* Input/ouptut Avro bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)). +* - Don't parse beyond the quotes when reading UUIDs. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)). +* Infer UInt64 in case of Int64 overflow and fix some transforms in schema inference. [#44696](https://github.com/ClickHouse/ClickHouse/pull/44696) ([Kruglov Pavel](https://github.com/Avogar)). +* Previously dependency resolving inside DatabaseReplicated was done in a hacky way and now it done right using an explicit graph. [#44697](https://github.com/ClickHouse/ClickHouse/pull/44697) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Support Bool type in Arrow/Parquet/ORC. Closes [#43970](https://github.com/ClickHouse/ClickHouse/issues/43970). [#44698](https://github.com/ClickHouse/ClickHouse/pull/44698) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix `output_format_pretty_row_numbers` does not preserve the counter across the blocks. Closes [#44815](https://github.com/ClickHouse/ClickHouse/issues/44815). [#44832](https://github.com/ClickHouse/ClickHouse/pull/44832) ([flynn](https://github.com/ucasfl)). +* Extend function "toDayOfWeek" with a mode argument describing if a) the week starts on Monday or Sunday and b) if counting starts at 0 or 1. [#44860](https://github.com/ClickHouse/ClickHouse/pull/44860) ([李扬](https://github.com/taiyang-li)). +* - Don't report errors in system.errors due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)). +* Optimize and fix metrics for Distributed async INSERT. [#44922](https://github.com/ClickHouse/ClickHouse/pull/44922) ([Azat Khuzhin](https://github.com/azat)). +* Added settings to disallow concurrent backups and restores resolves [#43891](https://github.com/ClickHouse/ClickHouse/issues/43891) Implementation: * Added server level settings to disallow concurrent backups and restores, which are read and set when BackupWorker is created in Context. * Settings are set to true by default. * Before starting backup or restores, added a check to see if any other backups/restores are running. For internal request it checks if its from the self node using backup_uuid. [#45072](https://github.com/ClickHouse/ClickHouse/pull/45072) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* add a cache for async block ids. This will reduce the requests of zookeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)). +* CRC32 changes to address the WeakHash collision issue in PowerPC. [#45144](https://github.com/ClickHouse/ClickHouse/pull/45144) ([MeenaRenganathan22](https://github.com/MeenaRenganathan22)). +* Optimize memory consumption during backup to S3: files to S3 now will be copied directly without using `WriteBufferFromS3` (which could use a lot of memory). [#45188](https://github.com/ClickHouse/ClickHouse/pull/45188) ([Vitaly Baranov](https://github.com/vitlibar)). +* Use structure from insertion table in generateRandom without arguments. [#45239](https://github.com/ClickHouse/ClickHouse/pull/45239) ([Kruglov Pavel](https://github.com/Avogar)). +* Use `GetObjectAttributes` request instead of `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit region, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add `` config parameter for system logs. [#45320](https://github.com/ClickHouse/ClickHouse/pull/45320) ([Stig Bakken](https://github.com/stigsb)). +* Allow to implicitly convert floats stored in string fields of JSON to integers in `JSONExtract` functions. E.g. `JSONExtract('{"a": "1000.111"}', 'a', 'UInt64')` -> `1000`, previously it returned 0. [#45432](https://github.com/ClickHouse/ClickHouse/pull/45432) ([Anton Popov](https://github.com/CurtizJ)). +* Added fields `supports_parallel_parsing` and `supports_parallel_formatting` to table `system.formats` for better introspection. [#45499](https://github.com/ClickHouse/ClickHouse/pull/45499) ([Anton Popov](https://github.com/CurtizJ)). + +#### Bug Fix +* Fix HTTP requests without path for AWS. After updating AWS SDK the sdk no longer adds a slash to requesting paths so we need to do it in our PocoHTTPClient to keep HTTP requests correct. [#45238](https://github.com/ClickHouse/ClickHouse/pull/45238) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix backup if mutations get killed during the backup process. [#45351](https://github.com/ClickHouse/ClickHouse/pull/45351) ([Vitaly Baranov](https://github.com/vitlibar)). #### Build/Testing/Packaging Improvement - -* Systemd integration now correctly notifies systemd that the service is really started and is ready to serve requests. [#43400](https://github.com/ClickHouse/ClickHouse/pull/43400) ([Коренберг Марк](https://github.com/socketpair)). -* Added the option to build ClickHouse with OpenSSL using the [OpenSSL FIPS Module](https://www.openssl.org/docs/man3.0/man7/fips_module.html). This build type has not been tested to validate security and is not supported. [#43991](https://github.com/ClickHouse/ClickHouse/pull/43991) ([Boris Kuschel](https://github.com/bkuschel)). -* Upgrade to the new `DeflateQpl` compression codec which has been implemented in a previous PR (details: https://github.com/ClickHouse/ClickHouse/pull/39494). This patch improves codec on below aspects: 1. QPL v0.2.0 to QPL v0.3.0 [Intel® Query Processing Library (QPL)](https://github.com/intel/qpl) 2. Improve CMake file for fixing QPL build issues for QPL v0.3.0. 3. Link the QPL library with libaccel-config at build time instead of runtime loading on QPL v0.2.0 (dlopen) 4. Fixed log print issue in CompressionCodecDeflateQpl.cpp. [#44024](https://github.com/ClickHouse/ClickHouse/pull/44024) ([jasperzhu](https://github.com/jinjunzh)). +* Builtin skim for fuzzy search in clickhouse client/local history. [#44239](https://github.com/ClickHouse/ClickHouse/pull/44239) ([Azat Khuzhin](https://github.com/azat)). +* Memory limit for server is set now in AST fuzz tests to avoid OOMs. [#44282](https://github.com/ClickHouse/ClickHouse/pull/44282) ([Nikita Taranov](https://github.com/nickitat)). +* In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* The "universal.sh" now fetches a SSE2 build on systems which don't have SSE4.2. [#44366](https://github.com/ClickHouse/ClickHouse/pull/44366) ([Robert Schulze](https://github.com/rschu1ze)). +* Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* ... 1. Added pytest-random by default in integration tests runner 2. Disable TSAN checks for tests with GPRC ( like https://s3.amazonaws.com/clickhouse-test-reports/42807/e9d7407a58f6e3f7d88c0c534685704f23560704/integration_tests__tsan__[4/6].html ) 3. Cleanup tables after tests in odbc. [#44711](https://github.com/ClickHouse/ClickHouse/pull/44711) ([Ilya Yatsishin](https://github.com/qoega)). +* We removed support for shared linking because of Rust. Actually, Rust is only an excuse for this removal, and we wanted to remove it nevertheless. [#44828](https://github.com/ClickHouse/ClickHouse/pull/44828) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Checks will try to download images before running integration tests. If image, proxy or whatever is broken in infrastructure it will not make tests flaky. Images will be cached locally and download time will not be added to random tests. Compose images are now changed to be used without correct environment from helpers/cluster.py. [#44848](https://github.com/ClickHouse/ClickHouse/pull/44848) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* The performance tests were silently broken because `Errors` wasn't detected in the status message. [#44867](https://github.com/ClickHouse/ClickHouse/pull/44867) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove the dependency on the `adduser` tool from the packages, because we don't use it. This fixes [#44934](https://github.com/ClickHouse/ClickHouse/issues/44934). [#45011](https://github.com/ClickHouse/ClickHouse/pull/45011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* SQLite library is updated to the latest. It is used for the SQLite database and table integration engines. Also, fixed a false-positive TSan report. This closes [#45027](https://github.com/ClickHouse/ClickHouse/issues/45027). [#45031](https://github.com/ClickHouse/ClickHouse/pull/45031) ([Alexey Milovidov](https://github.com/alexey-milovidov)). #### Bug Fix (user-visible misbehavior in official stable or prestable release) -* Fixed bug which could lead to deadlock while using asynchronous inserts. [#43233](https://github.com/ClickHouse/ClickHouse/pull/43233) ([Anton Popov](https://github.com/CurtizJ)). -* Fix some incorrect logic in AST level optimization `optimize_normalize_count_variants`. [#43873](https://github.com/ClickHouse/ClickHouse/pull/43873) ([Duc Canh Le](https://github.com/canhld94)). -* Fix a case when mutations are not making progress when checksums do not match between replicas (e.g. caused by a change in data format on an upgrade). [#36877](https://github.com/ClickHouse/ClickHouse/pull/36877) ([nvartolomei](https://github.com/nvartolomei)). -* Fix the `skip_unavailable_shards` optimization which did not work with the `hdfsCluster` table function. [#43236](https://github.com/ClickHouse/ClickHouse/pull/43236) ([chen](https://github.com/xiedeyantu)). -* Fix `s3` support for the `?` wildcard. Closes [#42731](https://github.com/ClickHouse/ClickHouse/issues/42731). [#43253](https://github.com/ClickHouse/ClickHouse/pull/43253) ([chen](https://github.com/xiedeyantu)). -* Fix functions `arrayFirstOrNull` and `arrayLastOrNull` or null when the array contains `Nullable` elements. [#43274](https://github.com/ClickHouse/ClickHouse/pull/43274) ([Duc Canh Le](https://github.com/canhld94)). -* Fix incorrect `UserTimeMicroseconds`/`SystemTimeMicroseconds` accounting related to Kafka tables. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)). -* Do not suppress exceptions in `web` disks. Fix retries for the `web` disk. [#42800](https://github.com/ClickHouse/ClickHouse/pull/42800) ([Azat Khuzhin](https://github.com/azat)). -* Fixed (logical) race condition between inserts and dropping materialized views. A race condition happened when a Materialized View was dropped at the same time as an INSERT, where the MVs were present as a dependency of the insert at the begining of the execution, but the table has been dropped by the time the insert chain tries to access it, producing either an `UNKNOWN_TABLE` or `TABLE_IS_DROPPED` exception, and stopping the insertion. After this change, we avoid these exceptions and just continue with the insert if the dependency is gone. [#43161](https://github.com/ClickHouse/ClickHouse/pull/43161) ([AlfVII](https://github.com/AlfVII)). -* Fix undefined behavior in the `quantiles` function, which might lead to uninitialized memory. Found by fuzzer. This closes [#44066](https://github.com/ClickHouse/ClickHouse/issues/44066). [#44067](https://github.com/ClickHouse/ClickHouse/pull/44067) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Additional check on zero uncompressed size is added to `CompressionCodecDelta`. [#43255](https://github.com/ClickHouse/ClickHouse/pull/43255) ([Nikita Taranov](https://github.com/nickitat)). -* Flatten arrays from Parquet to avoid an issue with inconsistent data in arrays. These incorrect files can be generated by Apache Iceberg. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). -* Fix bad cast from `LowCardinality` column when using short circuit function execution. [#43311](https://github.com/ClickHouse/ClickHouse/pull/43311) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)). -* Check and compare the content of the `format_version` file in `MergeTreeData` so that tables can be loaded even if the storage policy was changed. [#43328](https://github.com/ClickHouse/ClickHouse/pull/43328) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix possible (very unlikely) "No column to rollback" logical error during INSERT into `Buffer` tables. [#43336](https://github.com/ClickHouse/ClickHouse/pull/43336) ([Azat Khuzhin](https://github.com/azat)). -* Fix a bug that allowed the parser to parse an unlimited amount of round brackets into one function if `allow_function_parameters` is set. [#43350](https://github.com/ClickHouse/ClickHouse/pull/43350) ([Nikolay Degterinsky](https://github.com/evillique)). -* `MaterializeMySQL` (experimental feature) support DDL： `drop table t1, t2` and compatible with most of MySQL DROP DDL. [#43366](https://github.com/ClickHouse/ClickHouse/pull/43366) ([zzsmdfj](https://github.com/zzsmdfj)). -* `session_log` (experimental feature): Fixed the inability to log in (because of failure to create the session_log entry) in a very rare case of messed up setting profiles. [#42641](https://github.com/ClickHouse/ClickHouse/pull/42641) ([Vasily Nemkov](https://github.com/Enmk)). -* Fix possible `Cannot create non-empty column with type Nothing` in functions `if`/`multiIf`. Closes [#43356](https://github.com/ClickHouse/ClickHouse/issues/43356). [#43368](https://github.com/ClickHouse/ClickHouse/pull/43368) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix a bug when a row level filter uses the default value of a column. [#43387](https://github.com/ClickHouse/ClickHouse/pull/43387) ([Alexander Gololobov](https://github.com/davenger)). -* Query with `DISTINCT` + `LIMIT BY` + `LIMIT` can return fewer rows than expected. Fixes [#43377](https://github.com/ClickHouse/ClickHouse/issues/43377). [#43410](https://github.com/ClickHouse/ClickHouse/pull/43410) ([Igor Nikonov](https://github.com/devcrafter)). -* Fix `sumMap` for `Nullable(Decimal(...))`. [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)). -* Fix `date_diff` for hour/minute on macOS. Close [#42742](https://github.com/ClickHouse/ClickHouse/issues/42742). [#43466](https://github.com/ClickHouse/ClickHouse/pull/43466) ([zzsmdfj](https://github.com/zzsmdfj)). -* Fix incorrect memory accounting because of merges/mutations. [#43516](https://github.com/ClickHouse/ClickHouse/pull/43516) ([Azat Khuzhin](https://github.com/azat)). -* Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). This error has been found by @tisonkun. -* Ensure consistency when `clickhouse-copier` updates status and `attach_is_done` in Keeper after partition attach is done. [#43602](https://github.com/ClickHouse/ClickHouse/pull/43602) ([lzydmxy](https://github.com/lzydmxy)). -* During the recovery of a lost replica of a `Replicated` database (experimental feature), there could a situation where we need to atomically swap two table names (use EXCHANGE). Previously we tried to use two RENAME queries, which was obviously failing and moreover, failed the whole recovery process of the database replica. [#43628](https://github.com/ClickHouse/ClickHouse/pull/43628) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Fix the case when the `s3Cluster` function throws `NOT_FOUND_COLUMN_IN_BLOCK` error. Closes [#43534](https://github.com/ClickHouse/ClickHouse/issues/43534). [#43629](https://github.com/ClickHouse/ClickHouse/pull/43629) ([chen](https://github.com/xiedeyantu)). -* Fix possible logical error `Array sizes mismatched` while parsing JSON object with arrays with same key names but with different nesting level. Closes [#43569](https://github.com/ClickHouse/ClickHouse/issues/43569). [#43693](https://github.com/ClickHouse/ClickHouse/pull/43693) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed possible exception in the case of distributed `GROUP BY` with an `ALIAS` column among aggregation keys. [#43709](https://github.com/ClickHouse/ClickHouse/pull/43709) ([Nikita Taranov](https://github.com/nickitat)). -* Fix bug which can lead to broken projections if zero-copy replication (experimental feature) is enabled and used. [#43764](https://github.com/ClickHouse/ClickHouse/pull/43764) ([alesapin](https://github.com/alesapin)). -* Fix using multipart upload for very large S3 objects in AWS S3. [#43824](https://github.com/ClickHouse/ClickHouse/pull/43824) ([ianton-ru](https://github.com/ianton-ru)). -* Fixed `ALTER ... RESET SETTING` with `ON CLUSTER`. It could have been applied to one replica only. Fixes [#43843](https://github.com/ClickHouse/ClickHouse/issues/43843). [#43848](https://github.com/ClickHouse/ClickHouse/pull/43848) ([Elena Torró](https://github.com/elenatorro)). -* Fix a logical error in JOIN with `Join` table engine at right hand side, if `USING` is being used. [#43963](https://github.com/ClickHouse/ClickHouse/pull/43963) ([Vladimir C](https://github.com/vdimir)). Fix a bug with wrong order of keys in `Join` table engine. [#44012](https://github.com/ClickHouse/ClickHouse/pull/44012) ([Vladimir C](https://github.com/vdimir)). -* Keeper fix: throw if the interserver port for Raft is already in use. [#43984](https://github.com/ClickHouse/ClickHouse/pull/43984) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix ORDER BY positional argument (example: `ORDER BY 1, 2`) in case of unneeded columns pruning from subqueries. Closes [#43964](https://github.com/ClickHouse/ClickHouse/issues/43964). [#43987](https://github.com/ClickHouse/ClickHouse/pull/43987) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixed exception when a subquery contains HAVING but doesn't contain an actual aggregation. [#44051](https://github.com/ClickHouse/ClickHouse/pull/44051) ([Nikita Taranov](https://github.com/nickitat)). -* Fix race in s3 multipart upload. This race could cause the error `Part number must be an integer between 1 and 10000, inclusive. (S3_ERROR)` while restoring from a backup. [#44065](https://github.com/ClickHouse/ClickHouse/pull/44065) ([Vitaly Baranov](https://github.com/vitlibar)). - - -### ClickHouse release 22.11, 2022-11-17 - -#### Backward Incompatible Change -* `JSONExtract` family of functions will now attempt to coerce to the requested type. [#41502](https://github.com/ClickHouse/ClickHouse/pull/41502) ([Márcio Martins](https://github.com/marcioapm)). - -#### New Feature -* Adds support for retries during INSERTs into ReplicatedMergeTree when a session with ClickHouse Keeper is lost. Apart from fault tolerance, it aims to provide better user experience, - avoid returning a user an error during insert if keeper is restarted (for example, due to upgrade). This is controlled by the `insert_keeper_max_retries` setting, which is disabled by default. [#42607](https://github.com/ClickHouse/ClickHouse/pull/42607) ([Igor Nikonov](https://github.com/devcrafter)). -* Add `Hudi` and `DeltaLake` table engines, read-only, only for tables on S3. [#41054](https://github.com/ClickHouse/ClickHouse/pull/41054) ([Daniil Rubin](https://github.com/rubin-do), [Kseniia Sumarokova](https://github.com/kssenii)). -* Add table function `hudi` and `deltaLake`. [#43080](https://github.com/ClickHouse/ClickHouse/pull/43080) ([flynn](https://github.com/ucasfl)). -* Support for composite time intervals. 1. Add, subtract and negate operations are now available on Intervals. In the case where the types of Intervals are different, they will be transformed into the Tuple of those types. 2. A tuple of intervals can be added to or subtracted from a Date/DateTime field. 3. Added parsing of Intervals with different types, for example: `INTERVAL '1 HOUR 1 MINUTE 1 SECOND'`. [#42195](https://github.com/ClickHouse/ClickHouse/pull/42195) ([Nikolay Degterinsky](https://github.com/evillique)). -* Added `**` glob support for recursive directory traversal of the filesystem and S3. Resolves [#36316](https://github.com/ClickHouse/ClickHouse/issues/36316). [#42376](https://github.com/ClickHouse/ClickHouse/pull/42376) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* Introduce `s3_plain` disk type for write-once-read-many operations. Implement `ATTACH` of `MergeTree` table for `s3_plain` disk. [#42628](https://github.com/ClickHouse/ClickHouse/pull/42628) ([Azat Khuzhin](https://github.com/azat)). -* Added applied row-level policies to `system.query_log`. [#39819](https://github.com/ClickHouse/ClickHouse/pull/39819) ([Vladimir Chebotaryov](https://github.com/quickhouse)). -* Add four-letter command `csnp` for manually creating snapshots in ClickHouse Keeper. Additionally, `lgif` was added to get Raft information for a specific node (e.g. index of last created snapshot, last committed log index). [#41766](https://github.com/ClickHouse/ClickHouse/pull/41766) ([JackyWoo](https://github.com/JackyWoo)). -* Add function `ascii` like in Apache Spark: https://spark.apache.org/docs/latest/api/sql/#ascii. [#42670](https://github.com/ClickHouse/ClickHouse/pull/42670) ([李扬](https://github.com/taiyang-li)). -* Add function `positive_modulo` (`pmod`) which returns non-negative result based on modulo. [#42755](https://github.com/ClickHouse/ClickHouse/pull/42755) ([李扬](https://github.com/taiyang-li)). -* Add function `formatReadableDecimalSize`. [#42774](https://github.com/ClickHouse/ClickHouse/pull/42774) ([Alejandro](https://github.com/alexon1234)). -* Add function `randCanonical`, which is similar to the `rand` function in Apache Spark or Impala. The function generates pseudo random results with independent and identically distributed uniformly distributed values in [0, 1). [#43124](https://github.com/ClickHouse/ClickHouse/pull/43124) ([李扬](https://github.com/taiyang-li)). -* Add function `displayName`, closes [#36770](https://github.com/ClickHouse/ClickHouse/issues/36770). [#37681](https://github.com/ClickHouse/ClickHouse/pull/37681) ([hongbin](https://github.com/xlwh)). -* Add `min_age_to_force_merge_on_partition_only` setting to optimize old parts for the entire partition only. [#42659](https://github.com/ClickHouse/ClickHouse/pull/42659) ([Antonio Andelic](https://github.com/antonio2368)). -* Add generic implementation for arbitrary structured named collections, access type and `system.named_collections`. [#43147](https://github.com/ClickHouse/ClickHouse/pull/43147) ([Kseniia Sumarokova](https://github.com/kssenii)). - -#### Performance Improvement -* Parallelized merging of `uniqExact` states for aggregation without key, i.e. queries like `SELECT uniqExact(number) FROM table`. The improvement becomes noticeable when the number of unique keys approaches 10^6. Also `uniq` performance is slightly optimized. [#43072](https://github.com/ClickHouse/ClickHouse/pull/43072) ([Nikita Taranov](https://github.com/nickitat)). -* `match` function can use the index if it's a condition on string prefix. This closes [#37333](https://github.com/ClickHouse/ClickHouse/issues/37333). [#42458](https://github.com/ClickHouse/ClickHouse/pull/42458) ([clarkcaoliu](https://github.com/Clark0)). -* Speed up AND and OR operators when they are sequenced. [#42214](https://github.com/ClickHouse/ClickHouse/pull/42214) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). -* Support parallel parsing for `LineAsString` input format. This improves performance just slightly. This closes [#42502](https://github.com/ClickHouse/ClickHouse/issues/42502). [#42780](https://github.com/ClickHouse/ClickHouse/pull/42780) ([Kruglov Pavel](https://github.com/Avogar)). -* ClickHouse Keeper performance improvement: improve commit performance for cases when many different nodes have uncommitted states. This should help with cases when a follower node can't sync fast enough. [#42926](https://github.com/ClickHouse/ClickHouse/pull/42926) ([Antonio Andelic](https://github.com/antonio2368)). -* A condition like `NOT LIKE 'prefix%'` can use the primary index. [#42209](https://github.com/ClickHouse/ClickHouse/pull/42209) ([Duc Canh Le](https://github.com/canhld94)). - -#### Experimental Feature -* Support type `Object` inside other types, e.g. `Array(JSON)`. [#36969](https://github.com/ClickHouse/ClickHouse/pull/36969) ([Anton Popov](https://github.com/CurtizJ)). -* Ignore MySQL binlog SAVEPOINT event for MaterializedMySQL. [#42931](https://github.com/ClickHouse/ClickHouse/pull/42931) ([zzsmdfj](https://github.com/zzsmdfj)). Handle (ignore) SAVEPOINT queries in MaterializedMySQL. [#43086](https://github.com/ClickHouse/ClickHouse/pull/43086) ([Stig Bakken](https://github.com/stigsb)). - -#### Improvement -* Trivial queries with small LIMIT will properly determine the number of estimated rows to read, so that the threshold will be checked properly. Closes [#7071](https://github.com/ClickHouse/ClickHouse/issues/7071). [#42580](https://github.com/ClickHouse/ClickHouse/pull/42580) ([Han Fei](https://github.com/hanfei1991)). -* Add support for interactive parameters in INSERT VALUES queries. [#43077](https://github.com/ClickHouse/ClickHouse/pull/43077) ([Nikolay Degterinsky](https://github.com/evillique)). -* Added new field `allow_readonly` in `system.table_functions` to allow using table functions in readonly mode. Resolves [#42414](https://github.com/ClickHouse/ClickHouse/issues/42414) Implementation: * Added a new field allow_readonly to table system.table_functions. * Updated to use new field allow_readonly to allow using table functions in readonly mode. Testing: * Added a test for filesystem tests/queries/0_stateless/02473_functions_in_readonly_mode.sh Documentation: * Updated the english documentation for Table Functions. [#42708](https://github.com/ClickHouse/ClickHouse/pull/42708) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* The `system.asynchronous_metrics` gets embedded documentation. This documentation is also exported to Prometheus. Fixed an error with the metrics about `cache` disks - they were calculated only for one arbitrary cache disk instead all of them. This closes [#7644](https://github.com/ClickHouse/ClickHouse/issues/7644). [#43194](https://github.com/ClickHouse/ClickHouse/pull/43194) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Throttling algorithm changed to token bucket. [#42665](https://github.com/ClickHouse/ClickHouse/pull/42665) ([Sergei Trifonov](https://github.com/serxa)). -* Mask passwords and secret keys both in `system.query_log` and `/var/log/clickhouse-server/*.log` and also in error messages. [#42484](https://github.com/ClickHouse/ClickHouse/pull/42484) ([Vitaly Baranov](https://github.com/vitlibar)). -* Remove covered parts for fetched part (to avoid possible replication delay grows). [#39737](https://github.com/ClickHouse/ClickHouse/pull/39737) ([Azat Khuzhin](https://github.com/azat)). -* If `/dev/tty` is available, the progress in clickhouse-client and clickhouse-local will be rendered directly to the terminal, without writing to STDERR. It allows getting progress even if STDERR is redirected to a file, and the file will not be polluted by terminal escape sequences. The progress can be disabled by `--progress false`. This closes [#32238](https://github.com/ClickHouse/ClickHouse/issues/32238). [#42003](https://github.com/ClickHouse/ClickHouse/pull/42003) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add support for `FixedString` input to base64 coding functions. [#42285](https://github.com/ClickHouse/ClickHouse/pull/42285) ([ltrk2](https://github.com/ltrk2)). -* Add columns `bytes_on_disk` and `path` to `system.detached_parts`. Closes [#42264](https://github.com/ClickHouse/ClickHouse/issues/42264). [#42303](https://github.com/ClickHouse/ClickHouse/pull/42303) ([chen](https://github.com/xiedeyantu)). -* Improve using structure from insertion table in table functions, now setting `use_structure_from_insertion_table_in_table_functions` has new possible value - `2` that means that ClickHouse will try to determine if we can use structure from insertion table or not automatically. Closes [#40028](https://github.com/ClickHouse/ClickHouse/issues/40028). [#42320](https://github.com/ClickHouse/ClickHouse/pull/42320) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix no progress indication on INSERT FROM INFILE. Closes [#42548](https://github.com/ClickHouse/ClickHouse/issues/42548). [#42634](https://github.com/ClickHouse/ClickHouse/pull/42634) ([chen](https://github.com/xiedeyantu)). -* Refactor function `tokens` to enable max tokens returned for related functions (disabled by default). [#42673](https://github.com/ClickHouse/ClickHouse/pull/42673) ([李扬](https://github.com/taiyang-li)). -* Allow to use `Date32` arguments for `formatDateTime` and `FROM_UNIXTIME` functions. [#42737](https://github.com/ClickHouse/ClickHouse/pull/42737) ([Roman Vasin](https://github.com/rvasin)). -* Update tzdata to 2022f. Mexico will no longer observe DST except near the US border: https://www.timeanddate.com/news/time/mexico-abolishes-dst-2022.html. Chihuahua moves to year-round UTC-6 on 2022-10-30. Fiji no longer observes DST. See https://github.com/google/cctz/pull/235 and https://bugs.launchpad.net/ubuntu/+source/tzdata/+bug/1995209. [#42796](https://github.com/ClickHouse/ClickHouse/pull/42796) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add `FailedAsyncInsertQuery` event metric for async inserts. [#42814](https://github.com/ClickHouse/ClickHouse/pull/42814) ([Krzysztof Góralski](https://github.com/kgoralski)). -* Implement `read-in-order` optimization on top of query plan. It is enabled by default. Set `query_plan_read_in_order = 0` to use previous AST-based version. [#42829](https://github.com/ClickHouse/ClickHouse/pull/42829) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Increase the size of upload part exponentially for backup to S3 to avoid errors about max 10 000 parts limit of the multipart upload to s3. [#42833](https://github.com/ClickHouse/ClickHouse/pull/42833) ([Vitaly Baranov](https://github.com/vitlibar)). -* When the merge task is continuously busy and the disk space is insufficient, the completely expired parts cannot be selected and dropped, resulting in insufficient disk space. My idea is that when the entire Part expires, there is no need for additional disk space to guarantee, ensure the normal execution of TTL. [#42869](https://github.com/ClickHouse/ClickHouse/pull/42869) ([zhongyuankai](https://github.com/zhongyuankai)). -* Add `oss` function and `OSS` table engine (this is convenient for users). oss is fully compatible with s3. [#43155](https://github.com/ClickHouse/ClickHouse/pull/43155) ([zzsmdfj](https://github.com/zzsmdfj)). -* Improve error reporting in the collection of OS-related info for the `system.asynchronous_metrics` table. [#43192](https://github.com/ClickHouse/ClickHouse/pull/43192) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Modify the `INFORMATION_SCHEMA` tables in a way so that ClickHouse can connect to itself using the MySQL compatibility protocol. Add columns instead of aliases (related to [#9769](https://github.com/ClickHouse/ClickHouse/issues/9769)). It will improve the compatibility with various MySQL clients. [#43198](https://github.com/ClickHouse/ClickHouse/pull/43198) ([Filatenkov Artur](https://github.com/FArthur-cmd)). -* Add some functions for compatibility with PowerBI, when it connects using MySQL protocol [#42612](https://github.com/ClickHouse/ClickHouse/pull/42612) ([Filatenkov Artur](https://github.com/FArthur-cmd)). -* Better usability for Dashboard on changes [#42872](https://github.com/ClickHouse/ClickHouse/pull/42872) ([Vladimir C](https://github.com/vdimir)). - -#### Build/Testing/Packaging Improvement -* Run SQLancer for each pull request and commit to master. [SQLancer](https://github.com/sqlancer/sqlancer) is an OpenSource fuzzer that focuses on automatic detection of logical bugs. [#42397](https://github.com/ClickHouse/ClickHouse/pull/42397) ([Ilya Yatsishin](https://github.com/qoega)). -* Update to latest zlib-ng. [#42463](https://github.com/ClickHouse/ClickHouse/pull/42463) ([Boris Kuschel](https://github.com/bkuschel)). -* Add support for testing ClickHouse server with Jepsen. By the way, we already have support for testing ClickHouse Keeper with Jepsen. This pull request extends it to Replicated tables. [#42619](https://github.com/ClickHouse/ClickHouse/pull/42619) ([Antonio Andelic](https://github.com/antonio2368)). -* Use https://github.com/matus-chochlik/ctcache for clang-tidy results caching. [#42913](https://github.com/ClickHouse/ClickHouse/pull/42913) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Before the fix, the user-defined config was preserved by RPM in `$file.rpmsave`. The PR fixes it and won't replace the user's files from packages. [#42936](https://github.com/ClickHouse/ClickHouse/pull/42936) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Remove some libraries from Ubuntu Docker image. [#42622](https://github.com/ClickHouse/ClickHouse/pull/42622) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### Bug Fix (user-visible misbehavior in official stable or prestable release) - -* Updated normaliser to clone the alias ast. Resolves [#42452](https://github.com/ClickHouse/ClickHouse/issues/42452) Implementation: * Updated QueryNormalizer to clone alias ast, when its replaced. Previously just assigning the same leads to exception in LogicalExpressinsOptimizer as it would be the same parent being inserted again. * This bug is not seen with new analyser (allow_experimental_analyzer), so no changes for it. I added a test for the same. [#42827](https://github.com/ClickHouse/ClickHouse/pull/42827) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* Fix race for backup of tables in `Lazy` databases. [#43104](https://github.com/ClickHouse/ClickHouse/pull/43104) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix for `skip_unavailable_shards`: it did not work with the `s3Cluster` table function. [#43131](https://github.com/ClickHouse/ClickHouse/pull/43131) ([chen](https://github.com/xiedeyantu)). -* Fix schema inference in `s3Cluster` and improvement in `hdfsCluster`. [#41979](https://github.com/ClickHouse/ClickHouse/pull/41979) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix retries while reading from URL table engines / table function. (retriable errors could be retries more times than needed, non-retriable errors resulted in failed assertion in code). [#42224](https://github.com/ClickHouse/ClickHouse/pull/42224) ([Kseniia Sumarokova](https://github.com/kssenii)). -* A segmentation fault related to DNS & c-ares has been reported and fixed. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)). -* Fix `LOGICAL_ERROR` `Arguments of 'plus' have incorrect data types` which may happen in PK analysis (monotonicity check). Fix invalid PK analysis for monotonic binary functions with first constant argument. [#42410](https://github.com/ClickHouse/ClickHouse/pull/42410) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix incorrect key analysis when key types cannot be inside Nullable. This fixes [#42456](https://github.com/ClickHouse/ClickHouse/issues/42456). [#42469](https://github.com/ClickHouse/ClickHouse/pull/42469) ([Amos Bird](https://github.com/amosbird)). -* Fix typo in a setting name that led to bad usage of schema inference cache while using setting `input_format_csv_use_best_effort_in_schema_inference`. Closes [#41735](https://github.com/ClickHouse/ClickHouse/issues/41735). [#42536](https://github.com/ClickHouse/ClickHouse/pull/42536) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix creating a Set with wrong header when data type is LowCardinality. Closes [#42460](https://github.com/ClickHouse/ClickHouse/issues/42460). [#42579](https://github.com/ClickHouse/ClickHouse/pull/42579) ([flynn](https://github.com/ucasfl)). -* `(U)Int128` and `(U)Int256` values were correctly checked in `PREWHERE`. [#42605](https://github.com/ClickHouse/ClickHouse/pull/42605) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix a bug in functions parser that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix the locking in `truncate table`. [#42728](https://github.com/ClickHouse/ClickHouse/pull/42728) ([flynn](https://github.com/ucasfl)). -* Fix possible crash in `web` disks when file does not exist (or `OPTIMIZE TABLE FINAL`, that also can got the same error eventually). [#42767](https://github.com/ClickHouse/ClickHouse/pull/42767) ([Azat Khuzhin](https://github.com/azat)). -* Fix `auth_type` mapping in `system.session_log`, by including `SSL_CERTIFICATE` for the enum values. [#42782](https://github.com/ClickHouse/ClickHouse/pull/42782) ([Miel Donkers](https://github.com/mdonkers)). -* Fix stack-use-after-return under ASAN build in the Create User query parser. [#42804](https://github.com/ClickHouse/ClickHouse/pull/42804) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix `lowerUTF8`/`upperUTF8` in case of symbol was in between 16-byte boundary (very frequent case of you have strings > 16 bytes long). [#42812](https://github.com/ClickHouse/ClickHouse/pull/42812) ([Azat Khuzhin](https://github.com/azat)). -* Additional bound check was added to LZ4 decompression routine to fix misbehaviour in case of malformed input. [#42868](https://github.com/ClickHouse/ClickHouse/pull/42868) ([Nikita Taranov](https://github.com/nickitat)). -* Fix rare possible hang on query cancellation. [#42874](https://github.com/ClickHouse/ClickHouse/pull/42874) ([Azat Khuzhin](https://github.com/azat)). -* Fix incorrect behavior with multiple disjuncts in hash join, close [#42832](https://github.com/ClickHouse/ClickHouse/issues/42832). [#42876](https://github.com/ClickHouse/ClickHouse/pull/42876) ([Vladimir C](https://github.com/vdimir)). -* A null pointer will be generated when select if as from ‘three table join’ , For example, this SQL query: [#42883](https://github.com/ClickHouse/ClickHouse/pull/42883) ([zzsmdfj](https://github.com/zzsmdfj)). -* Fix memory sanitizer report in Cluster Discovery, close [#42763](https://github.com/ClickHouse/ClickHouse/issues/42763). [#42905](https://github.com/ClickHouse/ClickHouse/pull/42905) ([Vladimir C](https://github.com/vdimir)). -* Improve DateTime schema inference in case of empty string. [#42911](https://github.com/ClickHouse/ClickHouse/pull/42911) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix rare NOT_FOUND_COLUMN_IN_BLOCK error when projection is possible to use but there is no projection available. This fixes [#42771](https://github.com/ClickHouse/ClickHouse/issues/42771) . The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/25563. [#42938](https://github.com/ClickHouse/ClickHouse/pull/42938) ([Amos Bird](https://github.com/amosbird)). -* Fix ATTACH TABLE in `PostgreSQL` database engine if the table contains DATETIME data type. Closes [#42817](https://github.com/ClickHouse/ClickHouse/issues/42817). [#42960](https://github.com/ClickHouse/ClickHouse/pull/42960) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix lambda parsing. Closes [#41848](https://github.com/ClickHouse/ClickHouse/issues/41848). [#42979](https://github.com/ClickHouse/ClickHouse/pull/42979) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix incorrect key analysis when nullable keys appear in the middle of a hyperrectangle. This fixes [#43111](https://github.com/ClickHouse/ClickHouse/issues/43111) . [#43133](https://github.com/ClickHouse/ClickHouse/pull/43133) ([Amos Bird](https://github.com/amosbird)). -* Fix several buffer over-reads in deserialization of carefully crafted aggregate function states. [#43159](https://github.com/ClickHouse/ClickHouse/pull/43159) ([Raúl Marín](https://github.com/Algunenano)). -* Fix function `if` in case of NULL and const Nullable arguments. Closes [#43069](https://github.com/ClickHouse/ClickHouse/issues/43069). [#43178](https://github.com/ClickHouse/ClickHouse/pull/43178) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix decimal math overflow in parsing DateTime with the 'best effort' algorithm. Closes [#43061](https://github.com/ClickHouse/ClickHouse/issues/43061). [#43180](https://github.com/ClickHouse/ClickHouse/pull/43180) ([Kruglov Pavel](https://github.com/Avogar)). -* The `indent` field produced by the `git-import` tool was miscalculated. See https://clickhouse.com/docs/en/getting-started/example-datasets/github/. [#43191](https://github.com/ClickHouse/ClickHouse/pull/43191) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fixed unexpected behaviour of `Interval` types with subquery and casting. [#43193](https://github.com/ClickHouse/ClickHouse/pull/43193) ([jh0x](https://github.com/jh0x)). - -### ClickHouse release 22.10, 2022-10-26 - -#### Backward Incompatible Change -* Rename cache commands: `show caches` -> `show filesystem caches`, `describe cache` -> `describe filesystem cache`. [#41508](https://github.com/ClickHouse/ClickHouse/pull/41508) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Remove support for the `WITH TIMEOUT` section for `LIVE VIEW`. This closes [#40557](https://github.com/ClickHouse/ClickHouse/issues/40557). [#42173](https://github.com/ClickHouse/ClickHouse/pull/42173) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Remove support for the `{database}` macro from the client's prompt. It was displayed incorrectly if the database was unspecified and it was not updated on `USE` statements. This closes [#25891](https://github.com/ClickHouse/ClickHouse/issues/25891). [#42508](https://github.com/ClickHouse/ClickHouse/pull/42508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### New Feature -* Composable protocol configuration is added. Now different protocols can be set up with different listen hosts. Protocol wrappers such as PROXYv1 can be set up over any other protocols (TCP, TCP secure, MySQL, Postgres). [#41198](https://github.com/ClickHouse/ClickHouse/pull/41198) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Add `S3` as a new type of the destination of backups. Support BACKUP to S3 with as-is path/data structure. [#42333](https://github.com/ClickHouse/ClickHouse/pull/42333) ([Vitaly Baranov](https://github.com/vitlibar)), [#42232](https://github.com/ClickHouse/ClickHouse/pull/42232) ([Azat Khuzhin](https://github.com/azat)). -* Added functions (`randUniform`, `randNormal`, `randLogNormal`, `randExponential`, `randChiSquared`, `randStudentT`, `randFisherF`, `randBernoulli`, `randBinomial`, `randNegativeBinomial`, `randPoisson`) to generate random values according to the specified distributions. This closes [#21834](https://github.com/ClickHouse/ClickHouse/issues/21834). [#42411](https://github.com/ClickHouse/ClickHouse/pull/42411) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* An improvement for ClickHouse Keeper: add support for uploading snapshots to S3. S3 information can be defined inside `keeper_server.s3_snapshot`. [#41342](https://github.com/ClickHouse/ClickHouse/pull/41342) ([Antonio Andelic](https://github.com/antonio2368)). -* Added an aggregate function `analysisOfVariance` (`anova`) to perform a statistical test over several groups of normally distributed observations to find out whether all groups have the same mean or not. Original PR [#37872](https://github.com/ClickHouse/ClickHouse/issues/37872). [#42131](https://github.com/ClickHouse/ClickHouse/pull/42131) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Support limiting of temporary data stored on disk using settings `max_temporary_data_on_disk_size_for_user`/`max_temporary_data_on_disk_size_for_query` . [#40893](https://github.com/ClickHouse/ClickHouse/pull/40893) ([Vladimir C](https://github.com/vdimir)). -* Add setting `format_json_object_each_row_column_for_object_name` to write/parse object name as column value in JSONObjectEachRow format. [#41703](https://github.com/ClickHouse/ClickHouse/pull/41703) ([Kruglov Pavel](https://github.com/Avogar)). -* Add BLAKE3 hash-function to SQL. [#33435](https://github.com/ClickHouse/ClickHouse/pull/33435) ([BoloniniD](https://github.com/BoloniniD)). -* The function `javaHash` has been extended to integers. [#41131](https://github.com/ClickHouse/ClickHouse/pull/41131) ([JackyWoo](https://github.com/JackyWoo)). -* Add OpenTelemetry support to ON CLUSTER DDL (require `distributed_ddl_entry_format_version` to be set to 4). [#41484](https://github.com/ClickHouse/ClickHouse/pull/41484) ([Frank Chen](https://github.com/FrankChen021)). -* Added system table `asynchronous_insert_log`. It contains information about asynchronous inserts (including results of queries in fire-and-forget mode (with `wait_for_async_insert=0`)) for better introspection. [#42040](https://github.com/ClickHouse/ClickHouse/pull/42040) ([Anton Popov](https://github.com/CurtizJ)). -* Add support for methods `lz4`, `bz2`, `snappy` in HTTP's `Accept-Encoding` which is a non-standard extension to HTTP protocol. [#42071](https://github.com/ClickHouse/ClickHouse/pull/42071) ([Nikolay Degterinsky](https://github.com/evillique)). -* Adds Morton Coding (ZCurve) encode/decode functions. [#41753](https://github.com/ClickHouse/ClickHouse/pull/41753) ([Constantine Peresypkin](https://github.com/pkit)). -* Add support for `SET setting_name = DEFAULT`. [#42187](https://github.com/ClickHouse/ClickHouse/pull/42187) ([Filatenkov Artur](https://github.com/FArthur-cmd)). - -#### Experimental Feature -* Added new infrastructure for query analysis and planning under the `allow_experimental_analyzer` setting. [#31796](https://github.com/ClickHouse/ClickHouse/pull/31796) ([Maksim Kita](https://github.com/kitaisreal)). -* Initial implementation of Kusto Query Language. Please don't use it. [#37961](https://github.com/ClickHouse/ClickHouse/pull/37961) ([Yong Wang](https://github.com/kashwy)). - -#### Performance Improvement -* Relax the "Too many parts" threshold. This closes [#6551](https://github.com/ClickHouse/ClickHouse/issues/6551). Now ClickHouse will allow more parts in a partition if the average part size is large enough (at least 10 GiB). This allows to have up to petabytes of data in a single partition of a single table on a single server, which is possible using disk shelves or object storage. [#42002](https://github.com/ClickHouse/ClickHouse/pull/42002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Implement operator precedence element parser to make the required stack size smaller. [#34892](https://github.com/ClickHouse/ClickHouse/pull/34892) ([Nikolay Degterinsky](https://github.com/evillique)). -* DISTINCT in order optimization leverage sorting properties of data streams. This improvement will enable reading in order for DISTINCT if applicable (before it was necessary to provide ORDER BY for columns in DISTINCT). [#41014](https://github.com/ClickHouse/ClickHouse/pull/41014) ([Igor Nikonov](https://github.com/devcrafter)). -* ColumnVector: optimize UInt8 index with AVX512VBMI. [#41247](https://github.com/ClickHouse/ClickHouse/pull/41247) ([Guo Wangyang](https://github.com/guowangy)). -* Optimize the lock contentions for `ThreadGroupStatus::mutex`. The performance experiments of **SSB** (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could bring a **2.95x** improvement of the geomean of all subcases' QPS. [#41675](https://github.com/ClickHouse/ClickHouse/pull/41675) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). -* Add `ldapr` capabilities to AArch64 builds. This is supported from Graviton 2+, Azure and GCP instances. Only appeared in clang-15 [not so long ago](https://github.com/llvm/llvm-project/commit/9609b5daffe9fd28d83d83da895abc5113f76c24). [#41778](https://github.com/ClickHouse/ClickHouse/pull/41778) ([Daniel Kutenin](https://github.com/danlark1)). -* Improve performance when comparing strings and one argument is an empty constant string. [#41870](https://github.com/ClickHouse/ClickHouse/pull/41870) ([Jiebin Sun](https://github.com/jiebinn)). -* Optimize `insertFrom` of ColumnAggregateFunction to share Aggregate State in some cases. [#41960](https://github.com/ClickHouse/ClickHouse/pull/41960) ([flynn](https://github.com/ucasfl)). -* Make writing to `azure_blob_storage` disks faster (respect `max_single_part_upload_size` instead of writing a block per each buffer size). Inefficiency mentioned in [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42041](https://github.com/ClickHouse/ClickHouse/pull/42041) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Make thread ids in the process list and query_log unique to avoid waste. [#42180](https://github.com/ClickHouse/ClickHouse/pull/42180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Support skipping cache completely (both download to cache and reading cached data) in case the requested read range exceeds the threshold defined by cache setting `bypass_cache_threashold`, requires to be enabled with `enable_bypass_cache_with_threshold`). [#42418](https://github.com/ClickHouse/ClickHouse/pull/42418) ([Han Shukai](https://github.com/KinderRiven)). This helps on slow local disks. - -#### Improvement -* Add setting `allow_implicit_no_password`: in combination with `allow_no_password` it forbids creating a user with no password unless `IDENTIFIED WITH no_password` is explicitly specified. [#41341](https://github.com/ClickHouse/ClickHouse/pull/41341) ([Nikolay Degterinsky](https://github.com/evillique)). -* Embedded Keeper will always start in the background allowing ClickHouse to start without achieving quorum. [#40991](https://github.com/ClickHouse/ClickHouse/pull/40991) ([Antonio Andelic](https://github.com/antonio2368)). -* Made reestablishing a new connection to ZooKeeper more reactive in case of expiration of the previous one. Previously there was a task which spawns every minute by default and thus a table could be in readonly state for about this time. [#41092](https://github.com/ClickHouse/ClickHouse/pull/41092) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Now projections can be used with zero copy replication (zero-copy replication is a non-production feature). [#41147](https://github.com/ClickHouse/ClickHouse/pull/41147) ([alesapin](https://github.com/alesapin)). -* Support expression `(EXPLAIN SELECT ...)` in a subquery. Queries like `SELECT * FROM (EXPLAIN PIPELINE SELECT col FROM TABLE ORDER BY col)` became valid. [#40630](https://github.com/ClickHouse/ClickHouse/pull/40630) ([Vladimir C](https://github.com/vdimir)). -* Allow changing `async_insert_max_data_size` or `async_insert_busy_timeout_ms` in scope of query. E.g. user wants to insert data rarely and she doesn't have access to the server config to tune default settings. [#40668](https://github.com/ClickHouse/ClickHouse/pull/40668) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Improvements for reading from remote filesystems, made threadpool size for reads/writes configurable. Closes [#41070](https://github.com/ClickHouse/ClickHouse/issues/41070). [#41011](https://github.com/ClickHouse/ClickHouse/pull/41011) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support all combinators combination in WindowTransform/arratReduce*/initializeAggregation/aggregate functions versioning. Previously combinators like `ForEach/Resample/Map` didn't work in these places, using them led to exception like`State function ... inserts results into non-state column`. [#41107](https://github.com/ClickHouse/ClickHouse/pull/41107) ([Kruglov Pavel](https://github.com/Avogar)). -* Add function `tryDecrypt` that returns NULL when decrypt fails (e.g. decrypt with incorrect key) instead of throwing an exception. [#41206](https://github.com/ClickHouse/ClickHouse/pull/41206) ([Duc Canh Le](https://github.com/canhld94)). -* Add the `unreserved_space` column to the `system.disks` table to check how much space is not taken by reservations per disk. [#41254](https://github.com/ClickHouse/ClickHouse/pull/41254) ([filimonov](https://github.com/filimonov)). -* Support s3 authorization headers in table function arguments. [#41261](https://github.com/ClickHouse/ClickHouse/pull/41261) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add support for MultiRead in Keeper and internal ZooKeeper client (this is an extension to ZooKeeper protocol, only available in ClickHouse Keeper). [#41410](https://github.com/ClickHouse/ClickHouse/pull/41410) ([Antonio Andelic](https://github.com/antonio2368)). -* Add support for decimal type comparing with floating point literal in IN operator. [#41544](https://github.com/ClickHouse/ClickHouse/pull/41544) ([liang.huang](https://github.com/lhuang09287750)). -* Allow readable size values (like `1TB`) in cache config. [#41688](https://github.com/ClickHouse/ClickHouse/pull/41688) ([Kseniia Sumarokova](https://github.com/kssenii)). -* ClickHouse could cache stale DNS entries for some period of time (15 seconds by default) until the cache won't be updated asynchronously. During these periods ClickHouse can nevertheless try to establish a connection and produce errors. This behavior is fixed. [#41707](https://github.com/ClickHouse/ClickHouse/pull/41707) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Add interactive history search with fzf-like utility (fzf/sk) for `clickhouse-client`/`clickhouse-local` (note you can use `FZF_DEFAULT_OPTS`/`SKIM_DEFAULT_OPTIONS` to additionally configure the behavior). [#41730](https://github.com/ClickHouse/ClickHouse/pull/41730) ([Azat Khuzhin](https://github.com/azat)). -* Only allow clients connecting to a secure server with an invalid certificate only to proceed with the '--accept-certificate' flag. [#41743](https://github.com/ClickHouse/ClickHouse/pull/41743) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Add function `tryBase58Decode`, similar to the existing function `tryBase64Decode`. [#41824](https://github.com/ClickHouse/ClickHouse/pull/41824) ([Robert Schulze](https://github.com/rschu1ze)). -* Improve feedback when replacing partition with different primary key. Fixes [#34798](https://github.com/ClickHouse/ClickHouse/issues/34798). [#41838](https://github.com/ClickHouse/ClickHouse/pull/41838) ([Salvatore](https://github.com/tbsal)). -* Fix parallel parsing: segmentator now checks `max_block_size`. This fixed memory overallocation in case of parallel parsing and small LIMIT. [#41852](https://github.com/ClickHouse/ClickHouse/pull/41852) ([Vitaly Baranov](https://github.com/vitlibar)). -* Don't add "TABLE_IS_DROPPED" exception to `system.errors` if it's happened during SELECT from a system table and was ignored. [#41908](https://github.com/ClickHouse/ClickHouse/pull/41908) ([AlfVII](https://github.com/AlfVII)). -* Improve option `enable_extended_results_for_datetime_functions` to return results of type DateTime64 for functions `toStartOfDay`, `toStartOfHour`, `toStartOfFifteenMinutes`, `toStartOfTenMinutes`, `toStartOfFiveMinutes`, `toStartOfMinute` and `timeSlot`. [#41910](https://github.com/ClickHouse/ClickHouse/pull/41910) ([Roman Vasin](https://github.com/rvasin)). -* Improve `DateTime` type inference for text formats. Now it respects setting `date_time_input_format` and doesn't try to infer datetimes from numbers as timestamps. Closes [#41389](https://github.com/ClickHouse/ClickHouse/issues/41389) Closes [#42206](https://github.com/ClickHouse/ClickHouse/issues/42206). [#41912](https://github.com/ClickHouse/ClickHouse/pull/41912) ([Kruglov Pavel](https://github.com/Avogar)). -* Remove confusing warning when inserting with `perform_ttl_move_on_insert` = false. [#41980](https://github.com/ClickHouse/ClickHouse/pull/41980) ([Vitaly Baranov](https://github.com/vitlibar)). -* Allow user to write `countState(*)` similar to `count(*)`. This closes [#9338](https://github.com/ClickHouse/ClickHouse/issues/9338). [#41983](https://github.com/ClickHouse/ClickHouse/pull/41983) ([Amos Bird](https://github.com/amosbird)). -* Fix `rankCorr` size overflow. [#42020](https://github.com/ClickHouse/ClickHouse/pull/42020) ([Duc Canh Le](https://github.com/canhld94)). -* Added an option to specify an arbitrary string as an environment name in the Sentry's config for more handy reports. [#42037](https://github.com/ClickHouse/ClickHouse/pull/42037) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Fix parsing out-of-range Date from CSV. [#42044](https://github.com/ClickHouse/ClickHouse/pull/42044) ([Andrey Zvonov](https://github.com/zvonand)). -* `parseDataTimeBestEffort` now supports comma between date and time. Closes [#42038](https://github.com/ClickHouse/ClickHouse/issues/42038). [#42049](https://github.com/ClickHouse/ClickHouse/pull/42049) ([flynn](https://github.com/ucasfl)). -* Improved stale replica recovery process for `ReplicatedMergeTree`. If a lost replica has some parts which are absent from a healthy replica, but these parts should appear in the future according to the replication queue of the healthy replica, then the lost replica will keep such parts instead of detaching them. [#42134](https://github.com/ClickHouse/ClickHouse/pull/42134) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Add a possibility to use `Date32` arguments for date_diff function. Fix issue in date_diff function when using DateTime64 arguments with a start date before Unix epoch and end date after Unix epoch. [#42308](https://github.com/ClickHouse/ClickHouse/pull/42308) ([Roman Vasin](https://github.com/rvasin)). -* When uploading big parts to Minio, 'Complete Multipart Upload' can take a long time. Minio sends heartbeats every 10 seconds (see https://github.com/minio/minio/pull/7198). But clickhouse times out earlier, because the default send/receive timeout is [set](https://github.com/ClickHouse/ClickHouse/blob/cc24fcd6d5dfb67f5f66f5483e986bd1010ad9cf/src/IO/S3/PocoHTTPClient.cpp#L123) to 5 seconds. [#42321](https://github.com/ClickHouse/ClickHouse/pull/42321) ([filimonov](https://github.com/filimonov)). -* Fix rarely invalid cast of aggregate state types with complex types such as Decimal. This fixes [#42408](https://github.com/ClickHouse/ClickHouse/issues/42408). [#42417](https://github.com/ClickHouse/ClickHouse/pull/42417) ([Amos Bird](https://github.com/amosbird)). -* Allow to use `Date32` arguments for `dateName` function. [#42554](https://github.com/ClickHouse/ClickHouse/pull/42554) ([Roman Vasin](https://github.com/rvasin)). -* Now filters with NULL literals will be used during index analysis. [#34063](https://github.com/ClickHouse/ClickHouse/issues/34063). [#41842](https://github.com/ClickHouse/ClickHouse/pull/41842) ([Amos Bird](https://github.com/amosbird)). -* Merge parts if every part in the range is older than a certain threshold. The threshold can be set by using `min_age_to_force_merge_seconds`. This closes [#35836](https://github.com/ClickHouse/ClickHouse/issues/35836). [#42423](https://github.com/ClickHouse/ClickHouse/pull/42423) ([Antonio Andelic](https://github.com/antonio2368)). This is continuation of [#39550i](https://github.com/ClickHouse/ClickHouse/pull/39550) by [@fastio](https://github.com/fastio) who implemented most of the logic. -* Improve the time to recover lost keeper connections. [#42541](https://github.com/ClickHouse/ClickHouse/pull/42541) ([Raúl Marín](https://github.com/Algunenano)). - -#### Build/Testing/Packaging Improvement -* Add fuzzer for table definitions [#40096](https://github.com/ClickHouse/ClickHouse/pull/40096) ([Anton Popov](https://github.com/CurtizJ)). This represents the biggest advancement for ClickHouse testing in this year so far. -* Beta version of the ClickHouse Cloud service is released: [https://clickhouse.cloud/](https://clickhouse.cloud/). It provides the easiest way to use ClickHouse (even slightly easier than the single-command installation). -* Added support of WHERE clause generation to AST Fuzzer and possibility to add or remove ORDER BY and WHERE clause. [#38519](https://github.com/ClickHouse/ClickHouse/pull/38519) ([Ilya Yatsishin](https://github.com/qoega)). -* Aarch64 binaries now require at least ARMv8.2, released in 2016. Most notably, this enables use of ARM LSE, i.e. native atomic operations. Also, CMake build option "NO_ARMV81_OR_HIGHER" has been added to allow compilation of binaries for older ARMv8.0 hardware, e.g. Raspberry Pi 4. [#41610](https://github.com/ClickHouse/ClickHouse/pull/41610) ([Robert Schulze](https://github.com/rschu1ze)). -* Allow building ClickHouse with Musl (small changes after it was already supported but broken). [#41987](https://github.com/ClickHouse/ClickHouse/pull/41987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add the `$CLICKHOUSE_CRONFILE` file checking to avoid running the `sed` command to get the file not found error on install. [#42081](https://github.com/ClickHouse/ClickHouse/pull/42081) ([Chun-Sheng, Li](https://github.com/peter279k)). -* Update cctz to `2022e` to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)). [#42273](https://github.com/ClickHouse/ClickHouse/pull/42273) ([Dom Del Nano](https://github.com/ddelnano)). -* Add Rust code support into ClickHouse with BLAKE3 hash-function library as an example. [#33435](https://github.com/ClickHouse/ClickHouse/pull/33435) ([BoloniniD](https://github.com/BoloniniD)). - -#### Bug Fix (user-visible misbehavior in official stable or prestable release) - -* Choose correct aggregation method for `LowCardinality` with big integer types. [#42342](https://github.com/ClickHouse/ClickHouse/pull/42342) ([Duc Canh Le](https://github.com/canhld94)). -* Several fixes for `web` disk. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixes an issue that causes docker run to fail if `https_port` is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Mutations were not cancelled properly on server shutdown or `SYSTEM STOP MERGES` query and cancellation might take long time, it's fixed. [#41699](https://github.com/ClickHouse/ClickHouse/pull/41699) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix wrong result of queries with `ORDER BY` or `GROUP BY` by columns from prefix of sorting key, wrapped into monotonic functions, with enable "read in order" optimization (settings `optimize_read_in_order` and `optimize_aggregation_in_order`). [#41701](https://github.com/ClickHouse/ClickHouse/pull/41701) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible crash in `SELECT` from `Merge` table with enabled `optimize_monotonous_functions_in_order_by` setting. Fixes [#41269](https://github.com/ClickHouse/ClickHouse/issues/41269). [#41740](https://github.com/ClickHouse/ClickHouse/pull/41740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fixed "Part ... intersects part ..." error that might happen in extremely rare cases if replica was restarted just after detaching some part as broken. [#41741](https://github.com/ClickHouse/ClickHouse/pull/41741) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Don't allow to create or alter merge tree tables with column name `_row_exists`, which is reserved for lightweight delete. Fixed [#41716](https://github.com/ClickHouse/ClickHouse/issues/41716). [#41763](https://github.com/ClickHouse/ClickHouse/pull/41763) ([Jianmei Zhang](https://github.com/zhangjmruc)). -* Fix a bug that CORS headers are missing in some HTTP responses. [#41792](https://github.com/ClickHouse/ClickHouse/pull/41792) ([Frank Chen](https://github.com/FrankChen021)). -* 22.9 might fail to startup `ReplicatedMergeTree` table if that table was created by 20.3 or older version and was never altered, it's fixed. Fixes [#41742](https://github.com/ClickHouse/ClickHouse/issues/41742). [#41796](https://github.com/ClickHouse/ClickHouse/pull/41796) ([Alexander Tokmakov](https://github.com/tavplubix)). -* When the batch sending fails for some reason, it cannot be automatically recovered, and if it is not processed in time, it will lead to accumulation, and the printed error message will become longer and longer, which will cause the http thread to block. [#41813](https://github.com/ClickHouse/ClickHouse/pull/41813) ([zhongyuankai](https://github.com/zhongyuankai)). -* Fix compact parts with compressed marks setting. Fixes [#41783](https://github.com/ClickHouse/ClickHouse/issues/41783) and [#41746](https://github.com/ClickHouse/ClickHouse/issues/41746). [#41823](https://github.com/ClickHouse/ClickHouse/pull/41823) ([alesapin](https://github.com/alesapin)). -* Old versions of Replicated database don't have a special marker in [Zoo]Keeper. We need to check only whether the node contains come obscure data instead of special mark. [#41875](https://github.com/ClickHouse/ClickHouse/pull/41875) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Fix possible exception in fs cache. [#41884](https://github.com/ClickHouse/ClickHouse/pull/41884) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix `use_environment_credentials` for s3 table function. [#41970](https://github.com/ClickHouse/ClickHouse/pull/41970) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixed "Directory already exists and is not empty" error on detaching broken part that might prevent `ReplicatedMergeTree` table from starting replication. Fixes [#40957](https://github.com/ClickHouse/ClickHouse/issues/40957). [#41981](https://github.com/ClickHouse/ClickHouse/pull/41981) ([Alexander Tokmakov](https://github.com/tavplubix)). -* `toDateTime64` now returns the same output with negative integer and float arguments. [#42025](https://github.com/ClickHouse/ClickHouse/pull/42025) ([Robert Schulze](https://github.com/rschu1ze)). -* Fix write into `azure_blob_storage`. Partially closes [#41754](https://github.com/ClickHouse/ClickHouse/issues/41754). [#42034](https://github.com/ClickHouse/ClickHouse/pull/42034) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix the `bzip2` decoding issue for specific `bzip2` files. [#42046](https://github.com/ClickHouse/ClickHouse/pull/42046) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix SQL function `toLastDayOfMonth` with setting "enable_extended_results_for_datetime_functions = 1" at the beginning of the extended range (January 1900). - Fix SQL function "toRelativeWeekNum()" with setting "enable_extended_results_for_datetime_functions = 1" at the end of extended range (December 2299). - Improve the performance of for SQL functions "toISOYear()", "toFirstDayNumOfISOYearIndex()" and "toYearWeekOfNewyearMode()" by avoiding unnecessary index arithmetics. [#42084](https://github.com/ClickHouse/ClickHouse/pull/42084) ([Roman Vasin](https://github.com/rvasin)). -* The maximum size of fetches for each table accidentally was set to 8 while the pool size could be bigger. Now the maximum size of fetches for table is equal to the pool size. [#42090](https://github.com/ClickHouse/ClickHouse/pull/42090) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* A table might be shut down and a dictionary might be detached before checking if can be dropped without breaking dependencies between table, it's fixed. Fixes [#41982](https://github.com/ClickHouse/ClickHouse/issues/41982). [#42106](https://github.com/ClickHouse/ClickHouse/pull/42106) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix bad inefficiency of `remote_filesystem_read_method=read` with filesystem cache. Closes [#42125](https://github.com/ClickHouse/ClickHouse/issues/42125). [#42129](https://github.com/ClickHouse/ClickHouse/pull/42129) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possible timeout exception for distributed queries with use_hedged_requests = 0. [#42130](https://github.com/ClickHouse/ClickHouse/pull/42130) ([Azat Khuzhin](https://github.com/azat)). -* Fixed a minor bug inside function `runningDifference` in case of using it with `Date32` type. Previously `Date` was used and it may cause some logical errors like `Bad cast from type DB::ColumnVector to DB::ColumnVector'`. [#42143](https://github.com/ClickHouse/ClickHouse/pull/42143) ([Alfred Xu](https://github.com/sperlingxx)). -* Fix reusing of files > 4GB from base backup. [#42146](https://github.com/ClickHouse/ClickHouse/pull/42146) ([Azat Khuzhin](https://github.com/azat)). -* DISTINCT in order fails with LOGICAL_ERROR if first column in sorting key contains function. [#42186](https://github.com/ClickHouse/ClickHouse/pull/42186) ([Igor Nikonov](https://github.com/devcrafter)). -* Fix a bug with projections and the `aggregate_functions_null_for_empty` setting. This bug is very rare and appears only if you enable the `aggregate_functions_null_for_empty` setting in the server's config. This closes [#41647](https://github.com/ClickHouse/ClickHouse/issues/41647). [#42198](https://github.com/ClickHouse/ClickHouse/pull/42198) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix read from `Buffer` tables with read in order desc. [#42236](https://github.com/ClickHouse/ClickHouse/pull/42236) ([Duc Canh Le](https://github.com/canhld94)). -* Fix a bug which prevents ClickHouse to start when `background_pool_size setting` is set on default profile but `background_merges_mutations_concurrency_ratio` is not. [#42315](https://github.com/ClickHouse/ClickHouse/pull/42315) ([nvartolomei](https://github.com/nvartolomei)). -* `ALTER UPDATE` of attached part (with columns different from table schema) could create an invalid `columns.txt` metadata on disk. Reading from such part could fail with errors or return invalid data. Fixes [#42161](https://github.com/ClickHouse/ClickHouse/issues/42161). [#42319](https://github.com/ClickHouse/ClickHouse/pull/42319) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Setting `additional_table_filters` were not applied to `Distributed` storage. Fixes [#41692](https://github.com/ClickHouse/ClickHouse/issues/41692). [#42322](https://github.com/ClickHouse/ClickHouse/pull/42322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix a data race in query finish/cancel. This closes [#42346](https://github.com/ClickHouse/ClickHouse/issues/42346). [#42362](https://github.com/ClickHouse/ClickHouse/pull/42362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* This reverts [#40217](https://github.com/ClickHouse/ClickHouse/issues/40217) which introduced a regression in date/time functions. [#42367](https://github.com/ClickHouse/ClickHouse/pull/42367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix assert cast in join on falsy condition, Close [#42380](https://github.com/ClickHouse/ClickHouse/issues/42380). [#42407](https://github.com/ClickHouse/ClickHouse/pull/42407) ([Vladimir C](https://github.com/vdimir)). -* Fix buffer overflow in the processing of Decimal data types. This closes [#42451](https://github.com/ClickHouse/ClickHouse/issues/42451). [#42465](https://github.com/ClickHouse/ClickHouse/pull/42465) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* `AggregateFunctionQuantile` now correctly works with UInt128 columns. Previously, the quantile state interpreted `UInt128` columns as `Int128` which could have led to incorrect results. [#42473](https://github.com/ClickHouse/ClickHouse/pull/42473) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix bad_cast assert during INSERT into `Annoy` indexes over non-Float32 columns. `Annoy` indices is an experimental feature. [#42485](https://github.com/ClickHouse/ClickHouse/pull/42485) ([Robert Schulze](https://github.com/rschu1ze)). -* Arithmetic operator with Date or DateTime and 128 or 256-bit integer was referencing uninitialized memory. [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)). - - -### ClickHouse release 22.9, 2022-09-22 - -#### Backward Incompatible Change - -* Upgrade from 20.3 and older to 22.9 and newer should be done through an intermediate version if there are any `ReplicatedMergeTree` tables, otherwise server with the new version will not start. [#40641](https://github.com/ClickHouse/ClickHouse/pull/40641) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Remove the functions `accurate_Cast` and `accurate_CastOrNull` (they are different to `accurateCast` and `accurateCastOrNull` by underscore in the name and they are not affected by the value of `cast_keep_nullable` setting). These functions were undocumented, untested, unused, and unneeded. They appeared to be alive due to code generalization. [#40682](https://github.com/ClickHouse/ClickHouse/pull/40682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test to ensure that every new table function will be documented. See [#40649](https://github.com/ClickHouse/ClickHouse/issues/40649). Rename table function `MeiliSearch` to `meilisearch`. [#40709](https://github.com/ClickHouse/ClickHouse/pull/40709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test to ensure that every new function will be documented. See [#40649](https://github.com/ClickHouse/ClickHouse/pull/40649). The functions `lemmatize`, `synonyms`, `stem` were case-insensitive by mistake. Now they are case-sensitive. [#40711](https://github.com/ClickHouse/ClickHouse/pull/40711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Make interpretation of YAML configs to be more conventional. [#41044](https://github.com/ClickHouse/ClickHouse/pull/41044) ([Vitaly Baranov](https://github.com/vitlibar)). - -#### New Feature - -* Support `insert_quorum = 'auto'` to use majority number. [#39970](https://github.com/ClickHouse/ClickHouse/pull/39970) ([Sachin](https://github.com/SachinSetiya)). -* Add embedded dashboards to ClickHouse server. This is a demo project about how to achieve 90% results with 1% effort using ClickHouse features. [#40461](https://github.com/ClickHouse/ClickHouse/pull/40461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Added new settings constraint writability kind `changeable_in_readonly`. [#40631](https://github.com/ClickHouse/ClickHouse/pull/40631) ([Sergei Trifonov](https://github.com/serxa)). -* Add support for `INTERSECT DISTINCT` and `EXCEPT DISTINCT`. [#40792](https://github.com/ClickHouse/ClickHouse/pull/40792) ([Duc Canh Le](https://github.com/canhld94)). -* Add new input/output format `JSONObjectEachRow` - Support import for formats `JSON/JSONCompact/JSONColumnsWithMetadata`. Add new setting `input_format_json_validate_types_from_metadata` that controls whether we should check if data types from metadata match data types from the header. - Add new setting `input_format_json_validate_utf8`, when it's enabled, all `JSON` formats will validate UTF-8 sequences. It will be disabled by default. Note that this setting doesn't influence output formats `JSON/JSONCompact/JSONColumnsWithMetadata`, they always validate utf8 sequences (this exception was made because of compatibility reasons). - Add new setting `input_format_json_read_numbers_as_strings ` that allows to parse numbers in String column, the setting is disabled by default. - Add new setting `output_format_json_quote_decimals` that allows to output decimals in double quotes, disabled by default. - Allow to parse decimals in double quotes during data import. [#40910](https://github.com/ClickHouse/ClickHouse/pull/40910) ([Kruglov Pavel](https://github.com/Avogar)). -* Query parameters supported in DESCRIBE TABLE query. [#40952](https://github.com/ClickHouse/ClickHouse/pull/40952) ([Nikita Taranov](https://github.com/nickitat)). -* Add support to Parquet Time32/64 by converting it into DateTime64. Parquet time32/64 represents time elapsed since midnight, while DateTime32/64 represents an actual unix timestamp. Conversion simply offsets from `0`. [#41333](https://github.com/ClickHouse/ClickHouse/pull/41333) ([Arthur Passos](https://github.com/arthurpassos)). -* Implement set operations on Apache Datasketches. [#39919](https://github.com/ClickHouse/ClickHouse/pull/39919) ([Fangyuan Deng](https://github.com/pzhdfy)). Note: there is no point of using Apache Datasketches, they are inferiour than ClickHouse and only make sense for integration with other systems. -* Allow recording errors to specified file while reading text formats (`CSV`, `TSV`). [#40516](https://github.com/ClickHouse/ClickHouse/pull/40516) ([zjial](https://github.com/zjial)). - -#### Experimental Feature - -* Add ANN (approximate nearest neighbor) index based on `Annoy`. [#40818](https://github.com/ClickHouse/ClickHouse/pull/40818) ([Filatenkov Artur](https://github.com/FArthur-cmd)). [#37215](https://github.com/ClickHouse/ClickHouse/pull/37215) ([VVMak](https://github.com/VVMak)). -* Add new storage engine `KeeperMap`, that uses ClickHouse Keeper or ZooKeeper as a key-value store. [#39976](https://github.com/ClickHouse/ClickHouse/pull/39976) ([Antonio Andelic](https://github.com/antonio2368)). This storage engine is intended to store a small amount of metadata. -* Improvement for in-memory data parts: remove completely processed WAL files. [#40592](https://github.com/ClickHouse/ClickHouse/pull/40592) ([Azat Khuzhin](https://github.com/azat)). - -#### Performance Improvement - -* Implement compression of marks and primary key. Close [#34437](https://github.com/ClickHouse/ClickHouse/issues/34437). [#37693](https://github.com/ClickHouse/ClickHouse/pull/37693) ([zhongyuankai](https://github.com/zhongyuankai)). -* Allow to load marks with threadpool in advance. Regulated by setting `load_marks_asynchronously` (default: 0). [#40821](https://github.com/ClickHouse/ClickHouse/pull/40821) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Virtual filesystem over s3 will use random object names split into multiple path prefixes for better performance on AWS. [#40968](https://github.com/ClickHouse/ClickHouse/pull/40968) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Account `max_block_size` value while producing single-level aggregation results. Allows to execute following query plan steps using more threads. [#39138](https://github.com/ClickHouse/ClickHouse/pull/39138) ([Nikita Taranov](https://github.com/nickitat)). -* Software prefetching is used in aggregation to speed up operations with hash tables. Controlled by the setting `enable_software_prefetch_in_aggregation`, enabled by default. [#39304](https://github.com/ClickHouse/ClickHouse/pull/39304) ([Nikita Taranov](https://github.com/nickitat)). -* Better support of `optimize_read_in_order` in case when some of sorting key columns are always constant after applying `WHERE` clause. E.g. query like `SELECT ... FROM table WHERE a = 'x' ORDER BY a, b`, where `table` has storage definition: `MergeTree ORDER BY (a, b)`. [#38715](https://github.com/ClickHouse/ClickHouse/pull/38715) ([Anton Popov](https://github.com/CurtizJ)). -* Filter joined streams for `full_sorting_join` by each other before sorting. [#39418](https://github.com/ClickHouse/ClickHouse/pull/39418) ([Vladimir C](https://github.com/vdimir)). -* LZ4 decompression optimised by skipping empty literals processing. [#40142](https://github.com/ClickHouse/ClickHouse/pull/40142) ([Nikita Taranov](https://github.com/nickitat)). -* Speedup backup process using native `copy` when possible instead of copying through `clickhouse-server` memory. [#40395](https://github.com/ClickHouse/ClickHouse/pull/40395) ([alesapin](https://github.com/alesapin)). -* Do not obtain storage snapshot for each INSERT block (slightly improves performance). [#40638](https://github.com/ClickHouse/ClickHouse/pull/40638) ([Azat Khuzhin](https://github.com/azat)). -* Implement batch processing for aggregate functions with multiple nullable arguments. [#41058](https://github.com/ClickHouse/ClickHouse/pull/41058) ([Raúl Marín](https://github.com/Algunenano)). -* Speed up reading UniquesHashSet (`uniqState` from disk for example). [#41089](https://github.com/ClickHouse/ClickHouse/pull/41089) ([Raúl Marín](https://github.com/Algunenano)). -* Fixed high memory usage while executing mutations of compact parts in tables with huge number of columns. [#41122](https://github.com/ClickHouse/ClickHouse/pull/41122) ([lthaooo](https://github.com/lthaooo)). -* Enable the vectorscan library on ARM, this speeds up regexp evaluation. [#41033](https://github.com/ClickHouse/ClickHouse/pull/41033) ([Robert Schulze](https://github.com/rschu1ze)). -* Upgrade vectorscan to 5.4.8 which has many performance optimizations to speed up regexp evaluation. [#41270](https://github.com/ClickHouse/ClickHouse/pull/41270) ([Robert Schulze](https://github.com/rschu1ze)). -* Fix incorrect fallback to skip the local filesystem cache for VFS (like S3) which happened on very high concurrency level. [#40420](https://github.com/ClickHouse/ClickHouse/pull/40420) ([Kseniia Sumarokova](https://github.com/kssenii)). -* If row policy filter is always false, return empty result immediately without reading any data. This closes [#24012](https://github.com/ClickHouse/ClickHouse/issues/24012). [#40740](https://github.com/ClickHouse/ClickHouse/pull/40740) ([Amos Bird](https://github.com/amosbird)). -* Parallel hash JOIN for Float data types might be suboptimal. Make it better. [#41183](https://github.com/ClickHouse/ClickHouse/pull/41183) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### Improvement - -* During startup and ATTACH call, `ReplicatedMergeTree` tables will be readonly until the ZooKeeper connection is made and the setup is finished. [#40148](https://github.com/ClickHouse/ClickHouse/pull/40148) ([Antonio Andelic](https://github.com/antonio2368)). -* Add `enable_extended_results_for_datetime_functions` option to return results of type Date32 for functions toStartOfYear, toStartOfISOYear, toStartOfQuarter, toStartOfMonth, toStartOfWeek, toMonday and toLastDayOfMonth when argument is Date32 or DateTime64, otherwise results of Date type are returned. For compatibility reasons default value is ‘0’. [#41214](https://github.com/ClickHouse/ClickHouse/pull/41214) ([Roman Vasin](https://github.com/rvasin)). -* For security and stability reasons, CatBoost models are no longer evaluated within the ClickHouse server. Instead, the evaluation is now done in the clickhouse-library-bridge, a separate process that loads the catboost library and communicates with the server process via HTTP. [#40897](https://github.com/ClickHouse/ClickHouse/pull/40897) ([Robert Schulze](https://github.com/rschu1ze)). [#39629](https://github.com/ClickHouse/ClickHouse/pull/39629) ([Robert Schulze](https://github.com/rschu1ze)). -* Add more metrics for on-disk temporary data, close [#40206](https://github.com/ClickHouse/ClickHouse/issues/40206). [#40239](https://github.com/ClickHouse/ClickHouse/pull/40239) ([Vladimir C](https://github.com/vdimir)). -* Add config option `warning_supress_regexp`, close [#40330](https://github.com/ClickHouse/ClickHouse/issues/40330). [#40548](https://github.com/ClickHouse/ClickHouse/pull/40548) ([Vladimir C](https://github.com/vdimir)). -* Add setting to disable limit on kafka_num_consumers. Closes [#40331](https://github.com/ClickHouse/ClickHouse/issues/40331). [#40670](https://github.com/ClickHouse/ClickHouse/pull/40670) ([Kruglov Pavel](https://github.com/Avogar)). -* Support `SETTINGS` in `DELETE ...` query. [#41533](https://github.com/ClickHouse/ClickHouse/pull/41533) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Detailed S3 profile events `DiskS3*` per S3 API call split for S3 ObjectStorage. [#41532](https://github.com/ClickHouse/ClickHouse/pull/41532) ([Sergei Trifonov](https://github.com/serxa)). -* Two new metrics in `system.asynchronous_metrics`. `NumberOfDetachedParts` and `NumberOfDetachedByUserParts`. [#40779](https://github.com/ClickHouse/ClickHouse/pull/40779) ([Sema Checherinda](https://github.com/CheSema)). -* Allow CONSTRAINTs for ODBC and JDBC tables. [#34551](https://github.com/ClickHouse/ClickHouse/pull/34551) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Don't print `SETTINGS` more than once during query formatting if it didn't appear multiple times in the original query. [#38900](https://github.com/ClickHouse/ClickHouse/pull/38900) ([Raúl Marín](https://github.com/Algunenano)). -* Improve the tracing (OpenTelemetry) context propagation across threads. [#39010](https://github.com/ClickHouse/ClickHouse/pull/39010) ([Frank Chen](https://github.com/FrankChen021)). -* ClickHouse Keeper: add listeners for `interserver_listen_host` only in Keeper if specified. [#39973](https://github.com/ClickHouse/ClickHouse/pull/39973) ([Antonio Andelic](https://github.com/antonio2368)). -* Improve recovery of Replicated user access storage after errors. [#39977](https://github.com/ClickHouse/ClickHouse/pull/39977) ([Vitaly Baranov](https://github.com/vitlibar)). -* Add support for TTL in `EmbeddedRocksDB`. [#39986](https://github.com/ClickHouse/ClickHouse/pull/39986) ([Lloyd-Pottiger](https://github.com/Lloyd-Pottiger)). -* Add schema inference to `clickhouse-obfuscator`, so the `--structure` argument is no longer required. [#40120](https://github.com/ClickHouse/ClickHouse/pull/40120) ([Nikolay Degterinsky](https://github.com/evillique)). -* Improve and fix dictionaries in `Arrow` format. [#40173](https://github.com/ClickHouse/ClickHouse/pull/40173) ([Kruglov Pavel](https://github.com/Avogar)). -* More natural conversion of `Date32`, `DateTime64`, `Date` to narrower types: upper or lower normal value is considered when out of normal range. [#40217](https://github.com/ClickHouse/ClickHouse/pull/40217) ([Andrey Zvonov](https://github.com/zvonand)). -* Fix the case when `Merge` table over `View` cannot use index. [#40233](https://github.com/ClickHouse/ClickHouse/pull/40233) ([Duc Canh Le](https://github.com/canhld94)). -* Custom key names for JSON server logs. [#40251](https://github.com/ClickHouse/ClickHouse/pull/40251) ([Mallik Hassan](https://github.com/SadiHassan)). -* It is now possible to set a custom error code for the exception thrown by function `throwIf`. [#40319](https://github.com/ClickHouse/ClickHouse/pull/40319) ([Robert Schulze](https://github.com/rschu1ze)). -* Improve schema inference cache, respect format settings that can change the schema. [#40414](https://github.com/ClickHouse/ClickHouse/pull/40414) ([Kruglov Pavel](https://github.com/Avogar)). -* Allow parsing `Date` as `DateTime` and `DateTime64`. This implements the enhancement proposed in [#36949](https://github.com/ClickHouse/ClickHouse/issues/36949). [#40474](https://github.com/ClickHouse/ClickHouse/pull/40474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Allow conversion from `String` with `DateTime64` like `2022-08-22 01:02:03.456` to `Date` and `Date32`. Allow conversion from String with DateTime like `2022-08-22 01:02:03` to `Date32`. This closes [#39598](https://github.com/ClickHouse/ClickHouse/issues/39598). [#40475](https://github.com/ClickHouse/ClickHouse/pull/40475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Better support for nested data structures in Parquet format [#40485](https://github.com/ClickHouse/ClickHouse/pull/40485) ([Arthur Passos](https://github.com/arthurpassos)). -* Support reading Array(Record) into flatten nested table in Avro. [#40534](https://github.com/ClickHouse/ClickHouse/pull/40534) ([Kruglov Pavel](https://github.com/Avogar)). -* Add read-only support for `EmbeddedRocksDB`. [#40543](https://github.com/ClickHouse/ClickHouse/pull/40543) ([Lloyd-Pottiger](https://github.com/Lloyd-Pottiger)). -* Validate the compression method parameter of URL table engine. [#40600](https://github.com/ClickHouse/ClickHouse/pull/40600) ([Frank Chen](https://github.com/FrankChen021)). -* Better format detection for url table function/engine in presence of a query string after a file name. Closes [#40315](https://github.com/ClickHouse/ClickHouse/issues/40315). [#40636](https://github.com/ClickHouse/ClickHouse/pull/40636) ([Kruglov Pavel](https://github.com/Avogar)). -* Disable projection when grouping set is used. It generated wrong result. This fixes [#40635](https://github.com/ClickHouse/ClickHouse/issues/40635). [#40726](https://github.com/ClickHouse/ClickHouse/pull/40726) ([Amos Bird](https://github.com/amosbird)). -* Fix incorrect format of `APPLY` column transformer which can break metadata if used in table definition. This fixes [#37590](https://github.com/ClickHouse/ClickHouse/issues/37590). [#40727](https://github.com/ClickHouse/ClickHouse/pull/40727) ([Amos Bird](https://github.com/amosbird)). -* Support the `%z` descriptor for formatting the timezone offset in `formatDateTime`. [#40736](https://github.com/ClickHouse/ClickHouse/pull/40736) ([Cory Levy](https://github.com/LevyCory)). -* The interactive mode in `clickhouse-client` now interprets `.` and `/` as "run the last command". [#40750](https://github.com/ClickHouse/ClickHouse/pull/40750) ([Robert Schulze](https://github.com/rschu1ze)). -* Fix issue with passing MySQL timeouts for MySQL database engine and MySQL table function. Closes [#34168](https://github.com/ClickHouse/ClickHouse/issues/34168). [#40751](https://github.com/ClickHouse/ClickHouse/pull/40751) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Create status file for filesystem cache directory to make sure that cache directories are not shared between different servers or caches. [#40820](https://github.com/ClickHouse/ClickHouse/pull/40820) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add support for `DELETE` and `UPDATE` for `EmbeddedRocksDB` storage. [#40853](https://github.com/ClickHouse/ClickHouse/pull/40853) ([Antonio Andelic](https://github.com/antonio2368)). -* ClickHouse Keeper: fix shutdown during long commit and increase allowed request size. [#40941](https://github.com/ClickHouse/ClickHouse/pull/40941) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix race in WriteBufferFromS3, add TSA annotations. [#40950](https://github.com/ClickHouse/ClickHouse/pull/40950) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Grouping sets with group_by_use_nulls should only convert key columns to nullable. [#40997](https://github.com/ClickHouse/ClickHouse/pull/40997) ([Duc Canh Le](https://github.com/canhld94)). -* Improve the observability of INSERT on distributed table. [#41034](https://github.com/ClickHouse/ClickHouse/pull/41034) ([Frank Chen](https://github.com/FrankChen021)). -* More low-level metrics for S3 interaction. [#41039](https://github.com/ClickHouse/ClickHouse/pull/41039) ([mateng915](https://github.com/mateng0915)). -* Support relative path in Location header after HTTP redirect. Closes [#40985](https://github.com/ClickHouse/ClickHouse/issues/40985). [#41162](https://github.com/ClickHouse/ClickHouse/pull/41162) ([Kruglov Pavel](https://github.com/Avogar)). -* Apply changes to HTTP handlers on fly without server restart. [#41177](https://github.com/ClickHouse/ClickHouse/pull/41177) ([Azat Khuzhin](https://github.com/azat)). -* ClickHouse Keeper: properly close active sessions during shutdown. [#41215](https://github.com/ClickHouse/ClickHouse/pull/41215) ([Antonio Andelic](https://github.com/antonio2368)). This lowers the period of "table is read-only" errors. -* Add ability to automatically comment SQL queries in clickhouse-client/local (with `Alt-#`, like in readline). [#41224](https://github.com/ClickHouse/ClickHouse/pull/41224) ([Azat Khuzhin](https://github.com/azat)). -* Fix incompatibility of cache after switching setting `do_no_evict_index_and_mark_files` from 1 to 0, 0 to 1. [#41330](https://github.com/ClickHouse/ClickHouse/pull/41330) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add a setting `allow_suspicious_fixed_string_types` to prevent users from creating columns of type FixedString with size > 256. [#41495](https://github.com/ClickHouse/ClickHouse/pull/41495) ([Duc Canh Le](https://github.com/canhld94)). -* Add `has_lightweight_delete` to system.parts. [#41564](https://github.com/ClickHouse/ClickHouse/pull/41564) ([Kseniia Sumarokova](https://github.com/kssenii)). - -#### Build/Testing/Packaging Improvement - -* Enforce documentation for every setting. [#40644](https://github.com/ClickHouse/ClickHouse/pull/40644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Enforce documentation for every current metric. [#40645](https://github.com/ClickHouse/ClickHouse/pull/40645) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Enforce documentation for every profile event counter. Write the documentation where it was missing. [#40646](https://github.com/ClickHouse/ClickHouse/pull/40646) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Allow minimal `clickhouse-local` build by correcting some dependencies. [#40460](https://github.com/ClickHouse/ClickHouse/pull/40460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). It is less than 50 MiB. -* Calculate and report SQL function coverage in tests. [#40593](https://github.com/ClickHouse/ClickHouse/issues/40593). [#40647](https://github.com/ClickHouse/ClickHouse/pull/40647) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Enforce documentation for every MergeTree setting. [#40648](https://github.com/ClickHouse/ClickHouse/pull/40648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* A prototype of embedded reference documentation for high-level uniform server components. [#40649](https://github.com/ClickHouse/ClickHouse/pull/40649) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* We will check all queries from the changed perf tests to ensure that all changed queries were tested. [#40322](https://github.com/ClickHouse/ClickHouse/pull/40322) ([Nikita Taranov](https://github.com/nickitat)). -* Fix TGZ packages. [#40681](https://github.com/ClickHouse/ClickHouse/pull/40681) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Fix debug symbols. [#40873](https://github.com/ClickHouse/ClickHouse/pull/40873) ([Azat Khuzhin](https://github.com/azat)). -* Extended the CI configuration to create a x86 SSE2-only build. Useful for old or embedded hardware. [#40999](https://github.com/ClickHouse/ClickHouse/pull/40999) ([Robert Schulze](https://github.com/rschu1ze)). -* Switch to llvm/clang 15. [#41046](https://github.com/ClickHouse/ClickHouse/pull/41046) ([Azat Khuzhin](https://github.com/azat)). -* Continuation of [#40938](https://github.com/ClickHouse/ClickHouse/issues/40938). Fix ODR violation for `Loggers` class. Fixes [#40398](https://github.com/ClickHouse/ClickHouse/issues/40398), [#40937](https://github.com/ClickHouse/ClickHouse/issues/40937). [#41060](https://github.com/ClickHouse/ClickHouse/pull/41060) ([Dmitry Novik](https://github.com/novikd)). -* Add macOS binaries to GitHub release assets, it fixes [#37718](https://github.com/ClickHouse/ClickHouse/issues/37718). [#41088](https://github.com/ClickHouse/ClickHouse/pull/41088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* The c-ares library is now bundled with ClickHouse's build system. [#41239](https://github.com/ClickHouse/ClickHouse/pull/41239) ([Robert Schulze](https://github.com/rschu1ze)). -* Get rid of `dlopen` from the main ClickHouse code. It remains in the library-bridge and odbc-bridge. [#41428](https://github.com/ClickHouse/ClickHouse/pull/41428) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Don't allow `dlopen` in the main ClickHouse binary, because it is harmful and insecure. We don't use it. But it can be used by some libraries for the implementation of "plugins". We absolutely discourage the ancient technique of loading 3rd-party uncontrolled dangerous libraries into the process address space, because it is insane. [#41429](https://github.com/ClickHouse/ClickHouse/pull/41429) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add `source` field to deb packages, update `nfpm`. [#41531](https://github.com/ClickHouse/ClickHouse/pull/41531) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Support for DWARF-5 in the in-house DWARF parser. [#40710](https://github.com/ClickHouse/ClickHouse/pull/40710) ([Azat Khuzhin](https://github.com/azat)). -* Add fault injection in ZooKeeper client for testing [#30498](https://github.com/ClickHouse/ClickHouse/pull/30498) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Add stateless tests with s3 storage with debug and tsan [#35262](https://github.com/ClickHouse/ClickHouse/pull/35262) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Trying stress on top of S3 [#36837](https://github.com/ClickHouse/ClickHouse/pull/36837) ([alesapin](https://github.com/alesapin)). -* Enable `concurrency-mt-unsafe` in `clang-tidy` [#40224](https://github.com/ClickHouse/ClickHouse/pull/40224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### Bug Fix - -* Fix potential dataloss due to [a bug in AWS SDK](https://github.com/aws/aws-sdk-cpp/issues/658). Bug can be triggered only when clickhouse is used over S3. [#40506](https://github.com/ClickHouse/ClickHouse/pull/40506) ([alesapin](https://github.com/alesapin)). This bug has been open for 5 years in AWS SDK and is closed after our report. -* Malicious data in Native format might cause a crash. [#41441](https://github.com/ClickHouse/ClickHouse/pull/41441) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* The aggregate function `categorialInformationValue` was having incorrectly defined properties, which might cause a null pointer dereferencing at runtime. This closes [#41443](https://github.com/ClickHouse/ClickHouse/issues/41443). [#41449](https://github.com/ClickHouse/ClickHouse/pull/41449) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Writing data in Apache `ORC` format might lead to a buffer overrun. [#41458](https://github.com/ClickHouse/ClickHouse/pull/41458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix memory safety issues with functions `encrypt` and `contingency` if Array of Nullable is used as an argument. This fixes [#41004](https://github.com/ClickHouse/ClickHouse/issues/41004). [#40195](https://github.com/ClickHouse/ClickHouse/pull/40195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix bugs in MergeJoin when 'not_processed' is not null. [#40335](https://github.com/ClickHouse/ClickHouse/pull/40335) ([liql2007](https://github.com/liql2007)). -* Fix incorrect result in case of decimal precision loss in IN operator, ref [#41125](https://github.com/ClickHouse/ClickHouse/issues/41125). [#41130](https://github.com/ClickHouse/ClickHouse/pull/41130) ([Vladimir C](https://github.com/vdimir)). -* Fix filling of missed `Nested` columns with multiple levels. [#37152](https://github.com/ClickHouse/ClickHouse/pull/37152) ([Anton Popov](https://github.com/CurtizJ)). -* Fix SYSTEM UNFREEZE query for Ordinary (deprecated) database. Fix for https://github.com/ClickHouse/ClickHouse/pull/36424. [#38262](https://github.com/ClickHouse/ClickHouse/pull/38262) ([Vadim Volodin](https://github.com/PolyProgrammist)). -* Fix unused unknown columns introduced by WITH statement. This fixes [#37812](https://github.com/ClickHouse/ClickHouse/issues/37812) . [#39131](https://github.com/ClickHouse/ClickHouse/pull/39131) ([Amos Bird](https://github.com/amosbird)). -* Fix query analysis for ORDER BY in presence of window functions. Fixes [#38741](https://github.com/ClickHouse/ClickHouse/issues/38741) Fixes [#24892](https://github.com/ClickHouse/ClickHouse/issues/24892). [#39354](https://github.com/ClickHouse/ClickHouse/pull/39354) ([Dmitry Novik](https://github.com/novikd)). -* Fixed `Unknown identifier (aggregate-function)` exception which appears when a user tries to calculate WINDOW ORDER BY/PARTITION BY expressions over aggregate functions. [#39762](https://github.com/ClickHouse/ClickHouse/pull/39762) ([Vladimir Chebotaryov](https://github.com/quickhouse)). -* Limit number of analyze for one query with setting `max_analyze_depth`. It prevents exponential blow up of analysis time for queries with extraordinarily large number of subqueries. [#40334](https://github.com/ClickHouse/ClickHouse/pull/40334) ([Vladimir C](https://github.com/vdimir)). -* Fix rare bug with column TTL for MergeTree engines family: In case of repeated vertical merge the error `Cannot unlink file ColumnName.bin ... No such file or directory.` could happen. [#40346](https://github.com/ClickHouse/ClickHouse/pull/40346) ([alesapin](https://github.com/alesapin)). -* Use DNS entries for both IPv4 and IPv6 if present. [#40353](https://github.com/ClickHouse/ClickHouse/pull/40353) ([Maksim Kita](https://github.com/kitaisreal)). -* Allow to read snappy compressed files from Hadoop. [#40482](https://github.com/ClickHouse/ClickHouse/pull/40482) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix crash while parsing values of type `Object` (experimental feature) that contains arrays of variadic dimension. [#40483](https://github.com/ClickHouse/ClickHouse/pull/40483) ([Duc Canh Le](https://github.com/canhld94)). -* Fix settings `input_format_tsv_skip_first_lines`. [#40491](https://github.com/ClickHouse/ClickHouse/pull/40491) ([mini4](https://github.com/mini4)). -* Fix bug (race condition) when starting up MaterializedPostgreSQL database/table engine. [#40262](https://github.com/ClickHouse/ClickHouse/issues/40262). Fix error with reaching limit of relcache_callback_list slots. [#40511](https://github.com/ClickHouse/ClickHouse/pull/40511) ([Maksim Buren](https://github.com/maks-buren630501)). -* Fix possible error 'Decimal math overflow' while parsing DateTime64. [#40546](https://github.com/ClickHouse/ClickHouse/pull/40546) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix vertical merge of parts with lightweight deleted rows. [#40559](https://github.com/ClickHouse/ClickHouse/pull/40559) ([Alexander Gololobov](https://github.com/davenger)). -* Fix segment fault when writing data to URL table engine if it enables compression. [#40565](https://github.com/ClickHouse/ClickHouse/pull/40565) ([Frank Chen](https://github.com/FrankChen021)). -* Fix possible logical error `'Invalid Field get from type UInt64 to type String'` in arrayElement function with Map. [#40572](https://github.com/ClickHouse/ClickHouse/pull/40572) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix possible race in filesystem cache. [#40586](https://github.com/ClickHouse/ClickHouse/pull/40586) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Removed skipping of mutations in unaffected partitions of `MergeTree` tables, because this feature never worked correctly and might cause resurrection of finished mutations. [#40589](https://github.com/ClickHouse/ClickHouse/pull/40589) ([Alexander Tokmakov](https://github.com/tavplubix)). -* The clickhouse server will crash if we add a grpc port which has been occupied to the configuration in runtime. [#40597](https://github.com/ClickHouse/ClickHouse/pull/40597) ([何李夫](https://github.com/helifu)). -* Fix `base58Encode / base58Decode` handling leading 0 / '1'. [#40620](https://github.com/ClickHouse/ClickHouse/pull/40620) ([Andrey Zvonov](https://github.com/zvonand)). -* keeper-fix: fix race in accessing logs while snapshot is being installed. [#40627](https://github.com/ClickHouse/ClickHouse/pull/40627) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix short circuit execution of toFixedString function. Solves (partially) [#40622](https://github.com/ClickHouse/ClickHouse/issues/40622). [#40628](https://github.com/ClickHouse/ClickHouse/pull/40628) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixes SQLite int8 column conversion to int64 column in ClickHouse. Fixes [#40639](https://github.com/ClickHouse/ClickHouse/issues/40639). [#40642](https://github.com/ClickHouse/ClickHouse/pull/40642) ([Barum Rho](https://github.com/barumrho)). -* Fix stack overflow in recursive `Buffer` tables. This closes [#40637](https://github.com/ClickHouse/ClickHouse/issues/40637). [#40643](https://github.com/ClickHouse/ClickHouse/pull/40643) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* During insertion of a new query to the `ProcessList` allocations happen. If we reach the memory limit during these allocations we can not use `OvercommitTracker`, because `ProcessList::mutex` is already acquired. Fixes [#40611](https://github.com/ClickHouse/ClickHouse/issues/40611). [#40677](https://github.com/ClickHouse/ClickHouse/pull/40677) ([Dmitry Novik](https://github.com/novikd)). -* Fix LOGICAL_ERROR with max_read_buffer_size=0 during reading marks. [#40705](https://github.com/ClickHouse/ClickHouse/pull/40705) ([Azat Khuzhin](https://github.com/azat)). -* Fix memory leak while pushing to MVs w/o query context (from Kafka/...). [#40732](https://github.com/ClickHouse/ClickHouse/pull/40732) ([Azat Khuzhin](https://github.com/azat)). -* Fix possible error Attempt to read after eof in CSV schema inference. [#40746](https://github.com/ClickHouse/ClickHouse/pull/40746) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix logical error in write-through cache "File segment completion can be done only by downloader". Closes [#40748](https://github.com/ClickHouse/ClickHouse/issues/40748). [#40759](https://github.com/ClickHouse/ClickHouse/pull/40759) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Make the result of GROUPING function the same as in SQL and other DBMS. [#40762](https://github.com/ClickHouse/ClickHouse/pull/40762) ([Dmitry Novik](https://github.com/novikd)). -* In [#40595](https://github.com/ClickHouse/ClickHouse/issues/40595) it was reported that the `host_regexp` functionality was not working properly with a name to address resolution in `/etc/hosts`. It's fixed. [#40769](https://github.com/ClickHouse/ClickHouse/pull/40769) ([Arthur Passos](https://github.com/arthurpassos)). -* Fix incremental backups for Log family. [#40827](https://github.com/ClickHouse/ClickHouse/pull/40827) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix extremely rare bug which can lead to potential data loss in zero-copy replication. [#40844](https://github.com/ClickHouse/ClickHouse/pull/40844) ([alesapin](https://github.com/alesapin)). -* Fix key condition analyzing crashes when same set expression built from different column(s). [#40850](https://github.com/ClickHouse/ClickHouse/pull/40850) ([Duc Canh Le](https://github.com/canhld94)). -* Fix nested JSON Objects schema inference. [#40851](https://github.com/ClickHouse/ClickHouse/pull/40851) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix 3-digit prefix directory for filesystem cache files not being deleted if empty. Closes [#40797](https://github.com/ClickHouse/ClickHouse/issues/40797). [#40867](https://github.com/ClickHouse/ClickHouse/pull/40867) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix uncaught DNS_ERROR on failed connection to replicas. [#40881](https://github.com/ClickHouse/ClickHouse/pull/40881) ([Robert Coelho](https://github.com/coelho)). -* Fix bug when removing unneeded columns in subquery. [#40884](https://github.com/ClickHouse/ClickHouse/pull/40884) ([luocongkai](https://github.com/TKaxe)). -* Fix extra memory allocation for remote read buffers. [#40896](https://github.com/ClickHouse/ClickHouse/pull/40896) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixed a behaviour when user with explicitly revoked grant for dropping databases can still drop it. [#40906](https://github.com/ClickHouse/ClickHouse/pull/40906) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* A fix for ClickHouse Keeper: correctly compare paths in write requests to Keeper internal system node paths. [#40918](https://github.com/ClickHouse/ClickHouse/pull/40918) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix deadlock in WriteBufferFromS3. [#40943](https://github.com/ClickHouse/ClickHouse/pull/40943) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix access rights for `DESCRIBE TABLE url()` and some other `DESCRIBE TABLE ()`. [#40975](https://github.com/ClickHouse/ClickHouse/pull/40975) ([Vitaly Baranov](https://github.com/vitlibar)). -* Remove wrong parser logic for `WITH GROUPING SETS` which may lead to nullptr dereference. [#41049](https://github.com/ClickHouse/ClickHouse/pull/41049) ([Duc Canh Le](https://github.com/canhld94)). -* A fix for ClickHouse Keeper: fix possible segfault during Keeper shutdown. [#41075](https://github.com/ClickHouse/ClickHouse/pull/41075) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix possible segfaults, use-heap-after-free and memory leak in aggregate function combinators. Closes [#40848](https://github.com/ClickHouse/ClickHouse/issues/40848). [#41083](https://github.com/ClickHouse/ClickHouse/pull/41083) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix query_views_log with Window views. [#41132](https://github.com/ClickHouse/ClickHouse/pull/41132) ([Raúl Marín](https://github.com/Algunenano)). -* Disables optimize_monotonous_functions_in_order_by by default, mitigates: [#40094](https://github.com/ClickHouse/ClickHouse/issues/40094). [#41136](https://github.com/ClickHouse/ClickHouse/pull/41136) ([Denny Crane](https://github.com/den-crane)). -* Fixed "possible deadlock avoided" error on automatic conversion of database engine from Ordinary to Atomic. [#41146](https://github.com/ClickHouse/ClickHouse/pull/41146) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix SIGSEGV in SortedBlocksWriter in case of empty block (possible to get with `optimize_aggregation_in_order` and `join_algorithm=auto`). [#41154](https://github.com/ClickHouse/ClickHouse/pull/41154) ([Azat Khuzhin](https://github.com/azat)). -* Fix incorrect query result when trivial count optimization is in effect with array join. This fixes [#39431](https://github.com/ClickHouse/ClickHouse/issues/39431). [#41158](https://github.com/ClickHouse/ClickHouse/pull/41158) ([Denny Crane](https://github.com/den-crane)). -* Fix stack-use-after-return in GetPriorityForLoadBalancing::getPriorityFunc(). [#41159](https://github.com/ClickHouse/ClickHouse/pull/41159) ([Azat Khuzhin](https://github.com/azat)). -* Fix positional arguments exception Positional argument out of bounds. Closes [#40634](https://github.com/ClickHouse/ClickHouse/issues/40634). [#41189](https://github.com/ClickHouse/ClickHouse/pull/41189) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix background clean up of broken detached parts. [#41190](https://github.com/ClickHouse/ClickHouse/pull/41190) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix exponential query rewrite in case of lots of cross joins with where, close [#21557](https://github.com/ClickHouse/ClickHouse/issues/21557). [#41223](https://github.com/ClickHouse/ClickHouse/pull/41223) ([Vladimir C](https://github.com/vdimir)). -* Fix possible logical error in write-through cache, which happened because not all types of exception were handled as needed. Closes [#41208](https://github.com/ClickHouse/ClickHouse/issues/41208). [#41232](https://github.com/ClickHouse/ClickHouse/pull/41232) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix String log entry in system.filesystem_cache_log. [#41233](https://github.com/ClickHouse/ClickHouse/pull/41233) ([jmimbrero](https://github.com/josemimbrero-tinybird)). -* Queries with `OFFSET` clause in subquery and `WHERE` clause in outer query might return incorrect result, it's fixed. Fixes [#40416](https://github.com/ClickHouse/ClickHouse/issues/40416). [#41280](https://github.com/ClickHouse/ClickHouse/pull/41280) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix possible wrong query result with `query_plan_optimize_primary_key` enabled. Fixes [#40599](https://github.com/ClickHouse/ClickHouse/issues/40599). [#41281](https://github.com/ClickHouse/ClickHouse/pull/41281) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Do not allow invalid sequences influence other rows in lowerUTF8/upperUTF8. [#41286](https://github.com/ClickHouse/ClickHouse/pull/41286) ([Azat Khuzhin](https://github.com/azat)). -* Fix `ALTER ADD COLUMN` queries with columns of type `Object`. [#41290](https://github.com/ClickHouse/ClickHouse/pull/41290) ([Anton Popov](https://github.com/CurtizJ)). -* Fixed "No node" error when selecting from `system.distributed_ddl_queue` when there's no `distributed_ddl.path` in config. Fixes [#41096](https://github.com/ClickHouse/ClickHouse/issues/41096). [#41296](https://github.com/ClickHouse/ClickHouse/pull/41296) ([young scott](https://github.com/young-scott)). -* Fix incorrect logical error `Expected relative path` in disk object storage. Related to [#41246](https://github.com/ClickHouse/ClickHouse/issues/41246). [#41297](https://github.com/ClickHouse/ClickHouse/pull/41297) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add column type check before UUID insertion in MsgPack format. [#41309](https://github.com/ClickHouse/ClickHouse/pull/41309) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix possible crash after inserting asynchronously (with enabled setting `async_insert`) malformed data to columns of type `Object`. It could happen, if JSONs in all batches of async inserts were invalid and could not be parsed. [#41336](https://github.com/ClickHouse/ClickHouse/pull/41336) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible deadlock with async_socket_for_remote/use_hedged_requests and parallel KILL. [#41343](https://github.com/ClickHouse/ClickHouse/pull/41343) ([Azat Khuzhin](https://github.com/azat)). -* Disables optimize_rewrite_sum_if_to_count_if by default, mitigates: [#38605](https://github.com/ClickHouse/ClickHouse/issues/38605) [#38683](https://github.com/ClickHouse/ClickHouse/issues/38683). [#41388](https://github.com/ClickHouse/ClickHouse/pull/41388) ([Denny Crane](https://github.com/den-crane)). -* Since 22.8 `ON CLUSTER` clause is ignored if database is `Replicated` and cluster name and database name are the same. Because of this `DROP PARTITION ON CLUSTER` worked unexpected way with `Replicated`. It's fixed, now `ON CLUSTER` clause is ignored only for queries that are replicated on database level. Fixes [#41299](https://github.com/ClickHouse/ClickHouse/issues/41299). [#41390](https://github.com/ClickHouse/ClickHouse/pull/41390) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix possible hung/deadlock on query cancellation (`KILL QUERY` or server shutdown). [#41467](https://github.com/ClickHouse/ClickHouse/pull/41467) ([Azat Khuzhin](https://github.com/azat)). -* Fix possible server crash when using the JBOD feature. This fixes [#41365](https://github.com/ClickHouse/ClickHouse/issues/41365). [#41483](https://github.com/ClickHouse/ClickHouse/pull/41483) ([Amos Bird](https://github.com/amosbird)). -* Fix conversion from nullable fixed string to string. [#41541](https://github.com/ClickHouse/ClickHouse/pull/41541) ([Duc Canh Le](https://github.com/canhld94)). -* Prevent crash when passing wrong aggregation states to groupBitmap*. [#41563](https://github.com/ClickHouse/ClickHouse/pull/41563) ([Raúl Marín](https://github.com/Algunenano)). -* Queries with `ORDER BY` and `1500 <= LIMIT <= max_block_size` could return incorrect result with missing rows from top. Fixes [#41182](https://github.com/ClickHouse/ClickHouse/issues/41182). [#41576](https://github.com/ClickHouse/ClickHouse/pull/41576) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix read bytes/rows in X-ClickHouse-Summary with materialized views. [#41586](https://github.com/ClickHouse/ClickHouse/pull/41586) ([Raúl Marín](https://github.com/Algunenano)). -* Fix possible `pipeline stuck` exception for queries with `OFFSET`. The error was found with `enable_optimize_predicate_expression = 0` and always false condition in `WHERE`. Fixes [#41383](https://github.com/ClickHouse/ClickHouse/issues/41383). [#41588](https://github.com/ClickHouse/ClickHouse/pull/41588) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). - -### ClickHouse release 22.8-lts, 2022-08-18 - -#### Backward Incompatible Change - -* Extended range of `Date32` and `DateTime64` to support dates from the year 1900 to 2299. In previous versions, the supported interval was only from the year 1925 to 2283. The implementation is using the proleptic Gregorian calendar (which is conformant with [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601):2004 (clause 3.2.1 The Gregorian calendar)) instead of accounting for historical transitions from the Julian to the Gregorian calendar. This change affects implementation-specific behavior for out-of-range arguments. E.g. if in previous versions the value of `1899-01-01` was clamped to `1925-01-01`, in the new version it will be clamped to `1900-01-01`. It changes the behavior of rounding with `toStartOfInterval` if you pass `INTERVAL 3 QUARTER` up to one quarter because the intervals are counted from an implementation-specific point of time. Closes [#28216](https://github.com/ClickHouse/ClickHouse/issues/28216), improves [#38393](https://github.com/ClickHouse/ClickHouse/issues/38393). [#39425](https://github.com/ClickHouse/ClickHouse/pull/39425) ([Roman Vasin](https://github.com/rvasin)). -* Now, all relevant dictionary sources respect `remote_url_allow_hosts` setting. It was already done for HTTP, Cassandra, Redis. Added ClickHouse, MongoDB, MySQL, PostgreSQL. Host is checked only for dictionaries created from DDL. [#39184](https://github.com/ClickHouse/ClickHouse/pull/39184) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Make the remote filesystem cache composable, allow not to evict certain files (regarding idx, mrk, ..), delete old cache version. Now it is possible to configure cache over Azure blob storage disk, over Local disk, over StaticWeb disk, etc. This PR is marked backward incompatible because cache configuration changes and in order for cache to work need to update the config file. Old cache will still be used with new configuration. The server will startup fine with the old cache configuration. Closes https://github.com/ClickHouse/ClickHouse/issues/36140. Closes https://github.com/ClickHouse/ClickHouse/issues/37889. ([Kseniia Sumarokova](https://github.com/kssenii)). [#36171](https://github.com/ClickHouse/ClickHouse/pull/36171)) - -#### New Feature - -* Query parameters can be set in interactive mode as `SET param_abc = 'def'` and transferred via the native protocol as settings. [#39906](https://github.com/ClickHouse/ClickHouse/pull/39906) ([Nikita Taranov](https://github.com/nickitat)). -* Quota key can be set in the native protocol ([Yakov Olkhovsky](https://github.com/ClickHouse/ClickHouse/pull/39874)). -* Added a setting `exact_rows_before_limit` (0/1). When enabled, ClickHouse will provide exact value for `rows_before_limit_at_least` statistic, but with the cost that the data before limit will have to be read completely. This closes [#6613](https://github.com/ClickHouse/ClickHouse/issues/6613). [#25333](https://github.com/ClickHouse/ClickHouse/pull/25333) ([kevin wan](https://github.com/MaxWk)). -* Added support for parallel distributed insert select with `s3Cluster` table function into tables with `Distributed` and `Replicated` engine [#34670](https://github.com/ClickHouse/ClickHouse/issues/34670). [#39107](https://github.com/ClickHouse/ClickHouse/pull/39107) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Add new settings to control schema inference from text formats: - `input_format_try_infer_dates` - try infer dates from strings. - `input_format_try_infer_datetimes` - try infer datetimes from strings. - `input_format_try_infer_integers` - try infer `Int64` instead of `Float64`. - `input_format_json_try_infer_numbers_from_strings` - try infer numbers from json strings in JSON formats. [#39186](https://github.com/ClickHouse/ClickHouse/pull/39186) ([Kruglov Pavel](https://github.com/Avogar)). -* An option to provide JSON formatted log output. The purpose is to allow easier ingestion and query in log analysis tools. [#39277](https://github.com/ClickHouse/ClickHouse/pull/39277) ([Mallik Hassan](https://github.com/SadiHassan)). -* Add function `nowInBlock` which allows getting the current time during long-running and continuous queries. Closes [#39522](https://github.com/ClickHouse/ClickHouse/issues/39522). Notes: there are no functions `now64InBlock` neither `todayInBlock`. [#39533](https://github.com/ClickHouse/ClickHouse/pull/39533) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add ability to specify settings for an `executable()` table function. [#39681](https://github.com/ClickHouse/ClickHouse/pull/39681) ([Constantine Peresypkin](https://github.com/pkit)). -* Implemented automatic conversion of database engine from `Ordinary` to `Atomic`. Create empty `convert_ordinary_to_atomic` file in `flags` directory and all `Ordinary` databases will be converted automatically on next server start. Resolves [#39546](https://github.com/ClickHouse/ClickHouse/issues/39546). [#39933](https://github.com/ClickHouse/ClickHouse/pull/39933) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Support `SELECT ... INTO OUTFILE '...' AND STDOUT`. [#37490](https://github.com/ClickHouse/ClickHouse/issues/37490). [#39054](https://github.com/ClickHouse/ClickHouse/pull/39054) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* Add formats `PrettyMonoBlock`, `PrettyNoEscapesMonoBlock`, `PrettyCompactNoEscapes`, `PrettyCompactNoEscapesMonoBlock`, `PrettySpaceNoEscapes`, `PrettySpaceMonoBlock`, `PrettySpaceNoEscapesMonoBlock`. [#39646](https://github.com/ClickHouse/ClickHouse/pull/39646) ([Kruglov Pavel](https://github.com/Avogar)). -* Add new setting schema_inference_hints that allows to specify structure hints in schema inference for specific columns. Closes [#39569](https://github.com/ClickHouse/ClickHouse/issues/39569). [#40068](https://github.com/ClickHouse/ClickHouse/pull/40068) ([Kruglov Pavel](https://github.com/Avogar)). - -#### Experimental Feature - -* Support SQL standard DELETE FROM syntax on merge tree tables and lightweight delete implementation for merge tree families. [#37893](https://github.com/ClickHouse/ClickHouse/pull/37893) ([Jianmei Zhang](https://github.com/zhangjmruc)) ([Alexander Gololobov](https://github.com/davenger)). Note: this new feature does not make ClickHouse an HTAP DBMS. - -#### Performance Improvement - -* Improved memory usage during memory efficient merging of aggregation results. [#39429](https://github.com/ClickHouse/ClickHouse/pull/39429) ([Nikita Taranov](https://github.com/nickitat)). -* Added concurrency control logic to limit total number of concurrent threads created by queries. [#37558](https://github.com/ClickHouse/ClickHouse/pull/37558) ([Sergei Trifonov](https://github.com/serxa)). Add `concurrent_threads_soft_limit parameter` to increase performance in case of high QPS by means of limiting total number of threads for all queries. [#37285](https://github.com/ClickHouse/ClickHouse/pull/37285) ([Roman Vasin](https://github.com/rvasin)). -* Add `SLRU` cache policy for uncompressed cache and marks cache. ([Kseniia Sumarokova](https://github.com/kssenii)). [#34651](https://github.com/ClickHouse/ClickHouse/pull/34651) ([alexX512](https://github.com/alexX512)). Decoupling local cache function and cache algorithm [#38048](https://github.com/ClickHouse/ClickHouse/pull/38048) ([Han Shukai](https://github.com/KinderRiven)). -* Intel® In-Memory Analytics Accelerator (Intel® IAA) is a hardware accelerator available in the upcoming generation of Intel® Xeon® Scalable processors ("Sapphire Rapids"). Its goal is to speed up common operations in analytics like data (de)compression and filtering. ClickHouse gained the new "DeflateQpl" compression codec which utilizes the Intel® IAA offloading technology to provide a high-performance DEFLATE implementation. The codec uses the [Intel® Query Processing Library (QPL)](https://github.com/intel/qpl) which abstracts access to the hardware accelerator, respectively to a software fallback in case the hardware accelerator is not available. DEFLATE provides in general higher compression rates than ClickHouse's LZ4 default codec, and as a result, offers less disk I/O and lower main memory consumption. [#36654](https://github.com/ClickHouse/ClickHouse/pull/36654) ([jasperzhu](https://github.com/jinjunzh)). [#39494](https://github.com/ClickHouse/ClickHouse/pull/39494) ([Robert Schulze](https://github.com/rschu1ze)). -* `DISTINCT` in order with `ORDER BY`: Deduce way to sort based on input stream sort description. Skip sorting if input stream is already sorted. [#38719](https://github.com/ClickHouse/ClickHouse/pull/38719) ([Igor Nikonov](https://github.com/devcrafter)). Improve memory usage (significantly) and query execution time + use `DistinctSortedChunkTransform` for final distinct when `DISTINCT` columns match `ORDER BY` columns, but rename to `DistinctSortedStreamTransform` in `EXPLAIN PIPELINE` → this improves memory usage significantly + remove unnecessary allocations in hot loop in `DistinctSortedChunkTransform`. [#39432](https://github.com/ClickHouse/ClickHouse/pull/39432) ([Igor Nikonov](https://github.com/devcrafter)). Use `DistinctSortedTransform` only when sort description is applicable to DISTINCT columns, otherwise fall back to ordinary DISTINCT implementation + it allows making less checks during `DistinctSortedTransform` execution. [#39528](https://github.com/ClickHouse/ClickHouse/pull/39528) ([Igor Nikonov](https://github.com/devcrafter)). Fix: `DistinctSortedTransform` didn't take advantage of sorting. It never cleared HashSet since clearing_columns were detected incorrectly (always empty). So, it basically worked as ordinary `DISTINCT` (`DistinctTransform`). The fix reduces memory usage significantly. [#39538](https://github.com/ClickHouse/ClickHouse/pull/39538) ([Igor Nikonov](https://github.com/devcrafter)). -* Use local node as first priority to get structure of remote table when executing `cluster` and similar table functions. [#39440](https://github.com/ClickHouse/ClickHouse/pull/39440) ([Mingliang Pan](https://github.com/liangliangpan)). -* Optimize filtering by numeric columns with AVX512VBMI2 compress store. [#39633](https://github.com/ClickHouse/ClickHouse/pull/39633) ([Guo Wangyang](https://github.com/guowangy)). For systems with AVX512 VBMI2, this PR improves performance by ca. 6% for SSB benchmark queries queries 3.1, 3.2 and 3.3 (SF=100). Tested on Intel Icelake Xeon 8380 * 2 socket. [#40033](https://github.com/ClickHouse/ClickHouse/pull/40033) ([Robert Schulze](https://github.com/rschu1ze)). -* Optimize index analysis with functional expressions in multi-thread scenario. [#39812](https://github.com/ClickHouse/ClickHouse/pull/39812) ([Guo Wangyang](https://github.com/guowangy)). -* Optimizations for complex queries: Don't visit the AST for UDFs if none are registered. [#40069](https://github.com/ClickHouse/ClickHouse/pull/40069) ([Raúl Marín](https://github.com/Algunenano)). Optimize CurrentMemoryTracker alloc and free. [#40078](https://github.com/ClickHouse/ClickHouse/pull/40078) ([Raúl Marín](https://github.com/Algunenano)). -* Improved Base58 encoding/decoding. [#39292](https://github.com/ClickHouse/ClickHouse/pull/39292) ([Andrey Zvonov](https://github.com/zvonand)). -* Improve bytes to bits mask transform for SSE/AVX/AVX512. [#39586](https://github.com/ClickHouse/ClickHouse/pull/39586) ([Guo Wangyang](https://github.com/guowangy)). - -#### Improvement - -* Normalize `AggregateFunction` types and state representations because optimizations like [#35788](https://github.com/ClickHouse/ClickHouse/pull/35788) will treat `count(not null columns)` as `count()`, which might confuses distributed interpreters with the following error : `Conversion from AggregateFunction(count) to AggregateFunction(count, Int64) is not supported`. [#39420](https://github.com/ClickHouse/ClickHouse/pull/39420) ([Amos Bird](https://github.com/amosbird)). The functions with identical states can be used in materialized views interchangeably. -* Rework and simplify the `system.backups` table, remove the `internal` column, allow user to set the ID of operation, add columns `num_files`, `uncompressed_size`, `compressed_size`, `start_time`, `end_time`. [#39503](https://github.com/ClickHouse/ClickHouse/pull/39503) ([Vitaly Baranov](https://github.com/vitlibar)). -* Improved structure of DDL query result table for `Replicated` database (separate columns with shard and replica name, more clear status) - `CREATE TABLE ... ON CLUSTER` queries can be normalized on initiator first if `distributed_ddl_entry_format_version` is set to 3 (default value). It means that `ON CLUSTER` queries may not work if initiator does not belong to the cluster that specified in query. Fixes [#37318](https://github.com/ClickHouse/ClickHouse/issues/37318), [#39500](https://github.com/ClickHouse/ClickHouse/issues/39500) - Ignore `ON CLUSTER` clause if database is `Replicated` and cluster name equals to database name. Related to [#35570](https://github.com/ClickHouse/ClickHouse/issues/35570) - Miscellaneous minor fixes for `Replicated` database engine - Check metadata consistency when starting up `Replicated` database, start replica recovery in case of mismatch of local metadata and metadata in Keeper. Resolves [#24880](https://github.com/ClickHouse/ClickHouse/issues/24880). [#37198](https://github.com/ClickHouse/ClickHouse/pull/37198) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Add result_rows and result_bytes to progress reports (`X-ClickHouse-Summary`). [#39567](https://github.com/ClickHouse/ClickHouse/pull/39567) ([Raúl Marín](https://github.com/Algunenano)). -* Improve primary key analysis for MergeTree. [#25563](https://github.com/ClickHouse/ClickHouse/pull/25563) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* `timeSlots` now works with DateTime64; subsecond duration and slot size available when working with DateTime64. [#37951](https://github.com/ClickHouse/ClickHouse/pull/37951) ([Andrey Zvonov](https://github.com/zvonand)). -* Added support of `LEFT SEMI` and `LEFT ANTI` direct join with `EmbeddedRocksDB` tables. [#38956](https://github.com/ClickHouse/ClickHouse/pull/38956) ([Vladimir C](https://github.com/vdimir)). -* Add profile events for fsync operations. [#39179](https://github.com/ClickHouse/ClickHouse/pull/39179) ([Azat Khuzhin](https://github.com/azat)). -* Add the second argument to the ordinary function `file(path[, default])`, which function returns in the case when a file does not exists. [#39218](https://github.com/ClickHouse/ClickHouse/pull/39218) ([Nikolay Degterinsky](https://github.com/evillique)). -* Some small fixes for reading via http, allow to retry partial content in case if 200 OK. [#39244](https://github.com/ClickHouse/ClickHouse/pull/39244) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support queries `CREATE TEMPORARY TABLE ... () AS ...`. [#39462](https://github.com/ClickHouse/ClickHouse/pull/39462) ([Kruglov Pavel](https://github.com/Avogar)). -* Add support of `!`/`*` (exclamation/asterisk) in custom TLDs (`cutToFirstSignificantSubdomainCustom()`/`cutToFirstSignificantSubdomainCustomWithWWW()`/`firstSignificantSubdomainCustom()`). [#39496](https://github.com/ClickHouse/ClickHouse/pull/39496) ([Azat Khuzhin](https://github.com/azat)). -* Add support for TLS connections to NATS. Implements [#39525](https://github.com/ClickHouse/ClickHouse/issues/39525). [#39527](https://github.com/ClickHouse/ClickHouse/pull/39527) ([Constantine Peresypkin](https://github.com/pkit)). -* `clickhouse-obfuscator` (a tool for database obfuscation for testing and load generation) now has the new `--save` and `--load` parameters to work with pre-trained models. This closes [#39534](https://github.com/ClickHouse/ClickHouse/issues/39534). [#39541](https://github.com/ClickHouse/ClickHouse/pull/39541) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix incorrect behavior of log rotation during restart. [#39558](https://github.com/ClickHouse/ClickHouse/pull/39558) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix building aggregate projections when external aggregation is on. Mark as improvement because the case is rare and there exists easy workaround to fix it via changing settings. This fixes [#39667](https://github.com/ClickHouse/ClickHouse/issues/39667) . [#39671](https://github.com/ClickHouse/ClickHouse/pull/39671) ([Amos Bird](https://github.com/amosbird)). -* Allow to execute hash functions with arguments of type `Map`. [#39685](https://github.com/ClickHouse/ClickHouse/pull/39685) ([Anton Popov](https://github.com/CurtizJ)). -* Add a configuration parameter to hide addresses in stack traces. It may improve security a little but generally, it is harmful and should not be used. [#39690](https://github.com/ClickHouse/ClickHouse/pull/39690) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Change the prefix size of AggregateFunctionDistinct to make sure nested function data memory segment is aligned. [#39696](https://github.com/ClickHouse/ClickHouse/pull/39696) ([Pxl](https://github.com/BiteTheDDDDt)). -* Properly escape credentials passed to the `clickhouse-diagnostic` tool. [#39707](https://github.com/ClickHouse/ClickHouse/pull/39707) ([Dale McDiarmid](https://github.com/gingerwizard)). -* ClickHouse Keeper improvement: create a snapshot on exit. It can be controlled with the config `keeper_server.create_snapshot_on_exit`, `true` by default. [#39755](https://github.com/ClickHouse/ClickHouse/pull/39755) ([Antonio Andelic](https://github.com/antonio2368)). -* Support primary key analysis for `row_policy_filter` and `additional_filter`. It also helps fix issues like [#37454](https://github.com/ClickHouse/ClickHouse/issues/37454) . [#39826](https://github.com/ClickHouse/ClickHouse/pull/39826) ([Amos Bird](https://github.com/amosbird)). -* Fix two usability issues in Play UI: - it was non-pixel-perfect on iPad due to parasitic border radius and margins; - the progress indication did not display after the first query. This closes [#39957](https://github.com/ClickHouse/ClickHouse/issues/39957). This closes [#39960](https://github.com/ClickHouse/ClickHouse/issues/39960). [#39961](https://github.com/ClickHouse/ClickHouse/pull/39961) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Play UI: add row numbers; add cell selection on click; add hysteresis for table cells. [#39962](https://github.com/ClickHouse/ClickHouse/pull/39962) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Play UI: recognize tab key in textarea, but at the same time don't mess up with tab navigation. [#40053](https://github.com/ClickHouse/ClickHouse/pull/40053) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* The client will show server-side elapsed time. This is important for the performance comparison of ClickHouse services in remote datacenters. This closes [#38070](https://github.com/ClickHouse/ClickHouse/issues/38070). See also [this](https://github.com/ClickHouse/ClickBench/blob/main/hardware/benchmark-cloud.sh#L37) for motivation. [#39968](https://github.com/ClickHouse/ClickHouse/pull/39968) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Adds `parseDateTime64BestEffortUS`, `parseDateTime64BestEffortUSOrNull`, `parseDateTime64BestEffortUSOrZero` functions, closing [#37492](https://github.com/ClickHouse/ClickHouse/issues/37492). [#40015](https://github.com/ClickHouse/ClickHouse/pull/40015) ([Tanya Bragin](https://github.com/tbragin)). -* Extend the `system.processors_profile_log` with more information such as input rows. [#40121](https://github.com/ClickHouse/ClickHouse/pull/40121) ([Amos Bird](https://github.com/amosbird)). -* Display server-side time in `clickhouse-benchmark` by default if it is available (since ClickHouse version 22.8). This is needed to correctly compare the performance of clouds. This behavior can be changed with the new `--client-side-time` command line option. Change the `--randomize` command line option from `--randomize 1` to the form without argument. [#40193](https://github.com/ClickHouse/ClickHouse/pull/40193) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add counters (ProfileEvents) for cases when query complexity limitation has been set and has reached (a separate counter for `overflow_mode` = `break` and `throw`). For example, if you have set up `max_rows_to_read` with `read_overflow_mode = 'break'`, looking at the value of `OverflowBreak` counter will allow distinguishing incomplete results. [#40205](https://github.com/ClickHouse/ClickHouse/pull/40205) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix memory accounting in case of "Memory limit exceeded" errors (previously [peak] memory usage was takes failed allocations into account). [#40249](https://github.com/ClickHouse/ClickHouse/pull/40249) ([Azat Khuzhin](https://github.com/azat)). -* Add metrics for filesystem cache: `FilesystemCacheSize` and `FilesystemCacheElements`. [#40260](https://github.com/ClickHouse/ClickHouse/pull/40260) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support Hadoop secure RPC transfer (hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#39411](https://github.com/ClickHouse/ClickHouse/pull/39411) ([michael1589](https://github.com/michael1589)). -* Avoid continuously growing memory consumption of pattern cache when using functions multi(Fuzzy)Match(Any|AllIndices|AnyIndex)(). [#40264](https://github.com/ClickHouse/ClickHouse/pull/40264) ([Robert Schulze](https://github.com/rschu1ze)). -* Add cache for schema inference for file/s3/hdfs/url table functions. Now, schema inference will be performed only on the first query to the file, all subsequent queries to the same file will use the schema from the cache if data has not changed. Add system table system.schema_inference_cache with all current schemas in cache and system queries SYSTEM DROP SCHEMA CACHE [FOR FILE/S3/HDFS/URL] to drop schemas from cache. [#38286](https://github.com/ClickHouse/ClickHouse/pull/38286) ([Kruglov Pavel](https://github.com/Avogar)). -* Add support for LARGE_BINARY/LARGE_STRING with Arrow (Closes [#32401](https://github.com/ClickHouse/ClickHouse/issues/32401)). [#40293](https://github.com/ClickHouse/ClickHouse/pull/40293) ([Josh Taylor](https://github.com/joshuataylor)). - -#### Build/Testing/Packaging Improvement - -* [ClickFiddle](https://fiddle.clickhouse.com/): A new tool for testing ClickHouse versions in read/write mode (**Igor Baliuk**). -* ClickHouse binary is made self-extracting [#35775](https://github.com/ClickHouse/ClickHouse/pull/35775) ([Yakov Olkhovskiy, Arthur Filatenkov](https://github.com/yakov-olkhovskiy)). -* Update `tzdata` to 2022b to support the new timezone changes. See https://github.com/google/cctz/pull/226. Chile's 2022 DST start is delayed from September 4 to September 11. Iran plans to stop observing DST permanently after it falls back on 2022-09-21. There are corrections to the historical time zone of Asia/Tehran in the year 1977: Iran adopted standard time in 1935, not 1946. In 1977 it observed DST from 03-21 23:00 to 10-20 24:00; its 1978 transitions were on 03-24 and 08-05, not 03-20 and 10-20; and its spring 1979 transition was on 05-27, not 03-21 (https://data.iana.org/time-zones/tzdb/NEWS). ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and are not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Ensure LSan is effective. [#39430](https://github.com/ClickHouse/ClickHouse/pull/39430) ([Azat Khuzhin](https://github.com/azat)). -* TSAN has issues with clang-14 (https://github.com/google/sanitizers/issues/1552, https://github.com/google/sanitizers/issues/1540), so here we build the TSAN binaries with clang-15. [#39450](https://github.com/ClickHouse/ClickHouse/pull/39450) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Remove the option to build ClickHouse tools as separate executable programs. This fixes [#37847](https://github.com/ClickHouse/ClickHouse/issues/37847). [#39520](https://github.com/ClickHouse/ClickHouse/pull/39520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Small preparations for build on s390x (which is big-endian). [#39627](https://github.com/ClickHouse/ClickHouse/pull/39627) ([Harry Lee](https://github.com/HarryLeeIBM)). [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issue in BitHelpers for s390x. [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Implement a piece of code related to SipHash for s390x architecture (which is not supported by ClickHouse). [#39732](https://github.com/ClickHouse/ClickHouse/pull/39732) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed an Endian issue in the Coordination snapshot code for s390x architecture (which is not supported by ClickHouse). [#39931](https://github.com/ClickHouse/ClickHouse/pull/39931) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in Codec code for s390x architecture (which is not supported by ClickHouse). [#40008](https://github.com/ClickHouse/ClickHouse/pull/40008) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in reading/writing BigEndian binary data in ReadHelpers and WriteHelpers code for s390x architecture (which is not supported by ClickHouse). [#40179](https://github.com/ClickHouse/ClickHouse/pull/40179) ([Harry Lee](https://github.com/HarryLeeIBM)). -* Support build with `clang-16` (trunk). This closes [#39949](https://github.com/ClickHouse/ClickHouse/issues/39949). [#40181](https://github.com/ClickHouse/ClickHouse/pull/40181) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Prepare RISC-V 64 build to run in CI. This is for [#40141](https://github.com/ClickHouse/ClickHouse/issues/40141). [#40197](https://github.com/ClickHouse/ClickHouse/pull/40197) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Simplified function registration macro interface (`FUNCTION_REGISTER*`) to eliminate the step to add and call an extern function in the registerFunctions.cpp, it also makes incremental builds of a new function faster. [#38615](https://github.com/ClickHouse/ClickHouse/pull/38615) ([Li Yin](https://github.com/liyinsg)). -* Docker: Now entrypoint.sh in docker image creates and executes chown for all folders it finds in the config for multidisk setup [#17717](https://github.com/ClickHouse/ClickHouse/issues/17717). [#39121](https://github.com/ClickHouse/ClickHouse/pull/39121) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). - -#### Bug Fix - -* Fix possible segfault in `CapnProto` input format. This bug was found and sent in through the ClickHouse bug-bounty [program](https://github.com/ClickHouse/ClickHouse/issues/38986) by *kiojj*. [#40241](https://github.com/ClickHouse/ClickHouse/pull/40241) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix a very rare case of incorrect behavior of the array subscript operator. This closes [#28720](https://github.com/ClickHouse/ClickHouse/issues/28720). [#40185](https://github.com/ClickHouse/ClickHouse/pull/40185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix insufficient argument check for encryption functions (found by query fuzzer). This closes [#39987](https://github.com/ClickHouse/ClickHouse/issues/39987). [#40194](https://github.com/ClickHouse/ClickHouse/pull/40194) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix the case when the order of columns can be incorrect if the `IN` operator is used with a table with `ENGINE = Set` containing multiple columns. This fixes [#13014](https://github.com/ClickHouse/ClickHouse/issues/13014). [#40225](https://github.com/ClickHouse/ClickHouse/pull/40225) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix duplicate columns in join plan. Finally, solve [#26809](https://github.com/ClickHouse/ClickHouse/issues/26809). [#40009](https://github.com/ClickHouse/ClickHouse/pull/40009) ([Vladimir C](https://github.com/vdimir)). -* Fixed query hanging for SELECT with ORDER BY WITH FILL with different date/time types. [#37849](https://github.com/ClickHouse/ClickHouse/pull/37849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Fix ORDER BY that matches projections ORDER BY (before it simply returns unsorted result). [#38725](https://github.com/ClickHouse/ClickHouse/pull/38725) ([Azat Khuzhin](https://github.com/azat)). -* Do not optimise functions in GROUP BY statements if they shadow one of the table columns or expressions. Fixes [#37032](https://github.com/ClickHouse/ClickHouse/issues/37032). [#39103](https://github.com/ClickHouse/ClickHouse/pull/39103) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Fix wrong table name in logs after RENAME TABLE. This fixes [#38018](https://github.com/ClickHouse/ClickHouse/issues/38018). [#39227](https://github.com/ClickHouse/ClickHouse/pull/39227) ([Amos Bird](https://github.com/amosbird)). -* Fix positional arguments in case of columns pruning when optimising the query. Closes [#38433](https://github.com/ClickHouse/ClickHouse/issues/38433). [#39293](https://github.com/ClickHouse/ClickHouse/pull/39293) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix bug in schema inference in case of empty messages in Protobuf/CapnProto formats that allowed to create column with empty `Tuple` type. Closes [#39051](https://github.com/ClickHouse/ClickHouse/issues/39051) Add 2 new settings `input_format_{protobuf/capnproto}_skip_fields_with_unsupported_types_in_schema_inference` that allow to skip fields with unsupported types while schema inference for Protobuf and CapnProto formats. [#39357](https://github.com/ClickHouse/ClickHouse/pull/39357) ([Kruglov Pavel](https://github.com/Avogar)). -* (Window View is an experimental feature) Fix segmentation fault on `CREATE WINDOW VIEW .. ON CLUSTER ... INNER`. Closes [#39363](https://github.com/ClickHouse/ClickHouse/issues/39363). [#39384](https://github.com/ClickHouse/ClickHouse/pull/39384) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix WriteBuffer finalize when cancelling insert into function (in previous versions it may leat to std::terminate). [#39458](https://github.com/ClickHouse/ClickHouse/pull/39458) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix storing of columns of type `Object` in sparse serialization. [#39464](https://github.com/ClickHouse/ClickHouse/pull/39464) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible "Not found column in block" exception when using projections. This closes [#39469](https://github.com/ClickHouse/ClickHouse/issues/39469). [#39470](https://github.com/ClickHouse/ClickHouse/pull/39470) ([小路](https://github.com/nicelulu)). -* Fix exception on race between DROP and INSERT with materialized views. [#39477](https://github.com/ClickHouse/ClickHouse/pull/39477) ([Azat Khuzhin](https://github.com/azat)). -* A bug in Apache Avro library: fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix rare bug in asynchronous reading (with setting `local_filesystem_read_method='pread_threadpool'`) with enabled `O_DIRECT` (enabled by setting `min_bytes_to_use_direct_io`). [#39506](https://github.com/ClickHouse/ClickHouse/pull/39506) ([Anton Popov](https://github.com/CurtizJ)). -* (only on FreeBSD) Fixes "Code: 49. DB::Exception: FunctionFactory: the function name '' is not unique. (LOGICAL_ERROR)" observed on FreeBSD when starting clickhouse. [#39551](https://github.com/ClickHouse/ClickHouse/pull/39551) ([Alexander Gololobov](https://github.com/davenger)). -* Fix bug with the recently introduced "maxsplit" argument for `splitByChar`, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)). -* Fix bug in ASOF JOIN with `enable_optimize_predicate_expression`, close [#37813](https://github.com/ClickHouse/ClickHouse/issues/37813). [#39556](https://github.com/ClickHouse/ClickHouse/pull/39556) ([Vladimir C](https://github.com/vdimir)). -* Fixed `CREATE/DROP INDEX` query with `ON CLUSTER` or `Replicated` database and `ReplicatedMergeTree`. It used to be executed on all replicas (causing error or DDL queue stuck). Fixes [#39511](https://github.com/ClickHouse/ClickHouse/issues/39511). [#39565](https://github.com/ClickHouse/ClickHouse/pull/39565) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix "column not found" error for push down with join, close [#39505](https://github.com/ClickHouse/ClickHouse/issues/39505). [#39575](https://github.com/ClickHouse/ClickHouse/pull/39575) ([Vladimir C](https://github.com/vdimir)). -* Fix the wrong `REGEXP_REPLACE` alias. This fixes https://github.com/ClickHouse/ClickBench/issues/9. [#39592](https://github.com/ClickHouse/ClickHouse/pull/39592) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fixed point of origin for exponential decay window functions to the last value in window. Previously, decay was calculated by formula `exp((t - curr_row_t) / decay_length)`, which is incorrect when right boundary of window is not `CURRENT ROW`. It was changed to: `exp((t - last_row_t) / decay_length)`. There is no change in results for windows with `ROWS BETWEEN (smth) AND CURRENT ROW`. [#39593](https://github.com/ClickHouse/ClickHouse/pull/39593) ([Vladimir Chebotaryov](https://github.com/quickhouse)). -* Fix Decimal division overflow, which can be detected based on operands scale. [#39600](https://github.com/ClickHouse/ClickHouse/pull/39600) ([Andrey Zvonov](https://github.com/zvonand)). -* Fix settings `output_format_arrow_string_as_string` and `output_format_arrow_low_cardinality_as_dictionary` work in combination. Closes [#39624](https://github.com/ClickHouse/ClickHouse/issues/39624). [#39647](https://github.com/ClickHouse/ClickHouse/pull/39647) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed a bug in default database resolution in distributed table reads. [#39674](https://github.com/ClickHouse/ClickHouse/pull/39674) ([Anton Kozlov](https://github.com/tonickkozlov)). -* (Only with the obsolete Ordinary databases) Select might read data of dropped table if cache for mmap IO is used and database engine is Ordinary and new tables was created with the same name as dropped one had. It's fixed. [#39708](https://github.com/ClickHouse/ClickHouse/pull/39708) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix possible error `Invalid column type for ColumnUnique::insertRangeFrom. Expected String, got ColumnLowCardinality` Fixes [#38460](https://github.com/ClickHouse/ClickHouse/issues/38460). [#39716](https://github.com/ClickHouse/ClickHouse/pull/39716) ([Arthur Passos](https://github.com/arthurpassos)). -* Field names in the `meta` section of JSON format were erroneously double escaped. This closes [#39693](https://github.com/ClickHouse/ClickHouse/issues/39693). [#39747](https://github.com/ClickHouse/ClickHouse/pull/39747) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix wrong index analysis with tuples and operator `IN`, which could lead to wrong query result. [#39752](https://github.com/ClickHouse/ClickHouse/pull/39752) ([Anton Popov](https://github.com/CurtizJ)). -* Fix `EmbeddedRocksDB` tables filtering by key using params. [#39757](https://github.com/ClickHouse/ClickHouse/pull/39757) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix error `Invalid number of columns in chunk pushed to OutputPort` which was caused by ARRAY JOIN optimization. Fixes [#39164](https://github.com/ClickHouse/ClickHouse/issues/39164). [#39799](https://github.com/ClickHouse/ClickHouse/pull/39799) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* A workaround for a bug in Linux kernel. Fix `CANNOT_READ_ALL_DATA` exception with `local_filesystem_read_method=pread_threadpool`. This bug affected only Linux kernel version 5.9 and 5.10 according to [man](https://manpages.debian.org/testing/manpages-dev/preadv2.2.en.html#BUGS). [#39800](https://github.com/ClickHouse/ClickHouse/pull/39800) ([Anton Popov](https://github.com/CurtizJ)). -* (Only on NFS) Fix broken NFS mkdir for root-squashed volumes. [#39898](https://github.com/ClickHouse/ClickHouse/pull/39898) ([Constantine Peresypkin](https://github.com/pkit)). -* Remove dictionaries from prometheus metrics on DETACH/DROP. [#39926](https://github.com/ClickHouse/ClickHouse/pull/39926) ([Azat Khuzhin](https://github.com/azat)). -* Fix read of StorageFile with virtual columns. Closes [#39907](https://github.com/ClickHouse/ClickHouse/issues/39907). [#39943](https://github.com/ClickHouse/ClickHouse/pull/39943) ([flynn](https://github.com/ucasfl)). -* Fix big memory usage during fetches. Fixes [#39915](https://github.com/ClickHouse/ClickHouse/issues/39915). [#39990](https://github.com/ClickHouse/ClickHouse/pull/39990) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* (experimental feature) Fix `hashId` crash and salt parameter not being used. [#40002](https://github.com/ClickHouse/ClickHouse/pull/40002) ([Raúl Marín](https://github.com/Algunenano)). -* `EXCEPT` and `INTERSECT` operators may lead to crash if a specific combination of constant and non-constant columns were used. [#40020](https://github.com/ClickHouse/ClickHouse/pull/40020) ([Duc Canh Le](https://github.com/canhld94)). -* Fixed "Part directory doesn't exist" and "`tmp_` ... No such file or directory" errors during too slow INSERT or too long merge/mutation. Also fixed issue that may cause some replication queue entries to stuck without any errors or warnings in logs if previous attempt to fetch part failed, but `tmp-fetch_` directory was not cleaned up. [#40031](https://github.com/ClickHouse/ClickHouse/pull/40031) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Fix rare cases of parsing of arrays of tuples in format `Values`. [#40034](https://github.com/ClickHouse/ClickHouse/pull/40034) ([Anton Popov](https://github.com/CurtizJ)). -* Fixes ArrowColumn format Dictionary(X) & Dictionary(Nullable(X)) conversion to ClickHouse LowCardinality(X) & LowCardinality(Nullable(X)) respectively. [#40037](https://github.com/ClickHouse/ClickHouse/pull/40037) ([Arthur Passos](https://github.com/arthurpassos)). -* Fix potential deadlock in writing to S3 during task scheduling failure. [#40070](https://github.com/ClickHouse/ClickHouse/pull/40070) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix bug in collectFilesToSkip() by adding correct file extension (.idx or idx2) for indexes to be recalculated, avoid wrong hard links. Fixed [#39896](https://github.com/ClickHouse/ClickHouse/issues/39896). [#40095](https://github.com/ClickHouse/ClickHouse/pull/40095) ([Jianmei Zhang](https://github.com/zhangjmruc)). -* A fix for reverse DNS resolution. [#40134](https://github.com/ClickHouse/ClickHouse/pull/40134) ([Arthur Passos](https://github.com/arthurpassos)). -* Fix unexpected result `arrayDifference` of `Array(UInt32). [#40211](https://github.com/ClickHouse/ClickHouse/pull/40211) ([Duc Canh Le](https://github.com/canhld94)). - -### ClickHouse release 22.7, 2022-07-21 - -#### Upgrade Notes - -* Enable setting `enable_positional_arguments` by default. It allows queries like `SELECT ... ORDER BY 1, 2` where 1, 2 are the references to the select clause. If you need to return the old behavior, disable this setting. [#38204](https://github.com/ClickHouse/ClickHouse/pull/38204) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Disable `format_csv_allow_single_quotes` by default. See [#37096](https://github.com/ClickHouse/ClickHouse/issues/37096). ([Kruglov Pavel](https://github.com/Avogar)). -* `Ordinary` database engine and old storage definition syntax for `*MergeTree` tables are deprecated. By default it's not possible to create new databases with `Ordinary` engine. If `system` database has `Ordinary` engine it will be automatically converted to `Atomic` on server startup. There are settings to keep old behavior (`allow_deprecated_database_ordinary` and `allow_deprecated_syntax_for_merge_tree`), but these settings may be removed in future releases. [#38335](https://github.com/ClickHouse/ClickHouse/pull/38335) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Force rewriting comma join to inner by default (set default value `cross_to_inner_join_rewrite = 2`). To have old behavior set `cross_to_inner_join_rewrite = 1`. [#39326](https://github.com/ClickHouse/ClickHouse/pull/39326) ([Vladimir C](https://github.com/vdimir)). If you will face any incompatibilities, you can turn this setting back. - -#### New Feature - -* Support expressions with window functions. Closes [#19857](https://github.com/ClickHouse/ClickHouse/issues/19857). [#37848](https://github.com/ClickHouse/ClickHouse/pull/37848) ([Dmitry Novik](https://github.com/novikd)). -* Add new `direct` join algorithm for `EmbeddedRocksDB` tables, see [#33582](https://github.com/ClickHouse/ClickHouse/issues/33582). [#35363](https://github.com/ClickHouse/ClickHouse/pull/35363) ([Vladimir C](https://github.com/vdimir)). -* Added full sorting merge join algorithm. [#35796](https://github.com/ClickHouse/ClickHouse/pull/35796) ([Vladimir C](https://github.com/vdimir)). -* Implement NATS table engine, which allows to pub/sub to NATS. Closes [#32388](https://github.com/ClickHouse/ClickHouse/issues/32388). [#37171](https://github.com/ClickHouse/ClickHouse/pull/37171) ([tchepavel](https://github.com/tchepavel)). ([Kseniia Sumarokova](https://github.com/kssenii)) -* Implement table function `mongodb`. Allow writes into `MongoDB` storage / table function. [#37213](https://github.com/ClickHouse/ClickHouse/pull/37213) ([aaapetrenko](https://github.com/aaapetrenko)). ([Kseniia Sumarokova](https://github.com/kssenii)) -* Add `SQLInsert` output format. Closes [#38441](https://github.com/ClickHouse/ClickHouse/issues/38441). [#38477](https://github.com/ClickHouse/ClickHouse/pull/38477) ([Kruglov Pavel](https://github.com/Avogar)). -* Introduced settings `additional_table_filters`. Using this setting, you can specify additional filtering condition for a table which will be applied directly after reading. Example: `select number, x, y from (select number from system.numbers limit 5) f any left join (select x, y from table_1) s on f.number = s.x settings additional_table_filters={'system.numbers : 'number != 3', 'table_1' : 'x != 2'}`. Introduced setting `additional_result_filter` which specifies additional filtering condition for query result. Closes [#37918](https://github.com/ClickHouse/ClickHouse/issues/37918). [#38475](https://github.com/ClickHouse/ClickHouse/pull/38475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Add `compatibility` setting and `system.settings_changes` system table that contains information about changes in settings through ClickHouse versions. Closes [#35972](https://github.com/ClickHouse/ClickHouse/issues/35972). [#38957](https://github.com/ClickHouse/ClickHouse/pull/38957) ([Kruglov Pavel](https://github.com/Avogar)). -* Add functions `translate(string, from_string, to_string)` and `translateUTF8(string, from_string, to_string)`. It translates some characters to another. [#38935](https://github.com/ClickHouse/ClickHouse/pull/38935) ([Nikolay Degterinsky](https://github.com/evillique)). -* Support `parseTimeDelta` function. It can be used like ` ;-+,:` can be used as separators, eg. `1yr-2mo`, `2m:6s`: `SELECT parseTimeDelta('1yr-2mo-4w + 12 days, 3 hours : 1 minute ; 33 seconds')`. [#39071](https://github.com/ClickHouse/ClickHouse/pull/39071) ([jiahui-97](https://github.com/jiahui-97)). -* Added `CREATE TABLE ... EMPTY AS SELECT` query. It automatically deduces table structure from the SELECT query, but does not fill the table after creation. Resolves [#38049](https://github.com/ClickHouse/ClickHouse/issues/38049). [#38272](https://github.com/ClickHouse/ClickHouse/pull/38272) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Added options to limit IO operations with remote storage: `max_remote_read_network_bandwidth_for_server` and `max_remote_write_network_bandwidth_for_server`. [#39095](https://github.com/ClickHouse/ClickHouse/pull/39095) ([Sergei Trifonov](https://github.com/serxa)). -* Add `group_by_use_nulls` setting to make aggregation key columns nullable in the case of ROLLUP, CUBE and GROUPING SETS. Closes [#37359](https://github.com/ClickHouse/ClickHouse/issues/37359). [#38642](https://github.com/ClickHouse/ClickHouse/pull/38642) ([Dmitry Novik](https://github.com/novikd)). -* Add the ability to specify compression level during data export. [#38907](https://github.com/ClickHouse/ClickHouse/pull/38907) ([Nikolay Degterinsky](https://github.com/evillique)). -* Add an option to require explicit grants to SELECT from the `system` database. Details: [#38970](https://github.com/ClickHouse/ClickHouse/pull/38970) ([Vitaly Baranov](https://github.com/vitlibar)). -* Functions `multiMatchAny`, `multiMatchAnyIndex`, `multiMatchAllIndices` and their fuzzy variants now accept non-const pattern array argument. [#38485](https://github.com/ClickHouse/ClickHouse/pull/38485) ([Robert Schulze](https://github.com/rschu1ze)). SQL function `multiSearchAllPositions` now accepts non-const needle arguments. [#39167](https://github.com/ClickHouse/ClickHouse/pull/39167) ([Robert Schulze](https://github.com/rschu1ze)). -* Add a setting `zstd_window_log_max` to configure max memory usage on zstd decoding when importing external files. Closes [#35693](https://github.com/ClickHouse/ClickHouse/issues/35693). [#37015](https://github.com/ClickHouse/ClickHouse/pull/37015) ([wuxiaobai24](https://github.com/wuxiaobai24)). -* Add `send_logs_source_regexp` setting. Send server text logs with specified regexp to match log source name. Empty means all sources. [#39161](https://github.com/ClickHouse/ClickHouse/pull/39161) ([Amos Bird](https://github.com/amosbird)). -* Support `ALTER` for `Hive` tables. [#38214](https://github.com/ClickHouse/ClickHouse/pull/38214) ([lgbo](https://github.com/lgbo-ustc)). -* Support `isNullable` function. This function checks whether it's argument is nullable and return 1 or 0. Closes [#38611](https://github.com/ClickHouse/ClickHouse/issues/38611). [#38841](https://github.com/ClickHouse/ClickHouse/pull/38841) ([lokax](https://github.com/lokax)). -* Added functions for base58 encoding/decoding. [#38159](https://github.com/ClickHouse/ClickHouse/pull/38159) ([Andrey Zvonov](https://github.com/zvonand)). -* Add chart visualization to Play UI. [#38197](https://github.com/ClickHouse/ClickHouse/pull/38197) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Added L2 Squared distance and norm functions for both arrays and tuples. [#38545](https://github.com/ClickHouse/ClickHouse/pull/38545) ([Julian Gilyadov](https://github.com/israelg99)). -* Add ability to pass HTTP headers to the `url` table function / storage via SQL. Closes [#37897](https://github.com/ClickHouse/ClickHouse/issues/37897). [#38176](https://github.com/ClickHouse/ClickHouse/pull/38176) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add `clickhouse-diagnostics` binary to the packages. [#38647](https://github.com/ClickHouse/ClickHouse/pull/38647) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - -#### Experimental Feature - -* Adds new setting `implicit_transaction` to run standalone queries inside a transaction. It handles both creation and closing (via COMMIT if the query succeeded or ROLLBACK if it didn't) of the transaction automatically. [#38344](https://github.com/ClickHouse/ClickHouse/pull/38344) ([Raúl Marín](https://github.com/Algunenano)). - -#### Performance Improvement - -* Distinct optimization for sorted columns. Use specialized distinct transformation in case input stream is sorted by column(s) in distinct. Optimization can be applied to pre-distinct, final distinct, or both. Initial implementation by @dimarub2000. [#37803](https://github.com/ClickHouse/ClickHouse/pull/37803) ([Igor Nikonov](https://github.com/devcrafter)). -* Improve performance of `ORDER BY`, `MergeTree` merges, window functions using batch version of `BinaryHeap`. [#38022](https://github.com/ClickHouse/ClickHouse/pull/38022) ([Maksim Kita](https://github.com/kitaisreal)). -* More parallel execution for queries with `FINAL` [#36396](https://github.com/ClickHouse/ClickHouse/pull/36396) ([Nikita Taranov](https://github.com/nickitat)). -* Fix significant join performance regression which was introduced in [#35616](https://github.com/ClickHouse/ClickHouse/pull/35616). It's interesting that common join queries such as ssb queries have been 10 times slower for almost 3 months while no one complains. [#38052](https://github.com/ClickHouse/ClickHouse/pull/38052) ([Amos Bird](https://github.com/amosbird)). -* Migrate from the Intel hyperscan library to vectorscan, this speeds up many string matching on non-x86 platforms. [#38171](https://github.com/ClickHouse/ClickHouse/pull/38171) ([Robert Schulze](https://github.com/rschu1ze)). -* Increased parallelism of query plan steps executed after aggregation. [#38295](https://github.com/ClickHouse/ClickHouse/pull/38295) ([Nikita Taranov](https://github.com/nickitat)). -* Improve performance of insertion to columns of type `JSON`. [#38320](https://github.com/ClickHouse/ClickHouse/pull/38320) ([Anton Popov](https://github.com/CurtizJ)). -* Optimized insertion and lookups in the HashTable. [#38413](https://github.com/ClickHouse/ClickHouse/pull/38413) ([Nikita Taranov](https://github.com/nickitat)). -* Fix performance degradation from [#32493](https://github.com/ClickHouse/ClickHouse/issues/32493). [#38417](https://github.com/ClickHouse/ClickHouse/pull/38417) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Improve performance of joining with numeric columns using SIMD instructions. [#37235](https://github.com/ClickHouse/ClickHouse/pull/37235) ([zzachimed](https://github.com/zzachimed)). [#38565](https://github.com/ClickHouse/ClickHouse/pull/38565) ([Maksim Kita](https://github.com/kitaisreal)). -* Norm and Distance functions for arrays speed up 1.2-2 times. [#38740](https://github.com/ClickHouse/ClickHouse/pull/38740) ([Alexander Gololobov](https://github.com/davenger)). -* Add AVX-512 VBMI optimized `copyOverlap32Shuffle` for LZ4 decompression. In other words, LZ4 decompression performance is improved. [#37891](https://github.com/ClickHouse/ClickHouse/pull/37891) ([Guo Wangyang](https://github.com/guowangy)). -* `ORDER BY (a, b)` will use all the same benefits as `ORDER BY a, b`. [#38873](https://github.com/ClickHouse/ClickHouse/pull/38873) ([Igor Nikonov](https://github.com/devcrafter)). -* Align branches within a 32B boundary to make benchmark more stable. [#38988](https://github.com/ClickHouse/ClickHouse/pull/38988) ([Guo Wangyang](https://github.com/guowangy)). It improves performance 1..2% on average for Intel. -* Executable UDF, executable dictionaries, and Executable tables will avoid wasting one second during wait for subprocess termination. [#38929](https://github.com/ClickHouse/ClickHouse/pull/38929) ([Constantine Peresypkin](https://github.com/pkit)). -* Optimize accesses to `system.stack_trace` table if not all columns are selected. [#39177](https://github.com/ClickHouse/ClickHouse/pull/39177) ([Azat Khuzhin](https://github.com/azat)). -* Improve isNullable/isConstant/isNull/isNotNull performance for LowCardinality argument. [#39192](https://github.com/ClickHouse/ClickHouse/pull/39192) ([Kruglov Pavel](https://github.com/Avogar)). -* Optimized processing of ORDER BY in window functions. [#34632](https://github.com/ClickHouse/ClickHouse/pull/34632) ([Vladimir Chebotarev](https://github.com/excitoon)). -* The table `system.asynchronous_metric_log` is further optimized for storage space. This closes [#38134](https://github.com/ClickHouse/ClickHouse/issues/38134). See the [YouTube video](https://www.youtube.com/watch?v=0fSp9SF8N8A). [#38428](https://github.com/ClickHouse/ClickHouse/pull/38428) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### Improvement - -* Support SQL standard CREATE INDEX and DROP INDEX syntax. [#35166](https://github.com/ClickHouse/ClickHouse/pull/35166) ([Jianmei Zhang](https://github.com/zhangjmruc)). -* Send profile events for INSERT queries (previously only SELECT was supported). [#37391](https://github.com/ClickHouse/ClickHouse/pull/37391) ([Azat Khuzhin](https://github.com/azat)). -* Implement in order aggregation (`optimize_aggregation_in_order`) for fully materialized projections. [#37469](https://github.com/ClickHouse/ClickHouse/pull/37469) ([Azat Khuzhin](https://github.com/azat)). -* Remove subprocess run for Kerberos initialization. Added new integration test. Closes [#27651](https://github.com/ClickHouse/ClickHouse/issues/27651). [#38105](https://github.com/ClickHouse/ClickHouse/pull/38105) ([Roman Vasin](https://github.com/rvasin)). -* * Add setting `multiple_joins_try_to_keep_original_names` to not rewrite identifier name on multiple JOINs rewrite, close [#34697](https://github.com/ClickHouse/ClickHouse/issues/34697). [#38149](https://github.com/ClickHouse/ClickHouse/pull/38149) ([Vladimir C](https://github.com/vdimir)). -* Improved trace-visualizer UX. [#38169](https://github.com/ClickHouse/ClickHouse/pull/38169) ([Sergei Trifonov](https://github.com/serxa)). -* Enable stack trace collection and query profiler for AArch64. [#38181](https://github.com/ClickHouse/ClickHouse/pull/38181) ([Maksim Kita](https://github.com/kitaisreal)). -* Do not skip symlinks in `user_defined` directory during SQL user defined functions loading. Closes [#38042](https://github.com/ClickHouse/ClickHouse/issues/38042). [#38184](https://github.com/ClickHouse/ClickHouse/pull/38184) ([Maksim Kita](https://github.com/kitaisreal)). -* Added background cleanup of subdirectories in `store/`. In some cases clickhouse-server might left garbage subdirectories in `store/` (for example, on unsuccessful table creation) and those dirs were never been removed. Fixes [#33710](https://github.com/ClickHouse/ClickHouse/issues/33710). [#38265](https://github.com/ClickHouse/ClickHouse/pull/38265) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Add `DESCRIBE CACHE` query to show cache settings from config. Add `SHOW CACHES` query to show available filesystem caches list. [#38279](https://github.com/ClickHouse/ClickHouse/pull/38279) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add access check for `system drop filesystem cache`. Support ON CLUSTER. [#38319](https://github.com/ClickHouse/ClickHouse/pull/38319) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix PostgreSQL database engine incompatibility on upgrade from 21.3 to 22.3. Closes [#36659](https://github.com/ClickHouse/ClickHouse/issues/36659). [#38369](https://github.com/ClickHouse/ClickHouse/pull/38369) ([Kseniia Sumarokova](https://github.com/kssenii)). -* `filesystemAvailable` and similar functions now work in `clickhouse-local`. This closes [#38423](https://github.com/ClickHouse/ClickHouse/issues/38423). [#38424](https://github.com/ClickHouse/ClickHouse/pull/38424) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add `revision` function. [#38555](https://github.com/ClickHouse/ClickHouse/pull/38555) ([Azat Khuzhin](https://github.com/azat)). -* Fix GCS via proxy tunnel usage. [#38726](https://github.com/ClickHouse/ClickHouse/pull/38726) ([Azat Khuzhin](https://github.com/azat)). -* Support `\i file` in clickhouse client / local (similar to psql \i). [#38813](https://github.com/ClickHouse/ClickHouse/pull/38813) ([Kseniia Sumarokova](https://github.com/kssenii)). -* New option `optimize = 1` in `EXPLAIN AST`. If enabled, it shows AST after it's rewritten, otherwise AST of original query. Disabled by default. [#38910](https://github.com/ClickHouse/ClickHouse/pull/38910) ([Igor Nikonov](https://github.com/devcrafter)). -* Allow trailing comma in columns list. closes [#38425](https://github.com/ClickHouse/ClickHouse/issues/38425). [#38440](https://github.com/ClickHouse/ClickHouse/pull/38440) ([chen](https://github.com/xiedeyantu)). -* Bugfixes and performance improvements for `parallel_hash` JOIN method. [#37648](https://github.com/ClickHouse/ClickHouse/pull/37648) ([Vladimir C](https://github.com/vdimir)). -* Support hadoop secure RPC transfer (hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#37852](https://github.com/ClickHouse/ClickHouse/pull/37852) ([Peng Liu](https://github.com/michael1589)). -* Add struct type support in `StorageHive`. [#38118](https://github.com/ClickHouse/ClickHouse/pull/38118) ([lgbo](https://github.com/lgbo-ustc)). -* S3 single objects are now removed with `RemoveObjectRequest`. Implement compatibility with GCP which did not allow to use `removeFileIfExists` effectively breaking approximately half of `remove` functionality. Automatic detection for `DeleteObjects` S3 API, that is not supported by GCS. This will allow to use GCS without explicit `support_batch_delete=0` in configuration. [#37882](https://github.com/ClickHouse/ClickHouse/pull/37882) ([Vladimir Chebotarev](https://github.com/excitoon)). -* Expose basic ClickHouse Keeper related monitoring data (via ProfileEvents and CurrentMetrics). [#38072](https://github.com/ClickHouse/ClickHouse/pull/38072) ([lingpeng0314](https://github.com/lingpeng0314)). -* Support `auto_close` option for PostgreSQL engine connection. Closes [#31486](https://github.com/ClickHouse/ClickHouse/issues/31486). [#38363](https://github.com/ClickHouse/ClickHouse/pull/38363) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Allow `NULL` modifier in columns declaration for table functions. [#38816](https://github.com/ClickHouse/ClickHouse/pull/38816) ([Kruglov Pavel](https://github.com/Avogar)). -* Deactivate `mutations_finalizing_task` before shutdown to avoid benign `TABLE_IS_READ_ONLY` errors during shutdown. [#38851](https://github.com/ClickHouse/ClickHouse/pull/38851) ([Raúl Marín](https://github.com/Algunenano)). -* Eliminate unnecessary waiting of SELECT queries after ALTER queries in presence of INSERT queries if you use deprecated Ordinary databases. [#38864](https://github.com/ClickHouse/ClickHouse/pull/38864) ([Azat Khuzhin](https://github.com/azat)). -* New option `rewrite` in `EXPLAIN AST`. If enabled, it shows AST after it's rewritten, otherwise AST of original query. Disabled by default. [#38910](https://github.com/ClickHouse/ClickHouse/pull/38910) ([Igor Nikonov](https://github.com/devcrafter)). -* Stop reporting Zookeeper "Node exists" exceptions in system.errors when they are expected. [#38961](https://github.com/ClickHouse/ClickHouse/pull/38961) ([Raúl Marín](https://github.com/Algunenano)). -* `clickhouse-keeper`: add support for real-time digest calculation and verification. It is disabled by default. [#37555](https://github.com/ClickHouse/ClickHouse/pull/37555) ([Antonio Andelic](https://github.com/antonio2368)). -* Allow to specify globs `* or {expr1, expr2, expr3}` inside a key for `clickhouse-extract-from-config` tool. [#38966](https://github.com/ClickHouse/ClickHouse/pull/38966) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* clearOldLogs: Don't report KEEPER_EXCEPTION on concurrent deletes. [#39016](https://github.com/ClickHouse/ClickHouse/pull/39016) ([Raúl Marín](https://github.com/Algunenano)). -* clickhouse-keeper improvement: persist meta-information about keeper servers to disk. [#39069](https://github.com/ClickHouse/ClickHouse/pull/39069) ([Antonio Andelic](https://github.com/antonio2368)). This will make it easier to operate if you shutdown or restart all keeper nodes at the same time. -* Continue without exception when running out of disk space when using filesystem cache. [#39106](https://github.com/ClickHouse/ClickHouse/pull/39106) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Handling SIGTERM signals from k8s. [#39130](https://github.com/ClickHouse/ClickHouse/pull/39130) ([Timur Solodovnikov](https://github.com/tsolodov)). -* Add `merge_algorithm` column (Undecided, Horizontal, Vertical) to system.part_log. [#39181](https://github.com/ClickHouse/ClickHouse/pull/39181) ([Azat Khuzhin](https://github.com/azat)). -* Don't increment a counter in `system.errors` when the disk is not rotational. [#39216](https://github.com/ClickHouse/ClickHouse/pull/39216) ([Raúl Marín](https://github.com/Algunenano)). -* The metric `result_bytes` for `INSERT` queries in `system.query_log` shows number of bytes inserted. Previously value was incorrect and stored the same value as `result_rows`. [#39225](https://github.com/ClickHouse/ClickHouse/pull/39225) ([Ilya Yatsishin](https://github.com/qoega)). -* The CPU usage metric in clickhouse-client will be displayed in a better way. Fixes [#38756](https://github.com/ClickHouse/ClickHouse/issues/38756). [#39280](https://github.com/ClickHouse/ClickHouse/pull/39280) ([Sergei Trifonov](https://github.com/serxa)). -* Rethrow exception on filesystem cache initialization on server startup, better error message. [#39386](https://github.com/ClickHouse/ClickHouse/pull/39386) ([Kseniia Sumarokova](https://github.com/kssenii)). -* OpenTelemetry now collects traces without Processors spans by default (there are too many). To enable Processors spans collection `opentelemetry_trace_processors` setting. [#39170](https://github.com/ClickHouse/ClickHouse/pull/39170) ([Ilya Yatsishin](https://github.com/qoega)). -* Functions `multiMatch[Fuzzy](AllIndices/Any/AnyIndex)` - don't throw a logical error if the needle argument is empty. [#39012](https://github.com/ClickHouse/ClickHouse/pull/39012) ([Robert Schulze](https://github.com/rschu1ze)). -* Allow to declare `RabbitMQ` queue without default arguments `x-max-length` and `x-overflow`. [#39259](https://github.com/ClickHouse/ClickHouse/pull/39259) ([rnbondarenko](https://github.com/rnbondarenko)). - -#### Build/Testing/Packaging Improvement - -* Apply Clang Thread Safety Analysis (TSA) annotations to ClickHouse. [#38068](https://github.com/ClickHouse/ClickHouse/pull/38068) ([Robert Schulze](https://github.com/rschu1ze)). -* Adapt universal installation script for FreeBSD. [#39302](https://github.com/ClickHouse/ClickHouse/pull/39302) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Preparation for building on `s390x` platform. [#39193](https://github.com/ClickHouse/ClickHouse/pull/39193) ([Harry Lee](https://github.com/HarryLeeIBM)). -* Fix a bug in `jemalloc` library [#38757](https://github.com/ClickHouse/ClickHouse/pull/38757) ([Azat Khuzhin](https://github.com/azat)). -* Hardware benchmark now has support for automatic results uploading. [#38427](https://github.com/ClickHouse/ClickHouse/pull/38427) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* System table "system.licenses" is now correctly populated on Mac (Darwin). [#38294](https://github.com/ClickHouse/ClickHouse/pull/38294) ([Robert Schulze](https://github.com/rschu1ze)). -* Change `all|noarch` packages to architecture-dependent - Fix some documentation for it - Push aarch64|arm64 packages to artifactory and release assets - Fixes [#36443](https://github.com/ClickHouse/ClickHouse/issues/36443). [#38580](https://github.com/ClickHouse/ClickHouse/pull/38580) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - -#### Bug Fix (user-visible misbehavior in official stable or prestable release) - -* Fix rounding for `Decimal128/Decimal256` with more than 19-digits long scale. [#38027](https://github.com/ClickHouse/ClickHouse/pull/38027) ([Igor Nikonov](https://github.com/devcrafter)). -* Fixed crash caused by data race in storage `Hive` (integration table engine). [#38887](https://github.com/ClickHouse/ClickHouse/pull/38887) ([lgbo](https://github.com/lgbo-ustc)). -* Fix crash when executing GRANT ALL ON *.* with ON CLUSTER. It was broken in https://github.com/ClickHouse/ClickHouse/pull/35767. This closes [#38618](https://github.com/ClickHouse/ClickHouse/issues/38618). [#38674](https://github.com/ClickHouse/ClickHouse/pull/38674) ([Vitaly Baranov](https://github.com/vitlibar)). -* Correct glob expansion in case of `{0..10}` forms. Fixes [#38498](https://github.com/ClickHouse/ClickHouse/issues/38498) Current Implementation is similar to what shell does mentiond by @rschu1ze [here](https://github.com/ClickHouse/ClickHouse/pull/38502#issuecomment-1169057723). [#38502](https://github.com/ClickHouse/ClickHouse/pull/38502) ([Heena Bansal](https://github.com/HeenaBansal2009)). -* Fix crash for `mapUpdate`, `mapFilter` functions when using with constant map argument. Closes [#38547](https://github.com/ClickHouse/ClickHouse/issues/38547). [#38553](https://github.com/ClickHouse/ClickHouse/pull/38553) ([hexiaoting](https://github.com/hexiaoting)). -* Fix `toHour` monotonicity information for query optimization which can lead to incorrect query result (incorrect index analysis). This fixes [#38333](https://github.com/ClickHouse/ClickHouse/issues/38333). [#38675](https://github.com/ClickHouse/ClickHouse/pull/38675) ([Amos Bird](https://github.com/amosbird)). -* Fix checking whether s3 storage support parallel writes. It resulted in s3 parallel writes not working. [#38792](https://github.com/ClickHouse/ClickHouse/pull/38792) ([chen](https://github.com/xiedeyantu)). -* Fix s3 seekable reads with parallel read buffer. (Affected memory usage during query). Closes [#38258](https://github.com/ClickHouse/ClickHouse/issues/38258). [#38802](https://github.com/ClickHouse/ClickHouse/pull/38802) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Update `simdjson`. This fixes [#38621](https://github.com/ClickHouse/ClickHouse/issues/38621) - a buffer overflow on machines with the latest Intel CPUs with AVX-512 VBMI. [#38838](https://github.com/ClickHouse/ClickHouse/pull/38838) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix possible logical error for Vertical merges. [#38859](https://github.com/ClickHouse/ClickHouse/pull/38859) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix settings profile with seconds unit. [#38896](https://github.com/ClickHouse/ClickHouse/pull/38896) ([Raúl Marín](https://github.com/Algunenano)). -* Fix incorrect partition pruning when there is a nullable partition key. Note: most likely you don't use nullable partition keys - this is an obscure feature you should not use. Nullable keys are a nonsense and this feature is only needed for some crazy use-cases. This fixes [#38941](https://github.com/ClickHouse/ClickHouse/issues/38941). [#38946](https://github.com/ClickHouse/ClickHouse/pull/38946) ([Amos Bird](https://github.com/amosbird)). -* Improve `fsync_part_directory` for fetches. [#38993](https://github.com/ClickHouse/ClickHouse/pull/38993) ([Azat Khuzhin](https://github.com/azat)). -* Fix possible dealock inside `OvercommitTracker`. Fixes [#37794](https://github.com/ClickHouse/ClickHouse/issues/37794). [#39030](https://github.com/ClickHouse/ClickHouse/pull/39030) ([Dmitry Novik](https://github.com/novikd)). -* Fix bug in filesystem cache that could happen in some corner case which coincided with cache capacity hitting the limit. Closes [#39066](https://github.com/ClickHouse/ClickHouse/issues/39066). [#39070](https://github.com/ClickHouse/ClickHouse/pull/39070) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix some corner cases of interpretation of the arguments of window expressions. Fixes [#38538](https://github.com/ClickHouse/ClickHouse/issues/38538) Allow using of higher-order functions in window expressions. [#39112](https://github.com/ClickHouse/ClickHouse/pull/39112) ([Dmitry Novik](https://github.com/novikd)). -* Keep `LowCardinality` type in `tuple` function. Previously `LowCardinality` type was dropped and elements of created tuple had underlying type of `LowCardinality`. [#39113](https://github.com/ClickHouse/ClickHouse/pull/39113) ([Anton Popov](https://github.com/CurtizJ)). -* Fix error `Block structure mismatch` which could happen for INSERT into table with attached MATERIALIZED VIEW and enabled setting `extremes = 1`. Closes [#29759](https://github.com/ClickHouse/ClickHouse/issues/29759) and [#38729](https://github.com/ClickHouse/ClickHouse/issues/38729). [#39125](https://github.com/ClickHouse/ClickHouse/pull/39125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix unexpected query result when both `optimize_trivial_count_query` and `empty_result_for_aggregation_by_empty_set` are set to true. This fixes [#39140](https://github.com/ClickHouse/ClickHouse/issues/39140). [#39155](https://github.com/ClickHouse/ClickHouse/pull/39155) ([Amos Bird](https://github.com/amosbird)). -* Fixed error `Not found column Type in block` in selects with `PREWHERE` and read-in-order optimizations. [#39157](https://github.com/ClickHouse/ClickHouse/pull/39157) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Fix extremely rare race condition in during hardlinks for remote filesystem. The only way to reproduce it is concurrent run of backups. [#39190](https://github.com/ClickHouse/ClickHouse/pull/39190) ([alesapin](https://github.com/alesapin)). -* (zero-copy replication is an experimental feature that should not be used in production) Fix fetch of in-memory part with `allow_remote_fs_zero_copy_replication`. [#39214](https://github.com/ClickHouse/ClickHouse/pull/39214) ([Azat Khuzhin](https://github.com/azat)). -* (MaterializedPostgreSQL - experimental feature). Fix segmentation fault in MaterializedPostgreSQL database engine, which could happen if some exception occurred at replication initialisation. Closes [#36939](https://github.com/ClickHouse/ClickHouse/issues/36939). [#39272](https://github.com/ClickHouse/ClickHouse/pull/39272) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix incorrect fetch of table metadata from PostgreSQL database engine. Closes [#33502](https://github.com/ClickHouse/ClickHouse/issues/33502). [#39283](https://github.com/ClickHouse/ClickHouse/pull/39283) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix projection exception when aggregation keys are wrapped inside other functions. This fixes [#37151](https://github.com/ClickHouse/ClickHouse/issues/37151). [#37155](https://github.com/ClickHouse/ClickHouse/pull/37155) ([Amos Bird](https://github.com/amosbird)). -* Fix possible logical error `... with argument with type Nothing and default implementation for Nothing is expected to return result with type Nothing, got ...` in some functions. Closes: [#37610](https://github.com/ClickHouse/ClickHouse/issues/37610) Closes: [#37741](https://github.com/ClickHouse/ClickHouse/issues/37741). [#37759](https://github.com/ClickHouse/ClickHouse/pull/37759) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix incorrect columns order in subqueries of UNION (in case of duplicated columns in subselects may produce incorrect result). [#37887](https://github.com/ClickHouse/ClickHouse/pull/37887) ([Azat Khuzhin](https://github.com/azat)). -* Fix incorrect work of MODIFY ALTER Column with column names that contain dots. Closes [#37907](https://github.com/ClickHouse/ClickHouse/issues/37907). [#37971](https://github.com/ClickHouse/ClickHouse/pull/37971) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible crash in `Distributed` async insert in case of removing a replica from config. [#38029](https://github.com/ClickHouse/ClickHouse/pull/38029) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix "Missing columns" for GLOBAL JOIN with CTE without alias. [#38056](https://github.com/ClickHouse/ClickHouse/pull/38056) ([Azat Khuzhin](https://github.com/azat)). -* Rewrite tuple functions as literals in backwards-compatibility mode. [#38096](https://github.com/ClickHouse/ClickHouse/pull/38096) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Fix redundant memory reservation for output block during `ORDER BY`. [#38127](https://github.com/ClickHouse/ClickHouse/pull/38127) ([iyupeng](https://github.com/iyupeng)). -* Fix possible logical error `Bad cast from type DB::IColumn* to DB::ColumnNullable*` in array mapped functions. Closes [#38006](https://github.com/ClickHouse/ClickHouse/issues/38006). [#38132](https://github.com/ClickHouse/ClickHouse/pull/38132) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix temporary name clash in partial merge join, close [#37928](https://github.com/ClickHouse/ClickHouse/issues/37928). [#38135](https://github.com/ClickHouse/ClickHouse/pull/38135) ([Vladimir C](https://github.com/vdimir)). -* Some minr issue with queries like `CREATE TABLE nested_name_tuples (`a` Tuple(x String, y Tuple(i Int32, j String))) ENGINE = Memory;` [#38136](https://github.com/ClickHouse/ClickHouse/pull/38136) ([lgbo](https://github.com/lgbo-ustc)). -* Fix bug with nested short-circuit functions that led to execution of arguments even if condition is false. Closes [#38040](https://github.com/ClickHouse/ClickHouse/issues/38040). [#38173](https://github.com/ClickHouse/ClickHouse/pull/38173) ([Kruglov Pavel](https://github.com/Avogar)). -* (Window View is a experimental feature) Fix LOGICAL_ERROR for WINDOW VIEW with incorrect structure. [#38205](https://github.com/ClickHouse/ClickHouse/pull/38205) ([Azat Khuzhin](https://github.com/azat)). -* Update librdkafka submodule to fix crash when an OAUTHBEARER refresh callback is set. [#38225](https://github.com/ClickHouse/ClickHouse/pull/38225) ([Rafael Acevedo](https://github.com/racevedoo)). -* Fix INSERT into Distributed hung due to ProfileEvents. [#38307](https://github.com/ClickHouse/ClickHouse/pull/38307) ([Azat Khuzhin](https://github.com/azat)). -* Fix retries in PostgreSQL engine. [#38310](https://github.com/ClickHouse/ClickHouse/pull/38310) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix optimization in PartialSortingTransform (SIGSEGV and possible incorrect result). [#38324](https://github.com/ClickHouse/ClickHouse/pull/38324) ([Azat Khuzhin](https://github.com/azat)). -* Fix RabbitMQ with formats based on PeekableReadBuffer. Closes [#38061](https://github.com/ClickHouse/ClickHouse/issues/38061). [#38356](https://github.com/ClickHouse/ClickHouse/pull/38356) ([Kseniia Sumarokova](https://github.com/kssenii)). -* MaterializedPostgreSQL - experimentail feature. Fix possible `Invalid number of rows in Chunk` in MaterializedPostgreSQL. Closes [#37323](https://github.com/ClickHouse/ClickHouse/issues/37323). [#38360](https://github.com/ClickHouse/ClickHouse/pull/38360) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix RabbitMQ configuration with connection string setting. Closes [#36531](https://github.com/ClickHouse/ClickHouse/issues/36531). [#38365](https://github.com/ClickHouse/ClickHouse/pull/38365) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix PostgreSQL engine not using PostgreSQL schema when retrieving array dimension size. Closes [#36755](https://github.com/ClickHouse/ClickHouse/issues/36755). Closes [#36772](https://github.com/ClickHouse/ClickHouse/issues/36772). [#38366](https://github.com/ClickHouse/ClickHouse/pull/38366) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possibly incorrect result of distributed queries with `DISTINCT` and `LIMIT`. Fixes [#38282](https://github.com/ClickHouse/ClickHouse/issues/38282). [#38371](https://github.com/ClickHouse/ClickHouse/pull/38371) ([Anton Popov](https://github.com/CurtizJ)). -* Fix wrong results of countSubstrings() & position() on patterns with 0-bytes. [#38589](https://github.com/ClickHouse/ClickHouse/pull/38589) ([Robert Schulze](https://github.com/rschu1ze)). -* Now it's possible to start a clickhouse-server and attach/detach tables even for tables with the incorrect values of IPv4/IPv6 representation. Proper fix for issue [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#38590](https://github.com/ClickHouse/ClickHouse/pull/38590) ([alesapin](https://github.com/alesapin)). -* `rankCorr` function will work correctly if some arguments are NaNs. This closes [#38396](https://github.com/ClickHouse/ClickHouse/issues/38396). [#38722](https://github.com/ClickHouse/ClickHouse/pull/38722) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)). -* Fix use-after-free for aggregate functions with `Map` combinator that leads to incorrect result. [#38748](https://github.com/ClickHouse/ClickHouse/pull/38748) ([Azat Khuzhin](https://github.com/azat)). - -### ClickHouse release 22.6, 2022-06-16 - -#### Backward Incompatible Change - -* Remove support for octal number literals in SQL. In previous versions they were parsed as Float64. [#37765](https://github.com/ClickHouse/ClickHouse/pull/37765) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Changes how settings using `seconds` as type are parsed to support floating point values (for example: `max_execution_time=0.5`). Infinity or NaN values will throw an exception. [#37187](https://github.com/ClickHouse/ClickHouse/pull/37187) ([Raúl Marín](https://github.com/Algunenano)). -* Changed format of binary serialization of columns of experimental type `Object`. New format is more convenient to implement by third-party clients. [#37482](https://github.com/ClickHouse/ClickHouse/pull/37482) ([Anton Popov](https://github.com/CurtizJ)). -* Turn on setting `output_format_json_named_tuples_as_objects` by default. It allows to serialize named tuples as JSON objects in JSON formats. [#37756](https://github.com/ClickHouse/ClickHouse/pull/37756) ([Anton Popov](https://github.com/CurtizJ)). -* LIKE patterns with trailing escape symbol ('\\') are now disallowed (as mandated by the SQL standard). [#37764](https://github.com/ClickHouse/ClickHouse/pull/37764) ([Robert Schulze](https://github.com/rschu1ze)). -* If you run different ClickHouse versions on a cluster with AArch64 CPU or mix AArch64 and amd64 on a cluster, and use distributed queries with GROUP BY multiple keys of fixed-size type that fit in 256 bits but don't fit in 64 bits, and the size of the result is huge, the data will not be fully aggregated in the result of these queries during upgrade. Workaround: upgrade with downtime instead of a rolling upgrade. - -#### New Feature - -* Add `GROUPING` function. It allows to disambiguate the records in the queries with `ROLLUP`, `CUBE` or `GROUPING SETS`. Closes [#19426](https://github.com/ClickHouse/ClickHouse/issues/19426). [#37163](https://github.com/ClickHouse/ClickHouse/pull/37163) ([Dmitry Novik](https://github.com/novikd)). -* A new codec [FPC](https://userweb.cs.txstate.edu/~burtscher/papers/dcc07a.pdf) algorithm for floating point data compression. [#37553](https://github.com/ClickHouse/ClickHouse/pull/37553) ([Mikhail Guzov](https://github.com/koloshmet)). -* Add new columnar JSON formats: `JSONColumns`, `JSONCompactColumns`, `JSONColumnsWithMetadata`. Closes [#36338](https://github.com/ClickHouse/ClickHouse/issues/36338) Closes [#34509](https://github.com/ClickHouse/ClickHouse/issues/34509). [#36975](https://github.com/ClickHouse/ClickHouse/pull/36975) ([Kruglov Pavel](https://github.com/Avogar)). -* Added open telemetry traces visualizing tool based on d3js. [#37810](https://github.com/ClickHouse/ClickHouse/pull/37810) ([Sergei Trifonov](https://github.com/serxa)). -* Support INSERTs into `system.zookeeper` table. Closes [#22130](https://github.com/ClickHouse/ClickHouse/issues/22130). [#37596](https://github.com/ClickHouse/ClickHouse/pull/37596) ([Han Fei](https://github.com/hanfei1991)). -* Support non-constant pattern argument for `LIKE`, `ILIKE` and `match` functions. [#37251](https://github.com/ClickHouse/ClickHouse/pull/37251) ([Robert Schulze](https://github.com/rschu1ze)). -* Executable user defined functions now support parameters. Example: `SELECT test_function(parameters)(arguments)`. Closes [#37578](https://github.com/ClickHouse/ClickHouse/issues/37578). [#37720](https://github.com/ClickHouse/ClickHouse/pull/37720) ([Maksim Kita](https://github.com/kitaisreal)). -* Add `merge_reason` column to system.part_log table. [#36912](https://github.com/ClickHouse/ClickHouse/pull/36912) ([Sema Checherinda](https://github.com/CheSema)). -* Add support for Maps and Records in Avro format. Add new setting `input_format_avro_null_as_default ` that allow to insert null as default in Avro format. Closes [#18925](https://github.com/ClickHouse/ClickHouse/issues/18925) Closes [#37378](https://github.com/ClickHouse/ClickHouse/issues/37378) Closes [#32899](https://github.com/ClickHouse/ClickHouse/issues/32899). [#37525](https://github.com/ClickHouse/ClickHouse/pull/37525) ([Kruglov Pavel](https://github.com/Avogar)). -* Add `clickhouse-disks` tool to introspect and operate on virtual filesystems configured for ClickHouse. [#36060](https://github.com/ClickHouse/ClickHouse/pull/36060) ([Artyom Yurkov](https://github.com/Varinara)). -* Adds H3 unidirectional edge functions. [#36843](https://github.com/ClickHouse/ClickHouse/pull/36843) ([Bharat Nallan](https://github.com/bharatnc)). -* Add support for calculating [hashids](https://hashids.org/) from unsigned integers. [#37013](https://github.com/ClickHouse/ClickHouse/pull/37013) ([Michael Nutt](https://github.com/mnutt)). -* Explicit `SALT` specification is allowed for `CREATE USER IDENTIFIED WITH sha256_hash`. [#37377](https://github.com/ClickHouse/ClickHouse/pull/37377) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Add two new settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines` to allow skipping specified number of lines in the beginning of the file in CSV/TSV formats. [#37537](https://github.com/ClickHouse/ClickHouse/pull/37537) ([Kruglov Pavel](https://github.com/Avogar)). -* `showCertificate` function shows current server's SSL certificate. [#37540](https://github.com/ClickHouse/ClickHouse/pull/37540) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* HTTP source for Data Dictionaries in Named Collections is supported. [#37581](https://github.com/ClickHouse/ClickHouse/pull/37581) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Implemented changing the comment for `ReplicatedMergeTree` tables. [#37416](https://github.com/ClickHouse/ClickHouse/pull/37416) ([Vasily Nemkov](https://github.com/Enmk)). -* Added `SYSTEM UNFREEZE` query that deletes the whole backup regardless if the corresponding table is deleted or not. [#36424](https://github.com/ClickHouse/ClickHouse/pull/36424) ([Vadim Volodin](https://github.com/PolyProgrammist)). - -#### Experimental Feature - -* Enables `POPULATE` for `WINDOW VIEW`. [#36945](https://github.com/ClickHouse/ClickHouse/pull/36945) ([vxider](https://github.com/Vxider)). -* `ALTER TABLE ... MODIFY QUERY` support for `WINDOW VIEW`. [#37188](https://github.com/ClickHouse/ClickHouse/pull/37188) ([vxider](https://github.com/Vxider)). -* This PR changes the behavior of the `ENGINE` syntax in `WINDOW VIEW`, to make it like in `MATERIALIZED VIEW`. [#37214](https://github.com/ClickHouse/ClickHouse/pull/37214) ([vxider](https://github.com/Vxider)). - -#### Performance Improvement - -* Added numerous optimizations for ARM NEON [#38093](https://github.com/ClickHouse/ClickHouse/pull/38093)([Daniel Kutenin](https://github.com/danlark1)), ([Alexandra Pilipyuk](https://github.com/chalice19)) Note: if you run different ClickHouse versions on a cluster with ARM CPU and use distributed queries with GROUP BY multiple keys of fixed-size type that fit in 256 bits but don't fit in 64 bits, the result of the aggregation query will be wrong during upgrade. Workaround: upgrade with downtime instead of a rolling upgrade. -* Improve performance and memory usage for select of subset of columns for formats Native, Protobuf, CapnProto, JSONEachRow, TSKV, all formats with suffixes WithNames/WithNamesAndTypes. Previously while selecting only subset of columns from files in these formats all columns were read and stored in memory. Now only required columns are read. This PR enables setting `input_format_skip_unknown_fields` by default, because otherwise in case of select of subset of columns exception will be thrown. [#37192](https://github.com/ClickHouse/ClickHouse/pull/37192) ([Kruglov Pavel](https://github.com/Avogar)). -* Now more filters can be pushed down for join. [#37472](https://github.com/ClickHouse/ClickHouse/pull/37472) ([Amos Bird](https://github.com/amosbird)). -* Load marks for only necessary columns when reading wide parts. [#36879](https://github.com/ClickHouse/ClickHouse/pull/36879) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Improved performance of aggregation in case, when sparse columns (can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization` in `MergeTree` tables) are used as arguments in aggregate functions. [#37617](https://github.com/ClickHouse/ClickHouse/pull/37617) ([Anton Popov](https://github.com/CurtizJ)). -* Optimize function `COALESCE` with two arguments. [#37666](https://github.com/ClickHouse/ClickHouse/pull/37666) ([Anton Popov](https://github.com/CurtizJ)). -* Replace `multiIf` to `if` in case when `multiIf` has only one condition, because function `if` is more performant. [#37695](https://github.com/ClickHouse/ClickHouse/pull/37695) ([Anton Popov](https://github.com/CurtizJ)). -* Improve performance of `dictGetDescendants`, `dictGetChildren` functions, create temporary parent to children hierarchical index per query, not per function call during query. Allow to specify `BIDIRECTIONAL` for `HIERARHICAL` attributes, dictionary will maintain parent to children index in memory, that way functions `dictGetDescendants`, `dictGetChildren` will not create temporary index per query. Closes [#32481](https://github.com/ClickHouse/ClickHouse/issues/32481). [#37148](https://github.com/ClickHouse/ClickHouse/pull/37148) ([Maksim Kita](https://github.com/kitaisreal)). -* Aggregates state destruction now may be posted on a thread pool. For queries with LIMIT and big state it provides significant speedup, e.g. `select uniq(number) from numbers_mt(1e7) group by number limit 100` became around 2.5x faster. [#37855](https://github.com/ClickHouse/ClickHouse/pull/37855) ([Nikita Taranov](https://github.com/nickitat)). -* Improve sort performance by single column. [#37195](https://github.com/ClickHouse/ClickHouse/pull/37195) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of single column sorting using sorting queue specializations. [#37990](https://github.com/ClickHouse/ClickHouse/pull/37990) ([Maksim Kita](https://github.com/kitaisreal)). -* Improved performance on array norm and distance functions 2x-4x times. [#37394](https://github.com/ClickHouse/ClickHouse/pull/37394) ([Alexander Gololobov](https://github.com/davenger)). -* Improve performance of number comparison functions using dynamic dispatch. [#37399](https://github.com/ClickHouse/ClickHouse/pull/37399) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of ORDER BY with LIMIT. [#37481](https://github.com/ClickHouse/ClickHouse/pull/37481) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of `hasAll` function using dynamic dispatch infrastructure. [#37484](https://github.com/ClickHouse/ClickHouse/pull/37484) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of `greatCircleAngle`, `greatCircleDistance`, `geoDistance` functions. [#37524](https://github.com/ClickHouse/ClickHouse/pull/37524) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of insert into MergeTree if there are multiple columns in ORDER BY. [#35762](https://github.com/ClickHouse/ClickHouse/pull/35762) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix excessive CPU usage in background when there are a lot of tables. [#38028](https://github.com/ClickHouse/ClickHouse/pull/38028) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of `not` function using dynamic dispatch. [#38058](https://github.com/ClickHouse/ClickHouse/pull/38058) ([Maksim Kita](https://github.com/kitaisreal)). -* Optimized the internal caching of re2 patterns which occur e.g. in LIKE and MATCH functions. [#37544](https://github.com/ClickHouse/ClickHouse/pull/37544) ([Robert Schulze](https://github.com/rschu1ze)). -* Improve filter bitmask generator function all in one with AVX-512 instructions. [#37588](https://github.com/ClickHouse/ClickHouse/pull/37588) ([yaqi-zhao](https://github.com/yaqi-zhao)). -* Apply read method `threadpool` for Hive integration engine. This will significantly speed up reading. [#36328](https://github.com/ClickHouse/ClickHouse/pull/36328) ([李扬](https://github.com/taiyang-li)). -* When all the columns to read are partition keys, construct columns by the file's row number without real reading the Hive file. [#37103](https://github.com/ClickHouse/ClickHouse/pull/37103) ([lgbo](https://github.com/lgbo-ustc)). -* Support multi disks for caching hive files. [#37279](https://github.com/ClickHouse/ClickHouse/pull/37279) ([lgbo](https://github.com/lgbo-ustc)). -* Limiting the maximum cache usage per query can effectively prevent cache pool contamination. [Related Issues](https://github.com/ClickHouse/ClickHouse/issues/28961). [#37859](https://github.com/ClickHouse/ClickHouse/pull/37859) ([Han Shukai](https://github.com/KinderRiven)). -* Currently clickhouse directly downloads all remote files to the local cache (even if they are only read once), which will frequently cause IO of the local hard disk. In some scenarios, these IOs may not be necessary and may easily cause negative optimization. As shown in the figure below, when we run SSB Q1-Q4, the performance of the cache has caused negative optimization. [#37516](https://github.com/ClickHouse/ClickHouse/pull/37516) ([Han Shukai](https://github.com/KinderRiven)). -* Allow to prune the list of files via virtual columns such as `_file` and `_path` when reading from S3. This is for [#37174](https://github.com/ClickHouse/ClickHouse/issues/37174) , [#23494](https://github.com/ClickHouse/ClickHouse/issues/23494). [#37356](https://github.com/ClickHouse/ClickHouse/pull/37356) ([Amos Bird](https://github.com/amosbird)). -* In function: CompressedWriteBuffer::nextImpl(), there is an unnecessary write-copy step that would happen frequently during inserting data. Below shows the differentiation with this patch: - Before: 1. Compress "working_buffer" into "compressed_buffer" 2. write-copy into "out" - After: Directly Compress "working_buffer" into "out". [#37242](https://github.com/ClickHouse/ClickHouse/pull/37242) ([jasperzhu](https://github.com/jinjunzh)). - -#### Improvement - -* Support types with non-standard defaults in ROLLUP, CUBE, GROUPING SETS. Closes [#37360](https://github.com/ClickHouse/ClickHouse/issues/37360). [#37667](https://github.com/ClickHouse/ClickHouse/pull/37667) ([Dmitry Novik](https://github.com/novikd)). -* Fix stack traces collection on ARM. Closes [#37044](https://github.com/ClickHouse/ClickHouse/issues/37044). Closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#37797](https://github.com/ClickHouse/ClickHouse/pull/37797) ([Maksim Kita](https://github.com/kitaisreal)). -* Client will try every IP address returned by DNS resolution until successful connection. [#37273](https://github.com/ClickHouse/ClickHouse/pull/37273) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Allow to use String type instead of Binary in Arrow/Parquet/ORC formats. This PR introduces 3 new settings for it: `output_format_arrow_string_as_string`, `output_format_parquet_string_as_string`, `output_format_orc_string_as_string`. Default value for all settings is `false`. [#37327](https://github.com/ClickHouse/ClickHouse/pull/37327) ([Kruglov Pavel](https://github.com/Avogar)). -* Apply setting `input_format_max_rows_to_read_for_schema_inference` for all read rows in total from all files in globs. Previously setting `input_format_max_rows_to_read_for_schema_inference` was applied for each file in glob separately and in case of huge number of nulls we could read first `input_format_max_rows_to_read_for_schema_inference` rows from each file and get nothing. Also increase default value for this setting to 25000. [#37332](https://github.com/ClickHouse/ClickHouse/pull/37332) ([Kruglov Pavel](https://github.com/Avogar)). -* Add separate `CLUSTER` grant (and `access_control_improvements.on_cluster_queries_require_cluster_grant` configuration directive, for backward compatibility, default to `false`). [#35767](https://github.com/ClickHouse/ClickHouse/pull/35767) ([Azat Khuzhin](https://github.com/azat)). -* Added support for schema inference for `hdfsCluster`. [#35812](https://github.com/ClickHouse/ClickHouse/pull/35812) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Implement `least_used` load balancing algorithm for disks inside volume (multi disk configuration). [#36686](https://github.com/ClickHouse/ClickHouse/pull/36686) ([Azat Khuzhin](https://github.com/azat)). -* Modify the HTTP Endpoint to return the full stats under the `X-ClickHouse-Summary` header when `send_progress_in_http_headers=0` (before it would return all zeros). - Modify the HTTP Endpoint to return `X-ClickHouse-Exception-Code` header when progress has been sent before (`send_progress_in_http_headers=1`) - Modify the HTTP Endpoint to return `HTTP_REQUEST_TIMEOUT` (408) instead of `HTTP_INTERNAL_SERVER_ERROR` (500) on `TIMEOUT_EXCEEDED` errors. [#36884](https://github.com/ClickHouse/ClickHouse/pull/36884) ([Raúl Marín](https://github.com/Algunenano)). -* Allow a user to inspect grants from granted roles. [#36941](https://github.com/ClickHouse/ClickHouse/pull/36941) ([nvartolomei](https://github.com/nvartolomei)). -* Do not calculate an integral numerically but use CDF functions instead. This will speed up execution and will increase the precision. This fixes [#36714](https://github.com/ClickHouse/ClickHouse/issues/36714). [#36953](https://github.com/ClickHouse/ClickHouse/pull/36953) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Add default implementation for Nothing in functions. Now most of the functions will return column with type Nothing in case one of it's arguments is Nothing. It also solves problem with functions like arrayMap/arrayFilter and similar when they have empty array as an argument. Previously queries like `select arrayMap(x -> 2 * x, []);` failed because function inside lambda cannot work with type `Nothing`, now such queries return empty array with type `Array(Nothing)`. Also add support for arrays of nullable types in functions like arrayFilter/arrayFill. Previously, queries like `select arrayFilter(x -> x % 2, [1, NULL])` failed, now they work (if the result of lambda is NULL, then this value won't be included in the result). Closes [#37000](https://github.com/ClickHouse/ClickHouse/issues/37000). [#37048](https://github.com/ClickHouse/ClickHouse/pull/37048) ([Kruglov Pavel](https://github.com/Avogar)). -* Now if a shard has local replica we create a local plan and a plan to read from all remote replicas. They have shared initiator which coordinates reading. [#37204](https://github.com/ClickHouse/ClickHouse/pull/37204) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Do no longer abort server startup if configuration option "mark_cache_size" is not explicitly set. [#37326](https://github.com/ClickHouse/ClickHouse/pull/37326) ([Robert Schulze](https://github.com/rschu1ze)). -* Allows providing `NULL`/`NOT NULL` right after type in column declaration. [#37337](https://github.com/ClickHouse/ClickHouse/pull/37337) ([Igor Nikonov](https://github.com/devcrafter)). -* optimize file segment PARTIALLY_DOWNLOADED get read buffer. [#37338](https://github.com/ClickHouse/ClickHouse/pull/37338) ([xiedeyantu](https://github.com/xiedeyantu)). -* Try to improve short circuit functions processing to fix problems with stress tests. [#37384](https://github.com/ClickHouse/ClickHouse/pull/37384) ([Kruglov Pavel](https://github.com/Avogar)). -* Generate multiple columns with UUID (generateUUIDv4(1), generateUUIDv4(2)) [#37395](https://github.com/ClickHouse/ClickHouse/issues/37395). [#37415](https://github.com/ClickHouse/ClickHouse/pull/37415) ([Memo](https://github.com/Joeywzr)). -* Fix extremely rare deadlock during part fetch in zero-copy replication. Fixes [#37423](https://github.com/ClickHouse/ClickHouse/issues/37423). [#37424](https://github.com/ClickHouse/ClickHouse/pull/37424) ([metahys](https://github.com/metahys)). -* Don't allow to create storage with unknown data format. [#37450](https://github.com/ClickHouse/ClickHouse/pull/37450) ([Kruglov Pavel](https://github.com/Avogar)). -* Set `global_memory_usage_overcommit_max_wait_microseconds` default value to 5 seconds. Add info about `OvercommitTracker` to OOM exception message. Add `MemoryOvercommitWaitTimeMicroseconds` profile event. [#37460](https://github.com/ClickHouse/ClickHouse/pull/37460) ([Dmitry Novik](https://github.com/novikd)). -* Do not display `-0.0` CPU time in clickhouse-client. It can appear due to rounding errors. This closes [#38003](https://github.com/ClickHouse/ClickHouse/issues/38003). This closes [#38038](https://github.com/ClickHouse/ClickHouse/issues/38038). [#38064](https://github.com/ClickHouse/ClickHouse/pull/38064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Play UI: Keep controls in place when the page is scrolled horizontally. This makes edits comfortable even if the table is wide and it was scrolled far to the right. The feature proposed by Maksym Tereshchenko from CaspianDB. [#37470](https://github.com/ClickHouse/ClickHouse/pull/37470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Modify query div in play.html to be extendable beyond 20% height. In case of very long queries it is helpful to extend the textarea element, only today, since the div is fixed height, the extended textarea hides the data div underneath. With this fix, extending the textarea element will push the data div down/up such the extended textarea won't hide it. Also, keeps query box width 100% even when the user adjusting the size of the query textarea. [#37488](https://github.com/ClickHouse/ClickHouse/pull/37488) ([guyco87](https://github.com/guyco87)). -* Added `ProfileEvents` for introspection of type of written (inserted or merged) parts (`Inserted{Wide/Compact/InMemory}Parts`, `MergedInto{Wide/Compact/InMemory}Parts`. Added column `part_type` to `system.part_log`. Resolves [#37495](https://github.com/ClickHouse/ClickHouse/issues/37495). [#37536](https://github.com/ClickHouse/ClickHouse/pull/37536) ([Anton Popov](https://github.com/CurtizJ)). -* clickhouse-keeper improvement: move broken logs to a timestamped folder. [#37565](https://github.com/ClickHouse/ClickHouse/pull/37565) ([Antonio Andelic](https://github.com/antonio2368)). -* Do not write expired columns by TTL after subsequent merges (before only first merge/optimize of the part will not write expired by TTL columns, all other will do). [#37570](https://github.com/ClickHouse/ClickHouse/pull/37570) ([Azat Khuzhin](https://github.com/azat)). -* More precise result of the `dumpColumnStructure` miscellaneous function in presence of LowCardinality or Sparse columns. In previous versions, these functions were converting the argument to a full column before returning the result. This is needed to provide an answer in [#6935](https://github.com/ClickHouse/ClickHouse/issues/6935). [#37633](https://github.com/ClickHouse/ClickHouse/pull/37633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* clickhouse-keeper: store only unique session IDs for watches. [#37641](https://github.com/ClickHouse/ClickHouse/pull/37641) ([Azat Khuzhin](https://github.com/azat)). -* Fix possible "Cannot write to finalized buffer". [#37645](https://github.com/ClickHouse/ClickHouse/pull/37645) ([Azat Khuzhin](https://github.com/azat)). -* Add setting `support_batch_delete` for `DiskS3` to disable multiobject delete calls, which Google Cloud Storage doesn't support. [#37659](https://github.com/ClickHouse/ClickHouse/pull/37659) ([Fred Wulff](https://github.com/frew)). -* Add an option to disable connection pooling in ODBC bridge. [#37705](https://github.com/ClickHouse/ClickHouse/pull/37705) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Functions `dictGetHierarchy`, `dictIsIn`, `dictGetChildren`, `dictGetDescendants` added support nullable `HIERARCHICAL` attribute in dictionaries. Closes [#35521](https://github.com/ClickHouse/ClickHouse/issues/35521). [#37805](https://github.com/ClickHouse/ClickHouse/pull/37805) ([Maksim Kita](https://github.com/kitaisreal)). -* Expose BoringSSL version related info in the `system.build_options` table. [#37850](https://github.com/ClickHouse/ClickHouse/pull/37850) ([Bharat Nallan](https://github.com/bharatnc)). -* Now clickhouse-server removes `delete_tmp` directories on server start. Fixes [#26503](https://github.com/ClickHouse/ClickHouse/issues/26503). [#37906](https://github.com/ClickHouse/ClickHouse/pull/37906) ([alesapin](https://github.com/alesapin)). -* Clean up broken detached parts after timeout. Closes [#25195](https://github.com/ClickHouse/ClickHouse/issues/25195). [#37975](https://github.com/ClickHouse/ClickHouse/pull/37975) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Now in MergeTree table engines family failed-to-move parts will be removed instantly. [#37994](https://github.com/ClickHouse/ClickHouse/pull/37994) ([alesapin](https://github.com/alesapin)). -* Now if setting `always_fetch_merged_part` is enabled for ReplicatedMergeTree merges will try to find parts on other replicas rarely with smaller load for [Zoo]Keeper. [#37995](https://github.com/ClickHouse/ClickHouse/pull/37995) ([alesapin](https://github.com/alesapin)). -* Add implicit grants with grant option too. For example `GRANT CREATE TABLE ON test.* TO A WITH GRANT OPTION` now allows `A` to execute `GRANT CREATE VIEW ON test.* TO B`. [#38017](https://github.com/ClickHouse/ClickHouse/pull/38017) ([Vitaly Baranov](https://github.com/vitlibar)). - -#### Build/Testing/Packaging Improvement - -* Use `clang-14` and LLVM infrastructure version 14 for builds. This closes [#34681](https://github.com/ClickHouse/ClickHouse/issues/34681). [#34754](https://github.com/ClickHouse/ClickHouse/pull/34754) ([Alexey Milovidov](https://github.com/alexey-milovidov)). Note: `clang-14` has [a bug](https://github.com/google/sanitizers/issues/1540) in ThreadSanitizer that makes our CI work worse. -* Allow to drop privileges at startup. This simplifies Docker images. Closes [#36293](https://github.com/ClickHouse/ClickHouse/issues/36293). [#36341](https://github.com/ClickHouse/ClickHouse/pull/36341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add docs spellcheck to CI. [#37790](https://github.com/ClickHouse/ClickHouse/pull/37790) ([Vladimir C](https://github.com/vdimir)). -* Fix overly aggressive stripping which removed the embedded hash required for checking the consistency of the executable. [#37993](https://github.com/ClickHouse/ClickHouse/pull/37993) ([Robert Schulze](https://github.com/rschu1ze)). - -#### Bug Fix - -* Fix `SELECT ... INTERSECT` and `EXCEPT SELECT` statements with constant string types. [#37738](https://github.com/ClickHouse/ClickHouse/pull/37738) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix `GROUP BY` `AggregateFunction` (i.e. you `GROUP BY` by the column that has `AggregateFunction` type). [#37093](https://github.com/ClickHouse/ClickHouse/pull/37093) ([Azat Khuzhin](https://github.com/azat)). -* (experimental WINDOW VIEW) Fix `addDependency` in WindowView. This bug can be reproduced like [#37237](https://github.com/ClickHouse/ClickHouse/issues/37237). [#37224](https://github.com/ClickHouse/ClickHouse/pull/37224) ([vxider](https://github.com/Vxider)). -* Fix inconsistency in ORDER BY ... WITH FILL feature. Query, containing ORDER BY ... WITH FILL, can generate extra rows when multiple WITH FILL columns are present. [#38074](https://github.com/ClickHouse/ClickHouse/pull/38074) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* This PR moving `addDependency` from constructor to `startup()` to avoid adding dependency to a *dropped* table, fix [#37237](https://github.com/ClickHouse/ClickHouse/issues/37237). [#37243](https://github.com/ClickHouse/ClickHouse/pull/37243) ([vxider](https://github.com/Vxider)). -* Fix inserting defaults for missing values in columnar formats. Previously missing columns were filled with defaults for types, not for columns. [#37253](https://github.com/ClickHouse/ClickHouse/pull/37253) ([Kruglov Pavel](https://github.com/Avogar)). -* (experimental Object type) Fix some cases of insertion nested arrays to columns of type `Object`. [#37305](https://github.com/ClickHouse/ClickHouse/pull/37305) ([Anton Popov](https://github.com/CurtizJ)). -* Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). -* Fix projections with GROUP/ORDER BY in query and optimize_aggregation_in_order (before the result was incorrect since only finish sorting was performed). [#37342](https://github.com/ClickHouse/ClickHouse/pull/37342) ([Azat Khuzhin](https://github.com/azat)). -* Fixed error with symbols in key name in S3. Fixes [#33009](https://github.com/ClickHouse/ClickHouse/issues/33009). [#37344](https://github.com/ClickHouse/ClickHouse/pull/37344) ([Vladimir Chebotarev](https://github.com/excitoon)). -* Throw an exception when GROUPING SETS used with ROLLUP or CUBE. [#37367](https://github.com/ClickHouse/ClickHouse/pull/37367) ([Dmitry Novik](https://github.com/novikd)). -* Fix LOGICAL_ERROR in getMaxSourcePartsSizeForMerge during merges (in case of non standard, greater, values of `background_pool_size`/`background_merges_mutations_concurrency_ratio` has been specified in `config.xml` (new way) not in `users.xml` (deprecated way)). [#37413](https://github.com/ClickHouse/ClickHouse/pull/37413) ([Azat Khuzhin](https://github.com/azat)). -* Stop removing UTF-8 BOM in RowBinary format. [#37428](https://github.com/ClickHouse/ClickHouse/pull/37428) ([Paul Loyd](https://github.com/loyd)). [#37428](https://github.com/ClickHouse/ClickHouse/pull/37428) ([Paul Loyd](https://github.com/loyd)). -* clickhouse-keeper bugfix: fix force recovery for single node cluster. [#37440](https://github.com/ClickHouse/ClickHouse/pull/37440) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix cast lowcard of nullable in JoinSwitcher, close [#37385](https://github.com/ClickHouse/ClickHouse/issues/37385). [#37453](https://github.com/ClickHouse/ClickHouse/pull/37453) ([Vladimir C](https://github.com/vdimir)). -* Fix named tuples output in ORC/Arrow/Parquet formats. [#37458](https://github.com/ClickHouse/ClickHouse/pull/37458) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix optimization of monotonous functions in ORDER BY clause in presence of GROUPING SETS. Fixes [#37401](https://github.com/ClickHouse/ClickHouse/issues/37401). [#37493](https://github.com/ClickHouse/ClickHouse/pull/37493) ([Dmitry Novik](https://github.com/novikd)). -* Fix error on joining with dictionary on some conditions. Close [#37386](https://github.com/ClickHouse/ClickHouse/issues/37386). [#37530](https://github.com/ClickHouse/ClickHouse/pull/37530) ([Vladimir C](https://github.com/vdimir)). -* Prohibit `optimize_aggregation_in_order` with `GROUPING SETS` (fixes `LOGICAL_ERROR`). [#37542](https://github.com/ClickHouse/ClickHouse/pull/37542) ([Azat Khuzhin](https://github.com/azat)). -* Fix wrong dump information of ActionsDAG. [#37587](https://github.com/ClickHouse/ClickHouse/pull/37587) ([zhanglistar](https://github.com/zhanglistar)). -* Fix converting types for UNION queries (may produce LOGICAL_ERROR). [#37593](https://github.com/ClickHouse/ClickHouse/pull/37593) ([Azat Khuzhin](https://github.com/azat)). -* Fix `WITH FILL` modifier with negative intervals in `STEP` clause. Fixes [#37514](https://github.com/ClickHouse/ClickHouse/issues/37514). [#37600](https://github.com/ClickHouse/ClickHouse/pull/37600) ([Anton Popov](https://github.com/CurtizJ)). -* Fix illegal joinGet array usage when ` join_use_nulls = 1`. This fixes [#37562](https://github.com/ClickHouse/ClickHouse/issues/37562) . [#37650](https://github.com/ClickHouse/ClickHouse/pull/37650) ([Amos Bird](https://github.com/amosbird)). -* Fix columns number mismatch in cross join, close [#37561](https://github.com/ClickHouse/ClickHouse/issues/37561). [#37653](https://github.com/ClickHouse/ClickHouse/pull/37653) ([Vladimir C](https://github.com/vdimir)). -* Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix RabbitMQ Storage not being able to startup on server restart if storage was create without SETTINGS clause. Closes [#37463](https://github.com/ClickHouse/ClickHouse/issues/37463). [#37691](https://github.com/ClickHouse/ClickHouse/pull/37691) ([Kseniia Sumarokova](https://github.com/kssenii)). -* SQL user defined functions disable CREATE/DROP in readonly mode. Closes [#37280](https://github.com/ClickHouse/ClickHouse/issues/37280). [#37699](https://github.com/ClickHouse/ClickHouse/pull/37699) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix formatting of Nullable arguments for executable user defined functions. Closes [#35897](https://github.com/ClickHouse/ClickHouse/issues/35897). [#37711](https://github.com/ClickHouse/ClickHouse/pull/37711) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix optimization enabled by setting `optimize_monotonous_functions_in_order_by` in distributed queries. Fixes [#36037](https://github.com/ClickHouse/ClickHouse/issues/36037). [#37724](https://github.com/ClickHouse/ClickHouse/pull/37724) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible logical error: `Invalid Field get from type UInt64 to type Float64` in `values` table function. Closes [#37602](https://github.com/ClickHouse/ClickHouse/issues/37602). [#37754](https://github.com/ClickHouse/ClickHouse/pull/37754) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix possible segfault in schema inference in case of exception in SchemaReader constructor. Closes [#37680](https://github.com/ClickHouse/ClickHouse/issues/37680). [#37760](https://github.com/ClickHouse/ClickHouse/pull/37760) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix toString error on DatatypeDate32. [#37775](https://github.com/ClickHouse/ClickHouse/pull/37775) ([LiuNeng](https://github.com/liuneng1994)). -* The clickhouse-keeper setting `dead_session_check_period_ms` was transformed into microseconds (multiplied by 1000), which lead to dead sessions only being cleaned up after several minutes (instead of 500ms). [#37824](https://github.com/ClickHouse/ClickHouse/pull/37824) ([Michael Lex](https://github.com/mlex)). -* Fix possible "No more packets are available" for distributed queries (in case of `async_socket_for_remote`/`use_hedged_requests` is disabled). [#37826](https://github.com/ClickHouse/ClickHouse/pull/37826) ([Azat Khuzhin](https://github.com/azat)). -* (experimental WINDOW VIEW) Do not drop the inner target table when executing `ALTER TABLE … MODIFY QUERY` in WindowView. [#37879](https://github.com/ClickHouse/ClickHouse/pull/37879) ([vxider](https://github.com/Vxider)). -* Fix directory ownership of coordination dir in clickhouse-keeper Docker image. Fixes [#37914](https://github.com/ClickHouse/ClickHouse/issues/37914). [#37915](https://github.com/ClickHouse/ClickHouse/pull/37915) ([James Maidment](https://github.com/jamesmaidment)). -* Dictionaries fix custom query with update field and `{condition}`. Closes [#33746](https://github.com/ClickHouse/ClickHouse/issues/33746). [#37947](https://github.com/ClickHouse/ClickHouse/pull/37947) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix possible incorrect result of `SELECT ... WITH FILL` in the case when `ORDER BY` should be applied after `WITH FILL` result (e.g. for outer query). Incorrect result was caused by optimization for `ORDER BY` expressions ([#35623](https://github.com/ClickHouse/ClickHouse/issues/35623)). Closes [#37904](https://github.com/ClickHouse/ClickHouse/issues/37904). [#37959](https://github.com/ClickHouse/ClickHouse/pull/37959) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* (experimental WINDOW VIEW) Add missing default columns when pushing to the target table in WindowView, fix [#37815](https://github.com/ClickHouse/ClickHouse/issues/37815). [#37965](https://github.com/ClickHouse/ClickHouse/pull/37965) ([vxider](https://github.com/Vxider)). -* Fixed too large stack frame that would cause compilation to fail. [#37996](https://github.com/ClickHouse/ClickHouse/pull/37996) ([Han Shukai](https://github.com/KinderRiven)). -* When open enable_filesystem_query_cache_limit, throw Reserved cache size exceeds the remaining cache size. [#38004](https://github.com/ClickHouse/ClickHouse/pull/38004) ([xiedeyantu](https://github.com/xiedeyantu)). -* Fix converting types for UNION queries (may produce LOGICAL_ERROR). [#34775](https://github.com/ClickHouse/ClickHouse/pull/34775) ([Azat Khuzhin](https://github.com/azat)). -* TTL merge may not be scheduled again if BackgroundExecutor is busy. --merges_with_ttl_counter is increased in selectPartsToMerge() --merge task will be ignored if BackgroundExecutor is busy --merges_with_ttl_counter will not be decrease. [#36387](https://github.com/ClickHouse/ClickHouse/pull/36387) ([lthaooo](https://github.com/lthaooo)). -* Fix overridden settings value of `normalize_function_names`. [#36937](https://github.com/ClickHouse/ClickHouse/pull/36937) ([李扬](https://github.com/taiyang-li)). -* Fix for exponential time decaying window functions. Now respecting boundaries of the window. [#36944](https://github.com/ClickHouse/ClickHouse/pull/36944) ([Vladimir Chebotarev](https://github.com/excitoon)). -* Fix possible heap-use-after-free error when reading system.projection_parts and system.projection_parts_columns . This fixes [#37184](https://github.com/ClickHouse/ClickHouse/issues/37184). [#37185](https://github.com/ClickHouse/ClickHouse/pull/37185) ([Amos Bird](https://github.com/amosbird)). -* Fixed `DateTime64` fractional seconds behavior prior to Unix epoch. [#37697](https://github.com/ClickHouse/ClickHouse/pull/37697) ([Andrey Zvonov](https://github.com/zvonand)). [#37039](https://github.com/ClickHouse/ClickHouse/pull/37039) ([李扬](https://github.com/taiyang-li)). - -### ClickHouse release 22.5, 2022-05-19 - -#### Upgrade Notes - -* Now, background merges, mutations, and `OPTIMIZE` will not increment `SelectedRows` and `SelectedBytes` metrics. They (still) will increment `MergedRows` and `MergedUncompressedBytes` as it was before. This only affects the metric values and makes them better. This change does not introduce any incompatibility, but you may wonder about the changes to the metrics, so we put in this category. [#37040](https://github.com/ClickHouse/ClickHouse/pull/37040) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Updated the BoringSSL module to the official FIPS compliant version. This makes ClickHouse FIPS compliant in this area. [#35914](https://github.com/ClickHouse/ClickHouse/pull/35914) ([Meena-Renganathan](https://github.com/Meena-Renganathan)). The ciphers `aes-192-cfb128` and `aes-256-cfb128` were removed, because they are not included in the FIPS certified version of BoringSSL. -* `max_memory_usage` setting is removed from the default user profile in `users.xml`. This enables flexible memory limits for queries instead of the old rigid limit of 10 GB. -* Disable `log_query_threads` setting by default. It controls the logging of statistics about every thread participating in query execution. After supporting asynchronous reads, the total number of distinct thread ids became too large, and logging into the `query_thread_log` has become too heavy. [#37077](https://github.com/ClickHouse/ClickHouse/pull/37077) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Remove function `groupArraySorted` which has a bug. [#36822](https://github.com/ClickHouse/ClickHouse/pull/36822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### New Feature - -* Enable memory overcommit by default. [#35921](https://github.com/ClickHouse/ClickHouse/pull/35921) ([Dmitry Novik](https://github.com/novikd)). -* Add support of GROUPING SETS in GROUP BY clause. This implementation supports a parallel processing of grouping sets. [#33631](https://github.com/ClickHouse/ClickHouse/pull/33631) ([Dmitry Novik](https://github.com/novikd)). -* Added `system.certificates` table. [#37142](https://github.com/ClickHouse/ClickHouse/pull/37142) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Adds `h3Line`, `h3Distance` and `h3HexRing` functions. [#37030](https://github.com/ClickHouse/ClickHouse/pull/37030) ([Bharat Nallan](https://github.com/bharatnc)). -* New single binary based diagnostics tool (clickhouse-diagnostics). [#36705](https://github.com/ClickHouse/ClickHouse/pull/36705) ([Dale McDiarmid](https://github.com/gingerwizard)). -* Add output format `Prometheus` [#36051](https://github.com/ClickHouse/ClickHouse/issues/36051). [#36206](https://github.com/ClickHouse/ClickHouse/pull/36206) ([Vladimir C](https://github.com/vdimir)). -* Add `MySQLDump` input format. It reads all data from INSERT queries belonging to one table in dump. If there are more than one table, by default it reads data from the first one. [#36667](https://github.com/ClickHouse/ClickHouse/pull/36667) ([Kruglov Pavel](https://github.com/Avogar)). -* Show the `total_rows` and `total_bytes` fields in `system.tables` for temporary tables. [#36401](https://github.com/ClickHouse/ClickHouse/issues/36401). [#36439](https://github.com/ClickHouse/ClickHouse/pull/36439) ([xiedeyantu](https://github.com/xiedeyantu)). -* Allow to override `parts_to_delay_insert` and `parts_to_throw_insert` with query-level settings. If they are defined, they will override table-level settings. [#36371](https://github.com/ClickHouse/ClickHouse/pull/36371) ([Memo](https://github.com/Joeywzr)). - -#### Experimental Feature - -* Implemented L1, L2, Linf, Cosine distance functions for arrays and L1, L2, Linf norm functions for arrays. - [#37033](https://github.com/ClickHouse/ClickHouse/pull/37033) ([qieqieplus](https://github.com/qieqieplus)). Caveat: the functions will be renamed. -* Improve the `WATCH` query in WindowView: 1. Reduce the latency of providing query results by calling the `fire_condition` signal. 2. Makes the cancel query operation(ctrl-c) faster, by checking `isCancelled()` more frequently. [#37226](https://github.com/ClickHouse/ClickHouse/pull/37226) ([vxider](https://github.com/Vxider)). -* Introspection for remove filesystem cache. [#36802](https://github.com/ClickHouse/ClickHouse/pull/36802) ([Han Shukai](https://github.com/KinderRiven)). -* Added new hash function `wyHash64` for SQL. [#36467](https://github.com/ClickHouse/ClickHouse/pull/36467) ([olevino](https://github.com/olevino)). -* Improvement for replicated databases: Added `SYSTEM SYNC DATABASE REPLICA` query which allows to sync tables metadata inside Replicated database, because currently synchronisation is asynchronous. [#35944](https://github.com/ClickHouse/ClickHouse/pull/35944) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Improvement for remote filesystem cache: Better read from cache. [#37054](https://github.com/ClickHouse/ClickHouse/pull/37054) ([Kseniia Sumarokova](https://github.com/kssenii)). Improve `SYSTEM DROP FILESYSTEM CACHE` query: `` option and `FORCE` option. [#36639](https://github.com/ClickHouse/ClickHouse/pull/36639) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Improvement for semistructured data: Allow to cast columns of type `Object(...)` to `Object(Nullable(...))`. [#36564](https://github.com/ClickHouse/ClickHouse/pull/36564) ([awakeljw](https://github.com/awakeljw)). -* Improvement for parallel replicas: We create a local interpreter if we want to execute query on localhost replica. But for when executing query on multiple replicas we rely on the fact that a connection exists so replicas can talk to coordinator. It is now improved and localhost replica can talk to coordinator directly in the same process. [#36281](https://github.com/ClickHouse/ClickHouse/pull/36281) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). - -#### Performance Improvement - -* Improve performance of `avg`, `sum` aggregate functions if used without GROUP BY expression. [#37257](https://github.com/ClickHouse/ClickHouse/pull/37257) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of unary arithmetic functions (`bitCount`, `bitNot`, `abs`, `intExp2`, `intExp10`, `negate`, `roundAge`, `roundDuration`, `roundToExp2`, `sign`) using dynamic dispatch. [#37289](https://github.com/ClickHouse/ClickHouse/pull/37289) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of ORDER BY, MergeJoin, insertion into MergeTree using JIT compilation of sort columns comparator. [#34469](https://github.com/ClickHouse/ClickHouse/pull/34469) ([Maksim Kita](https://github.com/kitaisreal)). -* Change structure of `system.asynchronous_metric_log`. It will take about 10 times less space. This closes [#36357](https://github.com/ClickHouse/ClickHouse/issues/36357). The field `event_time_microseconds` was removed, because it is useless. [#36360](https://github.com/ClickHouse/ClickHouse/pull/36360) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Load marks for only necessary columns when reading wide parts. [#36879](https://github.com/ClickHouse/ClickHouse/pull/36879) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Improves performance of file descriptor cache by narrowing mutex scopes. [#36682](https://github.com/ClickHouse/ClickHouse/pull/36682) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Improve performance of reading from storage `File` and table functions `file` in case when path has globs and matched directory contains large number of files. [#36647](https://github.com/ClickHouse/ClickHouse/pull/36647) ([Anton Popov](https://github.com/CurtizJ)). -* Apply parallel parsing for input format `HiveText`, which can speed up HiveText parsing by 2x when reading local file. [#36650](https://github.com/ClickHouse/ClickHouse/pull/36650) ([李扬](https://github.com/taiyang-li)). -* The default `HashJoin` is not thread safe for inserting right table's rows and run it in a single thread. When the right table is large, the join process is too slow with low cpu utilization. [#36415](https://github.com/ClickHouse/ClickHouse/pull/36415) ([lgbo](https://github.com/lgbo-ustc)). -* Allow to rewrite `select countDistinct(a) from t` to `select count(1) from (select a from t groupBy a)`. [#35993](https://github.com/ClickHouse/ClickHouse/pull/35993) ([zhanglistar](https://github.com/zhanglistar)). -* Transform OR LIKE chain to multiMatchAny. Will enable once we have more confidence it works. [#34932](https://github.com/ClickHouse/ClickHouse/pull/34932) ([Daniel Kutenin](https://github.com/danlark1)). -* Improve performance of some functions with inlining. [#34544](https://github.com/ClickHouse/ClickHouse/pull/34544) ([Daniel Kutenin](https://github.com/danlark1)). -* Add a branch to avoid unnecessary memcpy in readBig. It improves performance somewhat. [#36095](https://github.com/ClickHouse/ClickHouse/pull/36095) ([jasperzhu](https://github.com/jinjunzh)). -* Implement partial GROUP BY key for optimize_aggregation_in_order. [#35111](https://github.com/ClickHouse/ClickHouse/pull/35111) ([Azat Khuzhin](https://github.com/azat)). - -#### Improvement - -* Show names of erroneous files in case of parsing errors while executing table functions `file`, `s3` and `url`. [#36314](https://github.com/ClickHouse/ClickHouse/pull/36314) ([Anton Popov](https://github.com/CurtizJ)). -* Allowed to increase the number of threads for executing background operations (merges, mutations, moves and fetches) at runtime if they are specified at top level config. [#36425](https://github.com/ClickHouse/ClickHouse/pull/36425) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Now date time conversion functions that generates time before 1970-01-01 00:00:00 with partial hours/minutes timezones will be saturated to zero instead of overflow. This is the continuation of https://github.com/ClickHouse/ClickHouse/pull/29953 which addresses https://github.com/ClickHouse/ClickHouse/pull/29953#discussion_r800550280 . Mark as improvement because it's implementation defined behavior (and very rare case) and we are allowed to break it. [#36656](https://github.com/ClickHouse/ClickHouse/pull/36656) ([Amos Bird](https://github.com/amosbird)). -* Add a warning if someone running clickhouse-server with log level "test". The log level "test" was added recently and cannot be used in production due to inevitable, unavoidable, fatal and life-threatening performance degradation. [#36824](https://github.com/ClickHouse/ClickHouse/pull/36824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Parse collations in CREATE TABLE, throw exception or ignore. closes [#35892](https://github.com/ClickHouse/ClickHouse/issues/35892). [#36271](https://github.com/ClickHouse/ClickHouse/pull/36271) ([yuuch](https://github.com/yuuch)). -* Option `compatibility_ignore_auto_increment_in_create_table` allows ignoring `AUTO_INCREMENT` keyword in a column declaration to simplify migration from MySQL. [#37178](https://github.com/ClickHouse/ClickHouse/pull/37178) ([Igor Nikonov](https://github.com/devcrafter)). -* Add aliases `JSONLines` and `NDJSON` for `JSONEachRow`. Closes [#36303](https://github.com/ClickHouse/ClickHouse/issues/36303). [#36327](https://github.com/ClickHouse/ClickHouse/pull/36327) ([flynn](https://github.com/ucasfl)). -* Limit the max partitions could be queried for each hive table. Avoid resource overruns. [#37281](https://github.com/ClickHouse/ClickHouse/pull/37281) ([lgbo](https://github.com/lgbo-ustc)). -* Added implicit cast for `h3kRing` function second argument to improve usability. Closes [#35432](https://github.com/ClickHouse/ClickHouse/issues/35432). [#37189](https://github.com/ClickHouse/ClickHouse/pull/37189) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix progress indication for `INSERT SELECT` in `clickhouse-local` for any query and for file progress in client, more correct file progress. [#37075](https://github.com/ClickHouse/ClickHouse/pull/37075) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix bug which can lead to forgotten outdated parts in MergeTree table engines family in case of filesystem failures during parts removal. Before fix they will be removed only after first server restart. [#37014](https://github.com/ClickHouse/ClickHouse/pull/37014) ([alesapin](https://github.com/alesapin)). -* Implemented a new mode of handling row policies which can be enabled in the main configuration which enables users without permissive row policies to read rows. [#36997](https://github.com/ClickHouse/ClickHouse/pull/36997) ([Vitaly Baranov](https://github.com/vitlibar)). -* Play UI: Nullable numbers will be aligned to the right in table cells. This closes [#36982](https://github.com/ClickHouse/ClickHouse/issues/36982). [#36988](https://github.com/ClickHouse/ClickHouse/pull/36988) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Play UI: If there is one row in result and more than a few columns, display the result vertically. Continuation of [#36811](https://github.com/ClickHouse/ClickHouse/issues/36811). [#36842](https://github.com/ClickHouse/ClickHouse/pull/36842) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Cleanup CSS in Play UI. The pixels are more evenly placed. Better usability for long content in table cells. [#36569](https://github.com/ClickHouse/ClickHouse/pull/36569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Finalize write buffers in case of exception to avoid doing it in destructors. Hope it fixes: [#36907](https://github.com/ClickHouse/ClickHouse/issues/36907). [#36979](https://github.com/ClickHouse/ClickHouse/pull/36979) ([Kruglov Pavel](https://github.com/Avogar)). -* After [#36425](https://github.com/ClickHouse/ClickHouse/issues/36425) settings like `background_fetches_pool_size` became obsolete and can appear in top level config, but clickhouse throws and exception like `Error updating configuration from '/etc/clickhouse-server/config.xml' config.: Code: 137. DB::Exception: A setting 'background_fetches_pool_size' appeared at top level in config /etc/clickhouse-server/config.xml.` This is fixed. [#36917](https://github.com/ClickHouse/ClickHouse/pull/36917) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Add extra diagnostic info (if applicable) when sending exception to other server. [#36872](https://github.com/ClickHouse/ClickHouse/pull/36872) ([tavplubix](https://github.com/tavplubix)). -* Allow to execute hash functions with arguments of type `Array(Tuple(..))`. [#36812](https://github.com/ClickHouse/ClickHouse/pull/36812) ([Anton Popov](https://github.com/CurtizJ)). -* Added `user_defined_path` config setting. [#36753](https://github.com/ClickHouse/ClickHouse/pull/36753) ([Maksim Kita](https://github.com/kitaisreal)). -* Allow cluster macro in `s3Cluster` table function. [#36726](https://github.com/ClickHouse/ClickHouse/pull/36726) ([Vadim Volodin](https://github.com/PolyProgrammist)). -* Properly cancel INSERT queries in `clickhouse-client`/`clickhouse-local`. [#36710](https://github.com/ClickHouse/ClickHouse/pull/36710) ([Azat Khuzhin](https://github.com/azat)). -* Allow to cancel a query while still keeping a decent query id in `MySQLHandler`. [#36699](https://github.com/ClickHouse/ClickHouse/pull/36699) ([Amos Bird](https://github.com/amosbird)). -* Add `is_all_data_sent` column into `system.processes`, and improve internal testing hardening check based on it. [#36649](https://github.com/ClickHouse/ClickHouse/pull/36649) ([Azat Khuzhin](https://github.com/azat)). -* The metrics about time spent reading from s3 now calculated correctly. Close [#35483](https://github.com/ClickHouse/ClickHouse/issues/35483). [#36572](https://github.com/ClickHouse/ClickHouse/pull/36572) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Allow file descriptors in table function file if it is run in clickhouse-local. [#36562](https://github.com/ClickHouse/ClickHouse/pull/36562) ([wuxiaobai24](https://github.com/wuxiaobai24)). -* Allow names of tuple elements that start from digits. [#36544](https://github.com/ClickHouse/ClickHouse/pull/36544) ([Anton Popov](https://github.com/CurtizJ)). -* Now clickhouse-benchmark can read authentication info from environment variables. [#36497](https://github.com/ClickHouse/ClickHouse/pull/36497) ([Anton Kozlov](https://github.com/tonickkozlov)). -* `clickhouse-keeper` improvement: add support for force recovery which allows you to reconfigure cluster without quorum. [#36258](https://github.com/ClickHouse/ClickHouse/pull/36258) ([Antonio Andelic](https://github.com/antonio2368)). -* Improve schema inference for JSON objects. [#36207](https://github.com/ClickHouse/ClickHouse/pull/36207) ([Kruglov Pavel](https://github.com/Avogar)). -* Refactor code around schema inference with globs. Try next file from glob only if it makes sense (previously we tried next file in case of any error). Also it fixes [#36317](https://github.com/ClickHouse/ClickHouse/issues/36317). [#36205](https://github.com/ClickHouse/ClickHouse/pull/36205) ([Kruglov Pavel](https://github.com/Avogar)). -* Add a separate `CLUSTER` grant (and `access_control_improvements.on_cluster_queries_require_cluster_grant` configuration directive, for backward compatibility, default to `false`). [#35767](https://github.com/ClickHouse/ClickHouse/pull/35767) ([Azat Khuzhin](https://github.com/azat)). -* If the required amount of memory is available before the selected query stopped, all waiting queries continue execution. Now we don't stop any query if memory is freed before the moment when the selected query knows about the cancellation. [#35637](https://github.com/ClickHouse/ClickHouse/pull/35637) ([Dmitry Novik](https://github.com/novikd)). -* Nullables detection in protobuf. In proto3, default values are not sent on the wire. This makes it non-trivial to distinguish between null and default values for Nullable columns. A standard way to deal with this problem is to use Google wrappers to nest the target value within an inner message (see https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/wrappers.proto). In this case, a missing field is interpreted as null value, a field with missing value if interpreted as default value, and a field with regular value is interpreted as regular value. However, ClickHouse interprets Google wrappers as nested columns. We propose to introduce special behaviour to detect Google wrappers and interpret them like in the description above. For example, to serialize values for a Nullable column `test`, we would use `google.protobuf.StringValue test` in our .proto schema. Note that these types are so called "well-known types" in Protobuf, implemented in the library itself. [#35149](https://github.com/ClickHouse/ClickHouse/pull/35149) ([Jakub Kuklis](https://github.com/jkuklis)). -* Added support for specifying `content_type` in predefined and static HTTP handler config. [#34916](https://github.com/ClickHouse/ClickHouse/pull/34916) ([Roman Nikonov](https://github.com/nic11)). -* Warn properly if use clickhouse-client --file without preceeding --external. Close [#34747](https://github.com/ClickHouse/ClickHouse/issues/34747). [#34765](https://github.com/ClickHouse/ClickHouse/pull/34765) ([李扬](https://github.com/taiyang-li)). -* Improve MySQL database engine to compatible with binary(0) dataType. [#37232](https://github.com/ClickHouse/ClickHouse/pull/37232) ([zzsmdfj](https://github.com/zzsmdfj)). -* Improve JSON report of clickhouse-benchmark. [#36473](https://github.com/ClickHouse/ClickHouse/pull/36473) ([Tian Xinhui](https://github.com/xinhuitian)). -* Server might refuse to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([tavplubix](https://github.com/tavplubix)). - -#### Build/Testing/Packaging Improvement - -* Now `clickhouse-keeper` for the `x86_64` architecture is statically linked with [musl](https://musl.libc.org/) and doesn't depend on any system libraries. [#31833](https://github.com/ClickHouse/ClickHouse/pull/31833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* ClickHouse builds for `PowerPC64LE` architecture are now available in universal installation script `curl https://clickhouse.com/ | sh` and by direct link `https://builds.clickhouse.com/master/powerpc64le/clickhouse`. [#37095](https://github.com/ClickHouse/ClickHouse/pull/37095) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Limit PowerPC code generation to Power8 for better compatibility. This closes [#36025](https://github.com/ClickHouse/ClickHouse/issues/36025). [#36529](https://github.com/ClickHouse/ClickHouse/pull/36529) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Simplify performance test. This will give a chance for us to use it. [#36769](https://github.com/ClickHouse/ClickHouse/pull/36769) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fail performance comparison on errors in the report. [#34797](https://github.com/ClickHouse/ClickHouse/pull/34797) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Add ZSTD support for Arrow. This fixes [#35283](https://github.com/ClickHouse/ClickHouse/issues/35283). [#35486](https://github.com/ClickHouse/ClickHouse/pull/35486) ([Sean Lafferty](https://github.com/seanlaff)). - -#### Bug Fix - -* Extracts Version ID if present from the URI and adds a request to the AWS HTTP URI. Closes [#31221](https://github.com/ClickHouse/ClickHouse/issues/31221). - [x] Extract `Version ID` from URI if present and reassemble without it. - [x] Configure `AWS HTTP URI` object with request. - [x] Unit Tests: [`gtest_s3_uri`](https://github.com/ClickHouse/ClickHouse/blob/2340a6c6849ebc05a8efbf97ba8de3ff9dc0eff4/src/IO/tests/gtest_s3_uri.cpp) - [x] Drop instrumentation commit. [#34571](https://github.com/ClickHouse/ClickHouse/pull/34571) ([Saad Ur Rahman](https://github.com/surahman)). -* Fix system.opentelemetry_span_log attribute.values alias to values instead of keys. [#37275](https://github.com/ClickHouse/ClickHouse/pull/37275) ([Aleksandr Razumov](https://github.com/ernado)). -* Fix Nullable(String) to Nullable(Bool/IPv4/IPv6) conversion Closes [#37221](https://github.com/ClickHouse/ClickHouse/issues/37221). [#37270](https://github.com/ClickHouse/ClickHouse/pull/37270) ([Kruglov Pavel](https://github.com/Avogar)). -* Experimental feature: Fix execution of mutations in tables, in which there exist columns of type `Object`. Using subcolumns of type `Object` in `WHERE` expression of `UPDATE` or `DELETE` queries is now allowed yet, as well as manipulating (`DROP`, `MODIFY`) of separate subcolumns. Fixes [#37205](https://github.com/ClickHouse/ClickHouse/issues/37205). [#37266](https://github.com/ClickHouse/ClickHouse/pull/37266) ([Anton Popov](https://github.com/CurtizJ)). -* Kafka does not need `group.id` on producer stage. In console log you can find Warning that describe this issue: ``` 2022.05.15 17:59:13.270227 [ 137 ] {} StorageKafka (topic-name): [rdk:CONFWARN] [thrd:app]: Configuration property group.id is a consumer property and will be ignored by this producer instance ```. [#37228](https://github.com/ClickHouse/ClickHouse/pull/37228) ([Mark Andreev](https://github.com/mrk-andreev)). -* Experimental feature (WindowView): Update `max_fired_watermark ` after blocks actually fired, in case delete data that hasn't been fired yet. [#37225](https://github.com/ClickHouse/ClickHouse/pull/37225) ([vxider](https://github.com/Vxider)). -* Fix "Cannot create column of type Set" for distributed queries with LIMIT BY. [#37193](https://github.com/ClickHouse/ClickHouse/pull/37193) ([Azat Khuzhin](https://github.com/azat)). -* Experimental feature: Now WindowView `WATCH EVENTS` query will not be terminated due to the nonempty Chunk created in `WindowViewSource.h:58`. [#37182](https://github.com/ClickHouse/ClickHouse/pull/37182) ([vxider](https://github.com/Vxider)). -* Enable `enable_global_with_statement` for subqueries, close [#37141](https://github.com/ClickHouse/ClickHouse/issues/37141). [#37166](https://github.com/ClickHouse/ClickHouse/pull/37166) ([Vladimir C](https://github.com/vdimir)). -* Fix implicit cast for optimize_skip_unused_shards_rewrite_in. [#37153](https://github.com/ClickHouse/ClickHouse/pull/37153) ([Azat Khuzhin](https://github.com/azat)). -* The ILIKE function on FixedString columns could have returned wrong results (i.e. match less than it should). [#37117](https://github.com/ClickHouse/ClickHouse/pull/37117) ([Robert Schulze](https://github.com/rschu1ze)). -* Fix `GROUP BY` `AggregateFunction` (i.e. you `GROUP BY` by the column that has `AggregateFunction` type). [#37093](https://github.com/ClickHouse/ClickHouse/pull/37093) ([Azat Khuzhin](https://github.com/azat)). -* Experimental feature: Fix optimize_aggregation_in_order with prefix GROUP BY and *Array aggregate functions. [#37050](https://github.com/ClickHouse/ClickHouse/pull/37050) ([Azat Khuzhin](https://github.com/azat)). -* Fixed performance degradation of some INSERT SELECT queries with implicit aggregation. Fixes [#36792](https://github.com/ClickHouse/ClickHouse/issues/36792). [#37047](https://github.com/ClickHouse/ClickHouse/pull/37047) ([tavplubix](https://github.com/tavplubix)). -* Experimental feature: Fix in-order `GROUP BY` (`optimize_aggregation_in_order=1`) with `*Array` (`groupArrayArray`/...) aggregate functions. [#37046](https://github.com/ClickHouse/ClickHouse/pull/37046) ([Azat Khuzhin](https://github.com/azat)). -* Fix LowCardinality->ArrowDictionary invalid output when type of indexes is not UInt8. Closes [#36832](https://github.com/ClickHouse/ClickHouse/issues/36832). [#37043](https://github.com/ClickHouse/ClickHouse/pull/37043) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). -* Fix sending external tables data in HedgedConnections with max_parallel_replicas != 1. [#36981](https://github.com/ClickHouse/ClickHouse/pull/36981) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed logical error on `TRUNCATE` query in `Replicated` database. Fixes [#33747](https://github.com/ClickHouse/ClickHouse/issues/33747). [#36976](https://github.com/ClickHouse/ClickHouse/pull/36976) ([tavplubix](https://github.com/tavplubix)). -* Experimental feature: Fix stuck when dropping source table in WindowView. Closes [#35678](https://github.com/ClickHouse/ClickHouse/issues/35678). [#36967](https://github.com/ClickHouse/ClickHouse/pull/36967) ([vxider](https://github.com/Vxider)). -* Experimental feature (rocksdb cache): Fix issue: [#36671](https://github.com/ClickHouse/ClickHouse/issues/36671). [#36929](https://github.com/ClickHouse/ClickHouse/pull/36929) ([李扬](https://github.com/taiyang-li)). -* Experimental feature: Fix bugs when using multiple columns in WindowView by adding converting actions to make it possible to call`writeIntoWindowView` with a slightly different schema. [#36928](https://github.com/ClickHouse/ClickHouse/pull/36928) ([vxider](https://github.com/Vxider)). -* Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). -* Fix incorrect query result when doing constant aggregation. This fixes [#36728](https://github.com/ClickHouse/ClickHouse/issues/36728) . [#36888](https://github.com/ClickHouse/ClickHouse/pull/36888) ([Amos Bird](https://github.com/amosbird)). -* Experimental feature: Fix `current_size` count in cache. [#36887](https://github.com/ClickHouse/ClickHouse/pull/36887) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Experimental feature: Fix fire in window view with hop window [#34044](https://github.com/ClickHouse/ClickHouse/issues/34044). [#36861](https://github.com/ClickHouse/ClickHouse/pull/36861) ([vxider](https://github.com/Vxider)). -* Experimental feature: Fix incorrect cast in cached buffer from remote fs. [#36809](https://github.com/ClickHouse/ClickHouse/pull/36809) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix creation of tables with `flatten_nested = 0`. Previously unflattened `Nested` columns could be flattened after server restart. [#36803](https://github.com/ClickHouse/ClickHouse/pull/36803) ([Anton Popov](https://github.com/CurtizJ)). -* Fix some issues with async reads from remote filesystem which happened when reading low cardinality. [#36763](https://github.com/ClickHouse/ClickHouse/pull/36763) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Experimental feature: Fix insertion to columns of type `Object` from multiple files, e.g. via table function `file` with globs. [#36762](https://github.com/ClickHouse/ClickHouse/pull/36762) ([Anton Popov](https://github.com/CurtizJ)). -* Fix timeouts in Hedged requests. Connection hang right after sending remote query could lead to eternal waiting. [#36749](https://github.com/ClickHouse/ClickHouse/pull/36749) ([Kruglov Pavel](https://github.com/Avogar)). -* Experimental feature: Fix a bug of `groupBitmapAndState`/`groupBitmapOrState`/`groupBitmapXorState` on distributed table. [#36739](https://github.com/ClickHouse/ClickHouse/pull/36739) ([Zhang Yifan](https://github.com/zhangyifan27)). -* Experimental feature: During the [test](https://s3.amazonaws.com/clickhouse-test-reports/36376/1cb1c7275cb53769ab826772db9b71361bb3e413/stress_test__thread__actions_/clickhouse-server.clean.log) in [PR](https://github.com/ClickHouse/ClickHouse/pull/36376), I found that the one cache class was initialized twice, it throws a exception. Although the cause of this problem is not clear, there should be code logic of repeatedly loading disk in ClickHouse, so we need to make special judgment for this situation. [#36737](https://github.com/ClickHouse/ClickHouse/pull/36737) ([Han Shukai](https://github.com/KinderRiven)). -* Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). -* Fix server reload on port change (do not wait for current connections from query context). [#36700](https://github.com/ClickHouse/ClickHouse/pull/36700) ([Azat Khuzhin](https://github.com/azat)). -* Experimental feature: In the previous [PR](https://github.com/ClickHouse/ClickHouse/pull/36376), I found that testing (stateless tests, flaky check (address, actions)) is timeout. Moreover, testing locally can also trigger unstable system deadlocks. This problem still exists when using the latest source code of master. [#36697](https://github.com/ClickHouse/ClickHouse/pull/36697) ([Han Shukai](https://github.com/KinderRiven)). -* Experimental feature: Fix server restart if cache configuration changed. [#36685](https://github.com/ClickHouse/ClickHouse/pull/36685) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possible heap-use-after-free in schema inference. Closes [#36661](https://github.com/ClickHouse/ClickHouse/issues/36661). [#36679](https://github.com/ClickHouse/ClickHouse/pull/36679) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed parsing of query settings in `CREATE` query when engine is not specified. Fixes https://github.com/ClickHouse/ClickHouse/pull/34187#issuecomment-1103812419. [#36642](https://github.com/ClickHouse/ClickHouse/pull/36642) ([tavplubix](https://github.com/tavplubix)). -* Experimental feature: Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). -* Fix format crash when default expression follow EPHEMERAL not literal. Closes [#36618](https://github.com/ClickHouse/ClickHouse/issues/36618). [#36633](https://github.com/ClickHouse/ClickHouse/pull/36633) ([flynn](https://github.com/ucasfl)). -* Fix `Missing column` exception which could happen while using `INTERPOLATE` with `ENGINE = MergeTree` table. [#36549](https://github.com/ClickHouse/ClickHouse/pull/36549) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Fix potential error with literals in `WHERE` for join queries. Close [#36279](https://github.com/ClickHouse/ClickHouse/issues/36279). [#36542](https://github.com/ClickHouse/ClickHouse/pull/36542) ([Vladimir C](https://github.com/vdimir)). -* Fix offset update ReadBufferFromEncryptedFile, which could cause undefined behaviour. [#36493](https://github.com/ClickHouse/ClickHouse/pull/36493) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix hostname sanity checks for Keeper cluster configuration. Add `keeper_server.host_checks_enabled` config to enable/disable those checks. [#36492](https://github.com/ClickHouse/ClickHouse/pull/36492) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix usage of executable user defined functions in GROUP BY. Before executable user defined functions cannot be used as expressions in GROUP BY. Closes [#36448](https://github.com/ClickHouse/ClickHouse/issues/36448). [#36486](https://github.com/ClickHouse/ClickHouse/pull/36486) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix possible exception with unknown packet from server in client. [#36481](https://github.com/ClickHouse/ClickHouse/pull/36481) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Experimental feature (please never use `system.session_log`, it is going to be removed): Add missing enum values in system.session_log table. Closes [#36474](https://github.com/ClickHouse/ClickHouse/issues/36474). [#36480](https://github.com/ClickHouse/ClickHouse/pull/36480) ([Memo](https://github.com/Joeywzr)). -* Fix bug in s3Cluster schema inference that let to the fact that not all data was read in the select from s3Cluster. The bug appeared in https://github.com/ClickHouse/ClickHouse/pull/35544. [#36434](https://github.com/ClickHouse/ClickHouse/pull/36434) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix nullptr dereference in JOIN and COLUMNS matcher. This fixes [#36416](https://github.com/ClickHouse/ClickHouse/issues/36416). This is for https://github.com/ClickHouse/ClickHouse/pull/36417. [#36430](https://github.com/ClickHouse/ClickHouse/pull/36430) ([Amos Bird](https://github.com/amosbird)). -* Fix dictionary reload for `ClickHouseDictionarySource` if it contains scalar subqueries. [#36390](https://github.com/ClickHouse/ClickHouse/pull/36390) ([lthaooo](https://github.com/lthaooo)). -* Fix assertion in JOIN, close [#36199](https://github.com/ClickHouse/ClickHouse/issues/36199). [#36201](https://github.com/ClickHouse/ClickHouse/pull/36201) ([Vladimir C](https://github.com/vdimir)). -* Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). -* Experimental feature: Fix insertion of complex JSONs with nested arrays to columns of type `Object`. [#36077](https://github.com/ClickHouse/ClickHouse/pull/36077) ([Anton Popov](https://github.com/CurtizJ)). -* Fix ALTER DROP COLUMN of nested column with compact parts (i.e. `ALTER TABLE x DROP COLUMN n`, when there is column `n.d`). [#35797](https://github.com/ClickHouse/ClickHouse/pull/35797) ([Azat Khuzhin](https://github.com/azat)). -* Fix substring function range error length when `offset` and `length` is negative constant and `s` is not constant. [#33861](https://github.com/ClickHouse/ClickHouse/pull/33861) ([RogerYK](https://github.com/RogerYK)). - -### ClickHouse release 22.4, 2022-04-19 - -#### Backward Incompatible Change - -* Do not allow SETTINGS after FORMAT for INSERT queries (there is compatibility setting `allow_settings_after_format_in_insert` to accept such queries, but it is turned OFF by default). [#35883](https://github.com/ClickHouse/ClickHouse/pull/35883) ([Azat Khuzhin](https://github.com/azat)). -* Function `yandexConsistentHash` (consistent hashing algorithm by Konstantin "kostik" Oblakov) is renamed to `kostikConsistentHash`. The old name is left as an alias for compatibility. Although this change is backward compatible, we may remove the alias in subsequent releases, that's why it's recommended to update the usages of this function in your apps. [#35553](https://github.com/ClickHouse/ClickHouse/pull/35553) ([Alexey Milovidov](https://github.com/alexey-milovidov)). - -#### New Feature - -* Added INTERPOLATE extension to the ORDER BY ... WITH FILL. Closes [#34903](https://github.com/ClickHouse/ClickHouse/issues/34903). [#35349](https://github.com/ClickHouse/ClickHouse/pull/35349) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Profiling on Processors level (under `log_processors_profiles` setting, ClickHouse will write time that processor spent during execution/waiting for data to `system.processors_profile_log` table). [#34355](https://github.com/ClickHouse/ClickHouse/pull/34355) ([Azat Khuzhin](https://github.com/azat)). -* Added functions makeDate(year, month, day), makeDate32(year, month, day). [#35628](https://github.com/ClickHouse/ClickHouse/pull/35628) ([Alexander Gololobov](https://github.com/davenger)). Implementation of makeDateTime() and makeDateTIme64(). [#35934](https://github.com/ClickHouse/ClickHouse/pull/35934) ([Alexander Gololobov](https://github.com/davenger)). -* Support new type of quota `WRITTEN BYTES` to limit amount of written bytes during insert queries. [#35736](https://github.com/ClickHouse/ClickHouse/pull/35736) ([Anton Popov](https://github.com/CurtizJ)). -* Added function `flattenTuple`. It receives nested named `Tuple` as an argument and returns a flatten `Tuple` which elements are the paths from the original `Tuple`. E.g.: `Tuple(a Int, Tuple(b Int, c Int)) -> Tuple(a Int, b Int, c Int)`. `flattenTuple` can be used to select all paths from type `Object` as separate columns. [#35690](https://github.com/ClickHouse/ClickHouse/pull/35690) ([Anton Popov](https://github.com/CurtizJ)). -* Added functions `arrayFirstOrNull`, `arrayLastOrNull`. Closes [#35238](https://github.com/ClickHouse/ClickHouse/issues/35238). [#35414](https://github.com/ClickHouse/ClickHouse/pull/35414) ([Maksim Kita](https://github.com/kitaisreal)). -* Added functions `minSampleSizeContinous` and `minSampleSizeConversion`. Author [achimbab](https://github.com/achimbab). [#35360](https://github.com/ClickHouse/ClickHouse/pull/35360) ([Maksim Kita](https://github.com/kitaisreal)). -* New functions minSampleSizeContinous and minSampleSizeConversion. [#34354](https://github.com/ClickHouse/ClickHouse/pull/34354) ([achimbab](https://github.com/achimbab)). -* Introduce format `ProtobufList` (all records as repeated messages in out Protobuf). Closes [#16436](https://github.com/ClickHouse/ClickHouse/issues/16436). [#35152](https://github.com/ClickHouse/ClickHouse/pull/35152) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Add `h3PointDistM`, `h3PointDistKm`, `h3PointDistRads`, `h3GetRes0Indexes`, `h3GetPentagonIndexes` functions. [#34568](https://github.com/ClickHouse/ClickHouse/pull/34568) ([Bharat Nallan](https://github.com/bharatnc)). -* Add `toLastDayOfMonth` function which rounds up a date or date with time to the last day of the month. [#33501](https://github.com/ClickHouse/ClickHouse/issues/33501). [#34394](https://github.com/ClickHouse/ClickHouse/pull/34394) ([Habibullah Oladepo](https://github.com/holadepo)). -* Added load balancing setting for \[Zoo\]Keeper client. Closes [#29617](https://github.com/ClickHouse/ClickHouse/issues/29617). [#30325](https://github.com/ClickHouse/ClickHouse/pull/30325) ([小路](https://github.com/nicelulu)). -* Add a new kind of row policies named `simple`. Before this PR we had two kinds or row policies: `permissive` and `restrictive`. A `simple` row policy adds a new filter on a table without any side-effects like it was for permissive and restrictive policies. [#35345](https://github.com/ClickHouse/ClickHouse/pull/35345) ([Vitaly Baranov](https://github.com/vitlibar)). -* Added an ability to specify cluster secret in replicated database. [#35333](https://github.com/ClickHouse/ClickHouse/pull/35333) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Added sanity checks on server startup (available memory and disk space, max thread count, etc). [#34566](https://github.com/ClickHouse/ClickHouse/pull/34566) ([Sergei Trifonov](https://github.com/serxa)). -* INTERVAL improvement - can be used with `[MILLI|MICRO|NANO]SECOND`. Added `toStartOf[Milli|Micro|Nano]second()` functions. Added `[add|subtract][Milli|Micro|Nano]seconds()`. [#34353](https://github.com/ClickHouse/ClickHouse/pull/34353) ([Andrey Zvonov](https://github.com/zvonand)). - -#### Experimental Feature - -* Added support for transactions for simple `MergeTree` tables. This feature is highly experimental and not recommended for production. Part of [#22086](https://github.com/ClickHouse/ClickHouse/issues/22086). [#24258](https://github.com/ClickHouse/ClickHouse/pull/24258) ([tavplubix](https://github.com/tavplubix)). -* Support schema inference for type `Object` in format `JSONEachRow`. Allow to convert columns of type `Map` to columns of type `Object`. [#35629](https://github.com/ClickHouse/ClickHouse/pull/35629) ([Anton Popov](https://github.com/CurtizJ)). -* Allow to write remote FS cache on all write operations. Add `system.remote_filesystem_cache` table. Add `drop remote filesystem cache` query. Add introspection for s3 metadata with `system.remote_data_paths` table. Closes [#34021](https://github.com/ClickHouse/ClickHouse/issues/34021). Add cache option for merges by adding mode `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` (turned on by default for merges and can also be turned on by query setting with the same name). Rename cache related settings (`remote_fs_enable_cache -> enable_filesystem_cache`, etc). [#35475](https://github.com/ClickHouse/ClickHouse/pull/35475) ([Kseniia Sumarokova](https://github.com/kssenii)). -* An option to store parts metadata in RocksDB. Speed up parts loading process of MergeTree to accelerate starting up of clickhouse-server. With this improvement, clickhouse-server was able to decrease starting up time from 75 minutes to 20 seconds, with 700k mergetree parts. [#32928](https://github.com/ClickHouse/ClickHouse/pull/32928) ([李扬](https://github.com/taiyang-li)). - -#### Performance Improvement - -* A new query plan optimization. Evaluate functions after `ORDER BY` when possible. As an example, for a query `SELECT sipHash64(number) FROM numbers(1e8) ORDER BY number LIMIT 5`, function `sipHash64` would be evaluated after `ORDER BY` and `LIMIT`, which gives ~20x speed up. [#35623](https://github.com/ClickHouse/ClickHouse/pull/35623) ([Nikita Taranov](https://github.com/nickitat)). -* Sizes of hash tables used during aggregation now collected and used in later queries to avoid hash tables resizes. [#33439](https://github.com/ClickHouse/ClickHouse/pull/33439) ([Nikita Taranov](https://github.com/nickitat)). -* Improvement for hasAll function using SIMD instructions (SSE and AVX2). [#27653](https://github.com/ClickHouse/ClickHouse/pull/27653) ([youennL-cs](https://github.com/youennL-cs)). [#35723](https://github.com/ClickHouse/ClickHouse/pull/35723) ([Maksim Kita](https://github.com/kitaisreal)). -* Multiple changes to improve ASOF JOIN performance (1.2 - 1.6x as fast). It also adds support to use big integers. [#34733](https://github.com/ClickHouse/ClickHouse/pull/34733) ([Raúl Marín](https://github.com/Algunenano)). -* Improve performance of ASOF JOIN if key is native integer. [#35525](https://github.com/ClickHouse/ClickHouse/pull/35525) ([Maksim Kita](https://github.com/kitaisreal)). -* Parallelization of multipart upload into S3 storage. [#35343](https://github.com/ClickHouse/ClickHouse/pull/35343) ([Sergei Trifonov](https://github.com/serxa)). -* URL storage engine now downloads multiple chunks in parallel if the endpoint supports HTTP Range. Two additional settings were added, `max_download_threads` and `max_download_buffer_size`, which control maximum number of threads a single query can use to download the file and the maximum number of bytes each thread can process. [#35150](https://github.com/ClickHouse/ClickHouse/pull/35150) ([Antonio Andelic](https://github.com/antonio2368)). -* Use multiple threads to download objects from S3. Downloading is controllable using `max_download_threads` and `max_download_buffer_size` settings. [#35571](https://github.com/ClickHouse/ClickHouse/pull/35571) ([Antonio Andelic](https://github.com/antonio2368)). -* Narrow mutex scope when interacting with HDFS. Related to [#35292](https://github.com/ClickHouse/ClickHouse/issues/35292). [#35646](https://github.com/ClickHouse/ClickHouse/pull/35646) ([shuchaome](https://github.com/shuchaome)). -* Require mutations for per-table TTL only when it had been changed. [#35953](https://github.com/ClickHouse/ClickHouse/pull/35953) ([Azat Khuzhin](https://github.com/azat)). - -#### Improvement - -* Multiple improvements for schema inference. Use some tweaks and heuristics to determine numbers, strings, arrays, tuples and maps in CSV, TSV and TSVRaw data formats. Add setting `input_format_csv_use_best_effort_in_schema_inference` for CSV format that enables/disables using these heuristics, if it's disabled, we treat everything as string. Add similar setting `input_format_tsv_use_best_effort_in_schema_inference` for TSV/TSVRaw format. These settings are enabled by default. - Add Maps support for schema inference in Values format. - Fix possible segfault in schema inference in Values format. - Allow to skip columns with unsupported types in Arrow/ORC/Parquet formats. Add corresponding settings for it: `input_format_{parquet|orc|arrow}_skip_columns_with_unsupported_types_in_schema_inference`. These settings are disabled by default. - Allow to convert a column with type Null to a Nullable column with all NULL values in Arrow/Parquet formats. - Allow to specify column names in schema inference via setting `column_names_for_schema_inference` for formats that don't contain column names (like CSV, TSV, JSONCompactEachRow, etc) - Fix schema inference in ORC/Arrow/Parquet formats in terms of working with Nullable columns. Previously all inferred types were not Nullable and it blocked reading Nullable columns from data, now it's fixed and all inferred types are always Nullable (because we cannot understand that column is Nullable or not by reading the schema). - Fix schema inference in Template format with CSV escaping rules. [#35582](https://github.com/ClickHouse/ClickHouse/pull/35582) ([Kruglov Pavel](https://github.com/Avogar)). -* Add parallel parsing and schema inference for format `JSONAsObject`. [#35592](https://github.com/ClickHouse/ClickHouse/pull/35592) ([Anton Popov](https://github.com/CurtizJ)). -* Added a support for automatic schema inference to `s3Cluster` table function. Synced the signatures of `s3 ` and `s3Cluster`. [#35544](https://github.com/ClickHouse/ClickHouse/pull/35544) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Added support for schema inference for `hdfsCluster`. [#35602](https://github.com/ClickHouse/ClickHouse/pull/35602) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Add new setting `input_format_json_read_bools_as_numbers` that allows to infer and parse bools as numbers in JSON input formats. It's enabled by default. Suggested by @alexey-milovidov. [#35735](https://github.com/ClickHouse/ClickHouse/pull/35735) ([Kruglov Pavel](https://github.com/Avogar)). -* Improve columns ordering in schema inference for formats TSKV and JSONEachRow, closes [#35640](https://github.com/ClickHouse/ClickHouse/issues/35640). Don't stop schema inference when reading empty row in schema inference for formats TSKV and JSONEachRow. [#35724](https://github.com/ClickHouse/ClickHouse/pull/35724) ([Kruglov Pavel](https://github.com/Avogar)). -* Add settings `input_format_orc_case_insensitive_column_matching`, `input_format_arrow_case_insensitive_column_matching`, and `input_format_parquet_case_insensitive_column_matching` which allows ClickHouse to use case insensitive matching of columns while reading data from ORC, Arrow or Parquet files. [#35459](https://github.com/ClickHouse/ClickHouse/pull/35459) ([Antonio Andelic](https://github.com/antonio2368)). -* Added `is_secure` column to `system.query_log` which denotes if the client is using a secure connection over TCP or HTTP. [#35705](https://github.com/ClickHouse/ClickHouse/pull/35705) ([Antonio Andelic](https://github.com/antonio2368)). -* Now `kafka_num_consumers` can be bigger than amount of physical cores in case of low resource machine (less than 16 cores). [#35926](https://github.com/ClickHouse/ClickHouse/pull/35926) ([alesapin](https://github.com/alesapin)). -* Add some basic metrics to monitor engine=Kafka tables. [#35916](https://github.com/ClickHouse/ClickHouse/pull/35916) ([filimonov](https://github.com/filimonov)). -* Now it's not allowed to `ALTER TABLE ... RESET SETTING` for non-existing settings for MergeTree engines family. Fixes [#35816](https://github.com/ClickHouse/ClickHouse/issues/35816). [#35884](https://github.com/ClickHouse/ClickHouse/pull/35884) ([alesapin](https://github.com/alesapin)). -* Now some `ALTER MODIFY COLUMN` queries for `Arrays` and `Nullable` types can be done at metadata level without mutations. For example, alter from `Array(Enum8('Option1'=1))` to `Array(Enum8('Option1'=1, 'Option2'=2))`. [#35882](https://github.com/ClickHouse/ClickHouse/pull/35882) ([alesapin](https://github.com/alesapin)). -* Added an animation to the hourglass icon to indicate to the user that a query is running. [#35860](https://github.com/ClickHouse/ClickHouse/pull/35860) ([peledni](https://github.com/peledni)). -* support ALTER TABLE t DETACH PARTITION (ALL). [#35794](https://github.com/ClickHouse/ClickHouse/pull/35794) ([awakeljw](https://github.com/awakeljw)). -* Improve projection analysis to optimize trivial queries such as `count()`. [#35788](https://github.com/ClickHouse/ClickHouse/pull/35788) ([Amos Bird](https://github.com/amosbird)). -* Support schema inference for insert select with using `input` table function. Get schema from insertion table instead of inferring it from the data in case of insert select from table functions that support schema inference. Closes [#35639](https://github.com/ClickHouse/ClickHouse/issues/35639). [#35760](https://github.com/ClickHouse/ClickHouse/pull/35760) ([Kruglov Pavel](https://github.com/Avogar)). -* Respect `remote_url_allow_hosts` for Hive tables. [#35743](https://github.com/ClickHouse/ClickHouse/pull/35743) ([李扬](https://github.com/taiyang-li)). -* Implement `send_logs_level` for clickhouse-local. Closes [#35653](https://github.com/ClickHouse/ClickHouse/issues/35653). [#35716](https://github.com/ClickHouse/ClickHouse/pull/35716) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Closes [#35641](https://github.com/ClickHouse/ClickHouse/issues/35641) Allow `EPHEMERAL` columns without explicit default expression. [#35706](https://github.com/ClickHouse/ClickHouse/pull/35706) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Add profile event counter `AsyncInsertBytes` about size of async INSERTs. [#35644](https://github.com/ClickHouse/ClickHouse/pull/35644) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Improve the pipeline description for JOIN. [#35612](https://github.com/ClickHouse/ClickHouse/pull/35612) ([何李夫](https://github.com/helifu)). -* Deduce absolute hdfs config path. [#35572](https://github.com/ClickHouse/ClickHouse/pull/35572) ([李扬](https://github.com/taiyang-li)). -* Improve pasting performance and compatibility of clickhouse-client. This helps [#35501](https://github.com/ClickHouse/ClickHouse/issues/35501). [#35541](https://github.com/ClickHouse/ClickHouse/pull/35541) ([Amos Bird](https://github.com/amosbird)). -* It was possible to get stack overflow in distributed queries if one of the settings `async_socket_for_remote` and `use_hedged_requests` is enabled while parsing very deeply nested data type (at least in debug build). Closes [#35509](https://github.com/ClickHouse/ClickHouse/issues/35509). [#35524](https://github.com/ClickHouse/ClickHouse/pull/35524) ([Kruglov Pavel](https://github.com/Avogar)). -* Add sizes of subcolumns to `system.parts_columns` table. [#35488](https://github.com/ClickHouse/ClickHouse/pull/35488) ([Anton Popov](https://github.com/CurtizJ)). -* Add explicit table info to the scan node of query plan and pipeline. [#35460](https://github.com/ClickHouse/ClickHouse/pull/35460) ([何李夫](https://github.com/helifu)). -* Allow server to bind to low-numbered ports (e.g. 443). ClickHouse installation script will set `cap_net_bind_service` to the binary file. [#35451](https://github.com/ClickHouse/ClickHouse/pull/35451) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix INSERT INTO table FROM INFILE: it did not display the progress bar. [#35429](https://github.com/ClickHouse/ClickHouse/pull/35429) ([xiedeyantu](https://github.com/xiedeyantu)). -* Add arguments `--user`, `--password`, `--host`, `--port` for `clickhouse-diagnostics` tool. [#35422](https://github.com/ClickHouse/ClickHouse/pull/35422) ([李扬](https://github.com/taiyang-li)). -* Support uuid for Postgres engines. Closes [#35384](https://github.com/ClickHouse/ClickHouse/issues/35384). [#35403](https://github.com/ClickHouse/ClickHouse/pull/35403) ([Kseniia Sumarokova](https://github.com/kssenii)). -* For table function `s3cluster` or `HDFSCluster` or `hive`, we can't get right `AccessType` by `StorageFactory::instance().getSourceAccessType(getStorageTypeName())`. This pr fix it. [#35365](https://github.com/ClickHouse/ClickHouse/pull/35365) ([李扬](https://github.com/taiyang-li)). -* Remove `--testmode` option for clickhouse-client, enable it unconditionally. [#35354](https://github.com/ClickHouse/ClickHouse/pull/35354) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Don't allow `wchc` operation (four letter command) for clickhouse-keeper. [#35320](https://github.com/ClickHouse/ClickHouse/pull/35320) ([zhangyuli1](https://github.com/zhangyuli1)). -* Add function `getTypeSerializationStreams`. For a specified type (which is detected from column), it returns an array with all the serialization substream paths. This function is useful mainly for developers. [#35290](https://github.com/ClickHouse/ClickHouse/pull/35290) ([李扬](https://github.com/taiyang-li)). -* If `port` is not specified in cluster configuration, default server port will be used. This closes [#34769](https://github.com/ClickHouse/ClickHouse/issues/34769). [#34772](https://github.com/ClickHouse/ClickHouse/pull/34772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Use `minmax` index for orc/parquet file in Hive Engine. Related PR: https://github.com/ClickHouse/arrow/pull/10. [#34631](https://github.com/ClickHouse/ClickHouse/pull/34631) ([李扬](https://github.com/taiyang-li)). -* System log tables now allow to specify COMMENT in ENGINE declaration. Closes [#33768](https://github.com/ClickHouse/ClickHouse/issues/33768). [#34536](https://github.com/ClickHouse/ClickHouse/pull/34536) ([Maksim Kita](https://github.com/kitaisreal)). -* Proper support of setting `max_rows_to_read` in case of reading in order of sorting key and specified limit. Previously the exception `Limit for rows or bytes to read exceeded` could be thrown even if query actually requires to read less amount of rows. [#33230](https://github.com/ClickHouse/ClickHouse/pull/33230) ([Anton Popov](https://github.com/CurtizJ)). -* Respect only quota & period from cgroups, ignore shares (which are not really limit the number of the cores which can be used). [#35815](https://github.com/ClickHouse/ClickHouse/pull/35815) ([filimonov](https://github.com/filimonov)). - -#### Build/Testing/Packaging Improvement - -* Add next batch of randomization settings in functional tests. [#35047](https://github.com/ClickHouse/ClickHouse/pull/35047) ([Kruglov Pavel](https://github.com/Avogar)). -* Add backward compatibility check in stress test. Closes [#25088](https://github.com/ClickHouse/ClickHouse/issues/25088). [#27928](https://github.com/ClickHouse/ClickHouse/pull/27928) ([Kruglov Pavel](https://github.com/Avogar)). -* Migrate package building to `nfpm` - Deprecate `release` script in favor of `packages/build` - Build everything in clickhouse/binary-builder image (cleanup: clickhouse/deb-builder) - Add symbol stripping to cmake (todo: use $prefix/lib/$bin_dir/clickhouse/$binary.debug) - Fix issue with DWARF symbols - Add Alpine APK packages - Rename `alien` to `additional_pkgs`. [#33664](https://github.com/ClickHouse/ClickHouse/pull/33664) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Add a night scan and upload for Coverity. [#34895](https://github.com/ClickHouse/ClickHouse/pull/34895) ([Boris Kuschel](https://github.com/bkuschel)). -* A dedicated small package for `clickhouse-keeper`. [#35308](https://github.com/ClickHouse/ClickHouse/pull/35308) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Running with podman was failing: it complains about specifying the same volume twice. [#35978](https://github.com/ClickHouse/ClickHouse/pull/35978) ([Roman Nikonov](https://github.com/nic11)). -* Minor improvement in contrib/krb5 build configuration. [#35832](https://github.com/ClickHouse/ClickHouse/pull/35832) ([Anton Kozlov](https://github.com/tonickkozlov)). -* Add a label to recognize a building task for every image. [#35583](https://github.com/ClickHouse/ClickHouse/pull/35583) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Apply `black` formatter to python code and add a per-commit check. [#35466](https://github.com/ClickHouse/ClickHouse/pull/35466) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Redo alpine image to use clean Dockerfile. Create a script in tests/ci to build both ubuntu and alpine images. Add clickhouse-keeper image (cc @nikitamikhaylov). Add build check to PullRequestCI. Add a job to a ReleaseCI. Add a job to MasterCI to build and push `clickhouse/clickhouse-server:head` and `clickhouse/clickhouse-keeper:head` images for each merged PR. [#35211](https://github.com/ClickHouse/ClickHouse/pull/35211) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Fix stress-test report in CI, now we upload the runlog with information about started stress tests only once. [#35093](https://github.com/ClickHouse/ClickHouse/pull/35093) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Switch to libcxx / libcxxabi from LLVM 14. [#34906](https://github.com/ClickHouse/ClickHouse/pull/34906) ([Raúl Marín](https://github.com/Algunenano)). -* Update unixodbc to mitigate CVE-2018-7485. Note: this CVE is not relevant for ClickHouse as it implements its own isolation layer for ODBC. [#35943](https://github.com/ClickHouse/ClickHouse/pull/35943) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). - -#### Bug Fix - -* Added settings `input_format_ipv4_default_on_conversion_error`, `input_format_ipv6_default_on_conversion_error` to allow insert of invalid ip address values as default into tables. Closes [#35726](https://github.com/ClickHouse/ClickHouse/issues/35726). [#35733](https://github.com/ClickHouse/ClickHouse/pull/35733) ([Maksim Kita](https://github.com/kitaisreal)). -* Avoid erasing columns from a block if it doesn't exist while reading data from Hive. [#35393](https://github.com/ClickHouse/ClickHouse/pull/35393) ([lgbo](https://github.com/lgbo-ustc)). -* Add type checking when creating materialized view. Close: [#23684](https://github.com/ClickHouse/ClickHouse/issues/23684). [#24896](https://github.com/ClickHouse/ClickHouse/pull/24896) ([hexiaoting](https://github.com/hexiaoting)). -* Fix formatting of INSERT INFILE queries (missing quotes). [#35886](https://github.com/ClickHouse/ClickHouse/pull/35886) ([Azat Khuzhin](https://github.com/azat)). -* Disable `session_log` because memory safety issue has been found by fuzzing. See [#35714](https://github.com/ClickHouse/ClickHouse/issues/35714). [#35873](https://github.com/ClickHouse/ClickHouse/pull/35873) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Avoid processing per-column TTL multiple times. [#35820](https://github.com/ClickHouse/ClickHouse/pull/35820) ([Azat Khuzhin](https://github.com/azat)). -* Fix inserts to columns of type `Object` in case when there is data related to several partitions in insert query. [#35806](https://github.com/ClickHouse/ClickHouse/pull/35806) ([Anton Popov](https://github.com/CurtizJ)). -* Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixes [#35751](https://github.com/ClickHouse/ClickHouse/issues/35751). [#35799](https://github.com/ClickHouse/ClickHouse/pull/35799) ([Nikolay Degterinsky](https://github.com/evillique)). -* Fix for reading from HDFS in Snappy format. [#35771](https://github.com/ClickHouse/ClickHouse/pull/35771) ([shuchaome](https://github.com/shuchaome)). -* Fix bug in conversion from custom types to string that could lead to segfault or unexpected error messages. Closes [#35752](https://github.com/ClickHouse/ClickHouse/issues/35752). [#35755](https://github.com/ClickHouse/ClickHouse/pull/35755) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix any/all (subquery) implementation. Closes [#35489](https://github.com/ClickHouse/ClickHouse/issues/35489). [#35727](https://github.com/ClickHouse/ClickHouse/pull/35727) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix dropping non-empty database in clickhouse-local. Closes [#35692](https://github.com/ClickHouse/ClickHouse/issues/35692). [#35711](https://github.com/ClickHouse/ClickHouse/pull/35711) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix bug in creating materialized view with subquery after server restart. Materialized view was not getting updated after inserts into underlying table after server restart. Closes [#35511](https://github.com/ClickHouse/ClickHouse/issues/35511). [#35691](https://github.com/ClickHouse/ClickHouse/pull/35691) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix possible `Can't adjust last granule` exception while reading subcolumns of experimental type `Object`. [#35687](https://github.com/ClickHouse/ClickHouse/pull/35687) ([Anton Popov](https://github.com/CurtizJ)). -* Enable build with JIT compilation by default. [#35683](https://github.com/ClickHouse/ClickHouse/pull/35683) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix possible loss of subcolumns in experimental type `Object`. [#35682](https://github.com/ClickHouse/ClickHouse/pull/35682) ([Anton Popov](https://github.com/CurtizJ)). -* Fix check ASOF JOIN key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). -* Fix part checking logic for parts with projections. Error happened when projection and main part had different types. This is similar to https://github.com/ClickHouse/ClickHouse/pull/33774 . The bug is addressed by @caoyang10. [#35667](https://github.com/ClickHouse/ClickHouse/pull/35667) ([Amos Bird](https://github.com/amosbird)). -* Fix server crash when large number of arguments are passed into `format` function. Please refer to the test file and see how to reproduce the crash. [#35651](https://github.com/ClickHouse/ClickHouse/pull/35651) ([Amos Bird](https://github.com/amosbird)). -* Fix usage of quotas with asynchronous inserts. [#35645](https://github.com/ClickHouse/ClickHouse/pull/35645) ([Anton Popov](https://github.com/CurtizJ)). -* Fix positional arguments with aliases. Closes [#35600](https://github.com/ClickHouse/ClickHouse/issues/35600). [#35620](https://github.com/ClickHouse/ClickHouse/pull/35620) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Check `remote_url_allow_hosts` before schema inference in URL engine Closes [#35064](https://github.com/ClickHouse/ClickHouse/issues/35064). [#35619](https://github.com/ClickHouse/ClickHouse/pull/35619) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix `HashJoin` when columns with `LowCardinality` type are used. This closes [#35548](https://github.com/ClickHouse/ClickHouse/issues/35548). [#35616](https://github.com/ClickHouse/ClickHouse/pull/35616) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix possible segfault in MaterializedPostgreSQL which happened if exception occurred when data, collected in memory, was synced into underlying tables. Closes [#35611](https://github.com/ClickHouse/ClickHouse/issues/35611). [#35614](https://github.com/ClickHouse/ClickHouse/pull/35614) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Setting `database_atomic_wait_for_drop_and_detach_synchronously` worked incorrectly for `ATTACH TABLE` query when previously detached table is still in use, It's fixed. [#35594](https://github.com/ClickHouse/ClickHouse/pull/35594) ([tavplubix](https://github.com/tavplubix)). -* Fix HTTP headers with named collections, add compression_method. Closes [#35273](https://github.com/ClickHouse/ClickHouse/issues/35273). Closes [#35269](https://github.com/ClickHouse/ClickHouse/issues/35269). [#35593](https://github.com/ClickHouse/ClickHouse/pull/35593) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix s3 engine getting virtual columns. Closes [#35411](https://github.com/ClickHouse/ClickHouse/issues/35411). [#35586](https://github.com/ClickHouse/ClickHouse/pull/35586) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fixed return type deduction for `caseWithExpression`. The type of the ELSE branch is now correctly taken into account. [#35576](https://github.com/ClickHouse/ClickHouse/pull/35576) ([Antonio Andelic](https://github.com/antonio2368)). -* Fix parsing of IPv6 addresses longer than 39 characters. Closes [#34022](https://github.com/ClickHouse/ClickHouse/issues/34022). [#35539](https://github.com/ClickHouse/ClickHouse/pull/35539) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix cast into IPv4, IPv6 address in IN section. Fixes [#35528](https://github.com/ClickHouse/ClickHouse/issues/35528). [#35534](https://github.com/ClickHouse/ClickHouse/pull/35534) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix crash during short circuit function evaluation when one of arguments is nullable constant. Closes [#35497](https://github.com/ClickHouse/ClickHouse/issues/35497). Closes [#35496](https://github.com/ClickHouse/ClickHouse/issues/35496). [#35502](https://github.com/ClickHouse/ClickHouse/pull/35502) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix crash for function `throwIf` with constant arguments. [#35500](https://github.com/ClickHouse/ClickHouse/pull/35500) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix bug in Keeper which can lead to unstable client connections. Introduced in [#35031](https://github.com/ClickHouse/ClickHouse/issues/35031). [#35498](https://github.com/ClickHouse/ClickHouse/pull/35498) ([alesapin](https://github.com/alesapin)). -* Fix bug in function `if` when resulting column type differs with resulting data type that led to logical errors like `Logical error: 'Bad cast from type DB::ColumnVector to DB::ColumnVector'.`. Closes [#35367](https://github.com/ClickHouse/ClickHouse/issues/35367). [#35476](https://github.com/ClickHouse/ClickHouse/pull/35476) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix excessive logging when using S3 as backend for MergeTree or as separate table engine/function. Fixes [#30559](https://github.com/ClickHouse/ClickHouse/issues/30559). [#35434](https://github.com/ClickHouse/ClickHouse/pull/35434) ([alesapin](https://github.com/alesapin)). -* Now merges executed with zero copy replication (experimental) will not spam logs with message `Found parts with the same min block and with the same max block as the missing part _ on replica _. Hoping that it will eventually appear as a result of a merge.`. [#35430](https://github.com/ClickHouse/ClickHouse/pull/35430) ([alesapin](https://github.com/alesapin)). -* Skip possible exception if empty chunks appear in GroupingAggregatedTransform. [#35417](https://github.com/ClickHouse/ClickHouse/pull/35417) ([Nikita Taranov](https://github.com/nickitat)). -* Fix working with columns that are not needed in query in Arrow/Parquet/ORC formats, it prevents possible errors like `Unsupported type of an input column ` when file contains column with unsupported type and we don't use it in query. [#35406](https://github.com/ClickHouse/ClickHouse/pull/35406) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix for local cache for remote filesystem (experimental feature) for high concurrency on corner cases. [#35381](https://github.com/ClickHouse/ClickHouse/pull/35381) ([Kseniia Sumarokova](https://github.com/kssenii)). Fix possible deadlock in cache. [#35378](https://github.com/ClickHouse/ClickHouse/pull/35378) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix partition pruning in case of comparison with constant in `WHERE`. If column and constant had different types, overflow was possible. Query could return an incorrect empty result. This fixes [#35304](https://github.com/ClickHouse/ClickHouse/issues/35304). [#35334](https://github.com/ClickHouse/ClickHouse/pull/35334) ([Amos Bird](https://github.com/amosbird)). -* Fix schema inference for TSKV format while using small max_read_buffer_size. [#35332](https://github.com/ClickHouse/ClickHouse/pull/35332) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix mutations in tables with enabled sparse columns. [#35284](https://github.com/ClickHouse/ClickHouse/pull/35284) ([Anton Popov](https://github.com/CurtizJ)). -* Do not delay final part writing by default (fixes possible `Memory limit exceeded` during `INSERT` by adding `max_insert_delayed_streams_for_parallel_write` with default to 1000 for writes to s3 and disabled as before otherwise). [#34780](https://github.com/ClickHouse/ClickHouse/pull/34780) ([Azat Khuzhin](https://github.com/azat)). - -### ClickHouse release v22.3-lts, 2022-03-17 - -#### Backward Incompatible Change - -* Make `arrayCompact` function behave as other higher-order functions: perform compaction not of lambda function results but on the original array. If you're using nontrivial lambda functions in arrayCompact you may restore old behaviour by wrapping `arrayCompact` arguments into `arrayMap`. Closes [#34010](https://github.com/ClickHouse/ClickHouse/issues/34010) [#18535](https://github.com/ClickHouse/ClickHouse/issues/18535) [#14778](https://github.com/ClickHouse/ClickHouse/issues/14778). [#34795](https://github.com/ClickHouse/ClickHouse/pull/34795) ([Alexandre Snarskii](https://github.com/snar)). -* Change implementation specific behavior on overflow of function `toDatetime`. It will be saturated to the nearest min/max supported instant of datetime instead of wraparound. This change is highlighted as "backward incompatible" because someone may unintentionally rely on the old behavior. [#32898](https://github.com/ClickHouse/ClickHouse/pull/32898) ([HaiBo Li](https://github.com/marising)). -* Make function `cast(value, 'IPv4')`, `cast(value, 'IPv6')` behave same as `toIPv4`, `toIPv6` functions. Changed behavior of incorrect IP address passed into functions `toIPv4`,` toIPv6`, now if invalid IP address passes into this functions exception will be raised, before this function return default value. Added functions `IPv4StringToNumOrDefault`, `IPv4StringToNumOrNull`, `IPv6StringToNumOrDefault`, `IPv6StringOrNull` `toIPv4OrDefault`, `toIPv4OrNull`, `toIPv6OrDefault`, `toIPv6OrNull`. Functions `IPv4StringToNumOrDefault `, `toIPv4OrDefault `, `toIPv6OrDefault ` should be used if previous logic relied on `IPv4StringToNum`, `toIPv4`, `toIPv6` returning default value for invalid address. Added setting `cast_ipv4_ipv6_default_on_conversion_error`, if this setting enabled, then IP address conversion functions will behave as before. Closes [#22825](https://github.com/ClickHouse/ClickHouse/issues/22825). Closes [#5799](https://github.com/ClickHouse/ClickHouse/issues/5799). Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#35240](https://github.com/ClickHouse/ClickHouse/pull/35240) ([Maksim Kita](https://github.com/kitaisreal)). - -#### New Feature - -* Support for caching data locally for remote filesystems. It can be enabled for `s3` disks. Closes [#28961](https://github.com/ClickHouse/ClickHouse/issues/28961). [#33717](https://github.com/ClickHouse/ClickHouse/pull/33717) ([Kseniia Sumarokova](https://github.com/kssenii)). In the meantime, we enabled the test suite on s3 filesystem and no more known issues exist, so it is started to be production ready. -* Add new table function `hive`. It can be used as follows `hive('', '', '', '', '')` for example `SELECT * FROM hive('thrift://hivetest:9083', 'test', 'demo', 'id Nullable(String), score Nullable(Int32), day Nullable(String)', 'day')`. [#34946](https://github.com/ClickHouse/ClickHouse/pull/34946) ([lgbo](https://github.com/lgbo-ustc)). -* Support authentication of users connected via SSL by their X.509 certificate. [#31484](https://github.com/ClickHouse/ClickHouse/pull/31484) ([eungenue](https://github.com/eungenue)). -* Support schema inference for inserting into table functions `file`/`hdfs`/`s3`/`url`. [#34732](https://github.com/ClickHouse/ClickHouse/pull/34732) ([Kruglov Pavel](https://github.com/Avogar)). -* Now you can read `system.zookeeper` table without restrictions on path or using `like` expression. This reads can generate quite heavy load for zookeeper so to enable this ability you have to enable setting `allow_unrestricted_reads_from_keeper`. [#34609](https://github.com/ClickHouse/ClickHouse/pull/34609) ([Sergei Trifonov](https://github.com/serxa)). -* Display CPU and memory metrics in clickhouse-local. Close [#34545](https://github.com/ClickHouse/ClickHouse/issues/34545). [#34605](https://github.com/ClickHouse/ClickHouse/pull/34605) ([李扬](https://github.com/taiyang-li)). -* Implement `startsWith` and `endsWith` function for arrays, closes [#33982](https://github.com/ClickHouse/ClickHouse/issues/33982). [#34368](https://github.com/ClickHouse/ClickHouse/pull/34368) ([usurai](https://github.com/usurai)). -* Add three functions for Map data type: 1. `mapReplace(map1, map2)` - replaces values for keys in map1 with the values of the corresponding keys in map2; adds keys from map2 that don't exist in map1. 2. `mapFilter` 3. `mapMap`. mapFilter and mapMap are higher order functions, accepting two arguments, the first argument is a lambda function with k, v pair as arguments, the second argument is a column of type Map. [#33698](https://github.com/ClickHouse/ClickHouse/pull/33698) ([hexiaoting](https://github.com/hexiaoting)). -* Allow getting default user and password for clickhouse-client from the `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables. Close [#34538](https://github.com/ClickHouse/ClickHouse/issues/34538). [#34947](https://github.com/ClickHouse/ClickHouse/pull/34947) ([DR](https://github.com/freedomDR)). - -#### Experimental Feature - -* New data type `Object()`, which supports storing of semi-structured data (for now JSON only). Data is written to such types as string. Then all paths are extracted according to format of semi-structured data and written as separate columns in most optimal types, that can store all their values. Those columns can be queried by names that match paths in source data. E.g `data.key1.key2` or with cast operator `data.key1.key2::Int64`. -* Add `database_replicated_allow_only_replicated_engine` setting. When enabled, it only allowed to only create `Replicated` tables or tables with stateless engines in `Replicated` databases. [#35214](https://github.com/ClickHouse/ClickHouse/pull/35214) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). Note that `Replicated` database is still an experimental feature. - -#### Performance Improvement - -* Improve performance of insertion into `MergeTree` tables by optimizing sorting. Up to 2x improvement is observed on realistic benchmarks. [#34750](https://github.com/ClickHouse/ClickHouse/pull/34750) ([Maksim Kita](https://github.com/kitaisreal)). -* Columns pruning when reading Parquet, ORC and Arrow files from URL and S3. Closes [#34163](https://github.com/ClickHouse/ClickHouse/issues/34163). [#34849](https://github.com/ClickHouse/ClickHouse/pull/34849) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Columns pruning when reading Parquet, ORC and Arrow files from Hive. [#34954](https://github.com/ClickHouse/ClickHouse/pull/34954) ([lgbo](https://github.com/lgbo-ustc)). -* A bunch of performance optimizations from a performance superhero. Improve performance of processing queries with large `IN` section. Improve performance of `direct` dictionary if its source is `ClickHouse`. Improve performance of `detectCharset `, `detectLanguageUnknown ` functions. [#34888](https://github.com/ClickHouse/ClickHouse/pull/34888) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of `any` aggregate function by using more batching. [#34760](https://github.com/ClickHouse/ClickHouse/pull/34760) ([Raúl Marín](https://github.com/Algunenano)). -* Multiple improvements for performance of `clickhouse-keeper`: less locking [#35010](https://github.com/ClickHouse/ClickHouse/pull/35010) ([zhanglistar](https://github.com/zhanglistar)), lower memory usage by streaming reading and writing of snapshot instead of full copy. [#34584](https://github.com/ClickHouse/ClickHouse/pull/34584) ([zhanglistar](https://github.com/zhanglistar)), optimizing compaction of log store in the RAFT implementation. [#34534](https://github.com/ClickHouse/ClickHouse/pull/34534) ([zhanglistar](https://github.com/zhanglistar)), versioning of the internal data structure [#34486](https://github.com/ClickHouse/ClickHouse/pull/34486) ([zhanglistar](https://github.com/zhanglistar)). - -#### Improvement - -* Allow asynchronous inserts to table functions. Fixes [#34864](https://github.com/ClickHouse/ClickHouse/issues/34864). [#34866](https://github.com/ClickHouse/ClickHouse/pull/34866) ([Anton Popov](https://github.com/CurtizJ)). -* Implicit type casting of the key argument for functions `dictGetHierarchy`, `dictIsIn`, `dictGetChildren`, `dictGetDescendants`. Closes [#34970](https://github.com/ClickHouse/ClickHouse/issues/34970). [#35027](https://github.com/ClickHouse/ClickHouse/pull/35027) ([Maksim Kita](https://github.com/kitaisreal)). -* `EXPLAIN AST` query can output AST in form of a graph in Graphviz format: `EXPLAIN AST graph = 1 SELECT * FROM system.parts`. [#35173](https://github.com/ClickHouse/ClickHouse/pull/35173) ([李扬](https://github.com/taiyang-li)). -* When large files were written with `s3` table function or table engine, the content type on the files was mistakenly set to `application/xml` due to a bug in the AWS SDK. This closes [#33964](https://github.com/ClickHouse/ClickHouse/issues/33964). [#34433](https://github.com/ClickHouse/ClickHouse/pull/34433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Change restrictive row policies a bit to make them an easier alternative to permissive policies in easy cases. If for a particular table only restrictive policies exist (without permissive policies) users will be able to see some rows. Also `SHOW CREATE ROW POLICY` will always show `AS permissive` or `AS restrictive` in row policy's definition. [#34596](https://github.com/ClickHouse/ClickHouse/pull/34596) ([Vitaly Baranov](https://github.com/vitlibar)). -* Improve schema inference with globs in File/S3/HDFS/URL engines. Try to use the next path for schema inference in case of error. [#34465](https://github.com/ClickHouse/ClickHouse/pull/34465) ([Kruglov Pavel](https://github.com/Avogar)). -* Play UI now correctly detects the preferred light/dark theme from the OS. [#35068](https://github.com/ClickHouse/ClickHouse/pull/35068) ([peledni](https://github.com/peledni)). -* Added `date_time_input_format = 'best_effort_us'`. Closes [#34799](https://github.com/ClickHouse/ClickHouse/issues/34799). [#34982](https://github.com/ClickHouse/ClickHouse/pull/34982) ([WenYao](https://github.com/Cai-Yao)). -* A new settings called `allow_plaintext_password` and `allow_no_password` are added in server configuration which turn on/off authentication types that can be potentially insecure in some environments. They are allowed by default. [#34738](https://github.com/ClickHouse/ClickHouse/pull/34738) ([Heena Bansal](https://github.com/HeenaBansal2009)). -* Support for `DateTime64` data type in `Arrow` format, closes [#8280](https://github.com/ClickHouse/ClickHouse/issues/8280) and closes [#28574](https://github.com/ClickHouse/ClickHouse/issues/28574). [#34561](https://github.com/ClickHouse/ClickHouse/pull/34561) ([李扬](https://github.com/taiyang-li)). -* Reload `remote_url_allow_hosts` (filtering of outgoing connections) on config update. [#35294](https://github.com/ClickHouse/ClickHouse/pull/35294) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Support `--testmode` parameter for `clickhouse-local`. This parameter enables interpretation of test hints that we use in functional tests. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add `distributed_depth` to query log. It is like a more detailed variant of `is_initial_query` [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)). -* Respect `remote_url_allow_hosts` for `MySQL` and `PostgreSQL` table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). -* Added `disk_name` field to `system.part_log`. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)). -* Do not retry non-rertiable errors when querying remote URLs. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support distributed INSERT SELECT queries (the setting `parallel_distributed_insert_select`) table function `view()`. [#35132](https://github.com/ClickHouse/ClickHouse/pull/35132) ([Azat Khuzhin](https://github.com/azat)). -* More precise memory tracking during `INSERT` into `Buffer` with `AggregateFunction`. [#35072](https://github.com/ClickHouse/ClickHouse/pull/35072) ([Azat Khuzhin](https://github.com/azat)). -* Avoid division by zero in Query Profiler if Linux kernel has a bug. Closes [#34787](https://github.com/ClickHouse/ClickHouse/issues/34787). [#35032](https://github.com/ClickHouse/ClickHouse/pull/35032) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add more sanity checks for keeper configuration: now mixing of localhost and non-local servers is not allowed, also add checks for same value of internal raft port and keeper client port. [#35004](https://github.com/ClickHouse/ClickHouse/pull/35004) ([alesapin](https://github.com/alesapin)). -* Currently, if the user changes the settings of the system tables there will be tons of logs and ClickHouse will rename the tables every minute. This fixes [#34929](https://github.com/ClickHouse/ClickHouse/issues/34929). [#34949](https://github.com/ClickHouse/ClickHouse/pull/34949) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Use connection pool for Hive metastore client. [#34940](https://github.com/ClickHouse/ClickHouse/pull/34940) ([lgbo](https://github.com/lgbo-ustc)). -* Ignore per-column `TTL` in `CREATE TABLE AS` if new table engine does not support it (i.e. if the engine is not of `MergeTree` family). [#34938](https://github.com/ClickHouse/ClickHouse/pull/34938) ([Azat Khuzhin](https://github.com/azat)). -* Allow `LowCardinality` strings for `ngrambf_v1`/`tokenbf_v1` indexes. Closes [#21865](https://github.com/ClickHouse/ClickHouse/issues/21865). [#34911](https://github.com/ClickHouse/ClickHouse/pull/34911) ([Lars Hiller Eidnes](https://github.com/larspars)). -* Allow opening empty sqlite db if the file doesn't exist. Closes [#33367](https://github.com/ClickHouse/ClickHouse/issues/33367). [#34907](https://github.com/ClickHouse/ClickHouse/pull/34907) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Implement memory statistics for FreeBSD - this is required for `max_server_memory_usage` to work correctly. [#34902](https://github.com/ClickHouse/ClickHouse/pull/34902) ([Alexandre Snarskii](https://github.com/snar)). -* In previous versions the progress bar in clickhouse-client can jump forward near 50% for no reason. This closes [#34324](https://github.com/ClickHouse/ClickHouse/issues/34324). [#34801](https://github.com/ClickHouse/ClickHouse/pull/34801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Now `ALTER TABLE DROP COLUMN columnX` queries for `MergeTree` table engines will work instantly when `columnX` is an `ALIAS` column. Fixes [#34660](https://github.com/ClickHouse/ClickHouse/issues/34660). [#34786](https://github.com/ClickHouse/ClickHouse/pull/34786) ([alesapin](https://github.com/alesapin)). -* Show hints when user mistyped the name of a data skipping index. Closes [#29698](https://github.com/ClickHouse/ClickHouse/issues/29698). [#34764](https://github.com/ClickHouse/ClickHouse/pull/34764) ([flynn](https://github.com/ucasfl)). -* Support `remote()`/`cluster()` table functions for `parallel_distributed_insert_select`. [#34728](https://github.com/ClickHouse/ClickHouse/pull/34728) ([Azat Khuzhin](https://github.com/azat)). -* Do not reset logging that configured via `--log-file`/`--errorlog-file` command line options in case of empty configuration in the config file. [#34718](https://github.com/ClickHouse/ClickHouse/pull/34718) ([Amos Bird](https://github.com/amosbird)). -* Extract schema only once on table creation and prevent reading from local files/external sources to extract schema on each server startup. [#34684](https://github.com/ClickHouse/ClickHouse/pull/34684) ([Kruglov Pavel](https://github.com/Avogar)). -* Allow specifying argument names for executable UDFs. This is necessary for formats where argument name is part of serialization, like `Native`, `JSONEachRow`. Closes [#34604](https://github.com/ClickHouse/ClickHouse/issues/34604). [#34653](https://github.com/ClickHouse/ClickHouse/pull/34653) ([Maksim Kita](https://github.com/kitaisreal)). -* `MaterializedMySQL` (experimental feature) now supports `materialized_mysql_tables_list` (a comma-separated list of MySQL database tables, which will be replicated by the MaterializedMySQL database engine. Default value: empty list — means all the tables will be replicated), mentioned at [#32977](https://github.com/ClickHouse/ClickHouse/issues/32977). [#34487](https://github.com/ClickHouse/ClickHouse/pull/34487) ([zzsmdfj](https://github.com/zzsmdfj)). -* Improve OpenTelemetry span logs for INSERT operation on distributed table. [#34480](https://github.com/ClickHouse/ClickHouse/pull/34480) ([Frank Chen](https://github.com/FrankChen021)). -* Make the znode `ctime` and `mtime` consistent between servers in ClickHouse Keeper. [#33441](https://github.com/ClickHouse/ClickHouse/pull/33441) ([小路](https://github.com/nicelulu)). - -#### Build/Testing/Packaging Improvement - -* Package repository is migrated to JFrog Artifactory (**Mikhail f. Shiryaev**). -* Randomize some settings in functional tests, so more possible combinations of settings will be tested. This is yet another fuzzing method to ensure better test coverage. This closes [#32268](https://github.com/ClickHouse/ClickHouse/issues/32268). [#34092](https://github.com/ClickHouse/ClickHouse/pull/34092) ([Kruglov Pavel](https://github.com/Avogar)). -* Drop PVS-Studio from our CI. [#34680](https://github.com/ClickHouse/ClickHouse/pull/34680) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Add an ability to build stripped binaries with CMake. In previous versions it was performed by dh-tools. [#35196](https://github.com/ClickHouse/ClickHouse/pull/35196) ([alesapin](https://github.com/alesapin)). -* Smaller "fat-free" `clickhouse-keeper` build. [#35031](https://github.com/ClickHouse/ClickHouse/pull/35031) ([alesapin](https://github.com/alesapin)). -* Use @robot-clickhouse as an author and committer for PRs like https://github.com/ClickHouse/ClickHouse/pull/34685. [#34793](https://github.com/ClickHouse/ClickHouse/pull/34793) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Limit DWARF version for debug info by 4 max, because our internal stack symbolizer cannot parse DWARF version 5. This makes sense if you compile ClickHouse with clang-15. [#34777](https://github.com/ClickHouse/ClickHouse/pull/34777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Remove `clickhouse-test` debian package as unneeded complication. CI use tests from repository and standalone testing via deb package is no longer supported. [#34606](https://github.com/ClickHouse/ClickHouse/pull/34606) ([Ilya Yatsishin](https://github.com/qoega)). - -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) - -* A fix for HDFS integration: When the inner buffer size is too small, NEED_MORE_INPUT in `HadoopSnappyDecoder` will run multi times (>=3) for one compressed block. This makes the input data be copied into the wrong place in `HadoopSnappyDecoder::buffer`. [#35116](https://github.com/ClickHouse/ClickHouse/pull/35116) ([lgbo](https://github.com/lgbo-ustc)). -* Ignore obsolete grants in ATTACH GRANT statements. This PR fixes [#34815](https://github.com/ClickHouse/ClickHouse/issues/34815). [#34855](https://github.com/ClickHouse/ClickHouse/pull/34855) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix segfault in Postgres database when getting create table query if database was created using named collections. Closes [#35312](https://github.com/ClickHouse/ClickHouse/issues/35312). [#35313](https://github.com/ClickHouse/ClickHouse/pull/35313) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix partial merge join duplicate rows bug, close [#31009](https://github.com/ClickHouse/ClickHouse/issues/31009). [#35311](https://github.com/ClickHouse/ClickHouse/pull/35311) ([Vladimir C](https://github.com/vdimir)). -* Fix possible `Assertion 'position() != working_buffer.end()' failed` while using bzip2 compression with small `max_read_buffer_size` setting value. The bug was found in https://github.com/ClickHouse/ClickHouse/pull/35047. [#35300](https://github.com/ClickHouse/ClickHouse/pull/35300) ([Kruglov Pavel](https://github.com/Avogar)). While using lz4 compression with a small max_read_buffer_size setting value. [#35296](https://github.com/ClickHouse/ClickHouse/pull/35296) ([Kruglov Pavel](https://github.com/Avogar)). While using lzma compression with small `max_read_buffer_size` setting value. [#35295](https://github.com/ClickHouse/ClickHouse/pull/35295) ([Kruglov Pavel](https://github.com/Avogar)). While using `brotli` compression with a small `max_read_buffer_size` setting value. The bug was found in https://github.com/ClickHouse/ClickHouse/pull/35047. [#35281](https://github.com/ClickHouse/ClickHouse/pull/35281) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix possible segfault in `JSONEachRow` schema inference. [#35291](https://github.com/ClickHouse/ClickHouse/pull/35291) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix `CHECK TABLE` query in case when sparse columns are enabled in table. [#35274](https://github.com/ClickHouse/ClickHouse/pull/35274) ([Anton Popov](https://github.com/CurtizJ)). -* Avoid std::terminate in case of exception in reading from remote VFS. [#35257](https://github.com/ClickHouse/ClickHouse/pull/35257) ([Azat Khuzhin](https://github.com/azat)). -* Fix reading port from config, close [#34776](https://github.com/ClickHouse/ClickHouse/issues/34776). [#35193](https://github.com/ClickHouse/ClickHouse/pull/35193) ([Vladimir C](https://github.com/vdimir)). -* Fix error in query with `WITH TOTALS` in case if `HAVING` returned empty result. This fixes [#33711](https://github.com/ClickHouse/ClickHouse/issues/33711). [#35186](https://github.com/ClickHouse/ClickHouse/pull/35186) ([Amos Bird](https://github.com/amosbird)). -* Fix a corner case of `replaceRegexpAll`, close [#35117](https://github.com/ClickHouse/ClickHouse/issues/35117). [#35182](https://github.com/ClickHouse/ClickHouse/pull/35182) ([Vladimir C](https://github.com/vdimir)). -* Schema inference didn't work properly on case of `INSERT INTO FUNCTION s3(...) FROM ...`, it tried to read schema from s3 file instead of from select query. [#35176](https://github.com/ClickHouse/ClickHouse/pull/35176) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix MaterializedPostgreSQL (experimental feature) `table overrides` for partition by, etc. Closes [#35048](https://github.com/ClickHouse/ClickHouse/issues/35048). [#35162](https://github.com/ClickHouse/ClickHouse/pull/35162) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix MaterializedPostgreSQL (experimental feature) adding new table to replication (ATTACH TABLE) after manually removing (DETACH TABLE). Closes [#33800](https://github.com/ClickHouse/ClickHouse/issues/33800). Closes [#34922](https://github.com/ClickHouse/ClickHouse/issues/34922). Closes [#34315](https://github.com/ClickHouse/ClickHouse/issues/34315). [#35158](https://github.com/ClickHouse/ClickHouse/pull/35158) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix partition pruning error when non-monotonic function is used with IN operator. This fixes [#35136](https://github.com/ClickHouse/ClickHouse/issues/35136). [#35146](https://github.com/ClickHouse/ClickHouse/pull/35146) ([Amos Bird](https://github.com/amosbird)). -* Fixed slightly incorrect translation of YAML configs to XML. [#35135](https://github.com/ClickHouse/ClickHouse/pull/35135) ([Miel Donkers](https://github.com/mdonkers)). -* Fix `optimize_skip_unused_shards_rewrite_in` for signed columns and negative values. [#35134](https://github.com/ClickHouse/ClickHouse/pull/35134) ([Azat Khuzhin](https://github.com/azat)). -* The `update_lag` external dictionary configuration option was unusable showing the error message ``Unexpected key `update_lag` in dictionary source configuration``. [#35089](https://github.com/ClickHouse/ClickHouse/pull/35089) ([Jason Chu](https://github.com/1lann)). -* Avoid possible deadlock on server shutdown. [#35081](https://github.com/ClickHouse/ClickHouse/pull/35081) ([Azat Khuzhin](https://github.com/azat)). -* Fix missing alias after function is optimized to a subcolumn when setting `optimize_functions_to_subcolumns` is enabled. Closes [#33798](https://github.com/ClickHouse/ClickHouse/issues/33798). [#35079](https://github.com/ClickHouse/ClickHouse/pull/35079) ([qieqieplus](https://github.com/qieqieplus)). -* Fix reading from `system.asynchronous_inserts` table if there exists asynchronous insert into table function. [#35050](https://github.com/ClickHouse/ClickHouse/pull/35050) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible exception `Reading for MergeTree family tables must be done with last position boundary` (relevant to operation on remote VFS). Closes [#34979](https://github.com/ClickHouse/ClickHouse/issues/34979). [#35001](https://github.com/ClickHouse/ClickHouse/pull/35001) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix unexpected result when use -State type aggregate function in window frame. [#34999](https://github.com/ClickHouse/ClickHouse/pull/34999) ([metahys](https://github.com/metahys)). -* Fix possible segfault in FileLog (experimental feature). Closes [#30749](https://github.com/ClickHouse/ClickHouse/issues/30749). [#34996](https://github.com/ClickHouse/ClickHouse/pull/34996) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possible rare error `Cannot push block to port which already has data`. [#34993](https://github.com/ClickHouse/ClickHouse/pull/34993) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix wrong schema inference for unquoted dates in CSV. Closes [#34768](https://github.com/ClickHouse/ClickHouse/issues/34768). [#34961](https://github.com/ClickHouse/ClickHouse/pull/34961) ([Kruglov Pavel](https://github.com/Avogar)). -* Integration with Hive: Fix unexpected result when use `in` in `where` in hive query. [#34945](https://github.com/ClickHouse/ClickHouse/pull/34945) ([lgbo](https://github.com/lgbo-ustc)). -* Avoid busy polling in ClickHouse Keeper while searching for changelog files to delete. [#34931](https://github.com/ClickHouse/ClickHouse/pull/34931) ([Azat Khuzhin](https://github.com/azat)). -* Fix DateTime64 conversion from PostgreSQL. Closes [#33364](https://github.com/ClickHouse/ClickHouse/issues/33364). [#34910](https://github.com/ClickHouse/ClickHouse/pull/34910) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possible "Part directory doesn't exist" during `INSERT` into MergeTree table backed by VFS over s3. [#34876](https://github.com/ClickHouse/ClickHouse/pull/34876) ([Azat Khuzhin](https://github.com/azat)). -* Support DDLs like CREATE USER to be executed on cross replicated cluster. [#34860](https://github.com/ClickHouse/ClickHouse/pull/34860) ([Jianmei Zhang](https://github.com/zhangjmruc)). -* Fix bugs for multiple columns group by in `WindowView` (experimental feature). [#34859](https://github.com/ClickHouse/ClickHouse/pull/34859) ([vxider](https://github.com/Vxider)). -* Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). -* Fix bug for H3 funcs containing const columns which cause queries to fail. [#34743](https://github.com/ClickHouse/ClickHouse/pull/34743) ([Bharat Nallan](https://github.com/bharatnc)). -* Fix `No such file or directory` with enabled `fsync_part_directory` and vertical merge. [#34739](https://github.com/ClickHouse/ClickHouse/pull/34739) ([Azat Khuzhin](https://github.com/azat)). -* Fix serialization/printing for system queries `RELOAD MODEL`, `RELOAD FUNCTION`, `RESTART DISK` when used `ON CLUSTER`. Closes [#34514](https://github.com/ClickHouse/ClickHouse/issues/34514). [#34696](https://github.com/ClickHouse/ClickHouse/pull/34696) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix `allow_experimental_projection_optimization` with `enable_global_with_statement` (before it may lead to `Stack size too large` error in case of multiple expressions in `WITH` clause, and also it executes scalar subqueries again and again, so not it will be more optimal). [#34650](https://github.com/ClickHouse/ClickHouse/pull/34650) ([Azat Khuzhin](https://github.com/azat)). -* Stop to select part for mutate when the other replica has already updated the transaction log for `ReplatedMergeTree` engine. [#34633](https://github.com/ClickHouse/ClickHouse/pull/34633) ([Jianmei Zhang](https://github.com/zhangjmruc)). -* Fix incorrect result of trivial count query when part movement feature is used [#34089](https://github.com/ClickHouse/ClickHouse/issues/34089). [#34385](https://github.com/ClickHouse/ClickHouse/pull/34385) ([nvartolomei](https://github.com/nvartolomei)). -* Fix inconsistency of `max_query_size` limitation in distributed subqueries. [#34078](https://github.com/ClickHouse/ClickHouse/pull/34078) ([Chao Ma](https://github.com/godliness)). - -### ClickHouse release v22.2, 2022-02-17 - -#### Upgrade Notes - -* Applying data skipping indexes for queries with FINAL may produce incorrect result. In this release we disabled data skipping indexes by default for queries with FINAL (a new setting `use_skip_indexes_if_final` is introduced and disabled by default). [#34243](https://github.com/ClickHouse/ClickHouse/pull/34243) ([Azat Khuzhin](https://github.com/azat)). - -#### New Feature - -* Projections are production ready. Set `allow_experimental_projection_optimization` by default and deprecate this setting. [#34456](https://github.com/ClickHouse/ClickHouse/pull/34456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* An option to create a new files on insert for `File`/`S3`/`HDFS` engines. Allow to overwrite a file in `HDFS`. Throw an exception in attempt to overwrite a file in `S3` by default. Throw an exception in attempt to append data to file in formats that have a suffix (and thus don't support appends, like `Parquet`, `ORC`). Closes [#31640](https://github.com/ClickHouse/ClickHouse/issues/31640) Closes [#31622](https://github.com/ClickHouse/ClickHouse/issues/31622) Closes [#23862](https://github.com/ClickHouse/ClickHouse/issues/23862) Closes [#15022](https://github.com/ClickHouse/ClickHouse/issues/15022) Closes [#16674](https://github.com/ClickHouse/ClickHouse/issues/16674). [#33302](https://github.com/ClickHouse/ClickHouse/pull/33302) ([Kruglov Pavel](https://github.com/Avogar)). -* Add a setting that allows a user to provide own deduplication semantic in `MergeTree`/`ReplicatedMergeTree` If provided, it's used instead of data digest to generate block ID. So, for example, by providing a unique value for the setting in each INSERT statement, the user can avoid the same inserted data being deduplicated. This closes: [#7461](https://github.com/ClickHouse/ClickHouse/issues/7461). [#32304](https://github.com/ClickHouse/ClickHouse/pull/32304) ([Igor Nikonov](https://github.com/devcrafter)). -* Add support of `DEFAULT` keyword for INSERT statements. Closes [#6331](https://github.com/ClickHouse/ClickHouse/issues/6331). [#33141](https://github.com/ClickHouse/ClickHouse/pull/33141) ([Andrii Buriachevskyi](https://github.com/1over)). -* `EPHEMERAL` column specifier is added to `CREATE TABLE` query. Closes [#9436](https://github.com/ClickHouse/ClickHouse/issues/9436). [#34424](https://github.com/ClickHouse/ClickHouse/pull/34424) ([yakov-olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Support `IF EXISTS` clause for `TTL expr TO [DISK|VOLUME] [IF EXISTS] 'xxx'` feature. Parts will be moved to disk or volume only if it exists on replica, so `MOVE TTL` rules will be able to behave differently on replicas according to the existing storage policies. Resolves [#34455](https://github.com/ClickHouse/ClickHouse/issues/34455). [#34504](https://github.com/ClickHouse/ClickHouse/pull/34504) ([Anton Popov](https://github.com/CurtizJ)). -* Allow set default table engine and to create tables without specifying ENGINE. [#34187](https://github.com/ClickHouse/ClickHouse/pull/34187) ([Ilya Yatsishin](https://github.com/qoega)). -* Add table function `format(format_name, data)`. [#34125](https://github.com/ClickHouse/ClickHouse/pull/34125) ([Kruglov Pavel](https://github.com/Avogar)). -* Detect format in `clickhouse-local` by file name even in the case when it is passed to stdin. [#33829](https://github.com/ClickHouse/ClickHouse/pull/33829) ([Kruglov Pavel](https://github.com/Avogar)). -* Add schema inference for `values` table function. Closes [#33811](https://github.com/ClickHouse/ClickHouse/issues/33811). [#34017](https://github.com/ClickHouse/ClickHouse/pull/34017) ([Kruglov Pavel](https://github.com/Avogar)). -* Dynamic reload of server TLS certificates on config reload. Closes [#15764](https://github.com/ClickHouse/ClickHouse/issues/15764). [#15765](https://github.com/ClickHouse/ClickHouse/pull/15765) ([johnskopis](https://github.com/johnskopis)). [#31257](https://github.com/ClickHouse/ClickHouse/pull/31257) ([Filatenkov Artur](https://github.com/FArthur-cmd)). -* Now ReplicatedMergeTree can recover data when some of its disks are broken. [#13544](https://github.com/ClickHouse/ClickHouse/pull/13544) ([Amos Bird](https://github.com/amosbird)). -* Fault-tolerant connections in clickhouse-client: `clickhouse-client ... --host host1 --host host2 --port port2 --host host3 --port port --host host4`. [#34490](https://github.com/ClickHouse/ClickHouse/pull/34490) ([Kruglov Pavel](https://github.com/Avogar)). [#33824](https://github.com/ClickHouse/ClickHouse/pull/33824) ([Filippov Denis](https://github.com/DF5HSE)). -* Add `DEGREES` and `RADIANS` functions for MySQL compatibility. [#33769](https://github.com/ClickHouse/ClickHouse/pull/33769) ([Bharat Nallan](https://github.com/bharatnc)). -* Add `h3ToCenterChild` function. [#33313](https://github.com/ClickHouse/ClickHouse/pull/33313) ([Bharat Nallan](https://github.com/bharatnc)). Add new h3 miscellaneous functions: `edgeLengthKm`,`exactEdgeLengthKm`,`exactEdgeLengthM`,`exactEdgeLengthRads`,`numHexagons`. [#33621](https://github.com/ClickHouse/ClickHouse/pull/33621) ([Bharat Nallan](https://github.com/bharatnc)). -* Add function `bitSlice` to extract bit subsequences from String/FixedString. [#33360](https://github.com/ClickHouse/ClickHouse/pull/33360) ([RogerYK](https://github.com/RogerYK)). -* Implemented `meanZTest` aggregate function. [#33354](https://github.com/ClickHouse/ClickHouse/pull/33354) ([achimbab](https://github.com/achimbab)). -* Add confidence intervals to T-tests aggregate functions. [#33260](https://github.com/ClickHouse/ClickHouse/pull/33260) ([achimbab](https://github.com/achimbab)). -* Add function `addressToLineWithInlines`. Close [#26211](https://github.com/ClickHouse/ClickHouse/issues/26211). [#33467](https://github.com/ClickHouse/ClickHouse/pull/33467) ([SuperDJY](https://github.com/cmsxbc)). -* Added `#!` and `# ` as a recognised start of a single line comment. Closes [#34138](https://github.com/ClickHouse/ClickHouse/issues/34138). [#34230](https://github.com/ClickHouse/ClickHouse/pull/34230) ([Aaron Katz](https://github.com/aaronstephenkatz)). - -#### Experimental Feature - -* Functions for text classification: language and charset detection. See [#23271](https://github.com/ClickHouse/ClickHouse/issues/23271). [#33314](https://github.com/ClickHouse/ClickHouse/pull/33314) ([Nikolay Degterinsky](https://github.com/evillique)). -* Add memory overcommit to `MemoryTracker`. Added `guaranteed` settings for memory limits which represent soft memory limits. In case when hard memory limit is reached, `MemoryTracker` tries to cancel the most overcommited query. New setting `memory_usage_overcommit_max_wait_microseconds` specifies how long queries may wait another query to stop. Closes [#28375](https://github.com/ClickHouse/ClickHouse/issues/28375). [#31182](https://github.com/ClickHouse/ClickHouse/pull/31182) ([Dmitry Novik](https://github.com/novikd)). -* Enable stream to table join in WindowView. [#33729](https://github.com/ClickHouse/ClickHouse/pull/33729) ([vxider](https://github.com/Vxider)). -* Support `SET`, `YEAR`, `TIME` and `GEOMETRY` data types in `MaterializedMySQL` (experimental feature). Fixes [#18091](https://github.com/ClickHouse/ClickHouse/issues/18091), [#21536](https://github.com/ClickHouse/ClickHouse/issues/21536), [#26361](https://github.com/ClickHouse/ClickHouse/issues/26361). [#33429](https://github.com/ClickHouse/ClickHouse/pull/33429) ([zzsmdfj](https://github.com/zzsmdfj)). -* Fix various issues when projection is enabled by default. Each issue is described in separate commit. This is for [#33678](https://github.com/ClickHouse/ClickHouse/issues/33678) . This fixes [#34273](https://github.com/ClickHouse/ClickHouse/issues/34273). [#34305](https://github.com/ClickHouse/ClickHouse/pull/34305) ([Amos Bird](https://github.com/amosbird)). - -#### Performance Improvement - -* Support `optimize_read_in_order` if prefix of sorting key is already sorted. E.g. if we have sorting key `ORDER BY (a, b)` in table and query with `WHERE a = const ORDER BY b` clauses, now it will be applied reading in order of sorting key instead of full sort. [#32748](https://github.com/ClickHouse/ClickHouse/pull/32748) ([Anton Popov](https://github.com/CurtizJ)). -* Improve performance of partitioned insert into table functions `URL`, `S3`, `File`, `HDFS`. Closes [#34348](https://github.com/ClickHouse/ClickHouse/issues/34348). [#34510](https://github.com/ClickHouse/ClickHouse/pull/34510) ([Maksim Kita](https://github.com/kitaisreal)). -* Multiple performance improvements of clickhouse-keeper. [#34484](https://github.com/ClickHouse/ClickHouse/pull/34484) [#34587](https://github.com/ClickHouse/ClickHouse/pull/34587) ([zhanglistar](https://github.com/zhanglistar)). -* `FlatDictionary` improve performance of dictionary data load. [#33871](https://github.com/ClickHouse/ClickHouse/pull/33871) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve performance of `mapPopulateSeries` function. Closes [#33944](https://github.com/ClickHouse/ClickHouse/issues/33944). [#34318](https://github.com/ClickHouse/ClickHouse/pull/34318) ([Maksim Kita](https://github.com/kitaisreal)). -* `_file` and `_path` virtual columns (in file-like table engines) are made `LowCardinality` - it will make queries for multiple files faster. Closes [#34300](https://github.com/ClickHouse/ClickHouse/issues/34300). [#34317](https://github.com/ClickHouse/ClickHouse/pull/34317) ([flynn](https://github.com/ucasfl)). -* Speed up loading of data parts. It was not parallelized before: the setting `part_loading_threads` did not have effect. See [#4699](https://github.com/ClickHouse/ClickHouse/issues/4699). [#34310](https://github.com/ClickHouse/ClickHouse/pull/34310) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Improve performance of `LineAsString` format. This closes [#34303](https://github.com/ClickHouse/ClickHouse/issues/34303). [#34306](https://github.com/ClickHouse/ClickHouse/pull/34306) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Optimize `quantilesExact{Low,High}` to use `nth_element` instead of `sort`. [#34287](https://github.com/ClickHouse/ClickHouse/pull/34287) ([Danila Kutenin](https://github.com/danlark1)). -* Slightly improve performance of `Regexp` format. [#34202](https://github.com/ClickHouse/ClickHouse/pull/34202) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Minor improvement for analysis of scalar subqueries. [#34128](https://github.com/ClickHouse/ClickHouse/pull/34128) ([Federico Rodriguez](https://github.com/fedrod)). -* Make ORDER BY tuple almost as fast as ORDER BY columns. We have special optimizations for multiple column ORDER BY: https://github.com/ClickHouse/ClickHouse/pull/10831 . It's beneficial to also apply to tuple columns. [#34060](https://github.com/ClickHouse/ClickHouse/pull/34060) ([Amos Bird](https://github.com/amosbird)). -* Rework and reintroduce the scalar subqueries cache to Materialized Views execution. [#33958](https://github.com/ClickHouse/ClickHouse/pull/33958) ([Raúl Marín](https://github.com/Algunenano)). -* Slightly improve performance of `ORDER BY` by adding x86-64 AVX-512 support for `memcmpSmall` functions to accelerate memory comparison. It works only if you compile ClickHouse by yourself. [#33706](https://github.com/ClickHouse/ClickHouse/pull/33706) ([hanqf-git](https://github.com/hanqf-git)). -* Improve `range_hashed` dictionary performance if for key there are a lot of intervals. Fixes [#23821](https://github.com/ClickHouse/ClickHouse/issues/23821). [#33516](https://github.com/ClickHouse/ClickHouse/pull/33516) ([Maksim Kita](https://github.com/kitaisreal)). -* For inserts and merges into S3, write files in parallel whenever possible (TODO: check if it's merged). [#33291](https://github.com/ClickHouse/ClickHouse/pull/33291) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Improve `clickhouse-keeper` performance and fix several memory leaks in NuRaft library. [#33329](https://github.com/ClickHouse/ClickHouse/pull/33329) ([alesapin](https://github.com/alesapin)). - -#### Improvement - -* Support asynchronous inserts in `clickhouse-client` for queries with inlined data. [#34267](https://github.com/ClickHouse/ClickHouse/pull/34267) ([Anton Popov](https://github.com/CurtizJ)). -* Functions `dictGet`, `dictHas` implicitly cast key argument to dictionary key structure, if they are different. [#33672](https://github.com/ClickHouse/ClickHouse/pull/33672) ([Maksim Kita](https://github.com/kitaisreal)). -* Improvements for `range_hashed` dictionaries. Improve performance of load time if there are multiple attributes. Allow to create a dictionary without attributes. Added option to specify strategy when intervals `start` and `end` have `Nullable` type `convert_null_range_bound_to_open` by default is `true`. Closes [#29791](https://github.com/ClickHouse/ClickHouse/issues/29791). Allow to specify `Float`, `Decimal`, `DateTime64`, `Int128`, `Int256`, `UInt128`, `UInt256` as range types. `RangeHashedDictionary` added support for range values that extend `Int64` type. Closes [#28322](https://github.com/ClickHouse/ClickHouse/issues/28322). Added option `range_lookup_strategy` to specify range lookup type `min`, `max` by default is `min` . Closes [#21647](https://github.com/ClickHouse/ClickHouse/issues/21647). Fixed allocated bytes calculations. Fixed type name in `system.dictionaries` in case of `ComplexKeyHashedDictionary`. [#33927](https://github.com/ClickHouse/ClickHouse/pull/33927) ([Maksim Kita](https://github.com/kitaisreal)). -* `flat`, `hashed`, `hashed_array` dictionaries now support creating with empty attributes, with support of reading the keys and using `dictHas`. Fixes [#33820](https://github.com/ClickHouse/ClickHouse/issues/33820). [#33918](https://github.com/ClickHouse/ClickHouse/pull/33918) ([Maksim Kita](https://github.com/kitaisreal)). -* Added support for `DateTime64` data type in dictionaries. [#33914](https://github.com/ClickHouse/ClickHouse/pull/33914) ([Maksim Kita](https://github.com/kitaisreal)). -* Allow to write `s3(url, access_key_id, secret_access_key)` (autodetect of data format and table structure, but with explicit credentials). [#34503](https://github.com/ClickHouse/ClickHouse/pull/34503) ([Kruglov Pavel](https://github.com/Avogar)). -* Added sending of the output format back to client like it's done in HTTP protocol as suggested in [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). Closes [#34362](https://github.com/ClickHouse/ClickHouse/issues/34362). [#34499](https://github.com/ClickHouse/ClickHouse/pull/34499) ([Vitaly Baranov](https://github.com/vitlibar)). -* Send ProfileEvents statistics in case of INSERT SELECT query (to display query metrics in `clickhouse-client` for this type of queries). [#34498](https://github.com/ClickHouse/ClickHouse/pull/34498) ([Dmitry Novik](https://github.com/novikd)). -* Recognize `.jsonl` extension for JSONEachRow format. [#34496](https://github.com/ClickHouse/ClickHouse/pull/34496) ([Kruglov Pavel](https://github.com/Avogar)). -* Improve schema inference in clickhouse-local. Allow to write just `clickhouse-local -q "select * from table" < data.format`. [#34495](https://github.com/ClickHouse/ClickHouse/pull/34495) ([Kruglov Pavel](https://github.com/Avogar)). -* Privileges CREATE/ALTER/DROP ROW POLICY now can be granted on a table or on `database.*` as well as globally `*.*`. [#34489](https://github.com/ClickHouse/ClickHouse/pull/34489) ([Vitaly Baranov](https://github.com/vitlibar)). -* Allow to export arbitrary large files to `s3`. Add two new settings: `s3_upload_part_size_multiply_factor` and `s3_upload_part_size_multiply_parts_count_threshold`. Now each time `s3_upload_part_size_multiply_parts_count_threshold` uploaded to S3 from a single query `s3_min_upload_part_size` multiplied by `s3_upload_part_size_multiply_factor`. Fixes [#34244](https://github.com/ClickHouse/ClickHouse/issues/34244). [#34422](https://github.com/ClickHouse/ClickHouse/pull/34422) ([alesapin](https://github.com/alesapin)). -* Allow to skip not found (404) URLs for globs when using URL storage / table function. Also closes [#34359](https://github.com/ClickHouse/ClickHouse/issues/34359). [#34392](https://github.com/ClickHouse/ClickHouse/pull/34392) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Default input and output formats for `clickhouse-local` that can be overriden by --input-format and --output-format. Close [#30631](https://github.com/ClickHouse/ClickHouse/issues/30631). [#34352](https://github.com/ClickHouse/ClickHouse/pull/34352) ([李扬](https://github.com/taiyang-li)). -* Add options for `clickhouse-format`. Which close [#30528](https://github.com/ClickHouse/ClickHouse/issues/30528) - `max_query_size` - `max_parser_depth`. [#34349](https://github.com/ClickHouse/ClickHouse/pull/34349) ([李扬](https://github.com/taiyang-li)). -* Better handling of pre-inputs before client start. This is for [#34308](https://github.com/ClickHouse/ClickHouse/issues/34308). [#34336](https://github.com/ClickHouse/ClickHouse/pull/34336) ([Amos Bird](https://github.com/amosbird)). -* `REGEXP_MATCHES` and `REGEXP_REPLACE` function aliases for compatibility with PostgreSQL. Close [#30885](https://github.com/ClickHouse/ClickHouse/issues/30885). [#34334](https://github.com/ClickHouse/ClickHouse/pull/34334) ([李扬](https://github.com/taiyang-li)). -* Some servers expect a User-Agent header in their HTTP requests. A `User-Agent` header entry has been added to HTTP requests of the form: User-Agent: ClickHouse/VERSION_STRING. [#34330](https://github.com/ClickHouse/ClickHouse/pull/34330) ([Saad Ur Rahman](https://github.com/surahman)). -* Cancel merges before acquiring table lock for `TRUNCATE` query to avoid `DEADLOCK_AVOIDED` error in some cases. Fixes [#34302](https://github.com/ClickHouse/ClickHouse/issues/34302). [#34304](https://github.com/ClickHouse/ClickHouse/pull/34304) ([tavplubix](https://github.com/tavplubix)). -* Change severity of the "Cancelled merging parts" message in logs, because it's not an error. This closes [#34148](https://github.com/ClickHouse/ClickHouse/issues/34148). [#34232](https://github.com/ClickHouse/ClickHouse/pull/34232) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Add ability to compose PostgreSQL-style cast operator `::` with expressions using `[]` and `.` operators (array and tuple indexing). [#34229](https://github.com/ClickHouse/ClickHouse/pull/34229) ([Nikolay Degterinsky](https://github.com/evillique)). -* Recognize `YYYYMMDD-hhmmss` format in `parseDateTimeBestEffort` function. This closes [#34206](https://github.com/ClickHouse/ClickHouse/issues/34206). [#34208](https://github.com/ClickHouse/ClickHouse/pull/34208) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Allow carriage return in the middle of the line while parsing by `Regexp` format. This closes [#34200](https://github.com/ClickHouse/ClickHouse/issues/34200). [#34205](https://github.com/ClickHouse/ClickHouse/pull/34205) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Allow to parse dictionary's `PRIMARY KEY` as `PRIMARY KEY (id, value)`; previously supported only `PRIMARY KEY id, value`. Closes [#34135](https://github.com/ClickHouse/ClickHouse/issues/34135). [#34141](https://github.com/ClickHouse/ClickHouse/pull/34141) ([Maksim Kita](https://github.com/kitaisreal)). -* An optional argument for `splitByChar` to limit the number of resulting elements. close [#34081](https://github.com/ClickHouse/ClickHouse/issues/34081). [#34140](https://github.com/ClickHouse/ClickHouse/pull/34140) ([李扬](https://github.com/taiyang-li)). -* Improving the experience of multiple line editing for clickhouse-client. This is a follow-up of [#31123](https://github.com/ClickHouse/ClickHouse/pull/31123). [#34114](https://github.com/ClickHouse/ClickHouse/pull/34114) ([Amos Bird](https://github.com/amosbird)). -* Add `UUID` suport in `MsgPack` input/output format. [#34065](https://github.com/ClickHouse/ClickHouse/pull/34065) ([Kruglov Pavel](https://github.com/Avogar)). -* Tracing context (for OpenTelemetry) is now propagated from GRPC client metadata (this change is relevant for GRPC client-server protocol). [#34064](https://github.com/ClickHouse/ClickHouse/pull/34064) ([andremarianiello](https://github.com/andremarianiello)). -* Supports all types of `SYSTEM` queries with `ON CLUSTER` clause. [#34005](https://github.com/ClickHouse/ClickHouse/pull/34005) ([小路](https://github.com/nicelulu)). -* Improve memory accounting for queries that are using less than `max_untracker_memory`. [#34001](https://github.com/ClickHouse/ClickHouse/pull/34001) ([Azat Khuzhin](https://github.com/azat)). -* Fixed UTF-8 string case-insensitive search when lowercase and uppercase characters are represented by different number of bytes. Example is `ẞ` and `ß`. This closes [#7334](https://github.com/ClickHouse/ClickHouse/issues/7334). [#33992](https://github.com/ClickHouse/ClickHouse/pull/33992) ([Harry Lee](https://github.com/HarryLeeIBM)). -* Detect format and schema from stdin in `clickhouse-local`. [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960) ([Kruglov Pavel](https://github.com/Avogar)). -* Correctly handle the case of misconfiguration when multiple disks are using the same path on the filesystem. [#29072](https://github.com/ClickHouse/ClickHouse/issues/29072). [#33905](https://github.com/ClickHouse/ClickHouse/pull/33905) ([zhongyuankai](https://github.com/zhongyuankai)). -* Try every resolved IP address while getting S3 proxy. S3 proxies are rarely used, mostly in Yandex Cloud. [#33862](https://github.com/ClickHouse/ClickHouse/pull/33862) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Support EXPLAIN AST CREATE FUNCTION query `EXPLAIN AST CREATE FUNCTION mycast AS (n) -> cast(n as String)` will return `EXPLAIN AST CREATE FUNCTION mycast AS n -> CAST(n, 'String')`. [#33819](https://github.com/ClickHouse/ClickHouse/pull/33819) ([李扬](https://github.com/taiyang-li)). -* Added support for cast from `Map(Key, Value)` to `Array(Tuple(Key, Value))`. [#33794](https://github.com/ClickHouse/ClickHouse/pull/33794) ([Maksim Kita](https://github.com/kitaisreal)). -* Add some improvements and fixes for `Bool` data type. Fixes [#33244](https://github.com/ClickHouse/ClickHouse/issues/33244). [#33737](https://github.com/ClickHouse/ClickHouse/pull/33737) ([Kruglov Pavel](https://github.com/Avogar)). -* Parse and store OpenTelemetry trace-id in big-endian order. [#33723](https://github.com/ClickHouse/ClickHouse/pull/33723) ([Frank Chen](https://github.com/FrankChen021)). -* Improvement for `fromUnixTimestamp64` family functions.. They now accept any integer value that can be converted to `Int64`. This closes: [#14648](https://github.com/ClickHouse/ClickHouse/issues/14648). [#33505](https://github.com/ClickHouse/ClickHouse/pull/33505) ([Andrey Zvonov](https://github.com/zvonand)). -* Reimplement `_shard_num` from constants (see [#7624](https://github.com/ClickHouse/ClickHouse/issues/7624)) with `shardNum()` function (seee [#27020](https://github.com/ClickHouse/ClickHouse/issues/27020)), to avoid possible issues (like those that had been found in [#16947](https://github.com/ClickHouse/ClickHouse/issues/16947)). [#33392](https://github.com/ClickHouse/ClickHouse/pull/33392) ([Azat Khuzhin](https://github.com/azat)). -* Enable binary arithmetic (plus, minus, multiply, division, least, greatest) between Decimal and Float. [#33355](https://github.com/ClickHouse/ClickHouse/pull/33355) ([flynn](https://github.com/ucasfl)). -* Respect cgroups limits in max_threads autodetection. [#33342](https://github.com/ClickHouse/ClickHouse/pull/33342) ([JaySon](https://github.com/JaySon-Huang)). -* Add new clickhouse-keeper setting `min_session_timeout_ms`. Now clickhouse-keeper will determine client session timeout according to `min_session_timeout_ms` and `session_timeout_ms` settings. [#33288](https://github.com/ClickHouse/ClickHouse/pull/33288) ([JackyWoo](https://github.com/JackyWoo)). -* Added `UUID` data type support for functions `hex` and `bin`. [#32170](https://github.com/ClickHouse/ClickHouse/pull/32170) ([Frank Chen](https://github.com/FrankChen021)). -* Fix reading of subcolumns with dots in their names. In particular fixed reading of `Nested` columns, if their element names contain dots (e.g ```Nested(`keys.name` String, `keys.id` UInt64, values UInt64)```). [#34228](https://github.com/ClickHouse/ClickHouse/pull/34228) ([Anton Popov](https://github.com/CurtizJ)). -* Fixes `parallel_view_processing = 0` not working when inserting into a table using `VALUES`. - Fixes `view_duration_ms` in the `query_views_log` not being set correctly for materialized views. [#34067](https://github.com/ClickHouse/ClickHouse/pull/34067) ([Raúl Marín](https://github.com/Algunenano)). -* Fix parsing tables structure from ZooKeeper: now metadata from ZooKeeper compared with local metadata in canonical form. It helps when canonical function names can change between ClickHouse versions. [#33933](https://github.com/ClickHouse/ClickHouse/pull/33933) ([sunny](https://github.com/sunny19930321)). -* Properly escape some characters for interaction with LDAP. [#33401](https://github.com/ClickHouse/ClickHouse/pull/33401) ([IlyaTsoi](https://github.com/IlyaTsoi)). - -#### Build/Testing/Packaging Improvement - -* Remove unbundled build support. [#33690](https://github.com/ClickHouse/ClickHouse/pull/33690) ([Azat Khuzhin](https://github.com/azat)). -* Ensure that tests don't depend on the result of non-stable sorting of equal elements. Added equal items ranges randomization in debug after sort to prevent issues when we rely on equal items sort order. [#34393](https://github.com/ClickHouse/ClickHouse/pull/34393) ([Maksim Kita](https://github.com/kitaisreal)). -* Add verbosity to a style check. [#34289](https://github.com/ClickHouse/ClickHouse/pull/34289) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Remove `clickhouse-test` debian package because it's obsolete. [#33948](https://github.com/ClickHouse/ClickHouse/pull/33948) ([Ilya Yatsishin](https://github.com/qoega)). -* Multiple improvements for build system to remove the possibility of occasionally using packages from the OS and to enforce hermetic builds. [#33695](https://github.com/ClickHouse/ClickHouse/pull/33695) ([Amos Bird](https://github.com/amosbird)). - -#### Bug Fix (user-visible misbehaviour in official stable or prestable release) - -* Fixed the assertion in case of using `allow_experimental_parallel_reading_from_replicas` with `max_parallel_replicas` equals to 1. This fixes [#34525](https://github.com/ClickHouse/ClickHouse/issues/34525). [#34613](https://github.com/ClickHouse/ClickHouse/pull/34613) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Fix rare bug while reading of empty arrays, which could lead to `Data compressed with different methods` error. It can reproduce if you have mostly empty arrays, but not always. And reading is performed in backward direction with ORDER BY ... DESC. This error is extremely unlikely to happen. [#34327](https://github.com/ClickHouse/ClickHouse/pull/34327) ([Anton Popov](https://github.com/CurtizJ)). -* Fix wrong result of `round`/`roundBankers` if integer values of small types are rounded. Closes [#33267](https://github.com/ClickHouse/ClickHouse/issues/33267). [#34562](https://github.com/ClickHouse/ClickHouse/pull/34562) ([李扬](https://github.com/taiyang-li)). -* Sometimes query cancellation did not work immediately when we were reading multiple files from s3 or HDFS. Fixes [#34301](https://github.com/ClickHouse/ClickHouse/issues/34301) Relates to [#34397](https://github.com/ClickHouse/ClickHouse/issues/34397). [#34539](https://github.com/ClickHouse/ClickHouse/pull/34539) ([Dmitry Novik](https://github.com/novikd)). -* Fix exception `Chunk should have AggregatedChunkInfo in MergingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1` and `distributed_aggregation_memory_efficient = 0`). Fixes [#34526](https://github.com/ClickHouse/ClickHouse/issues/34526). [#34532](https://github.com/ClickHouse/ClickHouse/pull/34532) ([Anton Popov](https://github.com/CurtizJ)). -* Fix comparison between integers and floats in index analysis. Previously it could lead to skipping some granules for reading by mistake. Fixes [#34493](https://github.com/ClickHouse/ClickHouse/issues/34493). [#34528](https://github.com/ClickHouse/ClickHouse/pull/34528) ([Anton Popov](https://github.com/CurtizJ)). -* Fix compression support in URL engine. [#34524](https://github.com/ClickHouse/ClickHouse/pull/34524) ([Frank Chen](https://github.com/FrankChen021)). -* Fix possible error 'file_size: Operation not supported' in files' schema autodetection. [#34479](https://github.com/ClickHouse/ClickHouse/pull/34479) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixes possible race with table deletion. [#34416](https://github.com/ClickHouse/ClickHouse/pull/34416) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix possible error `Cannot convert column Function to mask` in short circuit function evaluation. Closes [#34171](https://github.com/ClickHouse/ClickHouse/issues/34171). [#34415](https://github.com/ClickHouse/ClickHouse/pull/34415) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix potential crash when doing schema inference from url source. Closes [#34147](https://github.com/ClickHouse/ClickHouse/issues/34147). [#34405](https://github.com/ClickHouse/ClickHouse/pull/34405) ([Kruglov Pavel](https://github.com/Avogar)). -* For UDFs access permissions were checked for database level instead of global level as it should be. Closes [#34281](https://github.com/ClickHouse/ClickHouse/issues/34281). [#34404](https://github.com/ClickHouse/ClickHouse/pull/34404) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix wrong engine syntax in result of `SHOW CREATE DATABASE` query for databases with engine `Memory`. This closes [#34335](https://github.com/ClickHouse/ClickHouse/issues/34335). [#34345](https://github.com/ClickHouse/ClickHouse/pull/34345) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fixed a couple of extremely rare race conditions that might lead to broken state of replication queue and "intersecting parts" error. [#34297](https://github.com/ClickHouse/ClickHouse/pull/34297) ([tavplubix](https://github.com/tavplubix)). -* Fix progress bar width. It was incorrectly rounded to integer number of characters. [#34275](https://github.com/ClickHouse/ClickHouse/pull/34275) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix current_user/current_address client information fields for inter-server communication (before this patch current_user/current_address will be preserved from the previous query). [#34263](https://github.com/ClickHouse/ClickHouse/pull/34263) ([Azat Khuzhin](https://github.com/azat)). -* Fix memory leak in case of some Exception during query processing with `optimize_aggregation_in_order=1`. [#34234](https://github.com/ClickHouse/ClickHouse/pull/34234) ([Azat Khuzhin](https://github.com/azat)). -* Fix metric `Query`, which shows the number of executing queries. In last several releases it was always 0. [#34224](https://github.com/ClickHouse/ClickHouse/pull/34224) ([Anton Popov](https://github.com/CurtizJ)). -* Fix schema inference for table runction `s3`. [#34186](https://github.com/ClickHouse/ClickHouse/pull/34186) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix rare and benign race condition in `HDFS`, `S3` and `URL` storage engines which can lead to additional connections. [#34172](https://github.com/ClickHouse/ClickHouse/pull/34172) ([alesapin](https://github.com/alesapin)). -* Fix bug which can rarely lead to error "Cannot read all data" while reading LowCardinality columns of MergeTree table engines family which stores data on remote file system like S3 (virtual filesystem over s3 is an experimental feature that is not ready for production). [#34139](https://github.com/ClickHouse/ClickHouse/pull/34139) ([alesapin](https://github.com/alesapin)). -* Fix inserts to distributed tables in case of a change of native protocol. The last change was in the version 22.1, so there may be some failures of inserts to distributed tables after upgrade to that version. [#34132](https://github.com/ClickHouse/ClickHouse/pull/34132) ([Anton Popov](https://github.com/CurtizJ)). -* Fix possible data race in `File` table engine that was introduced in [#33960](https://github.com/ClickHouse/ClickHouse/pull/33960). Closes [#34111](https://github.com/ClickHouse/ClickHouse/issues/34111). [#34113](https://github.com/ClickHouse/ClickHouse/pull/34113) ([Kruglov Pavel](https://github.com/Avogar)). -* Fixed minor race condition that might cause "intersecting parts" error in extremely rare cases after ZooKeeper connection loss. [#34096](https://github.com/ClickHouse/ClickHouse/pull/34096) ([tavplubix](https://github.com/tavplubix)). -* Fix asynchronous inserts with `Native` format. [#34068](https://github.com/ClickHouse/ClickHouse/pull/34068) ([Anton Popov](https://github.com/CurtizJ)). -* Fix bug which lead to inability for server to start when both replicated access storage and keeper (embedded in clickhouse-server) are used. Introduced two settings for keeper socket timeout instead of settings from default user: `keeper_server.socket_receive_timeout_sec` and `keeper_server.socket_send_timeout_sec`. Fixes [#33973](https://github.com/ClickHouse/ClickHouse/issues/33973). [#33988](https://github.com/ClickHouse/ClickHouse/pull/33988) ([alesapin](https://github.com/alesapin)). -* Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix parsing IPv6 from query parameter (prepared statements) and fix IPv6 to string conversion. Closes [#33928](https://github.com/ClickHouse/ClickHouse/issues/33928). [#33971](https://github.com/ClickHouse/ClickHouse/pull/33971) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)). -* Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)). -* Aggregate function combinator `-If` did not correctly process `Nullable` filter argument. This closes [#27073](https://github.com/ClickHouse/ClickHouse/issues/27073). [#33920](https://github.com/ClickHouse/ClickHouse/pull/33920) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix potential race condition when doing remote disk read (virtual filesystem over s3 is an experimental feature that is not ready for production). [#33912](https://github.com/ClickHouse/ClickHouse/pull/33912) ([Amos Bird](https://github.com/amosbird)). -* Fix crash if SQL UDF is created with lambda with non identifier arguments. Closes [#33866](https://github.com/ClickHouse/ClickHouse/issues/33866). [#33868](https://github.com/ClickHouse/ClickHouse/pull/33868) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)). -* Fixed `replica is not readonly` logical error on `SYSTEM RESTORE REPLICA` query when replica is actually readonly. Fixes [#33806](https://github.com/ClickHouse/ClickHouse/issues/33806). [#33847](https://github.com/ClickHouse/ClickHouse/pull/33847) ([tavplubix](https://github.com/tavplubix)). -* Fix memory leak in `clickhouse-keeper` in case of compression is used (default). [#33840](https://github.com/ClickHouse/ClickHouse/pull/33840) ([Azat Khuzhin](https://github.com/azat)). -* Fix index analysis with no common types available. [#33833](https://github.com/ClickHouse/ClickHouse/pull/33833) ([Amos Bird](https://github.com/amosbird)). -* Fix schema inference for `JSONEachRow` and `JSONCompactEachRow`. [#33830](https://github.com/ClickHouse/ClickHouse/pull/33830) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix usage of external dictionaries with `redis` source and large number of keys. [#33804](https://github.com/ClickHouse/ClickHouse/pull/33804) ([Anton Popov](https://github.com/CurtizJ)). -* Fix bug in client that led to 'Connection reset by peer' in server. Closes [#33309](https://github.com/ClickHouse/ClickHouse/issues/33309). [#33790](https://github.com/ClickHouse/ClickHouse/pull/33790) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix parsing query INSERT INTO ... VALUES SETTINGS ... (...), ... [#33776](https://github.com/ClickHouse/ClickHouse/pull/33776) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix bug of check table when creating data part with wide format and projection. [#33774](https://github.com/ClickHouse/ClickHouse/pull/33774) ([李扬](https://github.com/taiyang-li)). -* Fix tiny race between count() and INSERT/merges/... in MergeTree (it is possible to return incorrect number of rows for SELECT with optimize_trivial_count_query). [#33753](https://github.com/ClickHouse/ClickHouse/pull/33753) ([Azat Khuzhin](https://github.com/azat)). -* Throw exception when directory listing request has failed in storage HDFS. [#33724](https://github.com/ClickHouse/ClickHouse/pull/33724) ([LiuNeng](https://github.com/liuneng1994)). -* Fix mutation when table contains projections. This fixes [#33010](https://github.com/ClickHouse/ClickHouse/issues/33010). This fixes [#33275](https://github.com/ClickHouse/ClickHouse/issues/33275). [#33679](https://github.com/ClickHouse/ClickHouse/pull/33679) ([Amos Bird](https://github.com/amosbird)). -* Correctly determine current database if `CREATE TEMPORARY TABLE AS SELECT` is queried inside a named HTTP session. This is a very rare use case. This closes [#8340](https://github.com/ClickHouse/ClickHouse/issues/8340). [#33676](https://github.com/ClickHouse/ClickHouse/pull/33676) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Allow some queries with sorting, LIMIT BY, ARRAY JOIN and lambda functions. This closes [#7462](https://github.com/ClickHouse/ClickHouse/issues/7462). [#33675](https://github.com/ClickHouse/ClickHouse/pull/33675) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix bug in "zero copy replication" (a feature that is under development and should not be used in production) which lead to data duplication in case of TTL move. Fixes [#33643](https://github.com/ClickHouse/ClickHouse/issues/33643). [#33642](https://github.com/ClickHouse/ClickHouse/pull/33642) ([alesapin](https://github.com/alesapin)). -* Fix `Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform` (in case of `optimize_aggregation_in_order = 1`). [#33637](https://github.com/ClickHouse/ClickHouse/pull/33637) ([Azat Khuzhin](https://github.com/azat)). -* Fix error `Bad cast from type ... to DB::DataTypeArray` which may happen when table has `Nested` column with dots in name, and default value is generated for it (e.g. during insert, when column is not listed). Continuation of [#28762](https://github.com/ClickHouse/ClickHouse/issues/28762). [#33588](https://github.com/ClickHouse/ClickHouse/pull/33588) ([Alexey Pavlenko](https://github.com/alexeypavlenko)). -* Export into `lz4` files has been fixed. Closes [#31421](https://github.com/ClickHouse/ClickHouse/issues/31421). [#31862](https://github.com/ClickHouse/ClickHouse/pull/31862) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix potential crash if `group_by_overflow_mode` was set to `any` (approximate GROUP BY) and aggregation was performed by single column of type `LowCardinality`. [#34506](https://github.com/ClickHouse/ClickHouse/pull/34506) ([DR](https://github.com/freedomDR)). -* Fix inserting to temporary tables via gRPC client-server protocol. Fixes [#34347](https://github.com/ClickHouse/ClickHouse/issues/34347), issue `#2`. [#34364](https://github.com/ClickHouse/ClickHouse/pull/34364) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix issue [#19429](https://github.com/ClickHouse/ClickHouse/issues/19429). [#34225](https://github.com/ClickHouse/ClickHouse/pull/34225) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix issue [#18206](https://github.com/ClickHouse/ClickHouse/issues/18206). [#33977](https://github.com/ClickHouse/ClickHouse/pull/33977) ([Vitaly Baranov](https://github.com/vitlibar)). -* This PR allows using multiple LDAP storages in the same list of user directories. It worked earlier but was broken because LDAP tests are disabled (they are part of the testflows tests). [#33574](https://github.com/ClickHouse/ClickHouse/pull/33574) ([Vitaly Baranov](https://github.com/vitlibar)). - -### ClickHouse release v22.1, 2022-01-18 - -#### Upgrade Notes - -* The functions `left` and `right` were previously implemented in parser and now full-featured. Distributed queries with `left` or `right` functions without aliases may throw exception if cluster contains different versions of clickhouse-server. If you are upgrading your cluster and encounter this error, you should finish upgrading your cluster to ensure all nodes have the same version. Also you can add aliases (`AS something`) to the columns in your queries to avoid this issue. [#33407](https://github.com/ClickHouse/ClickHouse/pull/33407) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Resource usage by scalar subqueries is fully accounted since this version. With this change, rows read in scalar subqueries are now reported in the query_log. If the scalar subquery is cached (repeated or called for several rows) the rows read are only counted once. This change allows KILLing queries and reporting progress while they are executing scalar subqueries. [#32271](https://github.com/ClickHouse/ClickHouse/pull/32271) ([Raúl Marín](https://github.com/Algunenano)). - -#### New Feature - -* Implement data schema inference for input formats. Allow to skip structure (or write just `auto`) in table functions `file`, `url`, `s3`, `hdfs` and in parameters of `clickhouse-local` . Allow to skip structure in create query for table engines `File`, `HDFS`, `S3`, `URL`, `Merge`, `Buffer`, `Distributed` and `ReplicatedMergeTree` (if we add new replicas). [#32455](https://github.com/ClickHouse/ClickHouse/pull/32455) ([Kruglov Pavel](https://github.com/Avogar)). -* Detect format by file extension in `file`/`hdfs`/`s3`/`url` table functions and `HDFS`/`S3`/`URL` table engines and also for `SELECT INTO OUTFILE` and `INSERT FROM INFILE` [#33565](https://github.com/ClickHouse/ClickHouse/pull/33565) ([Kruglov Pavel](https://github.com/Avogar)). Close [#30918](https://github.com/ClickHouse/ClickHouse/issues/30918). [#33443](https://github.com/ClickHouse/ClickHouse/pull/33443) ([OnePiece](https://github.com/zhongyuankai)). -* A tool for collecting diagnostics data if you need support. [#33175](https://github.com/ClickHouse/ClickHouse/pull/33175) ([Alexander Burmak](https://github.com/Alex-Burmak)). -* Automatic cluster discovery via Zoo/Keeper. It allows to add replicas to the cluster without changing configuration on every server. [#31442](https://github.com/ClickHouse/ClickHouse/pull/31442) ([vdimir](https://github.com/vdimir)). -* Implement hive table engine to access apache hive from clickhouse. This implements: [#29245](https://github.com/ClickHouse/ClickHouse/issues/29245). [#31104](https://github.com/ClickHouse/ClickHouse/pull/31104) ([taiyang-li](https://github.com/taiyang-li)). -* Add aggregate functions `cramersV`, `cramersVBiasCorrected`, `theilsU` and `contingency`. These functions calculate dependency (measure of association) between categorical values. All these functions are using cross-tab (histogram on pairs) for implementation. You can imagine it like a correlation coefficient but for any discrete values (not necessary numbers). [#33366](https://github.com/ClickHouse/ClickHouse/pull/33366) ([alexey-milovidov](https://github.com/alexey-milovidov)). Initial implementation by [Vanyok-All-is-OK](https://github.com/Vanyok-All-is-OK) and [antikvist](https://github.com/antikvist). -* Added table function `hdfsCluster` which allows processing files from HDFS in parallel from many nodes in a specified cluster, similarly to `s3Cluster`. [#32400](https://github.com/ClickHouse/ClickHouse/pull/32400) ([Zhichang Yu](https://github.com/yuzhichang)). -* Adding support for disks backed by Azure Blob Storage, in a similar way it has been done for disks backed by AWS S3. [#31505](https://github.com/ClickHouse/ClickHouse/pull/31505) ([Jakub Kuklis](https://github.com/jkuklis)). -* Allow `COMMENT` in `CREATE VIEW` (for all VIEW kinds). [#31062](https://github.com/ClickHouse/ClickHouse/pull/31062) ([Vasily Nemkov](https://github.com/Enmk)). -* Dynamically reinitialize listening ports and protocols when configuration changes. [#30549](https://github.com/ClickHouse/ClickHouse/pull/30549) ([Kevin Michel](https://github.com/kmichel-aiven)). -* Added `left`, `right`, `leftUTF8`, `rightUTF8` functions. Fix error in implementation of `substringUTF8` function with negative offset (offset from the end of string). [#33407](https://github.com/ClickHouse/ClickHouse/pull/33407) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Add new functions for `H3` coordinate system: `h3HexAreaKm2`, `h3CellAreaM2`, `h3CellAreaRads2`. [#33479](https://github.com/ClickHouse/ClickHouse/pull/33479) ([Bharat Nallan](https://github.com/bharatnc)). -* Add `MONTHNAME` function. [#33436](https://github.com/ClickHouse/ClickHouse/pull/33436) ([usurai](https://github.com/usurai)). -* Added function `arrayLast`. Closes [#33390](https://github.com/ClickHouse/ClickHouse/issues/33390). [#33415](https://github.com/ClickHouse/ClickHouse/pull/33415) Added function `arrayLastIndex`. [#33465](https://github.com/ClickHouse/ClickHouse/pull/33465) ([Maksim Kita](https://github.com/kitaisreal)). -* Add function `decodeURLFormComponent` slightly different to `decodeURLComponent`. Close [#10298](https://github.com/ClickHouse/ClickHouse/issues/10298). [#33451](https://github.com/ClickHouse/ClickHouse/pull/33451) ([SuperDJY](https://github.com/cmsxbc)). -* Allow to split `GraphiteMergeTree` rollup rules for plain/tagged metrics (optional rule_type field). [#33494](https://github.com/ClickHouse/ClickHouse/pull/33494) ([Michail Safronov](https://github.com/msaf1980)). - -#### Performance Improvement - -* Support moving conditions to `PREWHERE` (setting `optimize_move_to_prewhere`) for tables of `Merge` engine if its all underlying tables supports `PREWHERE`. [#33300](https://github.com/ClickHouse/ClickHouse/pull/33300) ([Anton Popov](https://github.com/CurtizJ)). -* More efficient handling of globs for URL storage. Now you can easily query million URLs in parallel with retries. Closes [#32866](https://github.com/ClickHouse/ClickHouse/issues/32866). [#32907](https://github.com/ClickHouse/ClickHouse/pull/32907) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Avoid exponential backtracking in parser. This closes [#20158](https://github.com/ClickHouse/ClickHouse/issues/20158). [#33481](https://github.com/ClickHouse/ClickHouse/pull/33481) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Abuse of `untuple` function was leading to exponential complexity of query analysis (found by fuzzer). This closes [#33297](https://github.com/ClickHouse/ClickHouse/issues/33297). [#33445](https://github.com/ClickHouse/ClickHouse/pull/33445) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Reduce allocated memory for dictionaries with string attributes. [#33466](https://github.com/ClickHouse/ClickHouse/pull/33466) ([Maksim Kita](https://github.com/kitaisreal)). -* Slight performance improvement of `reinterpret` function. [#32587](https://github.com/ClickHouse/ClickHouse/pull/32587) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Non significant change. In extremely rare cases when data part is lost on every replica, after merging of some data parts, the subsequent queries may skip less amount of partitions during partition pruning. This hardly affects anything. [#32220](https://github.com/ClickHouse/ClickHouse/pull/32220) ([Azat Khuzhin](https://github.com/azat)). -* Improve `clickhouse-keeper` writing performance by optimization the size calculation logic. [#32366](https://github.com/ClickHouse/ClickHouse/pull/32366) ([zhanglistar](https://github.com/zhanglistar)). -* Optimize single part projection materialization. This closes [#31669](https://github.com/ClickHouse/ClickHouse/issues/31669). [#31885](https://github.com/ClickHouse/ClickHouse/pull/31885) ([Amos Bird](https://github.com/amosbird)). -* Improve query performance of system tables. [#33312](https://github.com/ClickHouse/ClickHouse/pull/33312) ([OnePiece](https://github.com/zhongyuankai)). -* Optimize selecting of MergeTree parts that can be moved between volumes. [#33225](https://github.com/ClickHouse/ClickHouse/pull/33225) ([OnePiece](https://github.com/zhongyuankai)). -* Fix `sparse_hashed` dict performance with sequential keys (wrong hash function). [#32536](https://github.com/ClickHouse/ClickHouse/pull/32536) ([Azat Khuzhin](https://github.com/azat)). - -#### Experimental Feature - -* Parallel reading from multiple replicas within a shard during distributed query without using sample key. To enable this, set `allow_experimental_parallel_reading_from_replicas = 1` and `max_parallel_replicas` to any number. This closes [#26748](https://github.com/ClickHouse/ClickHouse/issues/26748). [#29279](https://github.com/ClickHouse/ClickHouse/pull/29279) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Implemented sparse serialization. It can reduce usage of disk space and improve performance of some queries for columns, which contain a lot of default (zero) values. It can be enabled by setting `ratio_for_sparse_serialization`. Sparse serialization will be chosen dynamically for column, if it has ratio of number of default values to number of all values above that threshold. Serialization (default or sparse) will be fixed for every column in part, but may varies between parts. [#22535](https://github.com/ClickHouse/ClickHouse/pull/22535) ([Anton Popov](https://github.com/CurtizJ)). -* Add "TABLE OVERRIDE" feature for customizing MaterializedMySQL table schemas. [#32325](https://github.com/ClickHouse/ClickHouse/pull/32325) ([Stig Bakken](https://github.com/stigsb)). -* Add `EXPLAIN TABLE OVERRIDE` query. [#32836](https://github.com/ClickHouse/ClickHouse/pull/32836) ([Stig Bakken](https://github.com/stigsb)). -* Support TABLE OVERRIDE clause for MaterializedPostgreSQL. RFC: [#31480](https://github.com/ClickHouse/ClickHouse/issues/31480). [#32749](https://github.com/ClickHouse/ClickHouse/pull/32749) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Change ZooKeeper path for zero-copy marks for shared data. Note that "zero-copy replication" is non-production feature (in early stages of development) that you shouldn't use anyway. But in case if you have used it, let you keep in mind this change. [#32061](https://github.com/ClickHouse/ClickHouse/pull/32061) ([ianton-ru](https://github.com/ianton-ru)). -* Events clause support for WINDOW VIEW watch query. [#32607](https://github.com/ClickHouse/ClickHouse/pull/32607) ([vxider](https://github.com/Vxider)). -* Fix ACL with explicit digit hash in `clickhouse-keeper`: now the behavior consistent with ZooKeeper and generated digest is always accepted. [#33249](https://github.com/ClickHouse/ClickHouse/pull/33249) ([小路](https://github.com/nicelulu)). [#33246](https://github.com/ClickHouse/ClickHouse/pull/33246). -* Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)). - -#### Improvement - -* Now date time conversion functions that generates time before `1970-01-01 00:00:00` will be saturated to zero instead of overflow. [#29953](https://github.com/ClickHouse/ClickHouse/pull/29953) ([Amos Bird](https://github.com/amosbird)). It also fixes a bug in index analysis if date truncation function would yield result before the Unix epoch. -* Always display resource usage (total CPU usage, total RAM usage and max RAM usage per host) in client. [#33271](https://github.com/ClickHouse/ClickHouse/pull/33271) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Improve `Bool` type serialization and deserialization, check the range of values. [#32984](https://github.com/ClickHouse/ClickHouse/pull/32984) ([Kruglov Pavel](https://github.com/Avogar)). -* If an invalid setting is defined using the `SET` query or using the query parameters in the HTTP request, error message will contain suggestions that are similar to the invalid setting string (if any exists). [#32946](https://github.com/ClickHouse/ClickHouse/pull/32946) ([Antonio Andelic](https://github.com/antonio2368)). -* Support hints for mistyped setting names for clickhouse-client and clickhouse-local. Closes [#32237](https://github.com/ClickHouse/ClickHouse/issues/32237). [#32841](https://github.com/ClickHouse/ClickHouse/pull/32841) ([凌涛](https://github.com/lingtaolf)). -* Allow to use virtual columns in Materialized Views. Close [#11210](https://github.com/ClickHouse/ClickHouse/issues/11210). [#33482](https://github.com/ClickHouse/ClickHouse/pull/33482) ([OnePiece](https://github.com/zhongyuankai)). -* Add config to disable IPv6 in clickhouse-keeper if needed. This close [#33381](https://github.com/ClickHouse/ClickHouse/issues/33381). [#33450](https://github.com/ClickHouse/ClickHouse/pull/33450) ([Wu Xueyang](https://github.com/wuxueyang96)). -* Add more info to `system.build_options` about current git revision. [#33431](https://github.com/ClickHouse/ClickHouse/pull/33431) ([taiyang-li](https://github.com/taiyang-li)). -* `clickhouse-local`: track memory under `--max_memory_usage_in_client` option. [#33341](https://github.com/ClickHouse/ClickHouse/pull/33341) ([Azat Khuzhin](https://github.com/azat)). -* Allow negative intervals in function `intervalLengthSum`. Their length will be added as well. This closes [#33323](https://github.com/ClickHouse/ClickHouse/issues/33323). [#33335](https://github.com/ClickHouse/ClickHouse/pull/33335) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* `LineAsString` can be used as output format. This closes [#30919](https://github.com/ClickHouse/ClickHouse/issues/30919). [#33331](https://github.com/ClickHouse/ClickHouse/pull/33331) ([Sergei Trifonov](https://github.com/serxa)). -* Support `` in cluster configuration, as an alternative form of `1`. Close [#33270](https://github.com/ClickHouse/ClickHouse/issues/33270). [#33330](https://github.com/ClickHouse/ClickHouse/pull/33330) ([SuperDJY](https://github.com/cmsxbc)). -* Pressing Ctrl+C twice will terminate `clickhouse-benchmark` immediately without waiting for in-flight queries. This closes [#32586](https://github.com/ClickHouse/ClickHouse/issues/32586). [#33303](https://github.com/ClickHouse/ClickHouse/pull/33303) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Support Unix timestamp with milliseconds in `parseDateTimeBestEffort` function. [#33276](https://github.com/ClickHouse/ClickHouse/pull/33276) ([Ben](https://github.com/benbiti)). -* Allow to cancel query while reading data from external table in the formats: `Arrow` / `Parquet` / `ORC` - it failed to be cancelled it case of big files and setting input_format_allow_seeks as false. Closes [#29678](https://github.com/ClickHouse/ClickHouse/issues/29678). [#33238](https://github.com/ClickHouse/ClickHouse/pull/33238) ([Kseniia Sumarokova](https://github.com/kssenii)). -* If table engine supports `SETTINGS` clause, allow to pass the settings as key-value or via config. Add this support for MySQL. [#33231](https://github.com/ClickHouse/ClickHouse/pull/33231) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Correctly prevent Nullable primary keys if necessary. This is for [#32780](https://github.com/ClickHouse/ClickHouse/issues/32780). [#33218](https://github.com/ClickHouse/ClickHouse/pull/33218) ([Amos Bird](https://github.com/amosbird)). -* Add retry for `PostgreSQL` connections in case nothing has been fetched yet. Closes [#33199](https://github.com/ClickHouse/ClickHouse/issues/33199). [#33209](https://github.com/ClickHouse/ClickHouse/pull/33209) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Validate config keys for external dictionaries. [#33095](https://github.com/ClickHouse/ClickHouse/issues/33095#issuecomment-1000577517). [#33130](https://github.com/ClickHouse/ClickHouse/pull/33130) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Send profile info inside `clickhouse-local`. Closes [#33093](https://github.com/ClickHouse/ClickHouse/issues/33093). [#33097](https://github.com/ClickHouse/ClickHouse/pull/33097) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Short circuit evaluation: support for function `throwIf`. Closes [#32969](https://github.com/ClickHouse/ClickHouse/issues/32969). [#32973](https://github.com/ClickHouse/ClickHouse/pull/32973) ([Maksim Kita](https://github.com/kitaisreal)). -* (This only happens in unofficial builds). Fixed segfault when inserting data into compressed Decimal, String, FixedString and Array columns. This closes [#32939](https://github.com/ClickHouse/ClickHouse/issues/32939). [#32940](https://github.com/ClickHouse/ClickHouse/pull/32940) ([N. Kolotov](https://github.com/nkolotov)). -* Added support for specifying subquery as SQL user defined function. Example: `CREATE FUNCTION test AS () -> (SELECT 1)`. Closes [#30755](https://github.com/ClickHouse/ClickHouse/issues/30755). [#32758](https://github.com/ClickHouse/ClickHouse/pull/32758) ([Maksim Kita](https://github.com/kitaisreal)). -* Improve gRPC compression support for [#28671](https://github.com/ClickHouse/ClickHouse/issues/28671). [#32747](https://github.com/ClickHouse/ClickHouse/pull/32747) ([Vitaly Baranov](https://github.com/vitlibar)). -* Flush all In-Memory data parts when WAL is not enabled while shutdown server or detaching table. [#32742](https://github.com/ClickHouse/ClickHouse/pull/32742) ([nauta](https://github.com/nautaa)). -* Allow to control connection timeouts for MySQL (previously was supported only for dictionary source). Closes [#16669](https://github.com/ClickHouse/ClickHouse/issues/16669). Previously default connect_timeout was rather small, now it is configurable. [#32734](https://github.com/ClickHouse/ClickHouse/pull/32734) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support `authSource` option for storage `MongoDB`. Closes [#32594](https://github.com/ClickHouse/ClickHouse/issues/32594). [#32702](https://github.com/ClickHouse/ClickHouse/pull/32702) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support `Date32` type in `genarateRandom` table function. [#32643](https://github.com/ClickHouse/ClickHouse/pull/32643) ([nauta](https://github.com/nautaa)). -* Add settings `max_concurrent_select_queries` and `max_concurrent_insert_queries` for control concurrent queries by query kind. Close [#3575](https://github.com/ClickHouse/ClickHouse/issues/3575). [#32609](https://github.com/ClickHouse/ClickHouse/pull/32609) ([SuperDJY](https://github.com/cmsxbc)). -* Improve handling nested structures with missing columns while reading data in `Protobuf` format. Follow-up to https://github.com/ClickHouse/ClickHouse/pull/31988. [#32531](https://github.com/ClickHouse/ClickHouse/pull/32531) ([Vitaly Baranov](https://github.com/vitlibar)). -* Allow empty credentials for `MongoDB` engine. Closes [#26267](https://github.com/ClickHouse/ClickHouse/issues/26267). [#32460](https://github.com/ClickHouse/ClickHouse/pull/32460) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Disable some optimizations for window functions that may lead to exceptions. Closes [#31535](https://github.com/ClickHouse/ClickHouse/issues/31535). Closes [#31620](https://github.com/ClickHouse/ClickHouse/issues/31620). [#32453](https://github.com/ClickHouse/ClickHouse/pull/32453) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Allows to connect to MongoDB 5.0. Closes [#31483](https://github.com/ClickHouse/ClickHouse/issues/31483),. [#32416](https://github.com/ClickHouse/ClickHouse/pull/32416) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Enable comparison between `Decimal` and `Float`. Closes [#22626](https://github.com/ClickHouse/ClickHouse/issues/22626). [#31966](https://github.com/ClickHouse/ClickHouse/pull/31966) ([flynn](https://github.com/ucasFL)). -* Added settings `command_read_timeout`, `command_write_timeout` for `StorageExecutable`, `StorageExecutablePool`, `ExecutableDictionary`, `ExecutablePoolDictionary`, `ExecutableUserDefinedFunctions`. Setting `command_read_timeout` controls timeout for reading data from command stdout in milliseconds. Setting `command_write_timeout` timeout for writing data to command stdin in milliseconds. Added settings `command_termination_timeout` for `ExecutableUserDefinedFunction`, `ExecutableDictionary`, `StorageExecutable`. Added setting `execute_direct` for `ExecutableUserDefinedFunction`, by default true. Added setting `execute_direct` for `ExecutableDictionary`, `ExecutablePoolDictionary`, by default false. [#30957](https://github.com/ClickHouse/ClickHouse/pull/30957) ([Maksim Kita](https://github.com/kitaisreal)). -* Bitmap aggregate functions will give correct result for out of range argument instead of wraparound. [#33127](https://github.com/ClickHouse/ClickHouse/pull/33127) ([DR](https://github.com/freedomDR)). -* Fix parsing incorrect queries with `FROM INFILE` statement. [#33521](https://github.com/ClickHouse/ClickHouse/pull/33521) ([Kruglov Pavel](https://github.com/Avogar)). -* Don't allow to write into `S3` if path contains globs. [#33142](https://github.com/ClickHouse/ClickHouse/pull/33142) ([Kruglov Pavel](https://github.com/Avogar)). -* `--echo` option was not used by `clickhouse-client` in batch mode with single query. [#32843](https://github.com/ClickHouse/ClickHouse/pull/32843) ([N. Kolotov](https://github.com/nkolotov)). -* Use `--database` option for clickhouse-local. [#32797](https://github.com/ClickHouse/ClickHouse/pull/32797) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix surprisingly bad code in SQL ordinary function `file`. Now it supports symlinks. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Updating `modification_time` for data part in `system.parts` after part movement [#32964](https://github.com/ClickHouse/ClickHouse/issues/32964). [#32965](https://github.com/ClickHouse/ClickHouse/pull/32965) ([save-my-heart](https://github.com/save-my-heart)). -* Potential issue, cannot be exploited: integer overflow may happen in array resize. [#33024](https://github.com/ClickHouse/ClickHouse/pull/33024) ([varadarajkumar](https://github.com/varadarajkumar)). - -#### Build/Testing/Packaging Improvement - -* Add packages, functional tests and Docker builds for AArch64 (ARM) version of ClickHouse. [#32911](https://github.com/ClickHouse/ClickHouse/pull/32911) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). [#32415](https://github.com/ClickHouse/ClickHouse/pull/32415) -* Prepare ClickHouse to be built with musl-libc. It is not enabled by default. [#33134](https://github.com/ClickHouse/ClickHouse/pull/33134) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Make installation script working on FreeBSD. This closes [#33384](https://github.com/ClickHouse/ClickHouse/issues/33384). [#33418](https://github.com/ClickHouse/ClickHouse/pull/33418) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Add `actionlint` for GitHub Actions workflows and verify workflow files via `act --list` to check the correct workflow syntax. [#33612](https://github.com/ClickHouse/ClickHouse/pull/33612) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Add more tests for the nullable primary key feature. Add more tests with different types and merge tree kinds, plus randomly generated data. [#33228](https://github.com/ClickHouse/ClickHouse/pull/33228) ([Amos Bird](https://github.com/amosbird)). -* Add a simple tool to visualize flaky tests in web browser. [#33185](https://github.com/ClickHouse/ClickHouse/pull/33185) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Enable hermetic build for shared builds. This is mainly for developers. [#32968](https://github.com/ClickHouse/ClickHouse/pull/32968) ([Amos Bird](https://github.com/amosbird)). -* Update `libc++` and `libc++abi` to the latest. [#32484](https://github.com/ClickHouse/ClickHouse/pull/32484) ([Raúl Marín](https://github.com/Algunenano)). -* Added integration test for external .NET client ([ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client)). [#23230](https://github.com/ClickHouse/ClickHouse/pull/23230) ([Oleg V. Kozlyuk](https://github.com/DarkWanderer)). -* Inject git information into clickhouse binary file. So we can get source code revision easily from clickhouse binary file. [#33124](https://github.com/ClickHouse/ClickHouse/pull/33124) ([taiyang-li](https://github.com/taiyang-li)). -* Remove obsolete code from ConfigProcessor. Yandex specific code is not used anymore. The code contained one minor defect. This defect was reported by [Mallik Hassan](https://github.com/SadiHassan) in [#33032](https://github.com/ClickHouse/ClickHouse/issues/33032). This closes [#33032](https://github.com/ClickHouse/ClickHouse/issues/33032). [#33026](https://github.com/ClickHouse/ClickHouse/pull/33026) ([alexey-milovidov](https://github.com/alexey-milovidov)). - -#### Bug Fix (user-visible misbehavior in official stable or prestable release) - -* Several fixes for format parsing. This is relevant if `clickhouse-server` is open for write access to adversary. Specifically crafted input data for `Native` format may lead to reading uninitialized memory or crash. This is relevant if `clickhouse-server` is open for write access to adversary. [#33050](https://github.com/ClickHouse/ClickHouse/pull/33050) ([Heena Bansal](https://github.com/HeenaBansal2009)). Fixed Apache Avro Union type index out of boundary issue in Apache Avro binary format. [#33022](https://github.com/ClickHouse/ClickHouse/pull/33022) ([Harry Lee](https://github.com/HarryLeeIBM)). Fix null pointer dereference in `LowCardinality` data when deserializing `LowCardinality` data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)). -* ClickHouse Keeper handler will correctly remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)). -* Potential off-by-one miscalculation of quotas: quota limit was not reached, but the limit was exceeded. This fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)). -* Fixed CASTing from String to IPv4 or IPv6 and back. Fixed error message in case of failed conversion. [#29224](https://github.com/ClickHouse/ClickHouse/pull/29224) ([Dmitry Novik](https://github.com/novikd)) [#27914](https://github.com/ClickHouse/ClickHouse/pull/27914) ([Vasily Nemkov](https://github.com/Enmk)). -* Fixed an exception like `Unknown aggregate function nothing` during an execution on a remote server. This fixes [#16689](https://github.com/ClickHouse/ClickHouse/issues/16689). [#26074](https://github.com/ClickHouse/ClickHouse/pull/26074) ([hexiaoting](https://github.com/hexiaoting)). -* Fix wrong database for JOIN without explicit database in distributed queries (Fixes: [#10471](https://github.com/ClickHouse/ClickHouse/issues/10471)). [#33611](https://github.com/ClickHouse/ClickHouse/pull/33611) ([Azat Khuzhin](https://github.com/azat)). -* Fix segfault in Apache `Avro` format that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix segfault in Apache `Arrow` format if schema contains `Dictionary` type. Closes [#33507](https://github.com/ClickHouse/ClickHouse/issues/33507). [#33529](https://github.com/ClickHouse/ClickHouse/pull/33529) ([Kruglov Pavel](https://github.com/Avogar)). -* Out of band `offset` and `limit` settings may be applied incorrectly for views. Close [#33289](https://github.com/ClickHouse/ClickHouse/issues/33289) [#33518](https://github.com/ClickHouse/ClickHouse/pull/33518) ([hexiaoting](https://github.com/hexiaoting)). -* Fix an exception `Block structure mismatch` which may happen during insertion into table with default nested `LowCardinality` column. Fixes [#33028](https://github.com/ClickHouse/ClickHouse/issues/33028). [#33504](https://github.com/ClickHouse/ClickHouse/pull/33504) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix dictionary expressions for `range_hashed` range min and range max attributes when created using DDL. Closes [#30809](https://github.com/ClickHouse/ClickHouse/issues/30809). [#33478](https://github.com/ClickHouse/ClickHouse/pull/33478) ([Maksim Kita](https://github.com/kitaisreal)). -* Fix possible use-after-free for INSERT into Materialized View with concurrent DROP ([Azat Khuzhin](https://github.com/azat)). -* Do not try to read pass EOF (to workaround for a bug in the Linux kernel), this bug can be reproduced on kernels (3.14..5.9), and requires `index_granularity_bytes=0` (i.e. turn off adaptive index granularity). [#33372](https://github.com/ClickHouse/ClickHouse/pull/33372) ([Azat Khuzhin](https://github.com/azat)). -* The commands `SYSTEM SUSPEND` and `SYSTEM ... THREAD FUZZER` missed access control. It is fixed. Author: Kevin Michel. [#33333](https://github.com/ClickHouse/ClickHouse/pull/33333) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix when `COMMENT` for dictionaries does not appear in `system.tables`, `system.dictionaries`. Allow to modify the comment for `Dictionary` engine. Closes [#33251](https://github.com/ClickHouse/ClickHouse/issues/33251). [#33261](https://github.com/ClickHouse/ClickHouse/pull/33261) ([Maksim Kita](https://github.com/kitaisreal)). -* Add asynchronous inserts (with enabled setting `async_insert`) to query log. Previously such queries didn't appear in the query log. [#33239](https://github.com/ClickHouse/ClickHouse/pull/33239) ([Anton Popov](https://github.com/CurtizJ)). -* Fix sending `WHERE 1 = 0` expressions for external databases query. Closes [#33152](https://github.com/ClickHouse/ClickHouse/issues/33152). [#33214](https://github.com/ClickHouse/ClickHouse/pull/33214) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix DDL validation for MaterializedPostgreSQL. Fix setting `materialized_postgresql_allow_automatic_update`. Closes [#29535](https://github.com/ClickHouse/ClickHouse/issues/29535). [#33200](https://github.com/ClickHouse/ClickHouse/pull/33200) ([Kseniia Sumarokova](https://github.com/kssenii)). Make sure unused replication slots are always removed. Found in [#26952](https://github.com/ClickHouse/ClickHouse/issues/26952). [#33187](https://github.com/ClickHouse/ClickHouse/pull/33187) ([Kseniia Sumarokova](https://github.com/kssenii)). Fix MaterializedPostreSQL detach/attach (removing / adding to replication) tables with non-default schema. Found in [#29535](https://github.com/ClickHouse/ClickHouse/issues/29535). [#33179](https://github.com/ClickHouse/ClickHouse/pull/33179) ([Kseniia Sumarokova](https://github.com/kssenii)). Fix DROP MaterializedPostgreSQL database. [#33468](https://github.com/ClickHouse/ClickHouse/pull/33468) ([Kseniia Sumarokova](https://github.com/kssenii)). -* The metric `StorageBufferBytes` sometimes was miscalculated. [#33159](https://github.com/ClickHouse/ClickHouse/pull/33159) ([xuyatian](https://github.com/xuyatian)). -* Fix error `Invalid version for SerializationLowCardinality key column` in case of reading from `LowCardinality` column with `local_filesystem_read_prefetch` or `remote_filesystem_read_prefetch` enabled. [#33046](https://github.com/ClickHouse/ClickHouse/pull/33046) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix `s3` table function reading empty file. Closes [#33008](https://github.com/ClickHouse/ClickHouse/issues/33008). [#33037](https://github.com/ClickHouse/ClickHouse/pull/33037) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix Context leak in case of cancel_http_readonly_queries_on_client_close (i.e. leaking of external tables that had been uploaded the the server and other resources). [#32982](https://github.com/ClickHouse/ClickHouse/pull/32982) ([Azat Khuzhin](https://github.com/azat)). -* Fix wrong tuple output in `CSV` format in case of custom csv delimiter. [#32981](https://github.com/ClickHouse/ClickHouse/pull/32981) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix HDFS URL check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix throwing exception like positional argument out of bounds for non-positional arguments. Closes [#31173](https://github.com/ClickHouse/ClickHouse/issues/31173)#event-5789668239. [#32961](https://github.com/ClickHouse/ClickHouse/pull/32961) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix UB in case of unexpected EOF during filling a set from HTTP query (i.e. if the client interrupted in the middle, i.e. `timeout 0.15s curl -Ss -F 's=@t.csv;' 'http://127.0.0.1:8123/?s_structure=key+Int&query=SELECT+dummy+IN+s'` and with large enough `t.csv`). [#32955](https://github.com/ClickHouse/ClickHouse/pull/32955) ([Azat Khuzhin](https://github.com/azat)). -* Fix a regression in `replaceRegexpAll` function. The function worked incorrectly when matched substring was empty. This closes [#32777](https://github.com/ClickHouse/ClickHouse/issues/32777). This closes [#30245](https://github.com/ClickHouse/ClickHouse/issues/30245). [#32945](https://github.com/ClickHouse/ClickHouse/pull/32945) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix `ORC` format stripe reading. [#32929](https://github.com/ClickHouse/ClickHouse/pull/32929) ([kreuzerkrieg](https://github.com/kreuzerkrieg)). -* `topKWeightedState` failed for some input types. [#32487](https://github.com/ClickHouse/ClickHouse/issues/32487). [#32914](https://github.com/ClickHouse/ClickHouse/pull/32914) ([vdimir](https://github.com/vdimir)). -* Fix exception `Single chunk is expected from view inner query (LOGICAL_ERROR)` in materialized view. Fixes [#31419](https://github.com/ClickHouse/ClickHouse/issues/31419). [#32862](https://github.com/ClickHouse/ClickHouse/pull/32862) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix optimization with lazy seek for async reads from remote filesystems. Closes [#32803](https://github.com/ClickHouse/ClickHouse/issues/32803). [#32835](https://github.com/ClickHouse/ClickHouse/pull/32835) ([Kseniia Sumarokova](https://github.com/kssenii)). -* `MergeTree` table engine might silently skip some mutations if there are too many running mutations or in case of high memory consumption, it's fixed. Fixes [#17882](https://github.com/ClickHouse/ClickHouse/issues/17882). [#32814](https://github.com/ClickHouse/ClickHouse/pull/32814) ([tavplubix](https://github.com/tavplubix)). -* Avoid reusing the scalar subquery cache when processing MV blocks. This fixes a bug when the scalar query reference the source table but it means that all subscalar queries in the MV definition will be calculated for each block. [#32811](https://github.com/ClickHouse/ClickHouse/pull/32811) ([Raúl Marín](https://github.com/Algunenano)). -* Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([tavplubix](https://github.com/tavplubix)). -* Fix crash when used `fuzzBits` function, close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). -* Fix error `Column is not under aggregate function` in case of MV with `GROUP BY (list of columns)` (which is pared as `GROUP BY tuple(...)`) over `Kafka`/`RabbitMQ`. Fixes [#32668](https://github.com/ClickHouse/ClickHouse/issues/32668) and [#32744](https://github.com/ClickHouse/ClickHouse/issues/32744). [#32751](https://github.com/ClickHouse/ClickHouse/pull/32751) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix `ALTER TABLE ... MATERIALIZE TTL` query with `TTL ... DELETE WHERE ...` and `TTL ... GROUP BY ...` modes. [#32695](https://github.com/ClickHouse/ClickHouse/pull/32695) ([Anton Popov](https://github.com/CurtizJ)). -* Fix `optimize_read_in_order` optimization in case when table engine is `Distributed` or `Merge` and its underlying `MergeTree` tables have monotonous function in prefix of sorting key. [#32670](https://github.com/ClickHouse/ClickHouse/pull/32670) ([Anton Popov](https://github.com/CurtizJ)). -* Fix LOGICAL_ERROR exception when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). -* Inserting into S3 with multipart upload to Google Cloud Storage may trigger abort. [#32504](https://github.com/ClickHouse/ClickHouse/issues/32504). [#32649](https://github.com/ClickHouse/ClickHouse/pull/32649) ([vdimir](https://github.com/vdimir)). -* Fix possible exception at `RabbitMQ` storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Fix table lifetime (i.e. possible use-after-free) in case of parallel DROP TABLE and INSERT. [#32572](https://github.com/ClickHouse/ClickHouse/pull/32572) ([Azat Khuzhin](https://github.com/azat)). -* Fix async inserts with formats `CustomSeparated`, `Template`, `Regexp`, `MsgPack` and `JSONAsString`. Previousely the async inserts with these formats didn't read any data. [#32530](https://github.com/ClickHouse/ClickHouse/pull/32530) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix `groupBitmapAnd` function on distributed table. [#32529](https://github.com/ClickHouse/ClickHouse/pull/32529) ([minhthucdao](https://github.com/dmthuc)). -* Fix crash in JOIN found by fuzzer, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([vdimir](https://github.com/vdimir)). -* Proper handling of the case with Apache Arrow column duplication. [#32507](https://github.com/ClickHouse/ClickHouse/pull/32507) ([Dmitriy Mokhnatkin](https://github.com/DMokhnatkin)). -* Fix issue with ambiguous query formatting in distributed queries that led to errors when some table columns were named `ALL` or `DISTINCT`. This closes [#32391](https://github.com/ClickHouse/ClickHouse/issues/32391). [#32490](https://github.com/ClickHouse/ClickHouse/pull/32490) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)). -* Fix broken select query when there are more than 2 row policies on same column, begin at second queries on the same session. [#31606](https://github.com/ClickHouse/ClickHouse/issues/31606). [#32291](https://github.com/ClickHouse/ClickHouse/pull/32291) ([SuperDJY](https://github.com/cmsxbc)). -* Fix fractional unix timestamp conversion to `DateTime64`, fractional part was reversed for negative unix timestamps (before 1970-01-01). [#32240](https://github.com/ClickHouse/ClickHouse/pull/32240) ([Ben](https://github.com/benbiti)). -* Some entries of replication queue might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([tavplubix](https://github.com/tavplubix)). -* Fix parsing of `APPLY lambda` column transformer which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix `base64Encode` adding trailing bytes on small strings. [#31797](https://github.com/ClickHouse/ClickHouse/pull/31797) ([Kevin Michel](https://github.com/kmichel-aiven)). -* Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix hang up with command `DROP TABLE system.query_log sync`. [#33293](https://github.com/ClickHouse/ClickHouse/pull/33293) ([zhanghuajie](https://github.com/zhanghuajieHIT)). - -## [Changelog for 2021](https://clickhouse.com/docs/en/whats-new/changelog/2021) +* #40651 [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). +* Fix possible use-of-unitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) CC: @nickitat. [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)). +* Better handling of NULL in aggregate combinators, fix possible segfault/logical error while using optimization `optimize_rewrite_sum_if_to_count_if`. Closes [#43758](https://github.com/ClickHouse/ClickHouse/issues/43758). [#43813](https://github.com/ClickHouse/ClickHouse/pull/43813) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix CREATE USER/ROLE query settings constraints. [#43993](https://github.com/ClickHouse/ClickHouse/pull/43993) ([Nikolay Degterinsky](https://github.com/evillique)). +* * Fix wrong behavior of `JOIN ON t1.x = t2.x AND 1 = 1`, forbid such queries. [#44016](https://github.com/ClickHouse/ClickHouse/pull/44016) ([Vladimir C](https://github.com/vdimir)). +* Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)). +* Bring interval subtraction from datetime in line with addition. [#44241](https://github.com/ClickHouse/ClickHouse/pull/44241) ([ltrk2](https://github.com/ltrk2)). +* Fix double-free in HashTable::clearAndShrink() with zero elements in it. [#44256](https://github.com/ClickHouse/ClickHouse/pull/44256) ([Azat Khuzhin](https://github.com/azat)). +* Remove limits on maximum size of the result for view. [#44261](https://github.com/ClickHouse/ClickHouse/pull/44261) ([lizhuoyu5](https://github.com/lzydmxy)). +* Fix possible logical error in cache if `do_not_evict_index_and_mrk_files=1`. Closes [#42142](https://github.com/ClickHouse/ClickHouse/issues/42142). [#44268](https://github.com/ClickHouse/ClickHouse/pull/44268) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible too early cache write interruption in write-through cache (caching could be stopped due to false assumption when it shouldn't have). [#44289](https://github.com/ClickHouse/ClickHouse/pull/44289) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible crash in case function `IN` with constant arguments was used as a constant argument together with `LowCardinality`. Fixes [#44221](https://github.com/ClickHouse/ClickHouse/issues/44221). [#44346](https://github.com/ClickHouse/ClickHouse/pull/44346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix support for complex parameters (like arrays) of parametric aggregate functions. This closes [#30975](https://github.com/ClickHouse/ClickHouse/issues/30975). The aggregate function `sumMapFiltered` was unusable in distributed queries before this change. [#44358](https://github.com/ClickHouse/ClickHouse/pull/44358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Fix possible nullptr deference in JoinSwitcher with `allow_experimental_analyzer`. [#44371](https://github.com/ClickHouse/ClickHouse/pull/44371) ([Vladimir C](https://github.com/vdimir)). +* Fix reading ObjectId in BSON schema inference. [#44382](https://github.com/ClickHouse/ClickHouse/pull/44382) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix race which can lead to premature temp parts removal before merge finished in ReplicatedMergeTree. This issue could lead to errors like `No such file or directory: xxx`. Fixes [#43983](https://github.com/ClickHouse/ClickHouse/issues/43983). [#44383](https://github.com/ClickHouse/ClickHouse/pull/44383) ([alesapin](https://github.com/alesapin)). +* Some invalid `SYSTEM ... ON CLUSTER` queries worked in an unexpected way if a cluster name was not specified. It's fixed, now invalid queries throw `SYNTAX_ERROR` as they should. Fixes [#44264](https://github.com/ClickHouse/ClickHouse/issues/44264). [#44387](https://github.com/ClickHouse/ClickHouse/pull/44387) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix reading Map type in ORC format. [#44400](https://github.com/ClickHouse/ClickHouse/pull/44400) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix reading columns that are not presented in input data in Parquet/ORC formats. Previously it could lead to error `INCORRECT_NUMBER_OF_COLUMNS`. Closes [#44333](https://github.com/ClickHouse/ClickHouse/issues/44333). [#44405](https://github.com/ClickHouse/ClickHouse/pull/44405) ([Kruglov Pavel](https://github.com/Avogar)). +* Previously bar() function used the same '▋' (U+258B "Left five eighths block") character to display both 5/8 and 6/8 bars. This change corrects this behavior by using '▊' (U+258A "Left three quarters block") for displaying 6/8 bar. [#44410](https://github.com/ClickHouse/ClickHouse/pull/44410) ([Alexander Gololobov](https://github.com/davenger)). +* Placing profile settings after profile settings constraints in the configuration file made constraints ineffective. [#44411](https://github.com/ClickHouse/ClickHouse/pull/44411) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix `SYNTAX_ERROR` while running `EXPLAIN AST INSERT` queries with data. Closes [#44207](https://github.com/ClickHouse/ClickHouse/issues/44207). [#44413](https://github.com/ClickHouse/ClickHouse/pull/44413) ([save-my-heart](https://github.com/save-my-heart)). +* Fix reading bool value with CRLF in CSV format. Closes [#44401](https://github.com/ClickHouse/ClickHouse/issues/44401). [#44442](https://github.com/ClickHouse/ClickHouse/pull/44442) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't execute and/or/if/multiIf on LowCardinality dictionary, so the result type cannot be LowCardinality. It could lead to error `Illegal column ColumnLowCardinality` in some cases. Fixes [#43603](https://github.com/ClickHouse/ClickHouse/issues/43603). [#44469](https://github.com/ClickHouse/ClickHouse/pull/44469) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix mutations with setting `max_streams_for_merge_tree_reading`. [#44472](https://github.com/ClickHouse/ClickHouse/pull/44472) ([Anton Popov](https://github.com/CurtizJ)). +* Fix potential null pointer dereference with GROUPING SETS in ASTSelectQuery::formatImpl ([#43049](https://github.com/ClickHouse/ClickHouse/issues/43049)). [#44479](https://github.com/ClickHouse/ClickHouse/pull/44479) ([Robert Schulze](https://github.com/rschu1ze)). +* Validate types in table function arguments, CAST function arguments, JSONAsObject schema inference according to settings. [#44501](https://github.com/ClickHouse/ClickHouse/pull/44501) ([Kruglov Pavel](https://github.com/Avogar)). +* - Fix IN function with LC and const column, close [#44503](https://github.com/ClickHouse/ClickHouse/issues/44503). [#44506](https://github.com/ClickHouse/ClickHouse/pull/44506) ([Duc Canh Le](https://github.com/canhld94)). +* Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Projections do not work in presence of WITH ROLLUP, WITH CUBE and WITH TOTALS. In previous versions, a query produced an exception instead of skipping the usage of projections. This closes [#44614](https://github.com/ClickHouse/ClickHouse/issues/44614). This closes [#42772](https://github.com/ClickHouse/ClickHouse/issues/42772). [#44615](https://github.com/ClickHouse/ClickHouse/pull/44615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* * Fix bug in experimental analyzer and `aggregate_functions_null_for_empty = 1`. Close [#44644](https://github.com/ClickHouse/ClickHouse/issues/44644). [#44648](https://github.com/ClickHouse/ClickHouse/pull/44648) ([Vladimir C](https://github.com/vdimir)). +* async blocks are not cleaned because the function `get all blocks sorted by time` didn't get async blocks. [#44651](https://github.com/ClickHouse/ClickHouse/pull/44651) ([Han Fei](https://github.com/hanfei1991)). +* Fix `LOGICAL_ERROR` `The top step of the right pipeline should be ExpressionStep` for JOIN with subquery, UNION, and TOTALS. Fixes [#43687](https://github.com/ClickHouse/ClickHouse/issues/43687). [#44673](https://github.com/ClickHouse/ClickHouse/pull/44673) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Avoid std::out_of_range exception in StorageExecutable. [#44681](https://github.com/ClickHouse/ClickHouse/pull/44681) ([Kruglov Pavel](https://github.com/Avogar)). +* Do not apply `optimize_syntax_fuse_functions` to quantiles on AST, close [#44712](https://github.com/ClickHouse/ClickHouse/issues/44712). [#44713](https://github.com/ClickHouse/ClickHouse/pull/44713) ([Vladimir C](https://github.com/vdimir)). +* Fix bug with wrong type in Merge table and PREWHERE, close [#43324](https://github.com/ClickHouse/ClickHouse/issues/43324). [#44716](https://github.com/ClickHouse/ClickHouse/pull/44716) ([Vladimir C](https://github.com/vdimir)). +* Fix possible crash during shutdown (while destroying TraceCollector). Fixes [#44757](https://github.com/ClickHouse/ClickHouse/issues/44757). [#44758](https://github.com/ClickHouse/ClickHouse/pull/44758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix a possible crash in distributed query processing. The crash could happen if a query with totals or extremes returned an empty result and there are mismatched types in the Distrubuted and the local tables. Fixes [#44738](https://github.com/ClickHouse/ClickHouse/issues/44738). [#44760](https://github.com/ClickHouse/ClickHouse/pull/44760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix fsync for fetches (`min_compressed_bytes_to_fsync_after_fetch`)/small files (ttl.txt, columns.txt) in mutations (`min_rows_to_fsync_after_merge`/`min_compressed_bytes_to_fsync_after_merge`). [#44781](https://github.com/ClickHouse/ClickHouse/pull/44781) ([Azat Khuzhin](https://github.com/azat)). +* A rare race condition was possible when querying the `system.parts` or `system.parts_columns` tables in the presence of parts being moved between disks. Introduced in [#41145](https://github.com/ClickHouse/ClickHouse/issues/41145). [#44809](https://github.com/ClickHouse/ClickHouse/pull/44809) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the error `Context has expired` which could appear with enabled projections optimization. Can be reproduced for queries with specific functions, like `dictHas/dictGet` which use context in runtime. Fixes [#44844](https://github.com/ClickHouse/ClickHouse/issues/44844). [#44850](https://github.com/ClickHouse/ClickHouse/pull/44850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* - Ignore hwmon sensors on label read issues. [#44895](https://github.com/ClickHouse/ClickHouse/pull/44895) ([Raúl Marín](https://github.com/Algunenano)). +* Use `max_delay_to_insert` value in case calculated time to delay INSERT exceeds the setting value. Related to [#44902](https://github.com/ClickHouse/ClickHouse/issues/44902). [#44916](https://github.com/ClickHouse/ClickHouse/pull/44916) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix error `Different order of columns in UNION subquery` for queries with `UNION`. Fixes [#44866](https://github.com/ClickHouse/ClickHouse/issues/44866). [#44920](https://github.com/ClickHouse/ClickHouse/pull/44920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Delay for INSERT can be calculated incorrectly, which can lead to always using `max_delay_to_insert` setting as delay instead of a correct value. Using simple formula `max_delay_to_insert * (parts_over_threshold/max_allowed_parts_over_threshold)` i.e. delay grows proportionally to parts over threshold. Closes [#44902](https://github.com/ClickHouse/ClickHouse/issues/44902). [#44954](https://github.com/ClickHouse/ClickHouse/pull/44954) ([Igor Nikonov](https://github.com/devcrafter)). +* fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)). +* Follow-up fix for Replace domain IP types (IPv4, IPv6) with native [#43221](https://github.com/ClickHouse/ClickHouse/issues/43221). [#45024](https://github.com/ClickHouse/ClickHouse/pull/45024) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Follow-up fix for Replace domain IP types (IPv4, IPv6) with native https://github.com/ClickHouse/ClickHouse/pull/43221. [#45043](https://github.com/ClickHouse/ClickHouse/pull/45043) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* A buffer overflow was possible in the parser. Found by fuzzer. [#45047](https://github.com/ClickHouse/ClickHouse/pull/45047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible cannot-read-all-data error in storage FileLog. Closes [#45051](https://github.com/ClickHouse/ClickHouse/issues/45051), [#38257](https://github.com/ClickHouse/ClickHouse/issues/38257). [#45057](https://github.com/ClickHouse/ClickHouse/pull/45057) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Memory efficient aggregation (setting `distributed_aggregation_memory_efficient`) is disabled when grouping sets are present in the query. [#45058](https://github.com/ClickHouse/ClickHouse/pull/45058) ([Nikita Taranov](https://github.com/nickitat)). +* Fix `RANGE_HASHED` dictionary to count range columns as part of primary key during updates when `update_field` is specified. Closes [#44588](https://github.com/ClickHouse/ClickHouse/issues/44588). [#45061](https://github.com/ClickHouse/ClickHouse/pull/45061) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix error `Cannot capture column` for `LowCardinality` captured argument of nested labmda. Fixes [#45028](https://github.com/ClickHouse/ClickHouse/issues/45028). [#45065](https://github.com/ClickHouse/ClickHouse/pull/45065) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix the wrong query result of `additional_table_filters` (additional filter was not applied) in case if minmax/count projection is used. [#45133](https://github.com/ClickHouse/ClickHouse/pull/45133) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* - Fixed bug in `histogram` function accepting negative values. [#45147](https://github.com/ClickHouse/ClickHouse/pull/45147) ([simpleton](https://github.com/rgzntrade)). +* Fix wrong column nullability in StoreageJoin, close [#44940](https://github.com/ClickHouse/ClickHouse/issues/44940). [#45184](https://github.com/ClickHouse/ClickHouse/pull/45184) ([Vladimir C](https://github.com/vdimir)). +* Fix `background_fetches_pool_size` settings reload (increase at runtime). [#45189](https://github.com/ClickHouse/ClickHouse/pull/45189) ([Raúl Marín](https://github.com/Algunenano)). +* Correctly process `SELECT` queries on KV engines (e.g. KeeperMap, EmbeddedRocksDB) using `IN` on the key with subquery producing different type. [#45215](https://github.com/ClickHouse/ClickHouse/pull/45215) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix logical error in SEMI JOIN & join_use_nulls in some cases, close [#45163](https://github.com/ClickHouse/ClickHouse/issues/45163), close [#45209](https://github.com/ClickHouse/ClickHouse/issues/45209). [#45230](https://github.com/ClickHouse/ClickHouse/pull/45230) ([Vladimir C](https://github.com/vdimir)). +* Fix heap-use-after-free in reading from s3. [#45253](https://github.com/ClickHouse/ClickHouse/pull/45253) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix bug when the Avro Union type is ['null', Nested type], closes [#45275](https://github.com/ClickHouse/ClickHouse/issues/45275). Fix bug that incorrectly infer `bytes` type to `Float`. [#45276](https://github.com/ClickHouse/ClickHouse/pull/45276) ([flynn](https://github.com/ucasfl)). +* Throw a correct exception when explicit PREWHERE cannot be used with table using storage engine `Merge`. [#45319](https://github.com/ClickHouse/ClickHouse/pull/45319) ([Antonio Andelic](https://github.com/antonio2368)). +* Under WSL1 Ubuntu self-extracting clickhouse fails to decompress due to inconsistency - /proc/self/maps reporting 32bit file's inode, while stat reporting 64bit inode. [#45339](https://github.com/ClickHouse/ClickHouse/pull/45339) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix race in Distributed table startup (that could lead to processing file of async INSERT multiple times). [#45360](https://github.com/ClickHouse/ClickHouse/pull/45360) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible crash while reading from storage `S3` and table function `s3` in case when `ListObject` request has failed. [#45371](https://github.com/ClickHouse/ClickHouse/pull/45371) ([Anton Popov](https://github.com/CurtizJ)). +* Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Fix s3Cluster schema inference when structure from insertion table is used in `INSERT INTO ... SELECT * FROM s3Cluster` queries. [#45422](https://github.com/ClickHouse/ClickHouse/pull/45422) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix bug in JSON/BSONEachRow parsing with HTTP that could lead to using default values for some columns instead of values from data. [#45424](https://github.com/ClickHouse/ClickHouse/pull/45424) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed bug (Code: 632. DB::Exception: Unexpected data ... after parsed IPv6 value ...) with typed parsing of IP types from text source. [#45425](https://github.com/ClickHouse/ClickHouse/pull/45425) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* close [#45297](https://github.com/ClickHouse/ClickHouse/issues/45297) Add check for empty regular expressions. [#45428](https://github.com/ClickHouse/ClickHouse/pull/45428) ([Han Fei](https://github.com/hanfei1991)). +* Fix possible (likely distributed) query hung. [#45448](https://github.com/ClickHouse/ClickHouse/pull/45448) ([Azat Khuzhin](https://github.com/azat)). +* Fix disabled two-level aggregation from HTTP. [#45450](https://github.com/ClickHouse/ClickHouse/pull/45450) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible deadlock with `allow_asynchronous_read_from_io_pool_for_merge_tree` enabled in case of exception from `ThreadPool::schedule`. [#45481](https://github.com/ClickHouse/ClickHouse/pull/45481) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible in-use table after DETACH. [#45493](https://github.com/ClickHouse/ClickHouse/pull/45493) ([Azat Khuzhin](https://github.com/azat)). +* Fix rare abort in case when query is canceled and parallel parsing was used during its execution. [#45498](https://github.com/ClickHouse/ClickHouse/pull/45498) ([Anton Popov](https://github.com/CurtizJ)). +* Fix a race between Distributed table creation and INSERT into it (could lead to CANNOT_LINK during INSERT into the table). [#45502](https://github.com/ClickHouse/ClickHouse/pull/45502) ([Azat Khuzhin](https://github.com/azat)). +* Add proper default (SLRU) to cache policy getter. Closes [#45514](https://github.com/ClickHouse/ClickHouse/issues/45514). [#45524](https://github.com/ClickHouse/ClickHouse/pull/45524) ([Kseniia Sumarokova](https://github.com/kssenii)). + +#### Bug-fix + +* Disallow arrayjoin in mutations closes [#42637](https://github.com/ClickHouse/ClickHouse/issues/42637) Implementation: * Added a new parameter to ActionsVisitor::Data disallow_arrayjoin, which is set by MutationsIterator when it appends expression. * ActionsVisitor uses disallow_arrayjoin and throws error when its used with mutations. Testing: * Added test for the same 02504_disallow_arrayjoin_in_mutations.sql. [#44447](https://github.com/ClickHouse/ClickHouse/pull/44447) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix for qualified asterisks with alias table name and column transformer resolves [#44736](https://github.com/ClickHouse/ClickHouse/issues/44736). [#44755](https://github.com/ClickHouse/ClickHouse/pull/44755) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). + +#### Build Improvement + +* crc32 fix for s390x. [#43706](https://github.com/ClickHouse/ClickHouse/pull/43706) ([Suzy Wang](https://github.com/SuzyWangIBMer)). + +#### Feature + +* Record server startup time in ProfileEvents resolves [#43188](https://github.com/ClickHouse/ClickHouse/issues/43188) Implementation: * Added ProfileEvents::ServerStartupMilliseconds. * Recorded time from start of main till listening to sockets. Testing: * Added a test 02532_profileevents_server_startup_time.sql. [#45250](https://github.com/ClickHouse/ClickHouse/pull/45250) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "If user only need virtual columns, we don't need to initialize ReadBufferFromS3"'. [#44939](https://github.com/ClickHouse/ClickHouse/pull/44939) ([Anton Popov](https://github.com/CurtizJ)). +* NO CL ENTRY: 'Revert "Custom reading for mutation"'. [#45121](https://github.com/ClickHouse/ClickHouse/pull/45121) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Revert "Custom reading for mutation""'. [#45122](https://github.com/ClickHouse/ClickHouse/pull/45122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* NO CL ENTRY: 'Revert "update function DAYOFWEEK and add new function WEEKDAY for mysql/spark compatiability"'. [#45221](https://github.com/ClickHouse/ClickHouse/pull/45221) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Validate function arguments in query tree"'. [#45299](https://github.com/ClickHouse/ClickHouse/pull/45299) ([Maksim Kita](https://github.com/kitaisreal)). +* NO CL ENTRY: 'Revert "Revert "Validate function arguments in query tree""'. [#45300](https://github.com/ClickHouse/ClickHouse/pull/45300) ([Maksim Kita](https://github.com/kitaisreal)). +* NO CL ENTRY: 'Revert "Support optimize_or_like_chain in QueryTreePassManager"'. [#45406](https://github.com/ClickHouse/ClickHouse/pull/45406) ([Anton Popov](https://github.com/CurtizJ)). +* NO CL ENTRY: 'Resubmit Support optimize_or_like_chain in QueryTreePassManager'. [#45410](https://github.com/ClickHouse/ClickHouse/pull/45410) ([Dmitry Novik](https://github.com/novikd)). +* NO CL ENTRY: 'Revert "Remove redundant sorting"'. [#45414](https://github.com/ClickHouse/ClickHouse/pull/45414) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix assertion in async read buffer from remote [#41231](https://github.com/ClickHouse/ClickHouse/pull/41231) ([Kseniia Sumarokova](https://github.com/kssenii)). +* add retries on ConnectionError [#42991](https://github.com/ClickHouse/ClickHouse/pull/42991) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update aws-c* submodules [#43020](https://github.com/ClickHouse/ClickHouse/pull/43020) ([Vitaly Baranov](https://github.com/vitlibar)). +* Replace domain IP types (IPv4, IPv6) with native [#43221](https://github.com/ClickHouse/ClickHouse/pull/43221) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix aggregate functions optimisation in AggregateFunctionsArithmericOperationsPass [#43372](https://github.com/ClickHouse/ClickHouse/pull/43372) ([Dmitry Novik](https://github.com/novikd)). +* Improve pytest --pdb experience by preserving dockerd on SIGINT [#43392](https://github.com/ClickHouse/ClickHouse/pull/43392) ([Azat Khuzhin](https://github.com/azat)). +* Followup fixes for systemd notification ([#43400](https://github.com/ClickHouse/ClickHouse/issues/43400)) [#43597](https://github.com/ClickHouse/ClickHouse/pull/43597) ([Alexander Gololobov](https://github.com/davenger)). +* Refactor FunctionNode [#43761](https://github.com/ClickHouse/ClickHouse/pull/43761) ([Dmitry Novik](https://github.com/novikd)). +* Some cleanup: grace hash join [#43851](https://github.com/ClickHouse/ClickHouse/pull/43851) ([Igor Nikonov](https://github.com/devcrafter)). +* Temporary files evict fs cache - 2nd approach [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). +* Randomize setting `enable_memory_bound_merging_of_aggregation_results` in tests [#43986](https://github.com/ClickHouse/ClickHouse/pull/43986) ([Nikita Taranov](https://github.com/nickitat)). +* Analyzer aggregate functions passes small fixes [#44013](https://github.com/ClickHouse/ClickHouse/pull/44013) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix wrong char in command [#44018](https://github.com/ClickHouse/ClickHouse/pull/44018) ([alesapin](https://github.com/alesapin)). +* Analyzer support Set index [#44097](https://github.com/ClickHouse/ClickHouse/pull/44097) ([Maksim Kita](https://github.com/kitaisreal)). +* Provide monotonicity info for `toUnixTimestamp64*` [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)). +* Avoid loading toolchain files multiple times [#44122](https://github.com/ClickHouse/ClickHouse/pull/44122) ([Azat Khuzhin](https://github.com/azat)). +* tests: exclude flaky columns from SHOW CLUSTERS test [#44123](https://github.com/ClickHouse/ClickHouse/pull/44123) ([Azat Khuzhin](https://github.com/azat)). +* Bump libdivide (to gain some new optimizations) [#44132](https://github.com/ClickHouse/ClickHouse/pull/44132) ([Azat Khuzhin](https://github.com/azat)). +* Make atomic counter relaxed in blockNumber() [#44193](https://github.com/ClickHouse/ClickHouse/pull/44193) ([Igor Nikonov](https://github.com/devcrafter)). +* Try fix flaky 01072_window_view_multiple_columns_groupby [#44195](https://github.com/ClickHouse/ClickHouse/pull/44195) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Apply new code of named collections (from [#43147](https://github.com/ClickHouse/ClickHouse/issues/43147)) to external table engines part 1 [#44204](https://github.com/ClickHouse/ClickHouse/pull/44204) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add some settings under `compatibility` [#44209](https://github.com/ClickHouse/ClickHouse/pull/44209) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Recommend Slack over Telegram in the "Question" issue template [#44222](https://github.com/ClickHouse/ClickHouse/pull/44222) ([Ivan Blinkov](https://github.com/blinkov)). +* Forbid paths in timezone names [#44225](https://github.com/ClickHouse/ClickHouse/pull/44225) ([Kruglov Pavel](https://github.com/Avogar)). +* Analyzer storage view crash fix [#44230](https://github.com/ClickHouse/ClickHouse/pull/44230) ([Maksim Kita](https://github.com/kitaisreal)). +* Add ThreadsInOvercommitTracker metric [#44233](https://github.com/ClickHouse/ClickHouse/pull/44233) ([Dmitry Novik](https://github.com/novikd)). +* Analyzer expired Context crash fix [#44234](https://github.com/ClickHouse/ClickHouse/pull/44234) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed use-after-free of BLAKE3 error message [#44242](https://github.com/ClickHouse/ClickHouse/pull/44242) ([Joanna Hulboj](https://github.com/jh0x)). +* Fix deadlock in StorageSystemDatabases [#44272](https://github.com/ClickHouse/ClickHouse/pull/44272) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of global Git object [#44273](https://github.com/ClickHouse/ClickHouse/pull/44273) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version after release [#44275](https://github.com/ClickHouse/ClickHouse/pull/44275) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version_date.tsv and changelogs after v22.12.1.1752-stable [#44281](https://github.com/ClickHouse/ClickHouse/pull/44281) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Do not hold data parts during insert [#44299](https://github.com/ClickHouse/ClickHouse/pull/44299) ([Anton Popov](https://github.com/CurtizJ)). +* Another fix `test_server_reload` [#44306](https://github.com/ClickHouse/ClickHouse/pull/44306) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv and changelogs after v22.9.7.34-stable [#44309](https://github.com/ClickHouse/ClickHouse/pull/44309) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* tests/perf: fix dependency check during DROP [#44312](https://github.com/ClickHouse/ClickHouse/pull/44312) ([Azat Khuzhin](https://github.com/azat)). +* (unused openssl integration, not for production) a follow-up [#44325](https://github.com/ClickHouse/ClickHouse/pull/44325) ([Boris Kuschel](https://github.com/bkuschel)). +* Replace old named collections code with new (from [#43147](https://github.com/ClickHouse/ClickHouse/issues/43147)) part 2 [#44327](https://github.com/ClickHouse/ClickHouse/pull/44327) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable "git-import" test in debug mode [#44328](https://github.com/ClickHouse/ClickHouse/pull/44328) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check s3 part upload settings [#44335](https://github.com/ClickHouse/ClickHouse/pull/44335) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix typo [#44337](https://github.com/ClickHouse/ClickHouse/pull/44337) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for PowerBI [#44338](https://github.com/ClickHouse/ClickHouse/pull/44338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#36038](https://github.com/ClickHouse/ClickHouse/issues/36038) [#44339](https://github.com/ClickHouse/ClickHouse/pull/44339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#29386](https://github.com/ClickHouse/ClickHouse/issues/29386) [#44340](https://github.com/ClickHouse/ClickHouse/pull/44340) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#22929](https://github.com/ClickHouse/ClickHouse/issues/22929) [#44341](https://github.com/ClickHouse/ClickHouse/pull/44341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#29883](https://github.com/ClickHouse/ClickHouse/issues/29883) [#44342](https://github.com/ClickHouse/ClickHouse/pull/44342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix Docker [#44343](https://github.com/ClickHouse/ClickHouse/pull/44343) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix flack test "02481_async_insert_dedup.python" [#44349](https://github.com/ClickHouse/ClickHouse/pull/44349) ([Han Fei](https://github.com/hanfei1991)). +* Add a test for [#22160](https://github.com/ClickHouse/ClickHouse/issues/22160) [#44355](https://github.com/ClickHouse/ClickHouse/pull/44355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#34708](https://github.com/ClickHouse/ClickHouse/issues/34708) [#44356](https://github.com/ClickHouse/ClickHouse/pull/44356) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#30679](https://github.com/ClickHouse/ClickHouse/issues/30679) [#44357](https://github.com/ClickHouse/ClickHouse/pull/44357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#34669](https://github.com/ClickHouse/ClickHouse/issues/34669) [#44359](https://github.com/ClickHouse/ClickHouse/pull/44359) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#34724](https://github.com/ClickHouse/ClickHouse/issues/34724) [#44360](https://github.com/ClickHouse/ClickHouse/pull/44360) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try restarting ZK cluster on failed connection in `test_keeper_zookeeper_converted` [#44363](https://github.com/ClickHouse/ClickHouse/pull/44363) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable grase_hash in test 00172_parallel_join [#44367](https://github.com/ClickHouse/ClickHouse/pull/44367) ([Vladimir C](https://github.com/vdimir)). +* Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Lock table for share during startup for database ordinary [#44393](https://github.com/ClickHouse/ClickHouse/pull/44393) ([alesapin](https://github.com/alesapin)). +* Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try fix some tests [#44406](https://github.com/ClickHouse/ClickHouse/pull/44406) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better ParserAllCollectionsOfLiterals [#44408](https://github.com/ClickHouse/ClickHouse/pull/44408) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix bug with merge/mutate pool size increase [#44436](https://github.com/ClickHouse/ClickHouse/pull/44436) ([alesapin](https://github.com/alesapin)). +* Update 01072_window_view_multiple_columns_groupby.sh [#44438](https://github.com/ClickHouse/ClickHouse/pull/44438) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable buggy tsan assertion for integration test [#44444](https://github.com/ClickHouse/ClickHouse/pull/44444) ([alesapin](https://github.com/alesapin)). +* Respect setting settings.schema_inference_make_columns_nullable in Parquet/ORC/Arrow formats [#44446](https://github.com/ClickHouse/ClickHouse/pull/44446) ([Kruglov Pavel](https://github.com/Avogar)). +* Add tests as examples with errors of date(time) and string comparison that we should eliminate [#44462](https://github.com/ClickHouse/ClickHouse/pull/44462) ([Ilya Yatsishin](https://github.com/qoega)). +* Parallel parts cleanup with zero copy replication [#44466](https://github.com/ClickHouse/ClickHouse/pull/44466) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix incorrect usages of `getPartName()` [#44468](https://github.com/ClickHouse/ClickHouse/pull/44468) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test `roaring_memory_tracking` [#44470](https://github.com/ClickHouse/ClickHouse/pull/44470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Clarify query_id in test 01092_memory_profiler [#44483](https://github.com/ClickHouse/ClickHouse/pull/44483) ([Vladimir C](https://github.com/vdimir)). +* Default value for optional in SortNode::updateTreeHashImpl [#44491](https://github.com/ClickHouse/ClickHouse/pull/44491) ([Vladimir C](https://github.com/vdimir)). +* Do not try to remove WAL/move broken parts for static storage [#44495](https://github.com/ClickHouse/ClickHouse/pull/44495) ([Azat Khuzhin](https://github.com/azat)). +* Removed parent pid check that breaks in containers [#44499](https://github.com/ClickHouse/ClickHouse/pull/44499) ([Alexander Gololobov](https://github.com/davenger)). +* Analyzer duplicate alias crash fix [#44508](https://github.com/ClickHouse/ClickHouse/pull/44508) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor code polishing [#44513](https://github.com/ClickHouse/ClickHouse/pull/44513) ([alesapin](https://github.com/alesapin)). +* Better error message if named collection does not exist [#44517](https://github.com/ClickHouse/ClickHouse/pull/44517) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add the lambda to collect data for workflow_jobs [#44520](https://github.com/ClickHouse/ClickHouse/pull/44520) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Introduce groupArrayLast() (useful to store last X values) [#44521](https://github.com/ClickHouse/ClickHouse/pull/44521) ([Azat Khuzhin](https://github.com/azat)). +* Infer numbers starting from zero as strings in TSV [#44522](https://github.com/ClickHouse/ClickHouse/pull/44522) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix wrong condition for enabling async reading from MergeTree. [#44530](https://github.com/ClickHouse/ClickHouse/pull/44530) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* tests: capture dmesg in integration tests [#44535](https://github.com/ClickHouse/ClickHouse/pull/44535) ([Azat Khuzhin](https://github.com/azat)). +* Analyzer support distributed queries processing [#44540](https://github.com/ClickHouse/ClickHouse/pull/44540) ([Maksim Kita](https://github.com/kitaisreal)). +* Followup [#43761](https://github.com/ClickHouse/ClickHouse/issues/43761) [#44541](https://github.com/ClickHouse/ClickHouse/pull/44541) ([Dmitry Novik](https://github.com/novikd)). +* Drop unused columns after join on/using [#44545](https://github.com/ClickHouse/ClickHouse/pull/44545) ([Vladimir C](https://github.com/vdimir)). +* Improve inferring arrays with nulls in JSON formats [#44550](https://github.com/ClickHouse/ClickHouse/pull/44550) ([Kruglov Pavel](https://github.com/Avogar)). +* Make BC check optional (if env var set) [#44564](https://github.com/ClickHouse/ClickHouse/pull/44564) ([alesapin](https://github.com/alesapin)). +* Fix extremely slow stack traces in debug build [#44569](https://github.com/ClickHouse/ClickHouse/pull/44569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better command line argument name in `clickhouse-benchmark` [#44570](https://github.com/ClickHouse/ClickHouse/pull/44570) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix HDFS test [#44572](https://github.com/ClickHouse/ClickHouse/pull/44572) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test_distributed_queries_stress [#44573](https://github.com/ClickHouse/ClickHouse/pull/44573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Switch "contrib/sysroot" back to master. [#44574](https://github.com/ClickHouse/ClickHouse/pull/44574) ([Vitaly Baranov](https://github.com/vitlibar)). +* Non-significant changes [#44575](https://github.com/ClickHouse/ClickHouse/pull/44575) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fuzzer HTML: fix trash [#44580](https://github.com/ClickHouse/ClickHouse/pull/44580) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better diagnostics on server stop for the stress test [#44593](https://github.com/ClickHouse/ClickHouse/pull/44593) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The position of the log message about the server environment was wrong [#44595](https://github.com/ClickHouse/ClickHouse/pull/44595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad punctuation in log [#44596](https://github.com/ClickHouse/ClickHouse/pull/44596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix misleading log message [#44598](https://github.com/ClickHouse/ClickHouse/pull/44598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad log message about MergeTree metadata cache. [#44599](https://github.com/ClickHouse/ClickHouse/pull/44599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly cleanup interactive line reader code [#44601](https://github.com/ClickHouse/ClickHouse/pull/44601) ([Azat Khuzhin](https://github.com/azat)). +* Rename `runlog.log` to `run.log` in tests [#44603](https://github.com/ClickHouse/ClickHouse/pull/44603) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hung query in stress test [#44604](https://github.com/ClickHouse/ClickHouse/pull/44604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve variable name [#44605](https://github.com/ClickHouse/ClickHouse/pull/44605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Faster server startup after stress test [#44606](https://github.com/ClickHouse/ClickHouse/pull/44606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix log messages in Coordination [#44607](https://github.com/ClickHouse/ClickHouse/pull/44607) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable Analyzer in fuzz and stress tests [#44609](https://github.com/ClickHouse/ClickHouse/pull/44609) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better log message [#44610](https://github.com/ClickHouse/ClickHouse/pull/44610) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe fix a bogus MSan error [#44611](https://github.com/ClickHouse/ClickHouse/pull/44611) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix "too large allocation" message from MSan [#44613](https://github.com/ClickHouse/ClickHouse/pull/44613) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not fail the AST fuzzer if sanitizer is out of memory [#44616](https://github.com/ClickHouse/ClickHouse/pull/44616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test `01111_create_drop_replicated_db_stress` [#44617](https://github.com/ClickHouse/ClickHouse/pull/44617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* tests/integration: suppress exceptions during logging (due to pytest) [#44618](https://github.com/ClickHouse/ClickHouse/pull/44618) ([Azat Khuzhin](https://github.com/azat)). +* Fix rust modules rebuild (previously ignores changes in cargo config.toml) [#44623](https://github.com/ClickHouse/ClickHouse/pull/44623) ([Azat Khuzhin](https://github.com/azat)). +* Sometimes spot instances fail more than 20 times in a row [#44626](https://github.com/ClickHouse/ClickHouse/pull/44626) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix restart after quorum insert [#44628](https://github.com/ClickHouse/ClickHouse/pull/44628) ([alesapin](https://github.com/alesapin)). +* Revert "Merge pull request [#38953](https://github.com/ClickHouse/ClickHouse/issues/38953) from ClickHouse/add-allocation-ptr-to-trace-log [#44629](https://github.com/ClickHouse/ClickHouse/pull/44629) ([Raúl Marín](https://github.com/Algunenano)). +* Fix lambdas parsing [#44639](https://github.com/ClickHouse/ClickHouse/pull/44639) ([Nikolay Degterinsky](https://github.com/evillique)). +* Function viewExplain accept SELECT and settings [#44641](https://github.com/ClickHouse/ClickHouse/pull/44641) ([Vladimir C](https://github.com/vdimir)). +* Fix test `02015_async_inserts_2` [#44642](https://github.com/ClickHouse/ClickHouse/pull/44642) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky test `test_keeper_multinode_simple` [#44645](https://github.com/ClickHouse/ClickHouse/pull/44645) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add +x flag for run-fuzzer.sh [#44649](https://github.com/ClickHouse/ClickHouse/pull/44649) ([alesapin](https://github.com/alesapin)). +* Custom reading for mutation [#44653](https://github.com/ClickHouse/ClickHouse/pull/44653) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test test_backup_restore_on_cluster [#44660](https://github.com/ClickHouse/ClickHouse/pull/44660) ([Vitaly Baranov](https://github.com/vitlibar)). +* tests/integration: add missing kazoo client termination [#44666](https://github.com/ClickHouse/ClickHouse/pull/44666) ([Azat Khuzhin](https://github.com/azat)). +* Move dmesg dumping out from runner to ci-runner.py [#44667](https://github.com/ClickHouse/ClickHouse/pull/44667) ([Azat Khuzhin](https://github.com/azat)). +* Remove questdb (it makes a little sense but the test was flaky) [#44669](https://github.com/ClickHouse/ClickHouse/pull/44669) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix minor typo: replace validate_bugix_check with validate_bugfix_check [#44672](https://github.com/ClickHouse/ClickHouse/pull/44672) ([Pradeep Chhetri](https://github.com/chhetripradeep)). +* Fix parsing of ANY operator [#44678](https://github.com/ClickHouse/ClickHouse/pull/44678) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix test `01130_in_memory_parts` [#44683](https://github.com/ClickHouse/ClickHouse/pull/44683) ([Anton Popov](https://github.com/CurtizJ)). +* Remove old code [#44685](https://github.com/ClickHouse/ClickHouse/pull/44685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test git-import [#44687](https://github.com/ClickHouse/ClickHouse/pull/44687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve odbc test [#44688](https://github.com/ClickHouse/ClickHouse/pull/44688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add retries to HTTP requests in ClickHouse test [#44689](https://github.com/ClickHouse/ClickHouse/pull/44689) ([alesapin](https://github.com/alesapin)). +* Fix flaky tests [#44690](https://github.com/ClickHouse/ClickHouse/pull/44690) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test "01502_long_log_tinylog_deadlock_race" [#44693](https://github.com/ClickHouse/ClickHouse/pull/44693) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve handling of old parts [#44694](https://github.com/ClickHouse/ClickHouse/pull/44694) ([Raúl Marín](https://github.com/Algunenano)). +* Update entrypoint.sh [#44699](https://github.com/ClickHouse/ClickHouse/pull/44699) ([Denny Crane](https://github.com/den-crane)). +* tests: more fixes for test_keeper_auth [#44702](https://github.com/ClickHouse/ClickHouse/pull/44702) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash on delete from materialized view [#44705](https://github.com/ClickHouse/ClickHouse/pull/44705) ([Alexander Gololobov](https://github.com/davenger)). +* Fix flaky filelog tests with database ordinary [#44706](https://github.com/ClickHouse/ClickHouse/pull/44706) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make lightweight deletes always synchronous [#44718](https://github.com/ClickHouse/ClickHouse/pull/44718) ([Alexander Gololobov](https://github.com/davenger)). +* Fix deadlock in attach thread [#44719](https://github.com/ClickHouse/ClickHouse/pull/44719) ([alesapin](https://github.com/alesapin)). +* A few improvements to AST Fuzzer [#44720](https://github.com/ClickHouse/ClickHouse/pull/44720) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test [#44721](https://github.com/ClickHouse/ClickHouse/pull/44721) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rename log in stress test [#44722](https://github.com/ClickHouse/ClickHouse/pull/44722) ([alesapin](https://github.com/alesapin)). +* Debug deadlock in stress test [#44723](https://github.com/ClickHouse/ClickHouse/pull/44723) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test "02102_row_binary_with_names_and_types.sh" [#44724](https://github.com/ClickHouse/ClickHouse/pull/44724) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly better some tests [#44725](https://github.com/ClickHouse/ClickHouse/pull/44725) ([alesapin](https://github.com/alesapin)). +* Fix cases when clickhouse-server takes long time to start in functional tests with MSan [#44726](https://github.com/ClickHouse/ClickHouse/pull/44726) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Perf test: Log the time spent waiting for file sync [#44737](https://github.com/ClickHouse/ClickHouse/pull/44737) ([Raúl Marín](https://github.com/Algunenano)). +* Fix flaky test 02448_clone_replica_lost_part [#44759](https://github.com/ClickHouse/ClickHouse/pull/44759) ([alesapin](https://github.com/alesapin)). +* Build rust modules from the binary directory [#44762](https://github.com/ClickHouse/ClickHouse/pull/44762) ([Azat Khuzhin](https://github.com/azat)). +* Remove database ordinary from stress test [#44763](https://github.com/ClickHouse/ClickHouse/pull/44763) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 02479_mysql_connect_to_self [#44768](https://github.com/ClickHouse/ClickHouse/pull/44768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Print fatal messages in Fuzzer [#44769](https://github.com/ClickHouse/ClickHouse/pull/44769) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect docs [#44795](https://github.com/ClickHouse/ClickHouse/pull/44795) ([Kruglov Pavel](https://github.com/Avogar)). +* Added table name to error message [#44806](https://github.com/ClickHouse/ClickHouse/pull/44806) ([Alexander Gololobov](https://github.com/davenger)). +* Retry packages download if GitHub returned HTTP 500. [#44807](https://github.com/ClickHouse/ClickHouse/pull/44807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Slightly better docs [#44808](https://github.com/ClickHouse/ClickHouse/pull/44808) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix total trash in stress test [#44810](https://github.com/ClickHouse/ClickHouse/pull/44810) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ASan builds for glibc 2.36+ [#44811](https://github.com/ClickHouse/ClickHouse/pull/44811) ([Azat Khuzhin](https://github.com/azat)). +* Remove the remainings of TestFlows [#44812](https://github.com/ClickHouse/ClickHouse/pull/44812) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `grep` [#44813](https://github.com/ClickHouse/ClickHouse/pull/44813) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad cast in monotonicity analysis [#44818](https://github.com/ClickHouse/ClickHouse/pull/44818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Modern tools, part 1 [#44819](https://github.com/ClickHouse/ClickHouse/pull/44819) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Modern tools in CI, part 2. [#44820](https://github.com/ClickHouse/ClickHouse/pull/44820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix data race in DDLWorker [#44821](https://github.com/ClickHouse/ClickHouse/pull/44821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests for bridges [#44822](https://github.com/ClickHouse/ClickHouse/pull/44822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test_multiple_disks::test_jbod_overflow [#44823](https://github.com/ClickHouse/ClickHouse/pull/44823) ([Azat Khuzhin](https://github.com/azat)). +* Less OOM in stress test [#44824](https://github.com/ClickHouse/ClickHouse/pull/44824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix misleading integration tests reports for parametrized tests [#44825](https://github.com/ClickHouse/ClickHouse/pull/44825) ([Azat Khuzhin](https://github.com/azat)). +* Fix two typos [#44826](https://github.com/ClickHouse/ClickHouse/pull/44826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust CSS [#44829](https://github.com/ClickHouse/ClickHouse/pull/44829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzzer report [#44830](https://github.com/ClickHouse/ClickHouse/pull/44830) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* check-style: check base for std::cerr/cout too [#44833](https://github.com/ClickHouse/ClickHouse/pull/44833) ([Azat Khuzhin](https://github.com/azat)). +* Try fixing `test_keeper_snapshot_small_distance` with ZK restart [#44834](https://github.com/ClickHouse/ClickHouse/pull/44834) ([Antonio Andelic](https://github.com/antonio2368)). +* Exclude cargo shared libraries from the artifacts [#44836](https://github.com/ClickHouse/ClickHouse/pull/44836) ([Azat Khuzhin](https://github.com/azat)). +* Add a tiny but important logging [#44837](https://github.com/ClickHouse/ClickHouse/pull/44837) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Escape submodules in style-check [#44838](https://github.com/ClickHouse/ClickHouse/pull/44838) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move `test_dies_with_parent` to another module [#44839](https://github.com/ClickHouse/ClickHouse/pull/44839) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove unneeded softlink to official dev docs [#44841](https://github.com/ClickHouse/ClickHouse/pull/44841) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix data race in StorageS3 [#44842](https://github.com/ClickHouse/ClickHouse/pull/44842) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix rare race which can lead to queue hang [#44847](https://github.com/ClickHouse/ClickHouse/pull/44847) ([alesapin](https://github.com/alesapin)). +* No more retries in integration tests [#44851](https://github.com/ClickHouse/ClickHouse/pull/44851) ([Ilya Yatsishin](https://github.com/qoega)). +* Document usage of check_cxx_source_compiles instead of check_cxx_source_runs [#44854](https://github.com/ClickHouse/ClickHouse/pull/44854) ([Robert Schulze](https://github.com/rschu1ze)). +* More cases of OOM in Fuzzer [#44855](https://github.com/ClickHouse/ClickHouse/pull/44855) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix: sorted DISTINCT with empty string [#44856](https://github.com/ClickHouse/ClickHouse/pull/44856) ([Igor Nikonov](https://github.com/devcrafter)). +* Try to fix MSan build [#44857](https://github.com/ClickHouse/ClickHouse/pull/44857) ([Nikolay Degterinsky](https://github.com/evillique)). +* Cleanup setup_minio.sh [#44858](https://github.com/ClickHouse/ClickHouse/pull/44858) ([Pradeep Chhetri](https://github.com/chhetripradeep)). +* Wait for ZK process to stop in tests using snapshot [#44859](https://github.com/ClickHouse/ClickHouse/pull/44859) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky test and several typos [#44870](https://github.com/ClickHouse/ClickHouse/pull/44870) ([alesapin](https://github.com/alesapin)). +* Upload status files to S3 report for bugfix check [#44871](https://github.com/ClickHouse/ClickHouse/pull/44871) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flaky test `02503_insert_storage_snapshot` [#44873](https://github.com/ClickHouse/ClickHouse/pull/44873) ([alesapin](https://github.com/alesapin)). +* Revert some changes from [#42777](https://github.com/ClickHouse/ClickHouse/issues/42777) to fix performance tests [#44876](https://github.com/ClickHouse/ClickHouse/pull/44876) ([Kruglov Pavel](https://github.com/Avogar)). +* Rewrite test_postgres_protocol test [#44880](https://github.com/ClickHouse/ClickHouse/pull/44880) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix ConcurrentBoundedQueue::emplace() return value in case of finished queue [#44881](https://github.com/ClickHouse/ClickHouse/pull/44881) ([Azat Khuzhin](https://github.com/azat)). +* Validate function arguments in query tree [#44882](https://github.com/ClickHouse/ClickHouse/pull/44882) ([Dmitry Novik](https://github.com/novikd)). +* Rework CI reports to have a class and clarify the logic [#44883](https://github.com/ClickHouse/ClickHouse/pull/44883) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* fix-typo [#44886](https://github.com/ClickHouse/ClickHouse/pull/44886) ([Enrique Herreros](https://github.com/eherrerosj)). +* Store ZK generated data in `test_keeper_snapshot_small_distance` [#44888](https://github.com/ClickHouse/ClickHouse/pull/44888) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix "AttributeError: 'BuildResult' object has no attribute 'libraries'" in BuilderReport and BuilderSpecialReport [#44890](https://github.com/ClickHouse/ClickHouse/pull/44890) ([Robert Schulze](https://github.com/rschu1ze)). +* Convert integration test_dictionaries_update_field to a stateless [#44891](https://github.com/ClickHouse/ClickHouse/pull/44891) ([Azat Khuzhin](https://github.com/azat)). +* Upgrade googletest to latest HEAD [#44894](https://github.com/ClickHouse/ClickHouse/pull/44894) ([Robert Schulze](https://github.com/rschu1ze)). +* Try fix rabbitmq potential leak [#44897](https://github.com/ClickHouse/ClickHouse/pull/44897) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix flaky `test_storage_kafka::test_kafka_produce_key_timestamp` [#44898](https://github.com/ClickHouse/ClickHouse/pull/44898) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky `test_concurrent_queries_restriction_by_query_kind` [#44903](https://github.com/ClickHouse/ClickHouse/pull/44903) ([Antonio Andelic](https://github.com/antonio2368)). +* Avoid Keeper crash on shutdown (fix `test_keeper_snapshot_on_exit`) [#44908](https://github.com/ClickHouse/ClickHouse/pull/44908) ([Antonio Andelic](https://github.com/antonio2368)). +* Do not merge over a gap with outdated undeleted parts [#44909](https://github.com/ClickHouse/ClickHouse/pull/44909) ([Sema Checherinda](https://github.com/CheSema)). +* Fix logging message in MergeTreeDataMergerMutator (about merged parts) [#44917](https://github.com/ClickHouse/ClickHouse/pull/44917) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test `test_lost_part` [#44921](https://github.com/ClickHouse/ClickHouse/pull/44921) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add fast and cancellable shared_mutex alternatives [#44924](https://github.com/ClickHouse/ClickHouse/pull/44924) ([Sergei Trifonov](https://github.com/serxa)). +* Fix deadlock in Keeper's changelog [#44937](https://github.com/ClickHouse/ClickHouse/pull/44937) ([Antonio Andelic](https://github.com/antonio2368)). +* Stop merges to avoid a race between merge and freeze. [#44938](https://github.com/ClickHouse/ClickHouse/pull/44938) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix memory leak in Aws::InitAPI [#44942](https://github.com/ClickHouse/ClickHouse/pull/44942) ([Vitaly Baranov](https://github.com/vitlibar)). +* Change error code on invalid background_pool_size config [#44947](https://github.com/ClickHouse/ClickHouse/pull/44947) ([Raúl Marín](https://github.com/Algunenano)). +* Fix exception fix in TraceCollector dtor [#44948](https://github.com/ClickHouse/ClickHouse/pull/44948) ([Robert Schulze](https://github.com/rschu1ze)). +* Parallel distributed insert select with s3Cluster [3] [#44955](https://github.com/ClickHouse/ClickHouse/pull/44955) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Do not check read result consistency when unwinding [#44956](https://github.com/ClickHouse/ClickHouse/pull/44956) ([Alexander Gololobov](https://github.com/davenger)). +* Up the log level of tables dependencies graphs [#44957](https://github.com/ClickHouse/ClickHouse/pull/44957) ([Vitaly Baranov](https://github.com/vitlibar)). +* Hipster's HTML [#44961](https://github.com/ClickHouse/ClickHouse/pull/44961) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Docs: Mention non-standard DOTALL behavior of ClickHouse's match() [#44977](https://github.com/ClickHouse/ClickHouse/pull/44977) ([Robert Schulze](https://github.com/rschu1ze)). +* tests: fix test_replicated_users flakiness [#44978](https://github.com/ClickHouse/ClickHouse/pull/44978) ([Azat Khuzhin](https://github.com/azat)). +* Check what if disable some checks in storage Merge. [#44983](https://github.com/ClickHouse/ClickHouse/pull/44983) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix check for not existing input in ActionsDAG [#44987](https://github.com/ClickHouse/ClickHouse/pull/44987) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update version_date.tsv and changelogs after v22.12.2.25-stable [#44988](https://github.com/ClickHouse/ClickHouse/pull/44988) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix test test_grpc_protocol/test.py::test_progress [#44996](https://github.com/ClickHouse/ClickHouse/pull/44996) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve S3 EC2 metadata tests [#45001](https://github.com/ClickHouse/ClickHouse/pull/45001) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix minmax_count_projection with _partition_value [#45003](https://github.com/ClickHouse/ClickHouse/pull/45003) ([Amos Bird](https://github.com/amosbird)). +* Fix strange trash in Fuzzer [#45006](https://github.com/ClickHouse/ClickHouse/pull/45006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add `dmesg.log` to Fuzzer [#45008](https://github.com/ClickHouse/ClickHouse/pull/45008) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `01961_roaring_memory_tracking` test, again [#45009](https://github.com/ClickHouse/ClickHouse/pull/45009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Recognize more ok cases for Fuzzer [#45012](https://github.com/ClickHouse/ClickHouse/pull/45012) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Supposedly fix the "Download script failed" error [#45013](https://github.com/ClickHouse/ClickHouse/pull/45013) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add snapshot creation retry in Keeper tests using ZooKeeper [#45016](https://github.com/ClickHouse/ClickHouse/pull/45016) ([Antonio Andelic](https://github.com/antonio2368)). +* test for [#20098](https://github.com/ClickHouse/ClickHouse/issues/20098) [#45017](https://github.com/ClickHouse/ClickHouse/pull/45017) ([Denny Crane](https://github.com/den-crane)). +* test for [#26473](https://github.com/ClickHouse/ClickHouse/issues/26473) [#45018](https://github.com/ClickHouse/ClickHouse/pull/45018) ([Denny Crane](https://github.com/den-crane)). +* Remove the remainings of Testflows (2). [#45021](https://github.com/ClickHouse/ClickHouse/pull/45021) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable the check that was commented [#45022](https://github.com/ClickHouse/ClickHouse/pull/45022) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix false positive in Fuzzer [#45025](https://github.com/ClickHouse/ClickHouse/pull/45025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix false positive in Fuzzer, alternative variant [#45026](https://github.com/ClickHouse/ClickHouse/pull/45026) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix function `range` (the bug was unreleased) [#45030](https://github.com/ClickHouse/ClickHouse/pull/45030) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix OOM in Fuzzer [#45032](https://github.com/ClickHouse/ClickHouse/pull/45032) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Less OOM in Stress test [#45033](https://github.com/ClickHouse/ClickHouse/pull/45033) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#31361](https://github.com/ClickHouse/ClickHouse/issues/31361) [#45034](https://github.com/ClickHouse/ClickHouse/pull/45034) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#38729](https://github.com/ClickHouse/ClickHouse/issues/38729) [#45035](https://github.com/ClickHouse/ClickHouse/pull/45035) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typos [#45036](https://github.com/ClickHouse/ClickHouse/pull/45036) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* I didn't understand the logic of this test, @azat [#45037](https://github.com/ClickHouse/ClickHouse/pull/45037) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small fixes for Coordination unit tests [#45039](https://github.com/ClickHouse/ClickHouse/pull/45039) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky test (hilarious) [#45042](https://github.com/ClickHouse/ClickHouse/pull/45042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Non significant changes [#45046](https://github.com/ClickHouse/ClickHouse/pull/45046) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't fix parallel formatting [#45050](https://github.com/ClickHouse/ClickHouse/pull/45050) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix (benign) data race in clickhouse-client [#45053](https://github.com/ClickHouse/ClickHouse/pull/45053) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Analyzer aggregation without column fix [#45055](https://github.com/ClickHouse/ClickHouse/pull/45055) ([Maksim Kita](https://github.com/kitaisreal)). +* Analyzer ARRAY JOIN crash fix [#45059](https://github.com/ClickHouse/ClickHouse/pull/45059) ([Maksim Kita](https://github.com/kitaisreal)). +* Analyzer function IN crash fix [#45064](https://github.com/ClickHouse/ClickHouse/pull/45064) ([Maksim Kita](https://github.com/kitaisreal)). +* JIT compilation float to bool conversion fix [#45067](https://github.com/ClickHouse/ClickHouse/pull/45067) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version_date.tsv and changelogs after v22.11.3.47-stable [#45069](https://github.com/ClickHouse/ClickHouse/pull/45069) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.10.5.54-stable [#45071](https://github.com/ClickHouse/ClickHouse/pull/45071) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.3.16.1190-lts [#45073](https://github.com/ClickHouse/ClickHouse/pull/45073) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Change the color of links in dark reports a little bit [#45077](https://github.com/ClickHouse/ClickHouse/pull/45077) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix Fuzzer script [#45082](https://github.com/ClickHouse/ClickHouse/pull/45082) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fixing KeeperMap tests [#45094](https://github.com/ClickHouse/ClickHouse/pull/45094) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv and changelogs after v22.8.12.45-lts [#45098](https://github.com/ClickHouse/ClickHouse/pull/45098) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Try to fix flaky test_create_user_and_login/test.py::test_login_as_dropped_user_xml [#45099](https://github.com/ClickHouse/ClickHouse/pull/45099) ([Ilya Yatsishin](https://github.com/qoega)). +* Update version_date.tsv and changelogs after v22.10.6.3-stable [#45107](https://github.com/ClickHouse/ClickHouse/pull/45107) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Docs: Make heading consistent with other headings in System Table docs [#45109](https://github.com/ClickHouse/ClickHouse/pull/45109) ([Robert Schulze](https://github.com/rschu1ze)). +* Update version_date.tsv and changelogs after v22.11.4.3-stable [#45110](https://github.com/ClickHouse/ClickHouse/pull/45110) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.12.3.5-stable [#45113](https://github.com/ClickHouse/ClickHouse/pull/45113) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Docs: Rewrite awkwardly phrased sentence about flush interval [#45114](https://github.com/ClickHouse/ClickHouse/pull/45114) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix data race in s3Cluster. [#45123](https://github.com/ClickHouse/ClickHouse/pull/45123) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Pull SQLancer image before check run [#45125](https://github.com/ClickHouse/ClickHouse/pull/45125) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flaky azure test [#45134](https://github.com/ClickHouse/ClickHouse/pull/45134) ([alesapin](https://github.com/alesapin)). +* Minor cleanup in stress/run.sh [#45136](https://github.com/ClickHouse/ClickHouse/pull/45136) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Performance report: "Partial queries" --> "Backward-incompatible queries [#45152](https://github.com/ClickHouse/ClickHouse/pull/45152) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix flaky test_tcp_handler_interserver_listen_host [#45156](https://github.com/ClickHouse/ClickHouse/pull/45156) ([Ilya Yatsishin](https://github.com/qoega)). +* Clean trash from changelog for v22.3.16.1190-lts [#45159](https://github.com/ClickHouse/ClickHouse/pull/45159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable `test_storage_rabbitmq` [#45161](https://github.com/ClickHouse/ClickHouse/pull/45161) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_ttl_move_memory_usage as too flaky. [#45162](https://github.com/ClickHouse/ClickHouse/pull/45162) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More logging to facilitate debugging of flaky test_ttl_replicated [#45165](https://github.com/ClickHouse/ClickHouse/pull/45165) ([Alexander Gololobov](https://github.com/davenger)). +* Try to fix flaky test_ttl_move_memory_usage [#45168](https://github.com/ClickHouse/ClickHouse/pull/45168) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test test_multiple_disks/test.py::test_rename [#45180](https://github.com/ClickHouse/ClickHouse/pull/45180) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Calculate only required columns in system.detached_parts [#45181](https://github.com/ClickHouse/ClickHouse/pull/45181) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Restart NightlyBuilds if the runner died [#45187](https://github.com/ClickHouse/ClickHouse/pull/45187) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix part ID generation for IP types for backward compatibility [#45191](https://github.com/ClickHouse/ClickHouse/pull/45191) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix integration test test_replicated_users::test_rename_replicated [#45192](https://github.com/ClickHouse/ClickHouse/pull/45192) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add CACHE_INVALIDATOR for sqlancer builds [#45201](https://github.com/ClickHouse/ClickHouse/pull/45201) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix possible stack-use-after-return in LimitReadBuffer [#45203](https://github.com/ClickHouse/ClickHouse/pull/45203) ([Kruglov Pavel](https://github.com/Avogar)). +* Disable check to make test_overcommit_tracker not flaky [#45206](https://github.com/ClickHouse/ClickHouse/pull/45206) ([Dmitry Novik](https://github.com/novikd)). +* Fix flaky test `01961_roaring_memory_tracking` (3) [#45208](https://github.com/ClickHouse/ClickHouse/pull/45208) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash from stress test [#45211](https://github.com/ClickHouse/ClickHouse/pull/45211) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove unused function [#45212](https://github.com/ClickHouse/ClickHouse/pull/45212) ([flynn](https://github.com/ucasfl)). +* Fix flaky `test_keeper_three_nodes_two_alive` [#45213](https://github.com/ClickHouse/ClickHouse/pull/45213) ([Antonio Andelic](https://github.com/antonio2368)). +* Fuzz PREWHERE clause [#45222](https://github.com/ClickHouse/ClickHouse/pull/45222) ([Alexander Gololobov](https://github.com/davenger)). +* Added a test for merge join key condition with big int & decimal [#45228](https://github.com/ClickHouse/ClickHouse/pull/45228) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix rare logical error: `Too large alignment` [#45229](https://github.com/ClickHouse/ClickHouse/pull/45229) ([Anton Popov](https://github.com/CurtizJ)). +* Update version_date.tsv and changelogs after v22.3.17.13-lts [#45234](https://github.com/ClickHouse/ClickHouse/pull/45234) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* More verbose logs about replication log entries [#45235](https://github.com/ClickHouse/ClickHouse/pull/45235) ([Alexander Tokmakov](https://github.com/tavplubix)). +* One more attempt to fix race in TCPHandler [#45240](https://github.com/ClickHouse/ClickHouse/pull/45240) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update clickhouse-test [#45251](https://github.com/ClickHouse/ClickHouse/pull/45251) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Planner small fixes [#45254](https://github.com/ClickHouse/ClickHouse/pull/45254) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix log level "Test" for send_logs_level in client [#45273](https://github.com/ClickHouse/ClickHouse/pull/45273) ([Azat Khuzhin](https://github.com/azat)). +* tests: fix clickhouse binaries detection [#45283](https://github.com/ClickHouse/ClickHouse/pull/45283) ([Azat Khuzhin](https://github.com/azat)). +* tests/ci: encode HTML entities in the reports [#45284](https://github.com/ClickHouse/ClickHouse/pull/45284) ([Azat Khuzhin](https://github.com/azat)). +* Disable `02151_hash_table_sizes_stats_distributed` under TSAN [#45287](https://github.com/ClickHouse/ClickHouse/pull/45287) ([Nikita Taranov](https://github.com/nickitat)). +* Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable 02028_create_select_settings with Ordinary [#45307](https://github.com/ClickHouse/ClickHouse/pull/45307) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Save message format strings for DB::Exception [#45342](https://github.com/ClickHouse/ClickHouse/pull/45342) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Slightly better output for glibc check [#45353](https://github.com/ClickHouse/ClickHouse/pull/45353) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add checks for compilation of regexps [#45356](https://github.com/ClickHouse/ClickHouse/pull/45356) ([Anton Popov](https://github.com/CurtizJ)). +* Analyzer compound identifier typo correction fix [#45357](https://github.com/ClickHouse/ClickHouse/pull/45357) ([Maksim Kita](https://github.com/kitaisreal)). +* Bump to newer version of debug-action [#45359](https://github.com/ClickHouse/ClickHouse/pull/45359) ([Ilya Yatsishin](https://github.com/qoega)). +* Improve failed kafka startup logging [#45369](https://github.com/ClickHouse/ClickHouse/pull/45369) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flaky ttl test [#45370](https://github.com/ClickHouse/ClickHouse/pull/45370) ([alesapin](https://github.com/alesapin)). +* Add detailed profile events for throttling [#45373](https://github.com/ClickHouse/ClickHouse/pull/45373) ([Sergei Trifonov](https://github.com/serxa)). +* Update .gitignore [#45378](https://github.com/ClickHouse/ClickHouse/pull/45378) ([Nikolay Degterinsky](https://github.com/evillique)). +* Make test simpler to see errors [#45402](https://github.com/ClickHouse/ClickHouse/pull/45402) ([Ilya Yatsishin](https://github.com/qoega)). +* Reduce an amount of trash in `tests_system_merges` [#45403](https://github.com/ClickHouse/ClickHouse/pull/45403) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix reading from encrypted disk with passed file size [#45418](https://github.com/ClickHouse/ClickHouse/pull/45418) ([Anton Popov](https://github.com/CurtizJ)). +* Add delete by ttl for zookeeper_log [#45419](https://github.com/ClickHouse/ClickHouse/pull/45419) ([Nikita Taranov](https://github.com/nickitat)). +* Minor improvements around reading from remote [#45442](https://github.com/ClickHouse/ClickHouse/pull/45442) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Docs: Beautify section on secondary index types [#45444](https://github.com/ClickHouse/ClickHouse/pull/45444) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix Buffer's offsets mismatch logical error in stress test [#45446](https://github.com/ClickHouse/ClickHouse/pull/45446) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better formatting for exception messages [#45449](https://github.com/ClickHouse/ClickHouse/pull/45449) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add default GRANULARITY argument for secondary indexes [#45451](https://github.com/ClickHouse/ClickHouse/pull/45451) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix typos [#45470](https://github.com/ClickHouse/ClickHouse/pull/45470) ([Robert Schulze](https://github.com/rschu1ze)). +* Add more retries to AST Fuzzer [#45479](https://github.com/ClickHouse/ClickHouse/pull/45479) ([Nikolay Degterinsky](https://github.com/evillique)). +* Remove unnecessary getTotalRowCount function calls [#45485](https://github.com/ClickHouse/ClickHouse/pull/45485) ([Maksim Kita](https://github.com/kitaisreal)). +* Forward declaration of ConcurrentBoundedQueue in ThreadStatus [#45489](https://github.com/ClickHouse/ClickHouse/pull/45489) ([Azat Khuzhin](https://github.com/azat)). +* Revert "Merge pull request [#44922](https://github.com/ClickHouse/ClickHouse/issues/44922) from azat/dist/async-INSERT-metrics" [#45492](https://github.com/ClickHouse/ClickHouse/pull/45492) ([Azat Khuzhin](https://github.com/azat)). +* Docs: Fix weird formatting [#45495](https://github.com/ClickHouse/ClickHouse/pull/45495) ([Robert Schulze](https://github.com/rschu1ze)). +* Docs: Fix link to writing guide [#45496](https://github.com/ClickHouse/ClickHouse/pull/45496) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve logging for TeePopen.timeout exceeded [#45504](https://github.com/ClickHouse/ClickHouse/pull/45504) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update test_system_merges/test.py [#45516](https://github.com/ClickHouse/ClickHouse/pull/45516) ([Alexander Tokmakov](https://github.com/tavplubix)). + +## [Changelog for 2022](https://clickhouse.com/docs/en/whats-new/changelog/2022) From 32b4bbf26c0ccc5c8099cb22886c6bca600e3af8 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 24 Jan 2023 14:32:49 +0100 Subject: [PATCH 083/439] LowCardinality insert fix --- src/Columns/ColumnUnique.h | 42 +++-------------- src/Columns/tests/gtest_low_cardinality.cpp | 50 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 src/Columns/tests/gtest_low_cardinality.cpp diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 8ba5abae1fd..8a95726d2be 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -331,46 +331,14 @@ size_t ColumnUnique::getNullValueIndex() const template size_t ColumnUnique::uniqueInsert(const Field & x) { - class FieldVisitorGetData : public StaticVisitor<> - { - public: - StringRef res; - - [[noreturn]] static void throwUnsupported() - { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsupported field type"); - } - - [[noreturn]] void operator() (const Null &) { throwUnsupported(); } - [[noreturn]] void operator() (const Array &) { throwUnsupported(); } - [[noreturn]] void operator() (const Tuple &) { throwUnsupported(); } - [[noreturn]] void operator() (const Map &) { throwUnsupported(); } - [[noreturn]] void operator() (const Object &) { throwUnsupported(); } - [[noreturn]] void operator() (const AggregateFunctionStateData &) { throwUnsupported(); } - void operator() (const String & x) { res = {x.data(), x.size()}; } - void operator() (const UInt64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const UInt128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const UInt256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const Int64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const Int128 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const Int256 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const UUID & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const IPv4 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const IPv6 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const Float64 & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const DecimalField & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - void operator() (const bool & x) { res = {reinterpret_cast(&x), sizeof(x)}; } - }; - if (x.isNull()) return getNullValueIndex(); - FieldVisitorGetData visitor; - applyVisitor(visitor, x); - return uniqueInsertData(visitor.res.data, visitor.res.size); + auto single_value_column = column_holder->cloneEmpty(); + single_value_column->insert(x); + auto single_value_data = single_value_column->getDataAt(0); + + return uniqueInsertData(single_value_data.data, single_value_data.size); } template diff --git a/src/Columns/tests/gtest_low_cardinality.cpp b/src/Columns/tests/gtest_low_cardinality.cpp new file mode 100644 index 00000000000..3ffc88f6a7d --- /dev/null +++ b/src/Columns/tests/gtest_low_cardinality.cpp @@ -0,0 +1,50 @@ +#include +#include + +#include +#include + +#include + +using namespace DB; + +template +void testLowCardinalityNumberInsert(const DataTypePtr & data_type) +{ + auto low_cardinality_type = std::make_shared(data_type); + auto column = low_cardinality_type->createColumn(); + + column->insert(static_cast(15)); + column->insert(static_cast(20)); + column->insert(static_cast(25)); + + Field value; + column->get(0, value); + ASSERT_EQ(value.get(), 15); + + column->get(1, value); + ASSERT_EQ(value.get(), 20); + + column->get(2, value); + ASSERT_EQ(value.get(), 25); +} + +TEST(ColumnLowCardinality, Insert) +{ + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); + + testLowCardinalityNumberInsert(std::make_shared()); + testLowCardinalityNumberInsert(std::make_shared()); +} From 6f7755ebcd763fd6b67714b12db417356f0a5e54 Mon Sep 17 00:00:00 2001 From: rfraposa Date: Tue, 24 Jan 2023 15:05:34 -0700 Subject: [PATCH 084/439] Added optimize using constraints settings --- .../settings/merge-tree-settings.md | 2 +- docs/en/operations/settings/settings.md | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index f806232fde5..58deb749a3f 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -234,7 +234,7 @@ Possible values: Default value: 100. -Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time. +Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time. ## max_replicated_logs_to_keep diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 97a3a0de5a0..2e6b3d14634 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3479,6 +3479,30 @@ Default value: `0`. - [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting +## optimize_using_constraints + +Use [constraints](../../sql-reference/statements/create/table#constraints) for query optimization. The default is `false`. + +Possible values: + +- true, false + +## optimize_append_index + +Use [constraints](../../sql-reference/statements/create/table#constraints) in order to append index condition. The default is `false`. + +Possible values: + +- true, false + +## optimize_substitute_columns + +Use [constraints](../../sql-reference/statements/create/table#constraints) for column substitution. The default is `false`. + +Possible values: + +- true, false + ## describe_include_subcolumns {#describe_include_subcolumns} Enables describing subcolumns for a [DESCRIBE](../../sql-reference/statements/describe-table.md) query. For example, members of a [Tuple](../../sql-reference/data-types/tuple.md) or subcolumns of a [Map](../../sql-reference/data-types/map.md/#map-subcolumns), [Nullable](../../sql-reference/data-types/nullable.md/#finding-null) or an [Array](../../sql-reference/data-types/array.md/#array-size) data type. From d1baa7300c043c4d1156d53e1df4d3b4951c2493 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 24 Jan 2023 23:21:29 +0100 Subject: [PATCH 085/439] reformat ParsingException --- src/Common/CounterInFile.h | 2 +- src/Common/Exception.h | 4 +-- .../Serializations/SerializationArray.cpp | 4 +-- .../Serializations/SerializationNullable.cpp | 30 ++++++++-------- src/Formats/JSONUtils.cpp | 9 +++-- src/Formats/NativeReader.cpp | 2 +- src/IO/ReadHelpers.cpp | 21 ++++++------ src/IO/ReadHelpers.h | 34 +++++++++---------- src/IO/parseDateTimeBestEffort.cpp | 4 +-- src/IO/readDecimalText.h | 4 +-- src/IO/readFloatText.h | 14 ++++---- .../Formats/Impl/AvroRowInputFormat.cpp | 2 +- .../Impl/JSONEachRowRowInputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockInputFormat.cpp | 5 +-- .../Formats/Impl/TSKVRowInputFormat.cpp | 4 +-- .../Formats/Impl/TemplateRowInputFormat.cpp | 4 +-- .../Formats/RowInputFormatWithNamesAndTypes.h | 4 +-- src/Server/ProxyV1Handler.cpp | 16 ++++----- ..._log_and_exception_messages_formatting.sql | 3 ++ .../02246_clickhouse_local_drop_database.sh | 2 ++ ...02352_interactive_queries_from_file.expect | 2 ++ 21 files changed, 89 insertions(+), 83 deletions(-) diff --git a/src/Common/CounterInFile.h b/src/Common/CounterInFile.h index c6ba355aac7..e3e98492c67 100644 --- a/src/Common/CounterInFile.h +++ b/src/Common/CounterInFile.h @@ -88,7 +88,7 @@ public: { /// A more understandable error message. if (e.code() == DB::ErrorCodes::CANNOT_READ_ALL_DATA || e.code() == DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) - throw DB::ParsingException("File " + path + " is empty. You must fill it manually with appropriate value.", e.code()); + throw DB::ParsingException(e.code(), "File {} is empty. You must fill it manually with appropriate value.", path); else throw; } diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 8177d92b59b..e548efce0d0 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -178,11 +178,11 @@ private: /// more convenient calculation of problem line number. class ParsingException : public Exception { + ParsingException(const std::string & msg, int code); public: ParsingException(); - ParsingException(const std::string & msg, int code); ParsingException(int code, const std::string & message); - ParsingException(int code, std::string && message) : Exception(message, code) {} + ParsingException(int code, std::string && message) : Exception(std::move(message), code) {} // Format message with fmt::format, like the logging functions. template diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp index c005793773e..24aa9e8320d 100644 --- a/src/DataTypes/Serializations/SerializationArray.cpp +++ b/src/DataTypes/Serializations/SerializationArray.cpp @@ -373,8 +373,8 @@ void SerializationArray::deserializeBinaryBulkWithMultipleStreams( /// Check consistency between offsets and elements subcolumns. /// But if elements column is empty - it's ok for columns of Nested types that was added by ALTER. if (!nested_column->empty() && nested_column->size() != last_offset) - throw ParsingException("Cannot read all array values: read just " + toString(nested_column->size()) + " of " + toString(last_offset), - ErrorCodes::CANNOT_READ_ALL_DATA); + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read all array values: read just {} of {}", + toString(nested_column->size()), toString(last_offset)); column = std::move(mutable_column); } diff --git a/src/DataTypes/Serializations/SerializationNullable.cpp b/src/DataTypes/Serializations/SerializationNullable.cpp index 7f494a694bd..8b0bdc05d00 100644 --- a/src/DataTypes/Serializations/SerializationNullable.cpp +++ b/src/DataTypes/Serializations/SerializationNullable.cpp @@ -360,19 +360,20 @@ ReturnType SerializationNullable::deserializeTextEscapedAndRawImpl(IColumn & col /// or if someone uses tab or LF in TSV null_representation. /// In the first case we cannot continue reading anyway. The second case seems to be unlikely. if (null_representation.find('\t') != std::string::npos || null_representation.find('\n') != std::string::npos) - throw DB::ParsingException("TSV custom null representation containing '\\t' or '\\n' may not work correctly " - "for large input.", ErrorCodes::CANNOT_READ_ALL_DATA); + throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "TSV custom null representation " + "containing '\\t' or '\\n' may not work correctly for large input."); WriteBufferFromOwnString parsed_value; if constexpr (escaped) nested_serialization->serializeTextEscaped(nested_column, nested_column.size() - 1, parsed_value, settings); else nested_serialization->serializeTextRaw(nested_column, nested_column.size() - 1, parsed_value, settings); - throw DB::ParsingException("Error while parsing \"" + std::string(pos, buf.buffer().end()) + std::string(istr.position(), std::min(size_t(10), istr.available())) + "\" as Nullable" - + " at position " + std::to_string(istr.count()) + ": got \"" + std::string(pos, buf.position() - pos) - + "\", which was deserialized as \"" - + parsed_value.str() + "\". It seems that input data is ill-formatted.", - ErrorCodes::CANNOT_READ_ALL_DATA); + throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while parsing \"{}{}\" as Nullable" + " at position {}: got \"{}\", which was deserialized as \"{}\". " + "It seems that input data is ill-formatted.", + std::string(pos, buf.buffer().end()), + std::string(istr.position(), std::min(size_t(10), istr.available())), + istr.count(), std::string(pos, buf.position() - pos), parsed_value.str()); }; return safeDeserialize(column, *nested_serialization, check_for_null, deserialize_nested); @@ -584,16 +585,17 @@ ReturnType SerializationNullable::deserializeTextCSVImpl(IColumn & column, ReadB /// In the first case we cannot continue reading anyway. The second case seems to be unlikely. if (null_representation.find(settings.csv.delimiter) != std::string::npos || null_representation.find('\r') != std::string::npos || null_representation.find('\n') != std::string::npos) - throw DB::ParsingException("CSV custom null representation containing format_csv_delimiter, '\\r' or '\\n' may not work correctly " - "for large input.", ErrorCodes::CANNOT_READ_ALL_DATA); + throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "CSV custom null representation containing " + "format_csv_delimiter, '\\r' or '\\n' may not work correctly for large input."); WriteBufferFromOwnString parsed_value; nested_serialization->serializeTextCSV(nested_column, nested_column.size() - 1, parsed_value, settings); - throw DB::ParsingException("Error while parsing \"" + std::string(pos, buf.buffer().end()) + std::string(istr.position(), std::min(size_t(10), istr.available())) + "\" as Nullable" - + " at position " + std::to_string(istr.count()) + ": got \"" + std::string(pos, buf.position() - pos) - + "\", which was deserialized as \"" - + parsed_value.str() + "\". It seems that input data is ill-formatted.", - ErrorCodes::CANNOT_READ_ALL_DATA); + throw DB::ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while parsing \"{}{}\" as Nullable" + " at position {}: got \"{}\", which was deserialized as \"{}\". " + "It seems that input data is ill-formatted.", + std::string(pos, buf.buffer().end()), + std::string(istr.position(), std::min(size_t(10), istr.available())), + istr.count(), std::string(pos, buf.position() - pos), parsed_value.str()); }; return safeDeserialize(column, *nested_serialization, check_for_null, deserialize_nested); diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 177a59c32be..148c51938fc 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -44,11 +44,10 @@ namespace JSONUtils { const auto current_object_size = memory.size() + static_cast(pos - in.position()); if (min_bytes != 0 && current_object_size > 10 * min_bytes) - throw ParsingException( - "Size of JSON object is extremely large. Expected not greater than " + std::to_string(min_bytes) - + " bytes, but current is " + std::to_string(current_object_size) - + " bytes per row. Increase the value setting 'min_chunk_bytes_for_parallel_parsing' or check your data manually, most likely JSON is malformed", - ErrorCodes::INCORRECT_DATA); + throw ParsingException(ErrorCodes::INCORRECT_DATA, + "Size of JSON object is extremely large. Expected not greater than {} bytes, but current is {} bytes per row. " + "Increase the value setting 'min_chunk_bytes_for_parallel_parsing' or check your data manually, " + "most likely JSON is malformed", min_bytes, current_object_size); if (quotes) { diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index 56caf7d7ff2..58baee5931b 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -103,7 +103,7 @@ Block NativeReader::read() if (istr.eof()) { if (use_index) - throw ParsingException("Input doesn't contain all data for index.", ErrorCodes::CANNOT_READ_ALL_DATA); + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Input doesn't contain all data for index."); return res; } diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index e80803f7557..edec22ce687 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -95,14 +95,14 @@ void parseUUIDWithoutSeparator(const UInt8 * src36, std::reverse_iterator; - auto error = [](const char * message [[maybe_unused]], int code [[maybe_unused]]) + auto error = [](T && message [[maybe_unused]], int code [[maybe_unused]]) { if constexpr (throw_exception) - throw ParsingException(message, code); + throw ParsingException(code, message); return ReturnType(false); }; @@ -990,10 +989,10 @@ ReturnType readJSONObjectPossiblyInvalid(Vector & s, ReadBuffer & buf) { static constexpr bool throw_exception = std::is_same_v; - auto error = [](const char * message [[maybe_unused]], int code [[maybe_unused]]) + auto error = [](T && message [[maybe_unused]], int code [[maybe_unused]]) { if constexpr (throw_exception) - throw ParsingException(message, code); + throw ParsingException(code, message); return ReturnType(false); }; @@ -1161,7 +1160,7 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D s_pos[size] = 0; if constexpr (throw_exception) - throw ParsingException(std::string("Cannot parse DateTime ") + s, ErrorCodes::CANNOT_PARSE_DATETIME); + throw ParsingException(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime {}", s); else return false; } @@ -1184,7 +1183,7 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D s_pos[size] = 0; if constexpr (throw_exception) - throw ParsingException(std::string("Cannot parse time component of DateTime ") + s, ErrorCodes::CANNOT_PARSE_DATETIME); + throw ParsingException(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse time component of DateTime {}", s); else return false; } @@ -1211,7 +1210,7 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D else { if constexpr (throw_exception) - throw ParsingException("Cannot parse datetime", ErrorCodes::CANNOT_PARSE_DATETIME); + throw ParsingException(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse datetime"); else return false; } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index aacf8fd9b73..fd547220069 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -287,7 +287,7 @@ inline void readBoolTextWord(bool & x, ReadBuffer & buf, bool support_upper_case [[fallthrough]]; } default: - throw ParsingException("Unexpected Bool value", ErrorCodes::CANNOT_PARSE_BOOL); + throw ParsingException(ErrorCodes::CANNOT_PARSE_BOOL, "Unexpected Bool value"); } } @@ -331,9 +331,8 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) if (has_sign) { if constexpr (throw_exception) - throw ParsingException( - "Cannot parse number with multiple sign (+/-) characters", - ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, + "Cannot parse number with multiple sign (+/-) characters"); else return ReturnType(false); } @@ -349,9 +348,8 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) if (has_sign) { if constexpr (throw_exception) - throw ParsingException( - "Cannot parse number with multiple sign (+/-) characters", - ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, + "Cannot parse number with multiple sign (+/-) characters"); else return ReturnType(false); } @@ -361,7 +359,7 @@ ReturnType readIntTextImpl(T & x, ReadBuffer & buf) else { if constexpr (throw_exception) - throw ParsingException("Unsigned type must not contain '-' symbol", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Unsigned type must not contain '-' symbol"); else return ReturnType(false); } @@ -423,8 +421,8 @@ end: if (has_sign && !has_number) { if constexpr (throw_exception) - throw ParsingException( - "Cannot parse number with a sign character but without any numeric character", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, + "Cannot parse number with a sign character but without any numeric character"); else return ReturnType(false); } @@ -808,7 +806,7 @@ inline ReturnType readUUIDTextImpl(UUID & uuid, ReadBuffer & buf) if constexpr (throw_exception) { - throw ParsingException(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + throw ParsingException(ErrorCodes::CANNOT_PARSE_UUID, "Cannot parse uuid {}", s); } else { @@ -829,7 +827,7 @@ inline ReturnType readUUIDTextImpl(UUID & uuid, ReadBuffer & buf) if constexpr (throw_exception) { - throw ParsingException(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + throw ParsingException(ErrorCodes::CANNOT_PARSE_UUID, "Cannot parse uuid {}", s); } else { @@ -855,7 +853,7 @@ inline ReturnType readIPv4TextImpl(IPv4 & ip, ReadBuffer & buf) return ReturnType(true); if constexpr (std::is_same_v) - throw ParsingException(std::string("Cannot parse IPv4 ").append(buf.position(), buf.available()), ErrorCodes::CANNOT_PARSE_IPV4); + throw ParsingException(ErrorCodes::CANNOT_PARSE_IPV4, "Cannot parse IPv4 {}", std::string_view(buf.position(), buf.available())); else return ReturnType(false); } @@ -877,7 +875,7 @@ inline ReturnType readIPv6TextImpl(IPv6 & ip, ReadBuffer & buf) return ReturnType(true); if constexpr (std::is_same_v) - throw ParsingException(std::string("Cannot parse IPv6 ").append(buf.position(), buf.available()), ErrorCodes::CANNOT_PARSE_IPV6); + throw ParsingException(ErrorCodes::CANNOT_PARSE_IPV6, "Cannot parse IPv6 {}", std::string_view(buf.position(), buf.available())); else return ReturnType(false); } @@ -1061,7 +1059,7 @@ inline void readDateTimeText(LocalDateTime & datetime, ReadBuffer & buf) if (10 != size) { s[size] = 0; - throw ParsingException(std::string("Cannot parse DateTime ") + s, ErrorCodes::CANNOT_PARSE_DATETIME); + throw ParsingException(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime {}", s); } datetime.year((s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0')); @@ -1077,7 +1075,7 @@ inline void readDateTimeText(LocalDateTime & datetime, ReadBuffer & buf) if (8 != size) { s[size] = 0; - throw ParsingException(std::string("Cannot parse time component of DateTime ") + s, ErrorCodes::CANNOT_PARSE_DATETIME); + throw ParsingException(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse time component of DateTime {}", s); } datetime.hour((s[0] - '0') * 10 + (s[1] - '0')); @@ -1300,7 +1298,7 @@ void readQuoted(std::vector & x, ReadBuffer & buf) if (*buf.position() == ',') ++buf.position(); else - throw ParsingException("Cannot read array from text", ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT); + throw ParsingException(ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT, "Cannot read array from text"); } first = false; @@ -1323,7 +1321,7 @@ void readDoubleQuoted(std::vector & x, ReadBuffer & buf) if (*buf.position() == ',') ++buf.position(); else - throw ParsingException("Cannot read array from text", ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT); + throw ParsingException(ErrorCodes::CANNOT_READ_ARRAY_FROM_TEXT, "Cannot read array from text"); } first = false; diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index e0cba169e81..c376df3d613 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -90,10 +90,10 @@ ReturnType parseDateTimeBestEffortImpl( const DateLUTImpl & utc_time_zone, DateTimeSubsecondPart * fractional) { - auto on_error = [](const std::string & message [[maybe_unused]], int code [[maybe_unused]]) + auto on_error = [](T && message [[maybe_unused]], int code [[maybe_unused]]) { if constexpr (std::is_same_v) - throw ParsingException(message, code); + throw ParsingException(code, message); else return false; }; diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 86fd45a8017..9fd9c439b87 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -121,7 +121,7 @@ inline bool readDigits(ReadBuffer & buf, T & x, uint32_t & digits, int32_t & exp if (!tryReadIntText(addition_exp, buf)) { if constexpr (_throw_on_error) - throw ParsingException("Cannot parse exponent while reading decimal", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot parse exponent while reading decimal"); else return false; } @@ -134,7 +134,7 @@ inline bool readDigits(ReadBuffer & buf, T & x, uint32_t & digits, int32_t & exp if (digits_only) { if constexpr (_throw_on_error) - throw ParsingException("Unexpected symbol while reading decimal", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected symbol while reading decimal"); return false; } stop = true; diff --git a/src/IO/readFloatText.h b/src/IO/readFloatText.h index 369ea4ab87c..c4cd46463a3 100644 --- a/src/IO/readFloatText.h +++ b/src/IO/readFloatText.h @@ -160,7 +160,7 @@ ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf) if (unlikely(res.ec != std::errc())) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value"); else return ReturnType(false); } @@ -243,7 +243,7 @@ ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf) if (unlikely(res.ec != std::errc())) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value"); else return ReturnType(false); } @@ -331,7 +331,7 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value"); else return false; } @@ -389,7 +389,7 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value: nothing after exponent", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value: nothing after exponent"); else return false; } @@ -427,7 +427,7 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value: no digits read", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value: no digits read"); else return false; } @@ -438,14 +438,14 @@ ReturnType readFloatTextFastImpl(T & x, ReadBuffer & in) if (in.eof()) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value: nothing after plus sign", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value: nothing after plus sign"); else return false; } else if (negative) { if constexpr (throw_exception) - throw ParsingException("Cannot read floating point value: plus after minus sign", ErrorCodes::CANNOT_PARSE_NUMBER); + throw ParsingException(ErrorCodes::CANNOT_PARSE_NUMBER, "Cannot read floating point value: plus after minus sign"); else return false; } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 935ea13a0c8..9a475efa195 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -193,7 +193,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { decoder.decodeString(tmp); if (tmp.length() != 36) - throw ParsingException(std::string("Cannot parse uuid ") + tmp, ErrorCodes::CANNOT_PARSE_UUID); + throw ParsingException(ErrorCodes::CANNOT_PARSE_UUID, "Cannot parse uuid {}", tmp); UUID uuid; parseUUID(reinterpret_cast(tmp.data()), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 148c639f939..22ac31c7824 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -143,7 +143,7 @@ inline bool JSONEachRowRowInputFormat::advanceToNextKey(size_t key_index) skipWhitespaceIfAny(*in); if (in->eof()) - throw ParsingException("Unexpected end of stream while parsing JSONEachRow format", ErrorCodes::CANNOT_READ_ALL_DATA); + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected end of stream while parsing JSONEachRow format"); else if (*in->position() == '}') { ++in->position(); diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 22a605dd5be..066f0b29407 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -62,12 +62,13 @@ Chunk ParquetBlockInputFormat::generate() arrow::Status get_batch_reader_status = file_reader->GetRecordBatchReader(row_group_indices, column_indices, &rbr); if (!get_batch_reader_status.ok()) - throw ParsingException{"Error while reading Parquet data: " + get_batch_reader_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while reading Parquet data: {}", + get_batch_reader_status.ToString()); arrow::Status read_status = rbr->ReadAll(&table); if (!read_status.ok()) - throw ParsingException{"Error while reading Parquet data: " + read_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while reading Parquet data: {}", read_status.ToString()); ++row_group_current; diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index 942d0511b1b..bf6d0ab88d2 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -92,7 +92,7 @@ static bool readName(ReadBuffer & buf, StringRef & ref, String & tmp) } } - throw ParsingException("Unexpected end of stream while reading key name from TSKV format", ErrorCodes::CANNOT_READ_ALL_DATA); + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected end of stream while reading key name from TSKV format"); } @@ -161,7 +161,7 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex if (in->eof()) { - throw ParsingException("Unexpected end of stream after field in TSKV format: " + name_ref.toString(), ErrorCodes::CANNOT_READ_ALL_DATA); + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected end of stream after field in TSKV format: {}", name_ref.toString()); } else if (*in->position() == '\t') { diff --git a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp index 9cffe316606..1a519fa977f 100644 --- a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp @@ -21,9 +21,9 @@ namespace ErrorCodes [[noreturn]] static void throwUnexpectedEof(size_t row_num) { - throw ParsingException("Unexpected EOF while parsing row " + std::to_string(row_num) + ". " + throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected EOF while parsing row {}. " "Maybe last row has wrong format or input doesn't contain specified suffix before EOF.", - ErrorCodes::CANNOT_READ_ALL_DATA); + std::to_string(row_num)); } static void updateFormatSettingsIfNeeded(FormatSettings::EscapingRule escaping_rule, FormatSettings & settings, const ParsedTemplateFormatString & row_format, char default_csv_delimiter, size_t file_column) diff --git a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h index 94ad9fac445..5648acd392d 100644 --- a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h +++ b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h @@ -99,7 +99,7 @@ public: /// Read row with raw values. virtual std::vector readRowForHeaderDetection() { - throw Exception("Method readRowAndGetFieldsAndDataTypes is not implemented for format reader", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method readRowAndGetFieldsAndDataTypes is not implemented for format reader"); } /// Skip single field, it's used to skip unknown columns. @@ -127,7 +127,7 @@ public: virtual FormatSettings::EscapingRule getEscapingRule() const { - throw Exception("Format reader doesn't have an escaping rule", ErrorCodes::NOT_IMPLEMENTED); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Format reader doesn't have an escaping rule"); } protected: diff --git a/src/Server/ProxyV1Handler.cpp b/src/Server/ProxyV1Handler.cpp index e40579ba821..cd5fe29112a 100644 --- a/src/Server/ProxyV1Handler.cpp +++ b/src/Server/ProxyV1Handler.cpp @@ -28,38 +28,38 @@ void ProxyV1Handler::run() // read "PROXY" if (!readWord(5, word, eol) || word != "PROXY" || eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); // read "TCP4" or "TCP6" or "UNKNOWN" if (!readWord(7, word, eol)) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); if (word != "TCP4" && word != "TCP6" && word != "UNKNOWN") - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); if (word == "UNKNOWN" && eol) return; if (eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); // read address if (!readWord(39, word, eol) || eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); stack_data.forwarded_for = std::move(word); // read address if (!readWord(39, word, eol) || eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); // read port if (!readWord(5, word, eol) || eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); // read port and "\r\n" if (!readWord(5, word, eol) || !eol) - throw ParsingException("PROXY protocol violation", ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED); + throw ParsingException(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "PROXY protocol violation"); if (!stack_data.forwarded_for.empty()) LOG_TRACE(log, "Forwarded client address from PROXY header: {}", stack_data.forwarded_for); diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index 45862bb1717..7ea531569a7 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -47,5 +47,8 @@ select 110, (select count() from logs where level = 'Warning' group by message_f select 120, count() < 3 from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.10); select 130, count() < 10 from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.05); +-- Each message matches its pattern (returns 0 rows) +select 140, message_format_string, any(message) from logs where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) + and message not like ('Code: %Exception: '||s||'%') group by message_format_string; drop table logs; diff --git a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh index 00f3904192f..7e25e099626 100755 --- a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh +++ b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh @@ -22,3 +22,5 @@ USE test; CREATE TABLE test (id Int32) ENGINE=MergeTree() ORDER BY id; DROP DATABASE test; """ + +rm -r $dir diff --git a/tests/queries/0_stateless/02352_interactive_queries_from_file.expect b/tests/queries/0_stateless/02352_interactive_queries_from_file.expect index d15b804b0b9..a34fc9909f8 100755 --- a/tests/queries/0_stateless/02352_interactive_queries_from_file.expect +++ b/tests/queries/0_stateless/02352_interactive_queries_from_file.expect @@ -43,3 +43,5 @@ expect ":) " send -- "exit\r" expect eof + +spawn bash -c "rm queries_02352" From fc4eba3b7ba9b95dce26c3134c755e1f8a9296fa Mon Sep 17 00:00:00 2001 From: Bharat Nallan Chakravarthy Date: Tue, 24 Jan 2023 14:22:32 -0800 Subject: [PATCH 086/439] add a test --- ...res_odbc_no_connection_pool_dictionary.xml | 41 +++++++++++++++++++ .../integration/test_odbc_interaction/test.py | 29 +++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_no_connection_pool_dictionary.xml diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_no_connection_pool_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_no_connection_pool_dictionary.xml new file mode 100644 index 00000000000..a8321b1bbf1 --- /dev/null +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_no_connection_pool_dictionary.xml @@ -0,0 +1,41 @@ + + + postgres_odbc_nopool + + +
clickhouse.test_table
+ DSN=postgresql_odbc + postgres + + + 0 + + + + 5 + 5 + + + + + + + + id + + + + column1 + Int64 + 1 + + + + column2 + String + '' + + + + + diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index ed925759114..946f97c2148 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -21,6 +21,7 @@ node1 = cluster.add_instance( "configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml", "configs/dictionaries/sqlite3_odbc_cached_dictionary.xml", "configs/dictionaries/postgres_odbc_hashed_dictionary.xml", + "configs/dictionaries/postgres_odbc_no_connection_pool_dictionary.xml", ], ) @@ -624,6 +625,34 @@ def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster): cursor.execute("truncate table clickhouse.test_table") +def test_no_connection_pooling(started_cluster): + skip_test_msan(node1) + + conn = get_postgres_conn(started_cluster) + cursor = conn.cursor() + cursor.execute("truncate table clickhouse.test_table") + cursor.execute( + "insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')" + ) + node1.exec_in_container(["ss", "-K", "dport", "5432"], privileged=True, user="root") + node1.query("SYSTEM RELOAD DICTIONARY postgres_odbc_nopool") + assert_eq_with_retry( + node1, + "select dictGetString('postgres_odbc_nopool', 'column2', toUInt64(1))", + "hello", + ) + assert_eq_with_retry( + node1, + "select dictGetString('postgres_odbc_nopool', 'column2', toUInt64(2))", + "world", + ) + + # No open connections should be left because we don't use connection pooling. + assert "" == node1.exec_in_container( + ["ss", "-H", "dport", "5432"], privileged=True, user="root" + ) + + def test_postgres_insert(started_cluster): skip_test_msan(node1) From b9097d2db23c8e1ba7ea730d3d6a88e281d06084 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 02:00:39 +0100 Subject: [PATCH 087/439] fix --- .../00002_log_and_exception_messages_formatting.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index 7ea531569a7..ca48813f54e 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -48,7 +48,11 @@ select 120, count() < 3 from (select count() / (select count() from logs) as fre select 130, count() < 10 from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.05); -- Each message matches its pattern (returns 0 rows) -select 140, message_format_string, any(message) from logs where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) - and message not like ('Code: %Exception: '||s||'%') group by message_format_string; +-- FIXME maybe we should make it stricter ('Code:%Exception: '||s||'%'), but it's not easy because of addMessage +select 140, message_format_string, any_message from ( + select message_format_string, any(message) as any_message from logs + where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) + and message not like ('%Exception: '||s||'%') group by message_format_string +) where any_message not like '%Poco::Exception%'; drop table logs; From e4d9d4c2dd2e252ba347df1c8e83b1af34d06745 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 25 Jan 2023 12:24:38 +0100 Subject: [PATCH 088/439] Editing trash, part 1 --- CHANGELOG.md | 224 ++++++++++++--------------------------------------- 1 file changed, 50 insertions(+), 174 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38862b72b0f..041f071c9ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,62 +10,65 @@ ### ClickHouse release master (2f1092e6d24) FIXME as compared to v22.12.1.1752-stable (688e488e930) -#### Backward Incompatible Change -* Remove query `SYSTEM RESTART DISK`. [#44647](https://github.com/ClickHouse/ClickHouse/pull/44647) ([alesapin](https://github.com/alesapin)). -* Disallow Gorilla compression on columns of non-Float32 or non-Float64 type. [#45252](https://github.com/ClickHouse/ClickHouse/pull/45252) ([Robert Schulze](https://github.com/rschu1ze)). -* Remove PREALLOCATE for HASHED/SPARSE_HASHED dictionaries. [#45388](https://github.com/ClickHouse/ClickHouse/pull/45388) ([Azat Khuzhin](https://github.com/azat)). -* Parallel quorum inserts might work incorrectly with `*MergeTree` tables created with deprecated syntax. Therefore, parallel quorum inserts support is completely disabled for such tables. It does not affect tables created with a new syntax. [#45430](https://github.com/ClickHouse/ClickHouse/pull/45430) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### Upgrade Notes +* The `SYSTEM RESTART DISK` query becomes a no-op. [#44647](https://github.com/ClickHouse/ClickHouse/pull/44647) ([alesapin](https://github.com/alesapin)). +* The `PREALLOCATE` option for `HASHED`/`SPARSE_HASHED` dictionaries becomes a no-op. [#45388](https://github.com/ClickHouse/ClickHouse/pull/45388) ([Azat Khuzhin](https://github.com/azat)). It does not give significant advantages anymore. +* Disallow `Gorilla` codec on columns of non-Float32 or non-Float64 type. [#45252](https://github.com/ClickHouse/ClickHouse/pull/45252) ([Robert Schulze](https://github.com/rschu1ze)). It was pointless and led to inconsistencies. +* Parallel quorum inserts might work incorrectly with `*MergeTree` tables created with the deprecated syntax. Therefore, parallel quorum inserts support is completely disabled for such tables. It does not affect tables created with a new syntax. [#45430](https://github.com/ClickHouse/ClickHouse/pull/45430) ([Alexander Tokmakov](https://github.com/tavplubix)). #### New Feature -* Add `quantileInterpolatedWeighted`/`quantilesInterpolatedWeighted` functions. [#38252](https://github.com/ClickHouse/ClickHouse/pull/38252) ([Bharat Nallan](https://github.com/bharatnc)). -* Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)). -* Add column `ptr` to `system.trace_log` for `trace_type = 'MemorySample'`. This column contains an address of allocation. Added function `flameGraph` which can build flamegraph containing allocated and not released memory. Reworking of [#38391](https://github.com/ClickHouse/ClickHouse/issues/38391). [#38953](https://github.com/ClickHouse/ClickHouse/pull/38953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Dictionary source for extracting keys by traversing regular expressions tree. [#40878](https://github.com/ClickHouse/ClickHouse/pull/40878) ([Vage Ogannisian](https://github.com/nooblose)). +* Dictionary source for extracting keys by traversing regular expressions tree. It can be used for User-Agent parsing. [#40878](https://github.com/ClickHouse/ClickHouse/pull/40878) ([Vage Ogannisian](https://github.com/nooblose)). [#43858](https://github.com/ClickHouse/ClickHouse/pull/43858) ([Han Fei](https://github.com/hanfei1991)). * Added parametrized view functionality, now it's possible to specify query parameters for View table engine. resolves [#40907](https://github.com/ClickHouse/ClickHouse/issues/40907). [#41687](https://github.com/ClickHouse/ClickHouse/pull/41687) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* added extendable and configurable scheduling subsystem for IO requests (not yet integrated with IO code itself). [#41840](https://github.com/ClickHouse/ClickHouse/pull/41840) ([Sergei Trifonov](https://github.com/serxa)). -* Added `SYSTEM DROP DATABASE REPLICA` that removes metadata of dead replica of `Replicated` database. Resolves [#41794](https://github.com/ClickHouse/ClickHouse/issues/41794). [#42807](https://github.com/ClickHouse/ClickHouse/pull/42807) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Array join support map type, like function explode in spark. [#43239](https://github.com/ClickHouse/ClickHouse/pull/43239) ([李扬](https://github.com/taiyang-li)). +* Add `quantileInterpolatedWeighted`/`quantilesInterpolatedWeighted` functions. [#38252](https://github.com/ClickHouse/ClickHouse/pull/38252) ([Bharat Nallan](https://github.com/bharatnc)). +* Add column `ptr` to `system.trace_log` for `trace_type = 'MemorySample'`. This column contains an address of allocation. Added function `flameGraph` which can build flamegraph containing allocated and not released memory. Reworking of [#38391](https://github.com/ClickHouse/ClickHouse/issues/38391). [#38953](https://github.com/ClickHouse/ClickHouse/pull/38953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Array join support Map type, like function explode in Spark. [#43239](https://github.com/ClickHouse/ClickHouse/pull/43239) ([李扬](https://github.com/taiyang-li)). * Support SQL standard binary and hex string literals. [#43785](https://github.com/ClickHouse/ClickHouse/pull/43785) ([Mo Xuan](https://github.com/mo-avatar)). -* Add experimental query result cache. [#43797](https://github.com/ClickHouse/ClickHouse/pull/43797) ([Robert Schulze](https://github.com/rschu1ze)). -* format datetime in joda datetime style. Refer to https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html. [#43818](https://github.com/ClickHouse/ClickHouse/pull/43818) ([李扬](https://github.com/taiyang-li)). -* to merge [#40878](https://github.com/ClickHouse/ClickHouse/issues/40878) , supporting regexp dictionary. [#43858](https://github.com/ClickHouse/ClickHouse/pull/43858) ([Han Fei](https://github.com/hanfei1991)). -* Implemented a fractional second formatter (`%f`) for formatDateTime. [#44060](https://github.com/ClickHouse/ClickHouse/pull/44060) ([ltrk2](https://github.com/ltrk2)). +* Allow to format DateTime in Joda-Time style. Refer to [the Joda-Time docs](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html). [#43818](https://github.com/ClickHouse/ClickHouse/pull/43818) ([李扬](https://github.com/taiyang-li)). +* Implemented a fractional second formatter (`%f`) for `formatDateTime`. [#44060](https://github.com/ClickHouse/ClickHouse/pull/44060) ([ltrk2](https://github.com/ltrk2)). [#44497](https://github.com/ClickHouse/ClickHouse/pull/44497) ([Alexander Gololobov](https://github.com/davenger)). * Added age function to calculate difference between two dates or dates with time values expressed as number of full units. Closes [#41115](https://github.com/ClickHouse/ClickHouse/issues/41115). [#44421](https://github.com/ClickHouse/ClickHouse/pull/44421) ([Robert Schulze](https://github.com/rschu1ze)). -* Implemented a fractional second formatter (%f) for formatDateTime. This is slightly modified PR [#44060](https://github.com/ClickHouse/ClickHouse/issues/44060) by @ltrk2. [#44497](https://github.com/ClickHouse/ClickHouse/pull/44497) ([Alexander Gololobov](https://github.com/davenger)). -* Add null source for dictionaries. Closes [#44240](https://github.com/ClickHouse/ClickHouse/issues/44240). [#44502](https://github.com/ClickHouse/ClickHouse/pull/44502) ([mayamika](https://github.com/mayamika)). -* We can use `s3_storage_class` to set different tier. Such as ``` s3 xxx xxx xxx STANDARD/INTELLIGENT_TIERING ``` Closes [#44443](https://github.com/ClickHouse/ClickHouse/issues/44443). [#44707](https://github.com/ClickHouse/ClickHouse/pull/44707) ([chen](https://github.com/xiedeyantu)). +* Add `Null` source for dictionaries. Closes [#44240](https://github.com/ClickHouse/ClickHouse/issues/44240). [#44502](https://github.com/ClickHouse/ClickHouse/pull/44502) ([mayamika](https://github.com/mayamika)). +* We can use `s3_storage_class` to set different tiers. Such as `STANDARD/INTELLIGENT_TIERING` Closes [#44443](https://github.com/ClickHouse/ClickHouse/issues/44443). [#44707](https://github.com/ClickHouse/ClickHouse/pull/44707) ([chen](https://github.com/xiedeyantu)). * Insert default values in case of missing elements in JSON object while parsing named tuple. Add setting `input_format_json_defaults_for_missing_elements_in_named_tuple` that controls this behaviour. Closes [#45142](https://github.com/ClickHouse/ClickHouse/issues/45142)#issuecomment-1380153217. [#45231](https://github.com/ClickHouse/ClickHouse/pull/45231) ([Kruglov Pavel](https://github.com/Avogar)). +* Record server startup time in ProfileEvents resolves [#43188](https://github.com/ClickHouse/ClickHouse/issues/43188) Implementation: * Added ProfileEvents::ServerStartupMilliseconds. * Recorded time from start of main till listening to sockets. Testing: * Added a test 02532_profileevents_server_startup_time.sql. [#45250](https://github.com/ClickHouse/ClickHouse/pull/45250) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). + +#### Experimental Feature +* Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)). +* Add experimental query result cache. [#43797](https://github.com/ClickHouse/ClickHouse/pull/43797) ([Robert Schulze](https://github.com/rschu1ze)). +* Added extendable and configurable scheduling subsystem for IO requests (not yet integrated with IO code itself). [#41840](https://github.com/ClickHouse/ClickHouse/pull/41840) ([Sergei Trifonov](https://github.com/serxa)). +* Added `SYSTEM DROP DATABASE REPLICA` that removes metadata of dead replica of `Replicated` database. Resolves [#41794](https://github.com/ClickHouse/ClickHouse/issues/41794). [#42807](https://github.com/ClickHouse/ClickHouse/pull/42807) ([Alexander Tokmakov](https://github.com/tavplubix)). #### Performance Improvement -* Added sharding support in HashedDictionary to allow parallel load (almost linear scaling based on number of shards). [#40003](https://github.com/ClickHouse/ClickHouse/pull/40003) ([Azat Khuzhin](https://github.com/azat)). * Do not load inactive parts at startup of `MergeTree` tables. [#42181](https://github.com/ClickHouse/ClickHouse/pull/42181) ([Anton Popov](https://github.com/CurtizJ)). +* Improved latency of reading from storage `S3` and table function `s3` with large number of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)). +* Added mmap support for StorageFile, which should improve the performance of clickhouse-local. [#43927](https://github.com/ClickHouse/ClickHouse/pull/43927) ([pufit](https://github.com/pufit)). +* Added sharding support in HashedDictionary to allow parallel load (almost linear scaling based on number of shards). [#40003](https://github.com/ClickHouse/ClickHouse/pull/40003) ([Azat Khuzhin](https://github.com/azat)). * - Speed up query parsing. [#42284](https://github.com/ClickHouse/ClickHouse/pull/42284) ([Raúl Marín](https://github.com/Algunenano)). * Always replace OR chain `expr = x1 OR ... OR expr = xN` to `expr IN (x1, ..., xN)` in case if `expr` is a `LowCardinality` column. Setting `optimize_min_equality_disjunction_chain_length` is ignored in this case. [#42889](https://github.com/ClickHouse/ClickHouse/pull/42889) ([Guo Wangyang](https://github.com/guowangy)). * > Original changelog In the original implementation, the memory of ThreadGroupStatus:: finished_threads_counters_memory is released by moving it to a temporary std::vector, which soon expired and gets destructed. This method is viable, however not straightforward enough. To enhance the code readability, this commit releases the memory in the vector by firstly resizing it to 0 and then shrinking the capacity accordingly. [#43586](https://github.com/ClickHouse/ClickHouse/pull/43586) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). * As a follow-up of [#42214](https://github.com/ClickHouse/ClickHouse/issues/42214), this PR tries to optimize the column-wise ternary logic evaluation by achieving auto-vectorization. In the performance test of this [microbenchmark](https://github.com/ZhiguoZh/ClickHouse/blob/20221123-ternary-logic-opt-example/src/Functions/examples/associative_applier_perf.cpp), we've observed a peak **performance gain** of **21x** on the ICX device (Intel Xeon Platinum 8380 CPU). [#43669](https://github.com/ClickHouse/ClickHouse/pull/43669) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). -* Improved latency of reading from storage `S3` and table function `s3` with large number of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)). -* - Avoid acquiring read locks in system.tables if possible. [#43840](https://github.com/ClickHouse/ClickHouse/pull/43840) ([Raúl Marín](https://github.com/Algunenano)). +* Avoid acquiring read locks in the `system.tables` table if possible. [#43840](https://github.com/ClickHouse/ClickHouse/pull/43840) ([Raúl Marín](https://github.com/Algunenano)). * The performance experiments of SSB (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could effectively decrease the lock contention for ThreadPoolImpl::mutex by **75%**, increasing the CPU utilization and improving the overall performance by **2.4%**. [#44308](https://github.com/ClickHouse/ClickHouse/pull/44308) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). -* Now optimisation is applied only if the cached HT size is sufficiently large (thresholds were determined empirically and hardcoded). [#44455](https://github.com/ClickHouse/ClickHouse/pull/44455) ([Nikita Taranov](https://github.com/nickitat)). -* ... The whole struct field will be loaded at current, even though we just want to read one field of the struct. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)). -* Small performance improvement for asynchronous reading from remote fs. [#44868](https://github.com/ClickHouse/ClickHouse/pull/44868) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add fast path for: - col like '%%' - col like '%' - col not like '%' - col not like '%' - match(col, '.*'). [#45244](https://github.com/ClickHouse/ClickHouse/pull/45244) ([李扬](https://github.com/taiyang-li)). +* Now optimisation for predicting the hash table size is applied only if the cached hash table size is sufficiently large (thresholds were determined empirically and hardcoded). [#44455](https://github.com/ClickHouse/ClickHouse/pull/44455) ([Nikita Taranov](https://github.com/nickitat)). +* TODO: edit this ... The whole struct field will be loaded at current, even though we just want to read one field of the struct. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)). +* Small performance improvement for asynchronous reading from remote filesystems. [#44868](https://github.com/ClickHouse/ClickHouse/pull/44868) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add fast path for: - `col like '%%'`; - `col like '%'`; - `col not like '%'`; - `col not like '%'`; - `match(col, '.*')`. [#45244](https://github.com/ClickHouse/ClickHouse/pull/45244) ([李扬](https://github.com/taiyang-li)). * todo. [#45289](https://github.com/ClickHouse/ClickHouse/pull/45289) ([Nikita Taranov](https://github.com/nickitat)). +* Provide monotonicity info for `toUnixTimestamp64*` [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)). #### Improvement * Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted complitely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)). +* Temporary files evict fs cache - 2nd approach [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). * Support `optimize_or_like_chain` in the new infrastructure. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42797](https://github.com/ClickHouse/ClickHouse/pull/42797) ([Dmitry Novik](https://github.com/novikd)). * Improve the Asterisk and ColumnMatcher parsers. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42884](https://github.com/ClickHouse/ClickHouse/pull/42884) ([Nikolay Degterinsky](https://github.com/evillique)). * Implement `optimize_redundant_functions_in_order_by` on top of QueryTree. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42970](https://github.com/ClickHouse/ClickHouse/pull/42970) ([Dmitry Novik](https://github.com/novikd)). * Support `optimize_group_by_function_keys` in the new analyzer architecture. Also, add support for optimizing GROUPING SETS keys. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43261](https://github.com/ClickHouse/ClickHouse/pull/43261) ([Dmitry Novik](https://github.com/novikd)). * Improve reading CSV field in CustomSeparated/Template format. Closes [#42352](https://github.com/ClickHouse/ClickHouse/issues/42352) Closes [#39620](https://github.com/ClickHouse/ClickHouse/issues/39620). [#43332](https://github.com/ClickHouse/ClickHouse/pull/43332) ([Kruglov Pavel](https://github.com/Avogar)). * Support reading/writing `Nested` tables as `List` of `Struct` in CapnProto format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)). -* - Unify query elapsed time measurements. [#43455](https://github.com/ClickHouse/ClickHouse/pull/43455) ([Raúl Marín](https://github.com/Algunenano)). +* Unify query elapsed time measurements. [#43455](https://github.com/ClickHouse/ClickHouse/pull/43455) ([Raúl Marín](https://github.com/Algunenano)). * Support scalar subqueries cache Implementation: * Added a map with hash of the node (without alias) and the evaluated value to Context. Testing: * Added a test-case with new analyser in 02174_cte_scalar_cache.sql. [#43640](https://github.com/ClickHouse/ClickHouse/pull/43640) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). * Improve automatic usage of structure from insertion table in table functions file/hdfs/s3 when virtual columns present in select query, it fixes possible error `Block structure mismatch` or `number of columns mismatch`. [#43695](https://github.com/ClickHouse/ClickHouse/pull/43695) ([Kruglov Pavel](https://github.com/Avogar)). * Add support for signed arguments in range(). Fixes [#43333](https://github.com/ClickHouse/ClickHouse/issues/43333). [#43733](https://github.com/ClickHouse/ClickHouse/pull/43733) ([sanyu](https://github.com/wineternity)). * Remove redundant sorting, for example, sorting related ORDER BY clauses in subqueries. Implemented on top of query plan. It does similar optimization as `optimize_duplicate_order_by_and_distinct` regarding `ORDER BY` clauses, but more generic, since it's applied to any redundant sorting steps (not only caused by ORDER BY clause) and applied to subqueries of any depth. Related to [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43905](https://github.com/ClickHouse/ClickHouse/pull/43905) ([Igor Nikonov](https://github.com/devcrafter)). -* Added mmap support for StorageFile, which should improve the performance of clickhouse-local. [#43927](https://github.com/ClickHouse/ClickHouse/pull/43927) ([pufit](https://github.com/pufit)). * Add ability to disable deduplication for BACKUP (for backups wiithout deduplication ATTACH can be used instead of full RESTORE), example `BACKUP foo TO S3(...) SETTINGS deduplicate_files=0` (default `deduplicate_files=1`). [#43947](https://github.com/ClickHouse/ClickHouse/pull/43947) ([Azat Khuzhin](https://github.com/azat)). * Make `system.replicas` table do parallel fetches of replicas statuses. Closes [#43918](https://github.com/ClickHouse/ClickHouse/issues/43918). [#43998](https://github.com/ClickHouse/ClickHouse/pull/43998) ([Nikolay Degterinsky](https://github.com/evillique)). * Refactor and improve schema inference for text formats. Add new setting `schema_inference_make_columns_nullable` that controls making result types `Nullable` (enabled by default);. [#44019](https://github.com/ClickHouse/ClickHouse/pull/44019) ([Kruglov Pavel](https://github.com/Avogar)). @@ -85,17 +88,16 @@ * Enable input_format_json_read_objects_as_strings by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)). * When users do duplicate async inserts, we should dedup inside the memory before we query keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)). * Input/ouptut Avro bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)). -* - Don't parse beyond the quotes when reading UUIDs. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)). +* Don't parse beyond the quotes when reading UUIDs. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)). * Infer UInt64 in case of Int64 overflow and fix some transforms in schema inference. [#44696](https://github.com/ClickHouse/ClickHouse/pull/44696) ([Kruglov Pavel](https://github.com/Avogar)). * Previously dependency resolving inside DatabaseReplicated was done in a hacky way and now it done right using an explicit graph. [#44697](https://github.com/ClickHouse/ClickHouse/pull/44697) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Support Bool type in Arrow/Parquet/ORC. Closes [#43970](https://github.com/ClickHouse/ClickHouse/issues/43970). [#44698](https://github.com/ClickHouse/ClickHouse/pull/44698) ([Kruglov Pavel](https://github.com/Avogar)). * Fix `output_format_pretty_row_numbers` does not preserve the counter across the blocks. Closes [#44815](https://github.com/ClickHouse/ClickHouse/issues/44815). [#44832](https://github.com/ClickHouse/ClickHouse/pull/44832) ([flynn](https://github.com/ucasfl)). * Extend function "toDayOfWeek" with a mode argument describing if a) the week starts on Monday or Sunday and b) if counting starts at 0 or 1. [#44860](https://github.com/ClickHouse/ClickHouse/pull/44860) ([李扬](https://github.com/taiyang-li)). -* - Don't report errors in system.errors due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)). +* Don't report errors in system.errors due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)). * Optimize and fix metrics for Distributed async INSERT. [#44922](https://github.com/ClickHouse/ClickHouse/pull/44922) ([Azat Khuzhin](https://github.com/azat)). * Added settings to disallow concurrent backups and restores resolves [#43891](https://github.com/ClickHouse/ClickHouse/issues/43891) Implementation: * Added server level settings to disallow concurrent backups and restores, which are read and set when BackupWorker is created in Context. * Settings are set to true by default. * Before starting backup or restores, added a check to see if any other backups/restores are running. For internal request it checks if its from the self node using backup_uuid. [#45072](https://github.com/ClickHouse/ClickHouse/pull/45072) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* add a cache for async block ids. This will reduce the requests of zookeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)). -* CRC32 changes to address the WeakHash collision issue in PowerPC. [#45144](https://github.com/ClickHouse/ClickHouse/pull/45144) ([MeenaRenganathan22](https://github.com/MeenaRenganathan22)). +* Add a cache for async block ids. This will reduce the requests of zookeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)). * Optimize memory consumption during backup to S3: files to S3 now will be copied directly without using `WriteBufferFromS3` (which could use a lot of memory). [#45188](https://github.com/ClickHouse/ClickHouse/pull/45188) ([Vitaly Baranov](https://github.com/vitlibar)). * Use structure from insertion table in generateRandom without arguments. [#45239](https://github.com/ClickHouse/ClickHouse/pull/45239) ([Kruglov Pavel](https://github.com/Avogar)). * Use `GetObjectAttributes` request instead of `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit region, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). @@ -103,41 +105,33 @@ * Allow to implicitly convert floats stored in string fields of JSON to integers in `JSONExtract` functions. E.g. `JSONExtract('{"a": "1000.111"}', 'a', 'UInt64')` -> `1000`, previously it returned 0. [#45432](https://github.com/ClickHouse/ClickHouse/pull/45432) ([Anton Popov](https://github.com/CurtizJ)). * Added fields `supports_parallel_parsing` and `supports_parallel_formatting` to table `system.formats` for better introspection. [#45499](https://github.com/ClickHouse/ClickHouse/pull/45499) ([Anton Popov](https://github.com/CurtizJ)). -#### Bug Fix -* Fix HTTP requests without path for AWS. After updating AWS SDK the sdk no longer adds a slash to requesting paths so we need to do it in our PocoHTTPClient to keep HTTP requests correct. [#45238](https://github.com/ClickHouse/ClickHouse/pull/45238) ([Vitaly Baranov](https://github.com/vitlibar)). -* Fix backup if mutations get killed during the backup process. [#45351](https://github.com/ClickHouse/ClickHouse/pull/45351) ([Vitaly Baranov](https://github.com/vitlibar)). - #### Build/Testing/Packaging Improvement * Builtin skim for fuzzy search in clickhouse client/local history. [#44239](https://github.com/ClickHouse/ClickHouse/pull/44239) ([Azat Khuzhin](https://github.com/azat)). -* Memory limit for server is set now in AST fuzz tests to avoid OOMs. [#44282](https://github.com/ClickHouse/ClickHouse/pull/44282) ([Nikita Taranov](https://github.com/nickitat)). -* In rare cases, we don't rebuild binaries, because another task with a similar prefix succeeded. E.g. `binary_darwin` didn't restart because `binary_darwin_aarch64`. [#44311](https://github.com/ClickHouse/ClickHouse/pull/44311) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* The "universal.sh" now fetches a SSE2 build on systems which don't have SSE4.2. [#44366](https://github.com/ClickHouse/ClickHouse/pull/44366) ([Robert Schulze](https://github.com/rschu1ze)). -* Retry the integration tests on compressing errors. [#44529](https://github.com/ClickHouse/ClickHouse/pull/44529) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* ... 1. Added pytest-random by default in integration tests runner 2. Disable TSAN checks for tests with GPRC ( like https://s3.amazonaws.com/clickhouse-test-reports/42807/e9d7407a58f6e3f7d88c0c534685704f23560704/integration_tests__tsan__[4/6].html ) 3. Cleanup tables after tests in odbc. [#44711](https://github.com/ClickHouse/ClickHouse/pull/44711) ([Ilya Yatsishin](https://github.com/qoega)). * We removed support for shared linking because of Rust. Actually, Rust is only an excuse for this removal, and we wanted to remove it nevertheless. [#44828](https://github.com/ClickHouse/ClickHouse/pull/44828) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Checks will try to download images before running integration tests. If image, proxy or whatever is broken in infrastructure it will not make tests flaky. Images will be cached locally and download time will not be added to random tests. Compose images are now changed to be used without correct environment from helpers/cluster.py. [#44848](https://github.com/ClickHouse/ClickHouse/pull/44848) ([Ilya Yatsishin](https://github.com/qoega)). -* Fix zookeeper downloading, update the version, and optimize the image size. [#44853](https://github.com/ClickHouse/ClickHouse/pull/44853) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* The performance tests were silently broken because `Errors` wasn't detected in the status message. [#44867](https://github.com/ClickHouse/ClickHouse/pull/44867) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Remove the dependency on the `adduser` tool from the packages, because we don't use it. This fixes [#44934](https://github.com/ClickHouse/ClickHouse/issues/44934). [#45011](https://github.com/ClickHouse/ClickHouse/pull/45011) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* SQLite library is updated to the latest. It is used for the SQLite database and table integration engines. Also, fixed a false-positive TSan report. This closes [#45027](https://github.com/ClickHouse/ClickHouse/issues/45027). [#45031](https://github.com/ClickHouse/ClickHouse/pull/45031) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The `SQLite` library is updated to the latest. It is used for the SQLite database and table integration engines. Also, fixed a false-positive TSan report. This closes [#45027](https://github.com/ClickHouse/ClickHouse/issues/45027). [#45031](https://github.com/ClickHouse/ClickHouse/pull/45031) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CRC32 changes to address the WeakHash collision issue in PowerPC. [#45144](https://github.com/ClickHouse/ClickHouse/pull/45144) ([MeenaRenganathan22](https://github.com/MeenaRenganathan22)). +* Update aws-c* submodules [#43020](https://github.com/ClickHouse/ClickHouse/pull/43020) ([Vitaly Baranov](https://github.com/vitlibar)). +* Replace domain IP types (IPv4, IPv6) with native [#43221](https://github.com/ClickHouse/ClickHouse/pull/43221) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Introduce a [website](https://aretestsgreenyet.com/) for the status of ClickHouse CI. [Source](https://github.com/ClickHouse/aretestsgreenyet). -#### Bug Fix (user-visible misbehavior in official stable or prestable release) +#### Bug Fix +* Fix backup if mutations get killed during the backup process. [#45351](https://github.com/ClickHouse/ClickHouse/pull/45351) ([Vitaly Baranov](https://github.com/vitlibar)). * #40651 [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)). -* Fix possible use-of-unitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) CC: @nickitat. [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible use-of-unitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)). * Better handling of NULL in aggregate combinators, fix possible segfault/logical error while using optimization `optimize_rewrite_sum_if_to_count_if`. Closes [#43758](https://github.com/ClickHouse/ClickHouse/issues/43758). [#43813](https://github.com/ClickHouse/ClickHouse/pull/43813) ([Kruglov Pavel](https://github.com/Avogar)). * Fix CREATE USER/ROLE query settings constraints. [#43993](https://github.com/ClickHouse/ClickHouse/pull/43993) ([Nikolay Degterinsky](https://github.com/evillique)). -* * Fix wrong behavior of `JOIN ON t1.x = t2.x AND 1 = 1`, forbid such queries. [#44016](https://github.com/ClickHouse/ClickHouse/pull/44016) ([Vladimir C](https://github.com/vdimir)). -* Fixed bug with non-parsable default value for EPHEMERAL column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix wrong behavior of `JOIN ON t1.x = t2.x AND 1 = 1`, forbid such queries. [#44016](https://github.com/ClickHouse/ClickHouse/pull/44016) ([Vladimir C](https://github.com/vdimir)). +* Fixed bug with non-parsable default value for `EPHEMERAL` column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)). * Bring interval subtraction from datetime in line with addition. [#44241](https://github.com/ClickHouse/ClickHouse/pull/44241) ([ltrk2](https://github.com/ltrk2)). -* Fix double-free in HashTable::clearAndShrink() with zero elements in it. [#44256](https://github.com/ClickHouse/ClickHouse/pull/44256) ([Azat Khuzhin](https://github.com/azat)). * Remove limits on maximum size of the result for view. [#44261](https://github.com/ClickHouse/ClickHouse/pull/44261) ([lizhuoyu5](https://github.com/lzydmxy)). * Fix possible logical error in cache if `do_not_evict_index_and_mrk_files=1`. Closes [#42142](https://github.com/ClickHouse/ClickHouse/issues/42142). [#44268](https://github.com/ClickHouse/ClickHouse/pull/44268) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix possible too early cache write interruption in write-through cache (caching could be stopped due to false assumption when it shouldn't have). [#44289](https://github.com/ClickHouse/ClickHouse/pull/44289) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix possible crash in case function `IN` with constant arguments was used as a constant argument together with `LowCardinality`. Fixes [#44221](https://github.com/ClickHouse/ClickHouse/issues/44221). [#44346](https://github.com/ClickHouse/ClickHouse/pull/44346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix support for complex parameters (like arrays) of parametric aggregate functions. This closes [#30975](https://github.com/ClickHouse/ClickHouse/issues/30975). The aggregate function `sumMapFiltered` was unusable in distributed queries before this change. [#44358](https://github.com/ClickHouse/ClickHouse/pull/44358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* * Fix possible nullptr deference in JoinSwitcher with `allow_experimental_analyzer`. [#44371](https://github.com/ClickHouse/ClickHouse/pull/44371) ([Vladimir C](https://github.com/vdimir)). * Fix reading ObjectId in BSON schema inference. [#44382](https://github.com/ClickHouse/ClickHouse/pull/44382) ([Kruglov Pavel](https://github.com/Avogar)). * Fix race which can lead to premature temp parts removal before merge finished in ReplicatedMergeTree. This issue could lead to errors like `No such file or directory: xxx`. Fixes [#43983](https://github.com/ClickHouse/ClickHouse/issues/43983). [#44383](https://github.com/ClickHouse/ClickHouse/pull/44383) ([alesapin](https://github.com/alesapin)). * Some invalid `SYSTEM ... ON CLUSTER` queries worked in an unexpected way if a cluster name was not specified. It's fixed, now invalid queries throw `SYNTAX_ERROR` as they should. Fixes [#44264](https://github.com/ClickHouse/ClickHouse/issues/44264). [#44387](https://github.com/ClickHouse/ClickHouse/pull/44387) ([Alexander Tokmakov](https://github.com/tavplubix)). @@ -151,13 +145,12 @@ * Fix mutations with setting `max_streams_for_merge_tree_reading`. [#44472](https://github.com/ClickHouse/ClickHouse/pull/44472) ([Anton Popov](https://github.com/CurtizJ)). * Fix potential null pointer dereference with GROUPING SETS in ASTSelectQuery::formatImpl ([#43049](https://github.com/ClickHouse/ClickHouse/issues/43049)). [#44479](https://github.com/ClickHouse/ClickHouse/pull/44479) ([Robert Schulze](https://github.com/rschu1ze)). * Validate types in table function arguments, CAST function arguments, JSONAsObject schema inference according to settings. [#44501](https://github.com/ClickHouse/ClickHouse/pull/44501) ([Kruglov Pavel](https://github.com/Avogar)). -* - Fix IN function with LC and const column, close [#44503](https://github.com/ClickHouse/ClickHouse/issues/44503). [#44506](https://github.com/ClickHouse/ClickHouse/pull/44506) ([Duc Canh Le](https://github.com/canhld94)). +* Fix IN function with LowCardinality and const column, close [#44503](https://github.com/ClickHouse/ClickHouse/issues/44503). [#44506](https://github.com/ClickHouse/ClickHouse/pull/44506) ([Duc Canh Le](https://github.com/canhld94)). * Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)). * Projections do not work in presence of WITH ROLLUP, WITH CUBE and WITH TOTALS. In previous versions, a query produced an exception instead of skipping the usage of projections. This closes [#44614](https://github.com/ClickHouse/ClickHouse/issues/44614). This closes [#42772](https://github.com/ClickHouse/ClickHouse/issues/42772). [#44615](https://github.com/ClickHouse/ClickHouse/pull/44615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* * Fix bug in experimental analyzer and `aggregate_functions_null_for_empty = 1`. Close [#44644](https://github.com/ClickHouse/ClickHouse/issues/44644). [#44648](https://github.com/ClickHouse/ClickHouse/pull/44648) ([Vladimir C](https://github.com/vdimir)). -* async blocks are not cleaned because the function `get all blocks sorted by time` didn't get async blocks. [#44651](https://github.com/ClickHouse/ClickHouse/pull/44651) ([Han Fei](https://github.com/hanfei1991)). +* Async blocks are not cleaned because the function `get all blocks sorted by time` didn't get async blocks. [#44651](https://github.com/ClickHouse/ClickHouse/pull/44651) ([Han Fei](https://github.com/hanfei1991)). * Fix `LOGICAL_ERROR` `The top step of the right pipeline should be ExpressionStep` for JOIN with subquery, UNION, and TOTALS. Fixes [#43687](https://github.com/ClickHouse/ClickHouse/issues/43687). [#44673](https://github.com/ClickHouse/ClickHouse/pull/44673) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Avoid std::out_of_range exception in StorageExecutable. [#44681](https://github.com/ClickHouse/ClickHouse/pull/44681) ([Kruglov Pavel](https://github.com/Avogar)). +* Avoid `std::out_of_range` exception in StorageExecutable. [#44681](https://github.com/ClickHouse/ClickHouse/pull/44681) ([Kruglov Pavel](https://github.com/Avogar)). * Do not apply `optimize_syntax_fuse_functions` to quantiles on AST, close [#44712](https://github.com/ClickHouse/ClickHouse/issues/44712). [#44713](https://github.com/ClickHouse/ClickHouse/pull/44713) ([Vladimir C](https://github.com/vdimir)). * Fix bug with wrong type in Merge table and PREWHERE, close [#43324](https://github.com/ClickHouse/ClickHouse/issues/43324). [#44716](https://github.com/ClickHouse/ClickHouse/pull/44716) ([Vladimir C](https://github.com/vdimir)). * Fix possible crash during shutdown (while destroying TraceCollector). Fixes [#44757](https://github.com/ClickHouse/ClickHouse/issues/44757). [#44758](https://github.com/ClickHouse/ClickHouse/pull/44758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). @@ -179,7 +172,7 @@ * Fix `RANGE_HASHED` dictionary to count range columns as part of primary key during updates when `update_field` is specified. Closes [#44588](https://github.com/ClickHouse/ClickHouse/issues/44588). [#45061](https://github.com/ClickHouse/ClickHouse/pull/45061) ([Maksim Kita](https://github.com/kitaisreal)). * Fix error `Cannot capture column` for `LowCardinality` captured argument of nested labmda. Fixes [#45028](https://github.com/ClickHouse/ClickHouse/issues/45028). [#45065](https://github.com/ClickHouse/ClickHouse/pull/45065) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix the wrong query result of `additional_table_filters` (additional filter was not applied) in case if minmax/count projection is used. [#45133](https://github.com/ClickHouse/ClickHouse/pull/45133) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* - Fixed bug in `histogram` function accepting negative values. [#45147](https://github.com/ClickHouse/ClickHouse/pull/45147) ([simpleton](https://github.com/rgzntrade)). +* Fixed bug in `histogram` function accepting negative values. [#45147](https://github.com/ClickHouse/ClickHouse/pull/45147) ([simpleton](https://github.com/rgzntrade)). * Fix wrong column nullability in StoreageJoin, close [#44940](https://github.com/ClickHouse/ClickHouse/issues/44940). [#45184](https://github.com/ClickHouse/ClickHouse/pull/45184) ([Vladimir C](https://github.com/vdimir)). * Fix `background_fetches_pool_size` settings reload (increase at runtime). [#45189](https://github.com/ClickHouse/ClickHouse/pull/45189) ([Raúl Marín](https://github.com/Algunenano)). * Correctly process `SELECT` queries on KV engines (e.g. KeeperMap, EmbeddedRocksDB) using `IN` on the key with subquery producing different type. [#45215](https://github.com/ClickHouse/ClickHouse/pull/45215) ([Antonio Andelic](https://github.com/antonio2368)). @@ -202,50 +195,19 @@ * Fix rare abort in case when query is canceled and parallel parsing was used during its execution. [#45498](https://github.com/ClickHouse/ClickHouse/pull/45498) ([Anton Popov](https://github.com/CurtizJ)). * Fix a race between Distributed table creation and INSERT into it (could lead to CANNOT_LINK during INSERT into the table). [#45502](https://github.com/ClickHouse/ClickHouse/pull/45502) ([Azat Khuzhin](https://github.com/azat)). * Add proper default (SLRU) to cache policy getter. Closes [#45514](https://github.com/ClickHouse/ClickHouse/issues/45514). [#45524](https://github.com/ClickHouse/ClickHouse/pull/45524) ([Kseniia Sumarokova](https://github.com/kssenii)). - -#### Bug-fix - -* Disallow arrayjoin in mutations closes [#42637](https://github.com/ClickHouse/ClickHouse/issues/42637) Implementation: * Added a new parameter to ActionsVisitor::Data disallow_arrayjoin, which is set by MutationsIterator when it appends expression. * ActionsVisitor uses disallow_arrayjoin and throws error when its used with mutations. Testing: * Added test for the same 02504_disallow_arrayjoin_in_mutations.sql. [#44447](https://github.com/ClickHouse/ClickHouse/pull/44447) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Disallow array join in mutations closes [#42637](https://github.com/ClickHouse/ClickHouse/issues/42637) [#44447](https://github.com/ClickHouse/ClickHouse/pull/44447) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). * Fix for qualified asterisks with alias table name and column transformer resolves [#44736](https://github.com/ClickHouse/ClickHouse/issues/44736). [#44755](https://github.com/ClickHouse/ClickHouse/pull/44755) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -#### Build Improvement - -* crc32 fix for s390x. [#43706](https://github.com/ClickHouse/ClickHouse/pull/43706) ([Suzy Wang](https://github.com/SuzyWangIBMer)). - -#### Feature - -* Record server startup time in ProfileEvents resolves [#43188](https://github.com/ClickHouse/ClickHouse/issues/43188) Implementation: * Added ProfileEvents::ServerStartupMilliseconds. * Recorded time from start of main till listening to sockets. Testing: * Added a test 02532_profileevents_server_startup_time.sql. [#45250](https://github.com/ClickHouse/ClickHouse/pull/45250) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). - -#### NO CL ENTRY - -* NO CL ENTRY: 'Revert "If user only need virtual columns, we don't need to initialize ReadBufferFromS3"'. [#44939](https://github.com/ClickHouse/ClickHouse/pull/44939) ([Anton Popov](https://github.com/CurtizJ)). -* NO CL ENTRY: 'Revert "Custom reading for mutation"'. [#45121](https://github.com/ClickHouse/ClickHouse/pull/45121) ([Alexander Tokmakov](https://github.com/tavplubix)). -* NO CL ENTRY: 'Revert "Revert "Custom reading for mutation""'. [#45122](https://github.com/ClickHouse/ClickHouse/pull/45122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* NO CL ENTRY: 'Revert "update function DAYOFWEEK and add new function WEEKDAY for mysql/spark compatiability"'. [#45221](https://github.com/ClickHouse/ClickHouse/pull/45221) ([Alexander Tokmakov](https://github.com/tavplubix)). -* NO CL ENTRY: 'Revert "Validate function arguments in query tree"'. [#45299](https://github.com/ClickHouse/ClickHouse/pull/45299) ([Maksim Kita](https://github.com/kitaisreal)). -* NO CL ENTRY: 'Revert "Revert "Validate function arguments in query tree""'. [#45300](https://github.com/ClickHouse/ClickHouse/pull/45300) ([Maksim Kita](https://github.com/kitaisreal)). -* NO CL ENTRY: 'Revert "Support optimize_or_like_chain in QueryTreePassManager"'. [#45406](https://github.com/ClickHouse/ClickHouse/pull/45406) ([Anton Popov](https://github.com/CurtizJ)). -* NO CL ENTRY: 'Resubmit Support optimize_or_like_chain in QueryTreePassManager'. [#45410](https://github.com/ClickHouse/ClickHouse/pull/45410) ([Dmitry Novik](https://github.com/novikd)). -* NO CL ENTRY: 'Revert "Remove redundant sorting"'. [#45414](https://github.com/ClickHouse/ClickHouse/pull/45414) ([Igor Nikonov](https://github.com/devcrafter)). - #### NOT FOR CHANGELOG / INSIGNIFICANT -* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Fix assertion in async read buffer from remote [#41231](https://github.com/ClickHouse/ClickHouse/pull/41231) ([Kseniia Sumarokova](https://github.com/kssenii)). * add retries on ConnectionError [#42991](https://github.com/ClickHouse/ClickHouse/pull/42991) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Update aws-c* submodules [#43020](https://github.com/ClickHouse/ClickHouse/pull/43020) ([Vitaly Baranov](https://github.com/vitlibar)). -* Replace domain IP types (IPv4, IPv6) with native [#43221](https://github.com/ClickHouse/ClickHouse/pull/43221) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). -* Fix aggregate functions optimisation in AggregateFunctionsArithmericOperationsPass [#43372](https://github.com/ClickHouse/ClickHouse/pull/43372) ([Dmitry Novik](https://github.com/novikd)). -* Improve pytest --pdb experience by preserving dockerd on SIGINT [#43392](https://github.com/ClickHouse/ClickHouse/pull/43392) ([Azat Khuzhin](https://github.com/azat)). -* Followup fixes for systemd notification ([#43400](https://github.com/ClickHouse/ClickHouse/issues/43400)) [#43597](https://github.com/ClickHouse/ClickHouse/pull/43597) ([Alexander Gololobov](https://github.com/davenger)). * Refactor FunctionNode [#43761](https://github.com/ClickHouse/ClickHouse/pull/43761) ([Dmitry Novik](https://github.com/novikd)). * Some cleanup: grace hash join [#43851](https://github.com/ClickHouse/ClickHouse/pull/43851) ([Igor Nikonov](https://github.com/devcrafter)). -* Temporary files evict fs cache - 2nd approach [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). * Randomize setting `enable_memory_bound_merging_of_aggregation_results` in tests [#43986](https://github.com/ClickHouse/ClickHouse/pull/43986) ([Nikita Taranov](https://github.com/nickitat)). * Analyzer aggregate functions passes small fixes [#44013](https://github.com/ClickHouse/ClickHouse/pull/44013) ([Maksim Kita](https://github.com/kitaisreal)). * Fix wrong char in command [#44018](https://github.com/ClickHouse/ClickHouse/pull/44018) ([alesapin](https://github.com/alesapin)). * Analyzer support Set index [#44097](https://github.com/ClickHouse/ClickHouse/pull/44097) ([Maksim Kita](https://github.com/kitaisreal)). -* Provide monotonicity info for `toUnixTimestamp64*` [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)). * Avoid loading toolchain files multiple times [#44122](https://github.com/ClickHouse/ClickHouse/pull/44122) ([Azat Khuzhin](https://github.com/azat)). * tests: exclude flaky columns from SHOW CLUSTERS test [#44123](https://github.com/ClickHouse/ClickHouse/pull/44123) ([Azat Khuzhin](https://github.com/azat)). * Bump libdivide (to gain some new optimizations) [#44132](https://github.com/ClickHouse/ClickHouse/pull/44132) ([Azat Khuzhin](https://github.com/azat)). @@ -271,84 +233,27 @@ * Replace old named collections code with new (from [#43147](https://github.com/ClickHouse/ClickHouse/issues/43147)) part 2 [#44327](https://github.com/ClickHouse/ClickHouse/pull/44327) ([Kseniia Sumarokova](https://github.com/kssenii)). * Disable "git-import" test in debug mode [#44328](https://github.com/ClickHouse/ClickHouse/pull/44328) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Check s3 part upload settings [#44335](https://github.com/ClickHouse/ClickHouse/pull/44335) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix typo [#44337](https://github.com/ClickHouse/ClickHouse/pull/44337) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for PowerBI [#44338](https://github.com/ClickHouse/ClickHouse/pull/44338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#36038](https://github.com/ClickHouse/ClickHouse/issues/36038) [#44339](https://github.com/ClickHouse/ClickHouse/pull/44339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#29386](https://github.com/ClickHouse/ClickHouse/issues/29386) [#44340](https://github.com/ClickHouse/ClickHouse/pull/44340) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#22929](https://github.com/ClickHouse/ClickHouse/issues/22929) [#44341](https://github.com/ClickHouse/ClickHouse/pull/44341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#29883](https://github.com/ClickHouse/ClickHouse/issues/29883) [#44342](https://github.com/ClickHouse/ClickHouse/pull/44342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix Docker [#44343](https://github.com/ClickHouse/ClickHouse/pull/44343) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* fix flack test "02481_async_insert_dedup.python" [#44349](https://github.com/ClickHouse/ClickHouse/pull/44349) ([Han Fei](https://github.com/hanfei1991)). -* Add a test for [#22160](https://github.com/ClickHouse/ClickHouse/issues/22160) [#44355](https://github.com/ClickHouse/ClickHouse/pull/44355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#34708](https://github.com/ClickHouse/ClickHouse/issues/34708) [#44356](https://github.com/ClickHouse/ClickHouse/pull/44356) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#30679](https://github.com/ClickHouse/ClickHouse/issues/30679) [#44357](https://github.com/ClickHouse/ClickHouse/pull/44357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#34669](https://github.com/ClickHouse/ClickHouse/issues/34669) [#44359](https://github.com/ClickHouse/ClickHouse/pull/44359) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Add a test for [#34724](https://github.com/ClickHouse/ClickHouse/issues/34724) [#44360](https://github.com/ClickHouse/ClickHouse/pull/44360) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Try restarting ZK cluster on failed connection in `test_keeper_zookeeper_converted` [#44363](https://github.com/ClickHouse/ClickHouse/pull/44363) ([Antonio Andelic](https://github.com/antonio2368)). -* Disable grase_hash in test 00172_parallel_join [#44367](https://github.com/ClickHouse/ClickHouse/pull/44367) ([Vladimir C](https://github.com/vdimir)). -* Add check for submodules sanity [#44386](https://github.com/ClickHouse/ClickHouse/pull/44386) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Lock table for share during startup for database ordinary [#44393](https://github.com/ClickHouse/ClickHouse/pull/44393) ([alesapin](https://github.com/alesapin)). -* Implement a custom central checkout action [#44399](https://github.com/ClickHouse/ClickHouse/pull/44399) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Try fix some tests [#44406](https://github.com/ClickHouse/ClickHouse/pull/44406) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Better ParserAllCollectionsOfLiterals [#44408](https://github.com/ClickHouse/ClickHouse/pull/44408) ([Nikolay Degterinsky](https://github.com/evillique)). * Fix bug with merge/mutate pool size increase [#44436](https://github.com/ClickHouse/ClickHouse/pull/44436) ([alesapin](https://github.com/alesapin)). * Update 01072_window_view_multiple_columns_groupby.sh [#44438](https://github.com/ClickHouse/ClickHouse/pull/44438) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Disable buggy tsan assertion for integration test [#44444](https://github.com/ClickHouse/ClickHouse/pull/44444) ([alesapin](https://github.com/alesapin)). * Respect setting settings.schema_inference_make_columns_nullable in Parquet/ORC/Arrow formats [#44446](https://github.com/ClickHouse/ClickHouse/pull/44446) ([Kruglov Pavel](https://github.com/Avogar)). * Add tests as examples with errors of date(time) and string comparison that we should eliminate [#44462](https://github.com/ClickHouse/ClickHouse/pull/44462) ([Ilya Yatsishin](https://github.com/qoega)). * Parallel parts cleanup with zero copy replication [#44466](https://github.com/ClickHouse/ClickHouse/pull/44466) ([Alexander Tokmakov](https://github.com/tavplubix)). * Fix incorrect usages of `getPartName()` [#44468](https://github.com/ClickHouse/ClickHouse/pull/44468) ([Alexander Tokmakov](https://github.com/tavplubix)). * Fix flaky test `roaring_memory_tracking` [#44470](https://github.com/ClickHouse/ClickHouse/pull/44470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Clarify query_id in test 01092_memory_profiler [#44483](https://github.com/ClickHouse/ClickHouse/pull/44483) ([Vladimir C](https://github.com/vdimir)). -* Default value for optional in SortNode::updateTreeHashImpl [#44491](https://github.com/ClickHouse/ClickHouse/pull/44491) ([Vladimir C](https://github.com/vdimir)). * Do not try to remove WAL/move broken parts for static storage [#44495](https://github.com/ClickHouse/ClickHouse/pull/44495) ([Azat Khuzhin](https://github.com/azat)). * Removed parent pid check that breaks in containers [#44499](https://github.com/ClickHouse/ClickHouse/pull/44499) ([Alexander Gololobov](https://github.com/davenger)). -* Analyzer duplicate alias crash fix [#44508](https://github.com/ClickHouse/ClickHouse/pull/44508) ([Maksim Kita](https://github.com/kitaisreal)). -* Minor code polishing [#44513](https://github.com/ClickHouse/ClickHouse/pull/44513) ([alesapin](https://github.com/alesapin)). -* Better error message if named collection does not exist [#44517](https://github.com/ClickHouse/ClickHouse/pull/44517) ([Kseniia Sumarokova](https://github.com/kssenii)). * Add the lambda to collect data for workflow_jobs [#44520](https://github.com/ClickHouse/ClickHouse/pull/44520) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). * Introduce groupArrayLast() (useful to store last X values) [#44521](https://github.com/ClickHouse/ClickHouse/pull/44521) ([Azat Khuzhin](https://github.com/azat)). * Infer numbers starting from zero as strings in TSV [#44522](https://github.com/ClickHouse/ClickHouse/pull/44522) ([Kruglov Pavel](https://github.com/Avogar)). * Fix wrong condition for enabling async reading from MergeTree. [#44530](https://github.com/ClickHouse/ClickHouse/pull/44530) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* tests: capture dmesg in integration tests [#44535](https://github.com/ClickHouse/ClickHouse/pull/44535) ([Azat Khuzhin](https://github.com/azat)). -* Analyzer support distributed queries processing [#44540](https://github.com/ClickHouse/ClickHouse/pull/44540) ([Maksim Kita](https://github.com/kitaisreal)). * Followup [#43761](https://github.com/ClickHouse/ClickHouse/issues/43761) [#44541](https://github.com/ClickHouse/ClickHouse/pull/44541) ([Dmitry Novik](https://github.com/novikd)). * Drop unused columns after join on/using [#44545](https://github.com/ClickHouse/ClickHouse/pull/44545) ([Vladimir C](https://github.com/vdimir)). * Improve inferring arrays with nulls in JSON formats [#44550](https://github.com/ClickHouse/ClickHouse/pull/44550) ([Kruglov Pavel](https://github.com/Avogar)). -* Make BC check optional (if env var set) [#44564](https://github.com/ClickHouse/ClickHouse/pull/44564) ([alesapin](https://github.com/alesapin)). -* Fix extremely slow stack traces in debug build [#44569](https://github.com/ClickHouse/ClickHouse/pull/44569) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Better command line argument name in `clickhouse-benchmark` [#44570](https://github.com/ClickHouse/ClickHouse/pull/44570) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix HDFS test [#44572](https://github.com/ClickHouse/ClickHouse/pull/44572) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix test_distributed_queries_stress [#44573](https://github.com/ClickHouse/ClickHouse/pull/44573) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Switch "contrib/sysroot" back to master. [#44574](https://github.com/ClickHouse/ClickHouse/pull/44574) ([Vitaly Baranov](https://github.com/vitlibar)). -* Non-significant changes [#44575](https://github.com/ClickHouse/ClickHouse/pull/44575) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fuzzer HTML: fix trash [#44580](https://github.com/ClickHouse/ClickHouse/pull/44580) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Better diagnostics on server stop for the stress test [#44593](https://github.com/ClickHouse/ClickHouse/pull/44593) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* The position of the log message about the server environment was wrong [#44595](https://github.com/ClickHouse/ClickHouse/pull/44595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix bad punctuation in log [#44596](https://github.com/ClickHouse/ClickHouse/pull/44596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix misleading log message [#44598](https://github.com/ClickHouse/ClickHouse/pull/44598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix bad log message about MergeTree metadata cache. [#44599](https://github.com/ClickHouse/ClickHouse/pull/44599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Slightly cleanup interactive line reader code [#44601](https://github.com/ClickHouse/ClickHouse/pull/44601) ([Azat Khuzhin](https://github.com/azat)). -* Rename `runlog.log` to `run.log` in tests [#44603](https://github.com/ClickHouse/ClickHouse/pull/44603) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix hung query in stress test [#44604](https://github.com/ClickHouse/ClickHouse/pull/44604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Improve variable name [#44605](https://github.com/ClickHouse/ClickHouse/pull/44605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Faster server startup after stress test [#44606](https://github.com/ClickHouse/ClickHouse/pull/44606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix log messages in Coordination [#44607](https://github.com/ClickHouse/ClickHouse/pull/44607) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Disable Analyzer in fuzz and stress tests [#44609](https://github.com/ClickHouse/ClickHouse/pull/44609) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Better log message [#44610](https://github.com/ClickHouse/ClickHouse/pull/44610) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Maybe fix a bogus MSan error [#44611](https://github.com/ClickHouse/ClickHouse/pull/44611) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix "too large allocation" message from MSan [#44613](https://github.com/ClickHouse/ClickHouse/pull/44613) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Do not fail the AST fuzzer if sanitizer is out of memory [#44616](https://github.com/ClickHouse/ClickHouse/pull/44616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix test `01111_create_drop_replicated_db_stress` [#44617](https://github.com/ClickHouse/ClickHouse/pull/44617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* tests/integration: suppress exceptions during logging (due to pytest) [#44618](https://github.com/ClickHouse/ClickHouse/pull/44618) ([Azat Khuzhin](https://github.com/azat)). -* Fix rust modules rebuild (previously ignores changes in cargo config.toml) [#44623](https://github.com/ClickHouse/ClickHouse/pull/44623) ([Azat Khuzhin](https://github.com/azat)). -* Sometimes spot instances fail more than 20 times in a row [#44626](https://github.com/ClickHouse/ClickHouse/pull/44626) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix restart after quorum insert [#44628](https://github.com/ClickHouse/ClickHouse/pull/44628) ([alesapin](https://github.com/alesapin)). -* Revert "Merge pull request [#38953](https://github.com/ClickHouse/ClickHouse/issues/38953) from ClickHouse/add-allocation-ptr-to-trace-log [#44629](https://github.com/ClickHouse/ClickHouse/pull/44629) ([Raúl Marín](https://github.com/Algunenano)). -* Fix lambdas parsing [#44639](https://github.com/ClickHouse/ClickHouse/pull/44639) ([Nikolay Degterinsky](https://github.com/evillique)). * Function viewExplain accept SELECT and settings [#44641](https://github.com/ClickHouse/ClickHouse/pull/44641) ([Vladimir C](https://github.com/vdimir)). -* Fix test `02015_async_inserts_2` [#44642](https://github.com/ClickHouse/ClickHouse/pull/44642) ([Anton Popov](https://github.com/CurtizJ)). -* Fix flaky test `test_keeper_multinode_simple` [#44645](https://github.com/ClickHouse/ClickHouse/pull/44645) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Add +x flag for run-fuzzer.sh [#44649](https://github.com/ClickHouse/ClickHouse/pull/44649) ([alesapin](https://github.com/alesapin)). * Custom reading for mutation [#44653](https://github.com/ClickHouse/ClickHouse/pull/44653) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix flaky test test_backup_restore_on_cluster [#44660](https://github.com/ClickHouse/ClickHouse/pull/44660) ([Vitaly Baranov](https://github.com/vitlibar)). @@ -358,12 +263,8 @@ * Fix minor typo: replace validate_bugix_check with validate_bugfix_check [#44672](https://github.com/ClickHouse/ClickHouse/pull/44672) ([Pradeep Chhetri](https://github.com/chhetripradeep)). * Fix parsing of ANY operator [#44678](https://github.com/ClickHouse/ClickHouse/pull/44678) ([Nikolay Degterinsky](https://github.com/evillique)). * Fix test `01130_in_memory_parts` [#44683](https://github.com/ClickHouse/ClickHouse/pull/44683) ([Anton Popov](https://github.com/CurtizJ)). -* Remove old code [#44685](https://github.com/ClickHouse/ClickHouse/pull/44685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix flaky test git-import [#44687](https://github.com/ClickHouse/ClickHouse/pull/44687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Improve odbc test [#44688](https://github.com/ClickHouse/ClickHouse/pull/44688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Add retries to HTTP requests in ClickHouse test [#44689](https://github.com/ClickHouse/ClickHouse/pull/44689) ([alesapin](https://github.com/alesapin)). * Fix flaky tests [#44690](https://github.com/ClickHouse/ClickHouse/pull/44690) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Fix flaky test "01502_long_log_tinylog_deadlock_race" [#44693](https://github.com/ClickHouse/ClickHouse/pull/44693) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Improve handling of old parts [#44694](https://github.com/ClickHouse/ClickHouse/pull/44694) ([Raúl Marín](https://github.com/Algunenano)). * Update entrypoint.sh [#44699](https://github.com/ClickHouse/ClickHouse/pull/44699) ([Denny Crane](https://github.com/den-crane)). * tests: more fixes for test_keeper_auth [#44702](https://github.com/ClickHouse/ClickHouse/pull/44702) ([Azat Khuzhin](https://github.com/azat)). @@ -383,33 +284,8 @@ * Build rust modules from the binary directory [#44762](https://github.com/ClickHouse/ClickHouse/pull/44762) ([Azat Khuzhin](https://github.com/azat)). * Remove database ordinary from stress test [#44763](https://github.com/ClickHouse/ClickHouse/pull/44763) ([alesapin](https://github.com/alesapin)). * Fix flaky test 02479_mysql_connect_to_self [#44768](https://github.com/ClickHouse/ClickHouse/pull/44768) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Print fatal messages in Fuzzer [#44769](https://github.com/ClickHouse/ClickHouse/pull/44769) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix incorrect docs [#44795](https://github.com/ClickHouse/ClickHouse/pull/44795) ([Kruglov Pavel](https://github.com/Avogar)). -* Added table name to error message [#44806](https://github.com/ClickHouse/ClickHouse/pull/44806) ([Alexander Gololobov](https://github.com/davenger)). -* Retry packages download if GitHub returned HTTP 500. [#44807](https://github.com/ClickHouse/ClickHouse/pull/44807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Slightly better docs [#44808](https://github.com/ClickHouse/ClickHouse/pull/44808) ([Kruglov Pavel](https://github.com/Avogar)). -* Fix total trash in stress test [#44810](https://github.com/ClickHouse/ClickHouse/pull/44810) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix ASan builds for glibc 2.36+ [#44811](https://github.com/ClickHouse/ClickHouse/pull/44811) ([Azat Khuzhin](https://github.com/azat)). -* Remove the remainings of TestFlows [#44812](https://github.com/ClickHouse/ClickHouse/pull/44812) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix `grep` [#44813](https://github.com/ClickHouse/ClickHouse/pull/44813) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix bad cast in monotonicity analysis [#44818](https://github.com/ClickHouse/ClickHouse/pull/44818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Modern tools, part 1 [#44819](https://github.com/ClickHouse/ClickHouse/pull/44819) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Modern tools in CI, part 2. [#44820](https://github.com/ClickHouse/ClickHouse/pull/44820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix data race in DDLWorker [#44821](https://github.com/ClickHouse/ClickHouse/pull/44821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix tests for bridges [#44822](https://github.com/ClickHouse/ClickHouse/pull/44822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix flaky test_multiple_disks::test_jbod_overflow [#44823](https://github.com/ClickHouse/ClickHouse/pull/44823) ([Azat Khuzhin](https://github.com/azat)). -* Less OOM in stress test [#44824](https://github.com/ClickHouse/ClickHouse/pull/44824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix misleading integration tests reports for parametrized tests [#44825](https://github.com/ClickHouse/ClickHouse/pull/44825) ([Azat Khuzhin](https://github.com/azat)). -* Fix two typos [#44826](https://github.com/ClickHouse/ClickHouse/pull/44826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Adjust CSS [#44829](https://github.com/ClickHouse/ClickHouse/pull/44829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* Fix fuzzer report [#44830](https://github.com/ClickHouse/ClickHouse/pull/44830) ([Alexey Milovidov](https://github.com/alexey-milovidov)). -* check-style: check base for std::cerr/cout too [#44833](https://github.com/ClickHouse/ClickHouse/pull/44833) ([Azat Khuzhin](https://github.com/azat)). -* Try fixing `test_keeper_snapshot_small_distance` with ZK restart [#44834](https://github.com/ClickHouse/ClickHouse/pull/44834) ([Antonio Andelic](https://github.com/antonio2368)). -* Exclude cargo shared libraries from the artifacts [#44836](https://github.com/ClickHouse/ClickHouse/pull/44836) ([Azat Khuzhin](https://github.com/azat)). -* Add a tiny but important logging [#44837](https://github.com/ClickHouse/ClickHouse/pull/44837) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Escape submodules in style-check [#44838](https://github.com/ClickHouse/ClickHouse/pull/44838) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Move `test_dies_with_parent` to another module [#44839](https://github.com/ClickHouse/ClickHouse/pull/44839) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Remove unneeded softlink to official dev docs [#44841](https://github.com/ClickHouse/ClickHouse/pull/44841) ([Robert Schulze](https://github.com/rschu1ze)). * Fix data race in StorageS3 [#44842](https://github.com/ClickHouse/ClickHouse/pull/44842) ([Antonio Andelic](https://github.com/antonio2368)). * Fix rare race which can lead to queue hang [#44847](https://github.com/ClickHouse/ClickHouse/pull/44847) ([alesapin](https://github.com/alesapin)). * No more retries in integration tests [#44851](https://github.com/ClickHouse/ClickHouse/pull/44851) ([Ilya Yatsishin](https://github.com/qoega)). From 527156c1044d62508fd4695a9e688ed57a88ac04 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 12:53:00 +0100 Subject: [PATCH 089/439] fix --- tests/clickhouse-test | 13 +++++++++++++ .../00002_log_and_exception_messages_formatting.sql | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3fec6b612b1..1130b3edede 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1869,6 +1869,19 @@ def reportLogStats(args): print(value) print("\n") + query = """ + SELECT message_format_string, count(), any(message) AS any_message + FROM system.text_log + WHERE (now() - toIntervalMinute(120)) < event_time + AND (message NOT LIKE (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') AS s)) + AND (message NOT LIKE concat('%Exception: ', s, '%')) + GROUP BY message_format_string ORDER BY count() DESC LIMIT 20 + """ + value = clickhouse_execute(args, query).decode(errors="replace") + print("\nTop messages that does not match its format string:\n") + print(value) + print("\n") + def main(args): global server_died diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index ca48813f54e..e5008871e4e 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -49,7 +49,7 @@ select 130, count() < 10 from (select count() / (select count() from logs) as fr -- Each message matches its pattern (returns 0 rows) -- FIXME maybe we should make it stricter ('Code:%Exception: '||s||'%'), but it's not easy because of addMessage -select 140, message_format_string, any_message from ( +select 140, countDistinct(message_format_string) < 15 from ( select message_format_string, any(message) as any_message from logs where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) and message not like ('%Exception: '||s||'%') group by message_format_string From 670a3f3e87faed66572eb9e319e2201204e09d26 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 25 Jan 2023 13:20:57 +0100 Subject: [PATCH 090/439] Editing trash, part 2 --- CHANGELOG.md | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 041f071c9ef..0a93c626fee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * The `PREALLOCATE` option for `HASHED`/`SPARSE_HASHED` dictionaries becomes a no-op. [#45388](https://github.com/ClickHouse/ClickHouse/pull/45388) ([Azat Khuzhin](https://github.com/azat)). It does not give significant advantages anymore. * Disallow `Gorilla` codec on columns of non-Float32 or non-Float64 type. [#45252](https://github.com/ClickHouse/ClickHouse/pull/45252) ([Robert Schulze](https://github.com/rschu1ze)). It was pointless and led to inconsistencies. * Parallel quorum inserts might work incorrectly with `*MergeTree` tables created with the deprecated syntax. Therefore, parallel quorum inserts support is completely disabled for such tables. It does not affect tables created with a new syntax. [#45430](https://github.com/ClickHouse/ClickHouse/pull/45430) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use `GetObjectAttributes` request instead of `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit region after updating the AWS SDK, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). AWS S3 and Minio are tested, but keep in mind that various S3-compatible services (GCS, R2, B2) may have subtle incompatibilities. #### New Feature * Dictionary source for extracting keys by traversing regular expressions tree. It can be used for User-Agent parsing. [#40878](https://github.com/ClickHouse/ClickHouse/pull/40878) ([Vage Ogannisian](https://github.com/nooblose)). [#43858](https://github.com/ClickHouse/ClickHouse/pull/43858) ([Han Fei](https://github.com/hanfei1991)). @@ -30,6 +31,9 @@ * We can use `s3_storage_class` to set different tiers. Such as `STANDARD/INTELLIGENT_TIERING` Closes [#44443](https://github.com/ClickHouse/ClickHouse/issues/44443). [#44707](https://github.com/ClickHouse/ClickHouse/pull/44707) ([chen](https://github.com/xiedeyantu)). * Insert default values in case of missing elements in JSON object while parsing named tuple. Add setting `input_format_json_defaults_for_missing_elements_in_named_tuple` that controls this behaviour. Closes [#45142](https://github.com/ClickHouse/ClickHouse/issues/45142)#issuecomment-1380153217. [#45231](https://github.com/ClickHouse/ClickHouse/pull/45231) ([Kruglov Pavel](https://github.com/Avogar)). * Record server startup time in ProfileEvents resolves [#43188](https://github.com/ClickHouse/ClickHouse/issues/43188) Implementation: * Added ProfileEvents::ServerStartupMilliseconds. * Recorded time from start of main till listening to sockets. Testing: * Added a test 02532_profileevents_server_startup_time.sql. [#45250](https://github.com/ClickHouse/ClickHouse/pull/45250) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted complitely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)). +* Support reading/writing `Nested` tables as `List` of `Struct` in CapnProto format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)). +* Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). This allows various analytics over ClickHouse own logs. #### Experimental Feature * Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)). @@ -40,67 +44,61 @@ #### Performance Improvement * Do not load inactive parts at startup of `MergeTree` tables. [#42181](https://github.com/ClickHouse/ClickHouse/pull/42181) ([Anton Popov](https://github.com/CurtizJ)). * Improved latency of reading from storage `S3` and table function `s3` with large number of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)). +* Optimization for reading struct fields in Parquet/ORC files. Only the required fields are loaded. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)). +* Two-level aggregation algorithm was mistakenly disabled for queries over HTTP interface. It was enabled back, and it leads to a major performance improvement. [#45450](https://github.com/ClickHouse/ClickHouse/pull/45450) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Added mmap support for StorageFile, which should improve the performance of clickhouse-local. [#43927](https://github.com/ClickHouse/ClickHouse/pull/43927) ([pufit](https://github.com/pufit)). * Added sharding support in HashedDictionary to allow parallel load (almost linear scaling based on number of shards). [#40003](https://github.com/ClickHouse/ClickHouse/pull/40003) ([Azat Khuzhin](https://github.com/azat)). -* - Speed up query parsing. [#42284](https://github.com/ClickHouse/ClickHouse/pull/42284) ([Raúl Marín](https://github.com/Algunenano)). +* Speed up query parsing. [#42284](https://github.com/ClickHouse/ClickHouse/pull/42284) ([Raúl Marín](https://github.com/Algunenano)). * Always replace OR chain `expr = x1 OR ... OR expr = xN` to `expr IN (x1, ..., xN)` in case if `expr` is a `LowCardinality` column. Setting `optimize_min_equality_disjunction_chain_length` is ignored in this case. [#42889](https://github.com/ClickHouse/ClickHouse/pull/42889) ([Guo Wangyang](https://github.com/guowangy)). -* > Original changelog In the original implementation, the memory of ThreadGroupStatus:: finished_threads_counters_memory is released by moving it to a temporary std::vector, which soon expired and gets destructed. This method is viable, however not straightforward enough. To enhance the code readability, this commit releases the memory in the vector by firstly resizing it to 0 and then shrinking the capacity accordingly. [#43586](https://github.com/ClickHouse/ClickHouse/pull/43586) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Original changelog In the original implementation, the memory of ThreadGroupStatus:: finished_threads_counters_memory is released by moving it to a temporary std::vector, which soon expired and gets destructed. This method is viable, however not straightforward enough. To enhance the code readability, this commit releases the memory in the vector by firstly resizing it to 0 and then shrinking the capacity accordingly. [#43586](https://github.com/ClickHouse/ClickHouse/pull/43586) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). * As a follow-up of [#42214](https://github.com/ClickHouse/ClickHouse/issues/42214), this PR tries to optimize the column-wise ternary logic evaluation by achieving auto-vectorization. In the performance test of this [microbenchmark](https://github.com/ZhiguoZh/ClickHouse/blob/20221123-ternary-logic-opt-example/src/Functions/examples/associative_applier_perf.cpp), we've observed a peak **performance gain** of **21x** on the ICX device (Intel Xeon Platinum 8380 CPU). [#43669](https://github.com/ClickHouse/ClickHouse/pull/43669) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). * Avoid acquiring read locks in the `system.tables` table if possible. [#43840](https://github.com/ClickHouse/ClickHouse/pull/43840) ([Raúl Marín](https://github.com/Algunenano)). * The performance experiments of SSB (Star Schema Benchmark) on the ICX device (Intel Xeon Platinum 8380 CPU, 80 cores, 160 threads) shows that this change could effectively decrease the lock contention for ThreadPoolImpl::mutex by **75%**, increasing the CPU utilization and improving the overall performance by **2.4%**. [#44308](https://github.com/ClickHouse/ClickHouse/pull/44308) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). * Now optimisation for predicting the hash table size is applied only if the cached hash table size is sufficiently large (thresholds were determined empirically and hardcoded). [#44455](https://github.com/ClickHouse/ClickHouse/pull/44455) ([Nikita Taranov](https://github.com/nickitat)). -* TODO: edit this ... The whole struct field will be loaded at current, even though we just want to read one field of the struct. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)). * Small performance improvement for asynchronous reading from remote filesystems. [#44868](https://github.com/ClickHouse/ClickHouse/pull/44868) ([Kseniia Sumarokova](https://github.com/kssenii)). * Add fast path for: - `col like '%%'`; - `col like '%'`; - `col not like '%'`; - `col not like '%'`; - `match(col, '.*')`. [#45244](https://github.com/ClickHouse/ClickHouse/pull/45244) ([李扬](https://github.com/taiyang-li)). -* todo. [#45289](https://github.com/ClickHouse/ClickHouse/pull/45289) ([Nikita Taranov](https://github.com/nickitat)). -* Provide monotonicity info for `toUnixTimestamp64*` [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)). +* Slightly improve happy path optimisation in filtering (WHERE clause). [#45289](https://github.com/ClickHouse/ClickHouse/pull/45289) ([Nikita Taranov](https://github.com/nickitat)). +* Provide monotonicity info for `toUnixTimestamp64*` to enable more algebraic optimizations for index analysis. [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to configure temporary data for query processing (spilling to disk) to cooperate with filesystem cache (taking up the space from the cache disk) [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). This mainly improves [ClickHouse Cloud](https://clickhouse.cloud/), but can be used for self-managed setups as well, if you know what to do. +* Make `system.replicas` table do parallel fetches of replicas statuses. Closes [#43918](https://github.com/ClickHouse/ClickHouse/issues/43918). [#43998](https://github.com/ClickHouse/ClickHouse/pull/43998) ([Nikolay Degterinsky](https://github.com/evillique)). +* Optimize memory consumption during backup to S3: files to S3 now will be copied directly without using `WriteBufferFromS3` (which could use a lot of memory). [#45188](https://github.com/ClickHouse/ClickHouse/pull/45188) ([Vitaly Baranov](https://github.com/vitlibar)). #### Improvement -* Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted complitely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)). -* Temporary files evict fs cache - 2nd approach [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). -* Support `optimize_or_like_chain` in the new infrastructure. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42797](https://github.com/ClickHouse/ClickHouse/pull/42797) ([Dmitry Novik](https://github.com/novikd)). + * Improve the Asterisk and ColumnMatcher parsers. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42884](https://github.com/ClickHouse/ClickHouse/pull/42884) ([Nikolay Degterinsky](https://github.com/evillique)). -* Implement `optimize_redundant_functions_in_order_by` on top of QueryTree. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#42970](https://github.com/ClickHouse/ClickHouse/pull/42970) ([Dmitry Novik](https://github.com/novikd)). -* Support `optimize_group_by_function_keys` in the new analyzer architecture. Also, add support for optimizing GROUPING SETS keys. Part of [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43261](https://github.com/ClickHouse/ClickHouse/pull/43261) ([Dmitry Novik](https://github.com/novikd)). * Improve reading CSV field in CustomSeparated/Template format. Closes [#42352](https://github.com/ClickHouse/ClickHouse/issues/42352) Closes [#39620](https://github.com/ClickHouse/ClickHouse/issues/39620). [#43332](https://github.com/ClickHouse/ClickHouse/pull/43332) ([Kruglov Pavel](https://github.com/Avogar)). -* Support reading/writing `Nested` tables as `List` of `Struct` in CapnProto format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)). * Unify query elapsed time measurements. [#43455](https://github.com/ClickHouse/ClickHouse/pull/43455) ([Raúl Marín](https://github.com/Algunenano)). -* Support scalar subqueries cache Implementation: * Added a map with hash of the node (without alias) and the evaluated value to Context. Testing: * Added a test-case with new analyser in 02174_cte_scalar_cache.sql. [#43640](https://github.com/ClickHouse/ClickHouse/pull/43640) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). * Improve automatic usage of structure from insertion table in table functions file/hdfs/s3 when virtual columns present in select query, it fixes possible error `Block structure mismatch` or `number of columns mismatch`. [#43695](https://github.com/ClickHouse/ClickHouse/pull/43695) ([Kruglov Pavel](https://github.com/Avogar)). * Add support for signed arguments in range(). Fixes [#43333](https://github.com/ClickHouse/ClickHouse/issues/43333). [#43733](https://github.com/ClickHouse/ClickHouse/pull/43733) ([sanyu](https://github.com/wineternity)). * Remove redundant sorting, for example, sorting related ORDER BY clauses in subqueries. Implemented on top of query plan. It does similar optimization as `optimize_duplicate_order_by_and_distinct` regarding `ORDER BY` clauses, but more generic, since it's applied to any redundant sorting steps (not only caused by ORDER BY clause) and applied to subqueries of any depth. Related to [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43905](https://github.com/ClickHouse/ClickHouse/pull/43905) ([Igor Nikonov](https://github.com/devcrafter)). * Add ability to disable deduplication for BACKUP (for backups wiithout deduplication ATTACH can be used instead of full RESTORE), example `BACKUP foo TO S3(...) SETTINGS deduplicate_files=0` (default `deduplicate_files=1`). [#43947](https://github.com/ClickHouse/ClickHouse/pull/43947) ([Azat Khuzhin](https://github.com/azat)). -* Make `system.replicas` table do parallel fetches of replicas statuses. Closes [#43918](https://github.com/ClickHouse/ClickHouse/issues/43918). [#43998](https://github.com/ClickHouse/ClickHouse/pull/43998) ([Nikolay Degterinsky](https://github.com/evillique)). * Refactor and improve schema inference for text formats. Add new setting `schema_inference_make_columns_nullable` that controls making result types `Nullable` (enabled by default);. [#44019](https://github.com/ClickHouse/ClickHouse/pull/44019) ([Kruglov Pavel](https://github.com/Avogar)). * Better support for PROXYv1. [#44135](https://github.com/ClickHouse/ClickHouse/pull/44135) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * Add information about the latest part check by cleanup thread into `system.parts` table. [#44244](https://github.com/ClickHouse/ClickHouse/pull/44244) ([Dmitry Novik](https://github.com/novikd)). -* Disable functions in readonly for inserts. [#44290](https://github.com/ClickHouse/ClickHouse/pull/44290) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Disable table functions in readonly for inserts. [#44290](https://github.com/ClickHouse/ClickHouse/pull/44290) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). * Add a setting `simultaneous_parts_removal_limit` to allow to limit the number of parts being processed by one iteration of CleanupThread. [#44461](https://github.com/ClickHouse/ClickHouse/pull/44461) ([Dmitry Novik](https://github.com/novikd)). * If user only need virtual columns, we don't need to initialize ReadBufferFromS3. May be helpful to [#44246](https://github.com/ClickHouse/ClickHouse/issues/44246). [#44493](https://github.com/ClickHouse/ClickHouse/pull/44493) ([chen](https://github.com/xiedeyantu)). * Prevent duplicate column names hints. Closes [#44130](https://github.com/ClickHouse/ClickHouse/issues/44130). [#44519](https://github.com/ClickHouse/ClickHouse/pull/44519) ([Joanna Hulboj](https://github.com/jh0x)). * Allow macro substitution in endpoint of disks resolve [#40951](https://github.com/ClickHouse/ClickHouse/issues/40951). [#44533](https://github.com/ClickHouse/ClickHouse/pull/44533) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). * Improve schema inference when `input_format_json_read_object_as_string` is enabled. [#44546](https://github.com/ClickHouse/ClickHouse/pull/44546) ([Kruglov Pavel](https://github.com/Avogar)). * Add user-level setting `database_replicated_allow_replicated_engine_arguments` which allow to ban creation of `ReplicatedMergeTree` tables with arguments in `DatabaseReplicated`. [#44566](https://github.com/ClickHouse/ClickHouse/pull/44566) ([alesapin](https://github.com/alesapin)). * Prevent users from mistakenly specifying zero (invalid) value for `index_granularity`. This closes [#44536](https://github.com/ClickHouse/ClickHouse/issues/44536). [#44578](https://github.com/ClickHouse/ClickHouse/pull/44578) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Added possibility to set path to service keytab file in `keytab` parameter in `kerberos` section of config.xml. [#44594](https://github.com/ClickHouse/ClickHouse/pull/44594) ([Roman Vasin](https://github.com/rvasin)). * Use already written part of the query for fuzzy search (pass to skim). [#44600](https://github.com/ClickHouse/ClickHouse/pull/44600) ([Azat Khuzhin](https://github.com/azat)). -* Enable input_format_json_read_objects_as_strings by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)). +* Enable `input_format_json_read_objects_as_strings` by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)). * When users do duplicate async inserts, we should dedup inside the memory before we query keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)). -* Input/ouptut Avro bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)). -* Don't parse beyond the quotes when reading UUIDs. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)). +* Input/ouptut `Avro` bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't greedy parse beyond the quotes when reading UUIDs - it may lead to mistakenly successful parsing of incorrect data. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)). * Infer UInt64 in case of Int64 overflow and fix some transforms in schema inference. [#44696](https://github.com/ClickHouse/ClickHouse/pull/44696) ([Kruglov Pavel](https://github.com/Avogar)). -* Previously dependency resolving inside DatabaseReplicated was done in a hacky way and now it done right using an explicit graph. [#44697](https://github.com/ClickHouse/ClickHouse/pull/44697) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Previously dependency resolving inside DatabaseReplicated was done in a hacky way, and now it's done right using an explicit graph. [#44697](https://github.com/ClickHouse/ClickHouse/pull/44697) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Support Bool type in Arrow/Parquet/ORC. Closes [#43970](https://github.com/ClickHouse/ClickHouse/issues/43970). [#44698](https://github.com/ClickHouse/ClickHouse/pull/44698) ([Kruglov Pavel](https://github.com/Avogar)). * Fix `output_format_pretty_row_numbers` does not preserve the counter across the blocks. Closes [#44815](https://github.com/ClickHouse/ClickHouse/issues/44815). [#44832](https://github.com/ClickHouse/ClickHouse/pull/44832) ([flynn](https://github.com/ucasfl)). * Extend function "toDayOfWeek" with a mode argument describing if a) the week starts on Monday or Sunday and b) if counting starts at 0 or 1. [#44860](https://github.com/ClickHouse/ClickHouse/pull/44860) ([李扬](https://github.com/taiyang-li)). -* Don't report errors in system.errors due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)). +* Don't report errors in `system.errors` due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)). * Optimize and fix metrics for Distributed async INSERT. [#44922](https://github.com/ClickHouse/ClickHouse/pull/44922) ([Azat Khuzhin](https://github.com/azat)). * Added settings to disallow concurrent backups and restores resolves [#43891](https://github.com/ClickHouse/ClickHouse/issues/43891) Implementation: * Added server level settings to disallow concurrent backups and restores, which are read and set when BackupWorker is created in Context. * Settings are set to true by default. * Before starting backup or restores, added a check to see if any other backups/restores are running. For internal request it checks if its from the self node using backup_uuid. [#45072](https://github.com/ClickHouse/ClickHouse/pull/45072) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). * Add a cache for async block ids. This will reduce the requests of zookeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)). -* Optimize memory consumption during backup to S3: files to S3 now will be copied directly without using `WriteBufferFromS3` (which could use a lot of memory). [#45188](https://github.com/ClickHouse/ClickHouse/pull/45188) ([Vitaly Baranov](https://github.com/vitlibar)). * Use structure from insertion table in generateRandom without arguments. [#45239](https://github.com/ClickHouse/ClickHouse/pull/45239) ([Kruglov Pavel](https://github.com/Avogar)). -* Use `GetObjectAttributes` request instead of `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit region, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). * Add `` config parameter for system logs. [#45320](https://github.com/ClickHouse/ClickHouse/pull/45320) ([Stig Bakken](https://github.com/stigsb)). * Allow to implicitly convert floats stored in string fields of JSON to integers in `JSONExtract` functions. E.g. `JSONExtract('{"a": "1000.111"}', 'a', 'UInt64')` -> `1000`, previously it returned 0. [#45432](https://github.com/ClickHouse/ClickHouse/pull/45432) ([Anton Popov](https://github.com/CurtizJ)). * Added fields `supports_parallel_parsing` and `supports_parallel_formatting` to table `system.formats` for better introspection. [#45499](https://github.com/ClickHouse/ClickHouse/pull/45499) ([Anton Popov](https://github.com/CurtizJ)). @@ -189,7 +187,6 @@ * Fixed bug (Code: 632. DB::Exception: Unexpected data ... after parsed IPv6 value ...) with typed parsing of IP types from text source. [#45425](https://github.com/ClickHouse/ClickHouse/pull/45425) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). * close [#45297](https://github.com/ClickHouse/ClickHouse/issues/45297) Add check for empty regular expressions. [#45428](https://github.com/ClickHouse/ClickHouse/pull/45428) ([Han Fei](https://github.com/hanfei1991)). * Fix possible (likely distributed) query hung. [#45448](https://github.com/ClickHouse/ClickHouse/pull/45448) ([Azat Khuzhin](https://github.com/azat)). -* Fix disabled two-level aggregation from HTTP. [#45450](https://github.com/ClickHouse/ClickHouse/pull/45450) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix possible deadlock with `allow_asynchronous_read_from_io_pool_for_merge_tree` enabled in case of exception from `ThreadPool::schedule`. [#45481](https://github.com/ClickHouse/ClickHouse/pull/45481) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix possible in-use table after DETACH. [#45493](https://github.com/ClickHouse/ClickHouse/pull/45493) ([Azat Khuzhin](https://github.com/azat)). * Fix rare abort in case when query is canceled and parallel parsing was used during its execution. [#45498](https://github.com/ClickHouse/ClickHouse/pull/45498) ([Anton Popov](https://github.com/CurtizJ)). From 3fb82ccd58195fc0cc5648f247e7710cccd6e697 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 14:20:40 +0100 Subject: [PATCH 091/439] fix --- .../00002_log_and_exception_messages_formatting.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference index 75dd0426f4a..9268fc859f4 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference @@ -11,3 +11,4 @@ 110 1 120 1 130 1 +140 1 From 55af44ca2fd434c5d648e55faeeffdf7cde8b8bb Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 25 Jan 2023 14:40:39 +0100 Subject: [PATCH 092/439] Better fix --- src/Storages/StorageReplicatedMergeTree.cpp | 24 +++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f1da18f7184..a77cce078ba 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7908,9 +7908,18 @@ String StorageReplicatedMergeTree::getTableSharedID() const { std::lock_guard lock(table_shared_id_mutex); - /// Can happen if table was partially initialized before drop by DatabaseCatalog - if (table_shared_id == UUIDHelpers::Nil) - createTableSharedID(); + /// If we has metadata or, we don't know about metadata -- try to create shared ID + /// Otherwise table is already dropped, doesn't make sense to do anything with shared ID + if (has_metadata_in_zookeeper.value_or(true)) + { + /// Can happen if table was partially initialized before drop by DatabaseCatalog + if (table_shared_id == UUIDHelpers::Nil) + createTableSharedID(); + } + else + { + return toString(UUIDHelpers::Nil); + } return toString(table_shared_id); } @@ -8101,6 +8110,13 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co return std::make_pair(true, NameSet{}); } + auto shared_id = getTableSharedID(); + if (shared_id == toString(UUIDHelpers::Nil)) + { + LOG_TRACE(log, "Part {} blobs can be removed, because table {} comletely dropped", part.name, getStorageID().getNameForLogs()); + return std::make_pair(true, NameSet{}); + } + /// If part is temporary refcount file may be absent if (part.getDataPartStorage().exists(IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK)) { @@ -8140,7 +8156,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co return std::make_pair(true, NameSet{}); return unlockSharedDataByID( - part.getUniqueId(), getTableSharedID(), part.name, replica_name, + part.getUniqueId(), shared_id, part.name, replica_name, part.getDataPartStorage().getDiskType(), zookeeper, *getSettings(), log, zookeeper_path, format_version); } From 70a3ffa0e6e857cc9db943b9696084c26b57026f Mon Sep 17 00:00:00 2001 From: Bharat Nallan Chakravarthy Date: Wed, 25 Jan 2023 06:46:19 -0800 Subject: [PATCH 093/439] fix test --- tests/integration/test_odbc_interaction/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 946f97c2148..14f5de17870 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -630,7 +630,6 @@ def test_no_connection_pooling(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() - cursor.execute("truncate table clickhouse.test_table") cursor.execute( "insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')" ) @@ -651,6 +650,7 @@ def test_no_connection_pooling(started_cluster): assert "" == node1.exec_in_container( ["ss", "-H", "dport", "5432"], privileged=True, user="root" ) + cursor.execute("truncate table clickhouse.test_table") def test_postgres_insert(started_cluster): From ae795d87b2ade7a5023d059eada2237da83f5fde Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 16:06:40 +0100 Subject: [PATCH 094/439] fix --- docker/test/fasttest/run.sh | 1 + src/Common/tests/gtest_rw_lock.cpp | 10 ++++------ tests/clickhouse-test | 5 ++++- utils/compressor/decompress_perf.cpp | 8 ++++---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 2df50a7934c..9d5a4db309f 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -234,6 +234,7 @@ function run_tests --check-zookeeper-session --order random --print-time + --report-logs-stats --jobs "${NPROC}" ) time clickhouse-test "${test_opts[@]}" -- "$FASTTEST_FOCUS" 2>&1 \ diff --git a/src/Common/tests/gtest_rw_lock.cpp b/src/Common/tests/gtest_rw_lock.cpp index 57f446ca249..5ea50f70d4e 100644 --- a/src/Common/tests/gtest_rw_lock.cpp +++ b/src/Common/tests/gtest_rw_lock.cpp @@ -171,9 +171,8 @@ TEST(Common, RWLockDeadlock) auto holder2 = lock2->getLock(RWLockImpl::Read, "q1", std::chrono::milliseconds(100)); if (!holder2) { - throw Exception( - "Locking attempt timed out! Possible deadlock avoided. Client should retry.", - ErrorCodes::DEADLOCK_AVOIDED); + throw Exception(ErrorCodes::DEADLOCK_AVOIDED, + "Locking attempt timed out! Possible deadlock avoided. Client should retry."); } } catch (const Exception & e) @@ -202,9 +201,8 @@ TEST(Common, RWLockDeadlock) auto holder1 = lock1->getLock(RWLockImpl::Read, "q3", std::chrono::milliseconds(100)); if (!holder1) { - throw Exception( - "Locking attempt timed out! Possible deadlock avoided. Client should retry.", - ErrorCodes::DEADLOCK_AVOIDED); + throw Exception(ErrorCodes::DEADLOCK_AVOIDED, + "Locking attempt timed out! Possible deadlock avoided. Client should retry."); } } catch (const Exception & e) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 10fc76ed795..5aad3756e02 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2017,7 +2017,10 @@ def main(args): print("All tests have finished.") if args.report_logs_stats: - reportLogStats(args) + try: + reportLogStats(args) + except Exception as e: + print(f"Failed to get stats about log messages: {e}") if args.report_coverage and not reportCoverage(args): exit_code.value = 1 diff --git a/utils/compressor/decompress_perf.cpp b/utils/compressor/decompress_perf.cpp index 891a6d3d1dd..cb98121c024 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/utils/compressor/decompress_perf.cpp @@ -74,10 +74,10 @@ protected: size_decompressed = unalignedLoad(&own_compressed_buffer[5]); } else - throw Exception("Unknown compression method: " + toString(method), ErrorCodes::UNKNOWN_COMPRESSION_METHOD); + throw Exception(ErrorCodes::UNKNOWN_COMPRESSION_METHOD, "Unknown compression method: {}", toString(method)); if (size_compressed > DBMS_MAX_COMPRESSED_SIZE) - throw Exception("Too large size_compressed. Most likely corrupted data.", ErrorCodes::TOO_LARGE_SIZE_COMPRESSED); + throw Exception(ErrorCodes::TOO_LARGE_SIZE_COMPRESSED, "Too large size_compressed. Most likely corrupted data."); /// Is whole compressed block located in 'compressed_in' buffer? if (compressed_in->offset() >= COMPRESSED_BLOCK_HEADER_SIZE && @@ -111,14 +111,14 @@ protected: compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, to, static_cast(size_decompressed)) < 0) { - throw Exception("Cannot LZ4_decompress_fast", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot LZ4_decompress_fast"); } } else LZ4::decompress(compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, to, size_compressed_without_checksum, size_decompressed, perf_stat); } else - throw Exception("Unknown compression method: " + toString(method), ErrorCodes::UNKNOWN_COMPRESSION_METHOD); + throw Exception(ErrorCodes::UNKNOWN_COMPRESSION_METHOD, "Unknown compression method: {}", toString(method)); } public: From fc5ce9a321b744c279e8314d72c85d8ab4829cca Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 25 Jan 2023 16:34:16 +0100 Subject: [PATCH 095/439] Add #44953 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a93c626fee..5663c545ff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ * Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted complitely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)). * Support reading/writing `Nested` tables as `List` of `Struct` in CapnProto format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)). * Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). This allows various analytics over ClickHouse own logs. +* Try to detect header with column names (and maybe types) for CSV/TSV/CustomSeparated input formats. +Add settings input_format_tsv/csv/custom_detect_header that enables this behaviour (enabled by default). Closes [#44640](https://github.com/ClickHouse/ClickHouse/issues/44640). [#44953](https://github.com/ClickHouse/ClickHouse/pull/44953) ([Kruglov Pavel](https://github.com/Avogar)). #### Experimental Feature * Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)). From 7bfdfa4bf78b17be4390880bb23810127ac16c00 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 25 Jan 2023 16:25:51 +0000 Subject: [PATCH 096/439] Preallocate more space --- src/Coordination/Changelog.cpp | 215 ++++++++---------- src/IO/WriteBufferFromVector.h | 12 +- src/IO/ZstdDeflatingAppendableWriteBuffer.cpp | 27 ++- src/IO/ZstdDeflatingAppendableWriteBuffer.h | 23 +- 4 files changed, 124 insertions(+), 153 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 25ba46ced45..036f40cb7da 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -10,6 +10,7 @@ #include #include #include +#include "IO/WriteBufferFromFile.h" namespace DB @@ -94,7 +95,6 @@ public: uint64_t rotate_interval_, uint64_t max_log_file_size_) : existing_changelogs(existing_changelogs_) - , memory_buf(std::make_unique(memory)) , compress_logs(compress_logs_) , force_fsync(force_fsync_) , rotate_interval(rotate_interval_) @@ -102,14 +102,6 @@ public: , changelogs_dir(changelogs_dir_) , log(&Poco::Logger::get("Changelog")) { - if (compress_logs) - { - compressed_buffer = std::make_unique(std::move(memory_buf), /* compression level = */ 3); - } - else - { - /// no compression, only memory buffer - } } void setFile(ChangelogFileDescriptionPtr file_description, WriteMode mode) @@ -124,9 +116,9 @@ public: file_description->expectedEntriesCountInLog()); // we have a file we need to finalize first - if (file_buf && prealloc_done) + if (tryGetFileBuffer() && prealloc_done) { - finalizeCurrentFile(/*final*/ false); + finalizeCurrentFile(); assert(current_file_description); // if we wrote at least 1 log in the log file we can rename the file to reflect correctly the @@ -151,12 +143,8 @@ public: last_index_written.reset(); current_file_description = std::move(file_description); - // try fixing ZstdStream for the file if we are appending to some old file - if (compress_logs && mode == WriteMode::Append && ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) - { - ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); - flush(); - } + if (compress_logs) + compressed_buffer = std::make_unique(std::move(file_buf), /* compression level = */ 3, /* append_to_existing_file_ = */ mode == WriteMode::Append); prealloc_done = false; } @@ -167,16 +155,27 @@ public: } } - bool isFileSet() const { return file_buf != nullptr; } + bool isFileSet() const { return tryGetFileBuffer() != nullptr; } bool appendRecord(ChangelogRecord && record) { + const auto * file_buffer = tryGetFileBuffer(); + assert(file_buffer && current_file_description); const bool log_is_complete = record.header.index - getStartIndex() == current_file_description->expectedEntriesCountInLog(); if (log_is_complete) rotate(record.header.index); + // writing at least 1 log is requirement - we don't want empty log files + const bool log_too_big = record.header.index != getStartIndex() && max_log_file_size != 0 && initial_file_size + file_buffer->count() > total_bytes_available; + + if (log_too_big) + { + LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", max_log_file_size); + rotate(record.header.index); + } + if (!prealloc_done) [[unlikely]] { tryPreallocateForFile(); @@ -185,62 +184,25 @@ public: return false; } - const auto write_record = [&] + auto & write_buffer = getBuffer(); + writeIntBinary(computeRecordChecksum(record), write_buffer); + + writeIntBinary(record.header.version, write_buffer); + + writeIntBinary(record.header.index, write_buffer); + writeIntBinary(record.header.term, write_buffer); + writeIntBinary(record.header.value_type, write_buffer); + writeIntBinary(record.header.blob_size, write_buffer); + + if (record.header.blob_size != 0) + write_buffer.write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); + + if (compressed_buffer) { - writeIntBinary(computeRecordChecksum(record), getBuffer()); - - writeIntBinary(record.header.version, getBuffer()); - - writeIntBinary(record.header.index, getBuffer()); - writeIntBinary(record.header.term, getBuffer()); - writeIntBinary(record.header.value_type, getBuffer()); - writeIntBinary(record.header.blob_size, getBuffer()); - - if (record.header.blob_size != 0) - getBuffer().write(reinterpret_cast(record.blob->data_begin()), record.blob->size()); - - if (compressed_buffer) - { - /// Flush compressed data to WriteBufferMemory working_buffer - compressed_buffer->next(); - } - }; - - // write record to an in-memory block - write_record(); - - const auto get_memory_buffer = [&]() -> WriteBufferMemory & - { - return compressed_buffer ? dynamic_cast(*compressed_buffer->getNestedBuffer()) : *memory_buf; - }; - - // if the file is too big, rotate - if (last_index_written.has_value() && max_log_file_size != 0 && total_bytes_written + get_memory_buffer().valuesWritten() > total_bytes_available) - { - LOG_TRACE(log, "Log file reached maximum allowed size ({} bytes), creating new log file", max_log_file_size); - rotate(record.header.index); - - // we need to preallocate space on disk for the new file - tryPreallocateForFile(); - if (!prealloc_done) - return false; - - // if we use compression we need to write again to the buffer because it's a new one - if (compress_logs) - write_record(); + /// Flush compressed data to file buffer + compressed_buffer->next(); } - // after possible rotation, we don't check if we have enough space - // because writing at least 1 log is requirement - we don't want empty log files - - auto & cur_memory_buf = get_memory_buffer(); - - // write the in-memory block to file - file_buf->write(memory.raw_data(), cur_memory_buf.valuesWritten()); - - total_bytes_written += cur_memory_buf.valuesWritten(); - - cur_memory_buf.restart(); last_index_written = record.header.index; return true; @@ -248,9 +210,10 @@ public: void flush() { + auto * file_buffer = tryGetFileBuffer(); /// Fsync file system if needed - if (force_fsync && file_buf) - file_buf->sync(); + if (file_buffer && force_fsync) + file_buffer->sync(); } uint64_t getStartIndex() const @@ -286,62 +249,74 @@ public: void finalize() { - if (file_buf && prealloc_done) - finalizeCurrentFile(/*final*/ true); + if (isFileSet() && prealloc_done) + finalizeCurrentFile(); } private: - void finalizeCurrentFile(bool final) + void finalizeCurrentFile() { - assert(file_buf && prealloc_done); + const auto * file_buffer = tryGetFileBuffer(); + assert(file_buffer && prealloc_done); assert(current_file_description); // compact can delete the file and we don't need to do anything - if (!current_file_description->deleted) + if (current_file_description->deleted) { - // finish the stream on disk if needed (finalize will write to in-memory block) - if (compress_logs) - { - if (ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(current_file_description->path)) - { - ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(*file_buf); - assert(max_log_file_size == 0 || memory.size() < total_bytes_available - total_bytes_written); - total_bytes_written += ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); - } - } - - flush(); - - if (max_log_file_size != 0) - ftruncate(file_buf->getFD(), total_bytes_written); - } - else - { - LOG_WARNING(log, "Log {} is already deleted", file_buf->getFileName()); + LOG_WARNING(log, "Log {} is already deleted", file_buffer->getFileName()); + return; } if (compress_logs) - { - WriteBufferMemory & cur_memory_buf - = dynamic_cast(*compressed_buffer->getNestedBuffer()); + compressed_buffer->finalize(); - // so finalize can be done - cur_memory_buf.restart(); + flush(); + + if (max_log_file_size != 0) + ftruncate(file_buffer->getFD(), initial_file_size + file_buffer->count()); + + if (compress_logs) compressed_buffer.reset(); - - if (!final) - compressed_buffer = std::make_unique(std::make_unique(memory), /* compression level = */ 3); - } - - file_buf.reset(); + else + file_buf.reset(); } WriteBuffer & getBuffer() { if (compressed_buffer) return *compressed_buffer; - return *memory_buf; + + if (file_buf) + return *file_buf; + + throw Exception(ErrorCodes::LOGICAL_ERROR, "Log writter wasn't initialized for any file"); + } + + WriteBufferFromFile & getFileBuffer() + { + auto * file_buffer = tryGetFileBuffer(); + + if (!file_buffer) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Log writter wasn't initialized for any file"); + + return *file_buffer; + } + + const WriteBufferFromFile * tryGetFileBuffer() const + { + return const_cast(this)->tryGetFileBuffer(); + } + + WriteBufferFromFile * tryGetFileBuffer() + { + if (compressed_buffer) + return dynamic_cast(compressed_buffer->getNestedBuffer()); + + if (file_buf) + return file_buf.get(); + + return nullptr; } void tryPreallocateForFile() @@ -349,15 +324,16 @@ private: if (max_log_file_size == 0) { total_bytes_available = 0; - total_bytes_written = 0; + initial_file_size = 0; prealloc_done = true; return; } + const auto & file_buffer = getFileBuffer(); bool fallocate_ok = false; #ifdef OS_LINUX { - int res = fallocate(file_buf->getFD(), FALLOC_FL_KEEP_SIZE, 0, max_log_file_size); + int res = fallocate(file_buffer.getFD(), FALLOC_FL_KEEP_SIZE, 0, 2 * max_log_file_size); if (res == ENOSPC) { LOG_FATAL(log, "Failed to allocate enough space on disk for logs"); @@ -370,34 +346,23 @@ private: struct stat buf; { - [[maybe_unused]] int res = fstat(file_buf->getFD(), &buf); + [[maybe_unused]] int res = fstat(file_buffer.getFD(), &buf); assert(res == 0); } - total_bytes_written = buf.st_size; + initial_file_size = buf.st_size; // if fallocate passed, we know exact allocated bytes on disk which can be larger than the max_log_file_size total_bytes_available = fallocate_ok ? buf.st_blocks * 512 : max_log_file_size; - // always leave space for last termination block - if (compress_logs) - total_bytes_available -= ZstdDeflatingAppendableWriteBuffer::ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); - prealloc_done = true; } std::map & existing_changelogs; - using MemoryBuffer = PODArray; - using WriteBufferMemory = WriteBufferFromVector; - MemoryBuffer memory; - // we write everything to memory buffer to see how many bytes are required - // when we have that information we can try to write to the disk - std::unique_ptr memory_buf; - ChangelogFileDescriptionPtr current_file_description{nullptr}; std::unique_ptr file_buf; std::optional last_index_written; - size_t total_bytes_written{0}; + size_t initial_file_size{0}; size_t total_bytes_available{0}; std::unique_ptr compressed_buffer; diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index c144959c59b..4b2a3581625 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -61,13 +61,6 @@ public: finalized = false; } - size_t valuesWritten() - { - return ((position() - reinterpret_cast(vector.data())) /// NOLINT - + sizeof(ValueType) - 1) /// Align up. - / sizeof(ValueType); - } - ~WriteBufferFromVector() override { finalize(); @@ -76,7 +69,10 @@ public: private: void finalizeImpl() override { - vector.resize(valuesWritten()); + vector.resize( + ((position() - reinterpret_cast(vector.data())) /// NOLINT + + sizeof(ValueType) - 1) /// Align up. + / sizeof(ValueType)); /// Prevent further writes. set(nullptr, 0); diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp index ed8b01c5d83..f8c4d0e2bac 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp @@ -11,13 +11,15 @@ namespace ErrorCodes } ZstdDeflatingAppendableWriteBuffer::ZstdDeflatingAppendableWriteBuffer( - std::unique_ptr out_, + std::unique_ptr out_, int compression_level, + bool append_to_existing_file_, size_t buf_size, char * existing_memory, size_t alignment) : BufferWithOwnMemory(buf_size, existing_memory, alignment) , out(std::move(out_)) + , append_to_existing_file(append_to_existing_file_) { cctx = ZSTD_createCCtx(); if (cctx == nullptr) @@ -41,6 +43,12 @@ void ZstdDeflatingAppendableWriteBuffer::nextImpl() input.size = offset(); input.pos = 0; + if (first_write && append_to_existing_file && isNeedToAddEmptyBlock()) + { + addEmptyBlock(); + first_write = false; + } + try { bool ended = false; @@ -145,7 +153,6 @@ void ZstdDeflatingAppendableWriteBuffer::finalizeBefore() output.pos = out->offset(); } } - out->position() = out->buffer().begin() + output.pos; } void ZstdDeflatingAppendableWriteBuffer::finalizeAfter() @@ -171,23 +178,23 @@ void ZstdDeflatingAppendableWriteBuffer::finalizeZstd() } } -void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock(WriteBuffer & write_buffer) +void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock() { /// HACK: https://github.com/facebook/zstd/issues/2090#issuecomment-620158967 - if (write_buffer.buffer().size() - write_buffer.offset() < ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()) - write_buffer.next(); + if (out->buffer().size() - out->offset() < ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()) + out->next(); - std::memcpy(write_buffer.buffer().begin() + write_buffer.offset(), + std::memcpy(out->buffer().begin() + out->offset(), ZSTD_CORRECT_TERMINATION_LAST_BLOCK.data(), ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size()); - write_buffer.position() = write_buffer.buffer().begin() + write_buffer.offset() + ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); + out->position() = out->buffer().begin() + out->offset() + ZSTD_CORRECT_TERMINATION_LAST_BLOCK.size(); } -bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(const std::string & file_name) +bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock() { - ReadBufferFromFile reader(file_name); + ReadBufferFromFile reader(out->getFileName()); auto fsize = reader.getFileSize(); if (fsize > 3) { @@ -205,7 +212,7 @@ bool ZstdDeflatingAppendableWriteBuffer::isNeedToAddEmptyBlock(const std::string throw Exception( ErrorCodes::ZSTD_ENCODER_FAILED, "Trying to write to non-empty file '{}' with tiny size {}. It can lead to data corruption", - file_name, fsize); + out->getFileName(), fsize); } return false; } diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.h b/src/IO/ZstdDeflatingAppendableWriteBuffer.h index 1d9cc127803..a0715480737 100644 --- a/src/IO/ZstdDeflatingAppendableWriteBuffer.h +++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.h @@ -29,8 +29,9 @@ public: static inline constexpr ZSTDLastBlock ZSTD_CORRECT_TERMINATION_LAST_BLOCK = {0x01, 0x00, 0x00}; ZstdDeflatingAppendableWriteBuffer( - std::unique_ptr out_, + std::unique_ptr out_, int compression_level, + bool append_to_existing_file_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0); @@ -45,13 +46,6 @@ public: WriteBuffer * getNestedBuffer() { return out.get(); } - /// Read three last bytes from non-empty compressed file and compares them with - /// ZSTD_CORRECT_TERMINATION_LAST_BLOCK. - static bool isNeedToAddEmptyBlock(const std::string & file_name); - - /// Adding zstd empty block (ZSTD_CORRECT_TERMINATION_LAST_BLOCK) to out.working_buffer - static void addEmptyBlock(WriteBuffer & write_buffer); - private: /// NOTE: will fill compressed data to the out.working_buffer, but will not call out.next method until the buffer is full void nextImpl() override; @@ -67,12 +61,21 @@ private: void finalizeAfter(); void finalizeZstd(); - std::unique_ptr out; + /// Read three last bytes from non-empty compressed file and compares them with + /// ZSTD_CORRECT_TERMINATION_LAST_BLOCK. + bool isNeedToAddEmptyBlock(); + /// Adding zstd empty block (ZSTD_CORRECT_TERMINATION_LAST_BLOCK) to out.working_buffer + void addEmptyBlock(); + + std::unique_ptr out; + + bool append_to_existing_file = false; ZSTD_CCtx * cctx; ZSTD_inBuffer input; ZSTD_outBuffer output; - bool first_write{true}; + /// Flipped on the first nextImpl call + bool first_write = true; }; } From fc23ff6ad2135542492d03fde50acf095c2a27b0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 25 Jan 2023 16:28:54 +0000 Subject: [PATCH 097/439] Add entity into setting changes history for compatibility mode --- src/Core/SettingsChangesHistory.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 534fcd42037..19fc4c5fab6 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -81,7 +81,8 @@ namespace SettingsChangesHistory static std::map settings_changes_history = { {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, - {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}}}, + {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, + {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, From 381c5a62b8c7968bd36004bbadb5e358aa3c4d9a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 17:33:34 +0100 Subject: [PATCH 098/439] fix --- tests/clickhouse-test | 20 +++++++++++++++++++ ..._log_and_exception_messages_formatting.sql | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 5aad3756e02..0b7c566d300 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1882,6 +1882,26 @@ def reportLogStats(args): print(value) print("\n") + query = """ + WITH ('', '({}) Keys: {}', '({}) {}', 'Aggregating', 'Became leader', 'Cleaning queue', 'Creating set.', + 'Cyclic aliases', 'Detaching {}', 'Executing {}', 'Fire events: {}', 'Found part {}', 'Loaded queue', + 'No tables', 'Query: {}', 'Removed', 'Removed part {}', 'Removing parts.', 'Request URI: {}', + 'Sending part {}', 'Sent handshake', 'Starting {}', 'Will mimic {}', 'Writing to {}', + 'dropIfEmpty', 'loadAll {}', '{} ({}:{})', '{} -> {}', '{} {}', '{}: {}' + ) AS known_short_messages + SELECT count() AS c, message_format_string, any(message) + FROM system.text_log + WHERE (now() - toIntervalMinute(120)) < event_time + AND (length(message_format_string) < 16 + OR (length(message_format_string) < 16 AND message ilike '%DB::Exception%')) + AND message_format_string NOT IN known_short_messages + GROUP BY message_format_string ORDER BY c DESC LIMIT 50 + """ + value = clickhouse_execute(args, query).decode(errors="replace") + print("\nTop short messages:\n") + print(value) + print("\n") + def main(args): global server_died diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index e5008871e4e..91926d2b7d1 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -22,7 +22,7 @@ select 30, countDistinct(message_format_string) < 10 from logs where length(mess select 40, countDistinct(message_format_string) < 35 from logs where length(message_format_string) < 16; -- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary -select 50, countDistinct(message_format_string) < 90 from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%'; +select 50, countDistinct(message_format_string) < 100 from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%'; -- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold From 8e3698c91f5187cddf5fee313fd9bda3550116b2 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 25 Jan 2023 17:34:09 +0000 Subject: [PATCH 099/439] refactoring of code near merge tree parts --- src/Disks/IDisk.h | 10 + src/IO/WriteSettings.h | 2 + ...Disk.cpp => DataPartStorageOnDiskBase.cpp} | 1017 ++++++++--------- ...geOnDisk.h => DataPartStorageOnDiskBase.h} | 124 +- .../MergeTree/DataPartStorageOnDiskFull.cpp | 199 ++++ .../MergeTree/DataPartStorageOnDiskFull.h | 62 + src/Storages/MergeTree/DataPartsExchange.cpp | 429 +++---- src/Storages/MergeTree/DataPartsExchange.h | 28 +- .../MergeTree/FutureMergedMutatedPart.cpp | 15 +- .../MergeTree/FutureMergedMutatedPart.h | 8 +- src/Storages/MergeTree/IDataPartStorage.h | 52 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 83 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 22 +- .../MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergeTask.cpp | 48 +- src/Storages/MergeTree/MergeTreeData.cpp | 176 ++- src/Storages/MergeTree/MergeTreeData.h | 20 +- .../MergeTree/MergeTreeDataPartBuilder.cpp | 201 ++++ .../MergeTree/MergeTreeDataPartBuilder.h | 69 ++ .../MergeTree/MergeTreeDataPartChecksum.cpp | 24 +- .../MergeTree/MergeTreeDataPartChecksum.h | 13 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 10 - .../MergeTree/MergeTreeDataPartCompact.h | 6 - .../MergeTree/MergeTreeDataPartInMemory.cpp | 61 +- .../MergeTree/MergeTreeDataPartInMemory.h | 6 - .../MergeTree/MergeTreeDataPartType.cpp | 27 - .../MergeTree/MergeTreeDataPartType.h | 86 +- .../MergeTree/MergeTreeDataPartWide.cpp | 10 - .../MergeTree/MergeTreeDataPartWide.h | 6 - .../MergeTreeDataPartWriterCompact.cpp | 9 +- .../MergeTree/MergeTreeDataWriter.cpp | 41 +- src/Storages/MergeTree/MergeTreeIOSettings.h | 9 - .../MergeTreeIndexGranularityInfo.cpp | 8 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 3 +- .../MergeTree/MergeTreeWriteAheadLog.cpp | 13 +- .../MergeTree/MutateFromLogEntryTask.cpp | 6 +- .../MergeTree/MutatePlainMergeTreeTask.cpp | 6 +- src/Storages/MergeTree/MutateTask.cpp | 28 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 26 +- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 2 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 3 +- src/Storages/StorageMergeTree.cpp | 6 +- src/Storages/StorageReplicatedMergeTree.cpp | 39 +- src/Storages/StorageReplicatedMergeTree.h | 4 +- .../System/StorageSystemDetachedParts.cpp | 2 +- 46 files changed, 1616 insertions(+), 1407 deletions(-) rename src/Storages/MergeTree/{DataPartStorageOnDisk.cpp => DataPartStorageOnDiskBase.cpp} (60%) rename src/Storages/MergeTree/{DataPartStorageOnDisk.h => DataPartStorageOnDiskBase.h} (57%) create mode 100644 src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp create mode 100644 src/Storages/MergeTree/DataPartStorageOnDiskFull.h create mode 100644 src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp create mode 100644 src/Storages/MergeTree/MergeTreeDataPartBuilder.h delete mode 100644 src/Storages/MergeTree/MergeTreeDataPartType.cpp diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index e831ef53b60..7907c0b2a74 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -487,3 +487,13 @@ inline String directoryPath(const String & path) } } + +template <> +struct fmt::formatter : fmt::formatter +{ + template + auto format(const fs::path & path, FormatCtx & ctx) const + { + return fmt::formatter::format(path.string(), ctx); + } +}; diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index 764d6c8992b..08f11e0bfd1 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -21,6 +21,8 @@ struct WriteSettings /// Monitoring bool for_object_storage = false; // to choose which profile events should be incremented + + bool operator==(const WriteSettings & other) const = default; }; } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp similarity index 60% rename from src/Storages/MergeTree/DataPartStorageOnDisk.cpp rename to src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index e4180a8e9f4..baa7fdbb418 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -1,17 +1,14 @@ -#include +#include #include -#include #include #include -#include +#include #include #include +#include #include #include -#include #include -#include -#include namespace DB { @@ -22,14 +19,15 @@ namespace ErrorCodes extern const int NOT_ENOUGH_SPACE; extern const int LOGICAL_ERROR; extern const int FILE_DOESNT_EXIST; + extern const int CORRUPTED_DATA; } -DataPartStorageOnDisk::DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_) +DataPartStorageOnDiskBase::DataPartStorageOnDiskBase(VolumePtr volume_, std::string root_path_, std::string part_dir_) : volume(std::move(volume_)), root_path(std::move(root_path_)), part_dir(std::move(part_dir_)) { } -DataPartStorageOnDisk::DataPartStorageOnDisk( +DataPartStorageOnDiskBase::DataPartStorageOnDiskBase( VolumePtr volume_, std::string root_path_, std::string part_dir_, DiskTransactionPtr transaction_) : volume(std::move(volume_)) , root_path(std::move(root_path_)) @@ -39,92 +37,132 @@ DataPartStorageOnDisk::DataPartStorageOnDisk( { } -std::string DataPartStorageOnDisk::getFullPath() const +DiskPtr DataPartStorageOnDiskBase::getDisk() const +{ + return volume->getDisk(); +} + +std::string DataPartStorageOnDiskBase::getFullPath() const { return fs::path(volume->getDisk()->getPath()) / root_path / part_dir / ""; } -std::string DataPartStorageOnDisk::getRelativePath() const +std::string DataPartStorageOnDiskBase::getRelativePath() const { return fs::path(root_path) / part_dir / ""; } -void DataPartStorageOnDisk::setRelativePath(const std::string & path) +std::optional DataPartStorageOnDiskBase::getRelativePathForPrefix(Poco::Logger * log, const String & prefix, bool detached, bool broken) const +{ + assert(!broken || detached); + String res; + + auto full_relative_path = fs::path(root_path); + if (detached) + full_relative_path /= "detached"; + + std::optional original_checksums_content; + std::optional original_files_list; + + for (int try_no = 0; try_no < 10; ++try_no) + { + if (prefix.empty()) + res = part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + else if (prefix.ends_with("_")) + res = prefix + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + else + res = prefix + "_" + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); + + if (!volume->getDisk()->exists(full_relative_path / res)) + return res; + + /// If part with compacted storage is broken then we probably + /// cannot read the single file with data and check its content. + if (broken + && isFullPartStorage(*this) + && looksLikeBrokenDetachedPartHasTheSameContent(res, original_checksums_content, original_files_list)) + { + LOG_WARNING(log, "Directory {} (to detach to) already exists, " + "but its content looks similar to content of the broken part which we are going to detach. " + "Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.", res); + return {}; + } + + LOG_WARNING(log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); + } + + return res; +} + +bool DataPartStorageOnDiskBase::looksLikeBrokenDetachedPartHasTheSameContent(const String & detached_part_path, + std::optional & original_checksums_content, + std::optional & original_files_list) const +{ + /// We cannot know for sure that content of detached part is the same, + /// but in most cases it's enough to compare checksums.txt and list of files. + + if (!exists("checksums.txt")) + return false; + + auto storage_from_detached = create(volume, fs::path(root_path) / "detached", detached_part_path, /*initialize=*/ true); + if (!storage_from_detached->exists("checksums.txt")) + return false; + + if (!original_checksums_content) + { + auto in = storage_from_detached->readFile("checksums.txt", /* settings */ {}, /* read_hint */ {}, /* file_size */ {}); + original_checksums_content.emplace(); + readStringUntilEOF(*original_checksums_content, *in); + } + + if (original_checksums_content->empty()) + return false; + + String detached_checksums_content; + { + auto in = readFile("checksums.txt", /* settings */ {}, /* read_hint */ {}, /* file_size */ {}); + readStringUntilEOF(detached_checksums_content, *in); + } + + if (original_checksums_content != detached_checksums_content) + return false; + + if (!original_files_list) + { + original_files_list.emplace(); + for (auto it = iterate(); it->isValid(); it->next()) + original_files_list->emplace_back(it->name()); + std::sort(original_files_list->begin(), original_files_list->end()); + } + + Strings detached_files_list; + for (auto it = storage_from_detached->iterate(); it->isValid(); it->next()) + detached_files_list.emplace_back(it->name()); + std::sort(detached_files_list.begin(), detached_files_list.end()); + + return original_files_list == detached_files_list; +} + +void DataPartStorageOnDiskBase::setRelativePath(const std::string & path) { part_dir = path; } -std::string DataPartStorageOnDisk::getFullRootPath() const +std::string DataPartStorageOnDiskBase::getPartDirectory() const +{ + return part_dir; +} + +std::string DataPartStorageOnDiskBase::getFullRootPath() const { return fs::path(volume->getDisk()->getPath()) / root_path / ""; } -MutableDataPartStoragePtr DataPartStorageOnDisk::getProjection(const std::string & name, bool use_parent_transaction) // NOLINT -{ - return std::shared_ptr(new DataPartStorageOnDisk(volume, std::string(fs::path(root_path) / part_dir), name, use_parent_transaction ? transaction : nullptr)); -} - -DataPartStoragePtr DataPartStorageOnDisk::getProjection(const std::string & name) const -{ - return std::make_shared(volume, std::string(fs::path(root_path) / part_dir), name); -} - -bool DataPartStorageOnDisk::exists() const -{ - return volume->getDisk()->exists(fs::path(root_path) / part_dir); -} - -bool DataPartStorageOnDisk::exists(const std::string & name) const -{ - return volume->getDisk()->exists(fs::path(root_path) / part_dir / name); -} - -bool DataPartStorageOnDisk::isDirectory(const std::string & name) const -{ - return volume->getDisk()->isDirectory(fs::path(root_path) / part_dir / name); -} - -Poco::Timestamp DataPartStorageOnDisk::getLastModified() const +Poco::Timestamp DataPartStorageOnDiskBase::getLastModified() const { return volume->getDisk()->getLastModified(fs::path(root_path) / part_dir); } -class DataPartStorageIteratorOnDisk final : public IDataPartStorageIterator -{ -public: - DataPartStorageIteratorOnDisk(DiskPtr disk_, DirectoryIteratorPtr it_) - : disk(std::move(disk_)), it(std::move(it_)) - { - } - - void next() override { it->next(); } - bool isValid() const override { return it->isValid(); } - bool isFile() const override { return isValid() && disk->isFile(it->path()); } - std::string name() const override { return it->name(); } - std::string path() const override { return it->path(); } - -private: - DiskPtr disk; - DirectoryIteratorPtr it; -}; - -DataPartStorageIteratorPtr DataPartStorageOnDisk::iterate() const -{ - return std::make_unique( - volume->getDisk(), - volume->getDisk()->iterateDirectory(fs::path(root_path) / part_dir)); -} - -size_t DataPartStorageOnDisk::getFileSize(const String & file_name) const -{ - return volume->getDisk()->getFileSize(fs::path(root_path) / part_dir / file_name); -} - -UInt32 DataPartStorageOnDisk::getRefCount(const String & file_name) const -{ - return volume->getDisk()->getRefCount(fs::path(root_path) / part_dir / file_name); -} - static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & from) { if (disk->isFile(from)) @@ -132,44 +170,339 @@ static UInt64 calculateTotalSizeOnDiskImpl(const DiskPtr & disk, const String & std::vector files; disk->listFiles(from, files); + UInt64 res = 0; for (const auto & file : files) res += calculateTotalSizeOnDiskImpl(disk, fs::path(from) / file); + return res; } -UInt64 DataPartStorageOnDisk::calculateTotalSizeOnDisk() const +UInt64 DataPartStorageOnDiskBase::calculateTotalSizeOnDisk() const { return calculateTotalSizeOnDiskImpl(volume->getDisk(), fs::path(root_path) / part_dir); } -std::unique_ptr DataPartStorageOnDisk::readFile( - const std::string & name, - const ReadSettings & settings, - std::optional read_hint, - std::optional file_size) const +std::string DataPartStorageOnDiskBase::getDiskName() const { - return volume->getDisk()->readFile(fs::path(root_path) / part_dir / name, settings, read_hint, file_size); + return volume->getDisk()->getName(); } -void DataPartStorageOnDisk::checkConsistency(const MergeTreeDataPartChecksums & checksums) const +std::string DataPartStorageOnDiskBase::getDiskType() const { - checksums.checkSizes(volume->getDisk(), getRelativePath()); + return toString(volume->getDisk()->getDataSourceDescription().type); } -void DataPartStorageOnDisk::remove( +bool DataPartStorageOnDiskBase::isStoredOnRemoteDisk() const +{ + return volume->getDisk()->isRemote(); +} + +bool DataPartStorageOnDiskBase::supportZeroCopyReplication() const +{ + return volume->getDisk()->supportZeroCopyReplication(); +} + +bool DataPartStorageOnDiskBase::supportParallelWrite() const +{ + return volume->getDisk()->supportParallelWrite(); +} + +bool DataPartStorageOnDiskBase::isBroken() const +{ + return volume->getDisk()->isBroken(); +} + +void DataPartStorageOnDiskBase::syncRevision(UInt64 revision) const +{ + volume->getDisk()->syncRevision(revision); +} + +UInt64 DataPartStorageOnDiskBase::getRevision() const +{ + return volume->getDisk()->getRevision(); +} + +std::string DataPartStorageOnDiskBase::getDiskPath() const +{ + return volume->getDisk()->getPath(); +} + +ReservationPtr DataPartStorageOnDiskBase::reserve(UInt64 bytes) const +{ + auto res = volume->reserve(bytes); + if (!res) + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Cannot reserve {}, not enough space", ReadableSize(bytes)); + + return res; +} + +ReservationPtr DataPartStorageOnDiskBase::tryReserve(UInt64 bytes) const +{ + return volume->reserve(bytes); +} + +IDataPartStorage::ReplicatedFilesDescription +DataPartStorageOnDiskBase::getReplicatedFilesDescription(const NameSet & file_names) const +{ + ReplicatedFilesDescription description; + auto relative_path = fs::path(root_path) / part_dir; + auto disk = volume->getDisk(); + + auto actual_file_names = getActualFileNamesOnDisk(file_names); + for (const auto & name : actual_file_names) + { + auto path = relative_path / name; + size_t file_size = disk->getFileSize(path); + + auto & file_desc = description.files[name]; + + file_desc.file_size = file_size; + file_desc.input_buffer_getter = [disk, path, file_size] + { + return disk->readFile(path, ReadSettings{}.adjustBufferSize(file_size), file_size, file_size); + }; + } + + return description; +} + +IDataPartStorage::ReplicatedFilesDescription +DataPartStorageOnDiskBase::getReplicatedFilesDescriptionForRemoteDisk(const NameSet & file_names) const +{ + ReplicatedFilesDescription description; + auto relative_path = fs::path(root_path) / part_dir; + + auto disk = volume->getDisk(); + if (!disk->supportZeroCopyReplication()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Disk {} doesn't support zero-copy replication", disk->getName()); + + description.unique_id = getUniqueId(); + + Names paths; + auto actual_file_names = getActualFileNamesOnDisk(file_names); + + for (const auto & name : actual_file_names) + { + /// Just some additional checks + auto metadata_full_file_path = fs::path(getFullPath()) / name; + if (!fs::exists(metadata_full_file_path)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", name); + if (!fs::is_regular_file(metadata_full_file_path)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", name); + + paths.emplace_back(relative_path / name); + } + + auto serialized_metadata = disk->getSerializedMetadata(paths); + for (const auto & name : actual_file_names) + { + auto & file_desc = description.files[name]; + const auto & metadata_str = serialized_metadata.at(relative_path / name); + + file_desc.file_size = metadata_str.size(); + file_desc.input_buffer_getter = [metadata_str] + { + return std::make_unique(metadata_str); + }; + } + + return description; +} + +void DataPartStorageOnDiskBase::backup( + const MergeTreeDataPartChecksums & checksums, + const NameSet & files_without_checksums, + const String & path_in_backup, + BackupEntries & backup_entries, + bool make_temporary_hard_links, + TemporaryFilesOnDisks * temp_dirs) const +{ + fs::path part_path_on_disk = fs::path{root_path} / part_dir; + fs::path part_path_in_backup = fs::path{path_in_backup} / part_dir; + + auto disk = volume->getDisk(); + + fs::path temp_part_dir; + std::shared_ptr temp_dir_owner; + if (make_temporary_hard_links) + { + assert(temp_dirs); + auto temp_dir_it = temp_dirs->find(disk); + if (temp_dir_it == temp_dirs->end()) + temp_dir_it = temp_dirs->emplace(disk, std::make_shared(disk, "tmp/")).first; + + temp_dir_owner = temp_dir_it->second; + fs::path temp_dir = temp_dir_owner->getPath(); + temp_part_dir = temp_dir / part_path_in_backup.relative_path(); + disk->createDirectories(temp_part_dir); + } + + /// For example, + /// part_path_in_backup = /data/test/table/0_1_1_0 + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0 + /// tmp_part_dir = tmp/1aaaaaa/data/test/table/0_1_1_0 + /// Or, for projections: + /// part_path_in_backup = /data/test/table/0_1_1_0/prjmax.proj + /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0/prjmax.proj + /// tmp_part_dir = tmp/1aaaaaa/data/test/table/0_1_1_0/prjmax.proj + + auto files_to_backup = files_without_checksums; + for (const auto & [name, _] : checksums.files) + { + if (!name.ends_with(".proj")) + files_to_backup.insert(name); + } + + files_to_backup = getActualFileNamesOnDisk(files_to_backup); + + for (const auto & filepath : files_to_backup) + { + auto filepath_on_disk = part_path_on_disk / filepath; + auto filepath_in_backup = part_path_in_backup / filepath; + + if (files_without_checksums.contains(filepath)) + { + backup_entries.emplace_back(filepath_in_backup, std::make_unique(disk, filepath_on_disk)); + continue; + } + + if (make_temporary_hard_links) + { + String hardlink_filepath = temp_part_dir / filepath; + disk->createHardLink(filepath_on_disk, hardlink_filepath); + filepath_on_disk = hardlink_filepath; + } + + std::optional file_size; + std::optional file_hash; + + auto it = checksums.files.find(filepath); + if (it != checksums.files.end()) + { + file_size = it->second.file_size; + file_hash = {it->second.file_hash.first, it->second.file_hash.second}; + } + + backup_entries.emplace_back( + filepath_in_backup, + std::make_unique(disk, filepath_on_disk, file_size, file_hash, temp_dir_owner)); + } +} + +MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( + const std::string & to, + const std::string & dir_path, + bool make_source_readonly, + std::function save_metadata_callback, + bool copy_instead_of_hardlink, + const NameSet & files_to_copy_instead_of_hardlinks) const +{ + auto disk = volume->getDisk(); + disk->createDirectories(to); + + localBackup(disk, getRelativePath(), fs::path(to) / dir_path, make_source_readonly, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks); + + if (save_metadata_callback) + save_metadata_callback(disk); + + disk->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt"); + disk->removeFileIfExists(fs::path(to) / dir_path / "txn_version.txt"); + + auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); + + /// Do not initialize storage in case of DETACH because part may be broken. + bool to_detached = dir_path.starts_with("detached/"); + return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached); +} + +MutableDataPartStoragePtr DataPartStorageOnDiskBase::clonePart( + const std::string & to, + const std::string & dir_path, + const DiskPtr & disk, + Poco::Logger * log) const +{ + String path_to_clone = fs::path(to) / dir_path / ""; + + if (disk->exists(path_to_clone)) + { + LOG_WARNING(log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone)); + disk->removeRecursive(path_to_clone); + } + + disk->createDirectories(to); + volume->getDisk()->copy(getRelativePath(), disk, to); + volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / "delete-on-destroy.txt"); + + auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); + return create(single_disk_volume, to, dir_path, /*initialize=*/ true); +} + +void DataPartStorageOnDiskBase::rename( + std::string new_root_path, + std::string new_part_dir, + Poco::Logger * log, + bool remove_new_dir_if_exists, + bool fsync_part_dir) +{ + if (new_root_path.ends_with('/')) + new_root_path.pop_back(); + if (new_part_dir.ends_with('/')) + new_part_dir.pop_back(); + + String to = fs::path(new_root_path) / new_part_dir / ""; + + if (volume->getDisk()->exists(to)) + { + if (remove_new_dir_if_exists) + { + Names files; + volume->getDisk()->listFiles(to, files); + + if (log) + LOG_WARNING(log, + "Part directory {} already exists and contains {} files. Removing it.", + fullPath(volume->getDisk(), to), files.size()); + + executeWriteOperation([&](auto & disk) { disk.removeRecursive(to); }); + } + else + { + throw Exception( + ErrorCodes::DIRECTORY_ALREADY_EXISTS, + "Part directory {} already exists", + fullPath(volume->getDisk(), to)); + } + } + + String from = getRelativePath(); + + /// Why? + executeWriteOperation([&](auto & disk) + { + disk.setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); + disk.moveDirectory(from, to); + }); + + part_dir = new_part_dir; + root_path = new_root_path; + + SyncGuardPtr sync_guard; + if (fsync_part_dir) + sync_guard = volume->getDisk()->getDirectorySyncGuard(getRelativePath()); +} + +void DataPartStorageOnDiskBase::remove( CanRemoveCallback && can_remove_callback, const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, - MergeTreeDataPartState state, Poco::Logger * log) { /// NOTE We rename part to delete_tmp_ instead of delete_tmp_ to avoid race condition /// when we try to remove two parts with the same name, but different relative paths, /// for example all_1_2_1 (in Deleting state) and tmp_merge_all_1_2_1 (in Temporary state). fs::path from = fs::path(root_path) / part_dir; - // fs::path to = fs::path(root_path) / ("delete_tmp_" + part_dir); // TODO directory delete_tmp_ is never removed if server crashes before returning from this function /// Cut last "/" if it exists (it shouldn't). Otherwise fs::path behave differently. @@ -262,15 +595,18 @@ void DataPartStorageOnDisk::remove( NameSet files_not_to_remove_for_projection; for (const auto & file_name : can_remove_description->files_not_to_remove) - { if (file_name.starts_with(proj_dir_name)) files_not_to_remove_for_projection.emplace(fs::path(file_name).filename()); - } + LOG_DEBUG(log, "Will not remove files [{}] for projection {}", fmt::join(files_not_to_remove_for_projection, ", "), projection.name); - clearDirectory( - fs::path(to) / proj_dir_name, - can_remove_description->can_remove_anything, files_not_to_remove_for_projection, projection.checksums, {}, is_temp, state, log, true); + CanRemoveDescription proj_description + { + can_remove_description->can_remove_anything, + std::move(files_not_to_remove_for_projection), + }; + + clearDirectory(fs::path(to) / proj_dir_name, proj_description, projection.checksums, is_temp, log); } /// It is possible that we are removing the part which have a written but not loaded projection. @@ -279,55 +615,49 @@ void DataPartStorageOnDisk::remove( /// See test 01701_clear_projection_and_part. for (const auto & [name, _] : checksums.files) { - if (endsWith(name, proj_suffix) && !projection_directories.contains(name) && disk->isDirectory(fs::path(to) / name)) + if (endsWith(name, proj_suffix) && !projection_directories.contains(name)) { + static constexpr auto checksums_name = "checksums.txt"; + auto projection_storage = create(volume, to, name, /*initialize=*/ true); /// If we have a directory with suffix '.proj' it is likely a projection. /// Try to load checksums for it (to avoid recursive removing fallback). - std::string checksum_path = fs::path(to) / name / "checksums.txt"; - if (disk->exists(checksum_path)) + if (projection_storage->exists(checksums_name)) { try { MergeTreeDataPartChecksums tmp_checksums; - auto in = disk->readFile(checksum_path, {}); + auto in = projection_storage->readFile(checksums_name, {}, {}, {}); tmp_checksums.read(*in); - projection_directories.emplace(name); - - clearDirectory( - fs::path(to) / name, - can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, tmp_checksums, {}, is_temp, state, log, true); + clearDirectory(fs::path(to) / name, *can_remove_description, tmp_checksums, is_temp, log); } catch (...) { - LOG_ERROR(log, "Cannot load checksums from {}", checksum_path); + LOG_ERROR(log, "Cannot load checksums from {}", fs::path(projection_storage->getRelativePath()) / checksums_name); } } } } - clearDirectory(to, can_remove_description->can_remove_anything, can_remove_description->files_not_to_remove, checksums, projection_directories, is_temp, state, log, false); + clearDirectory(to, *can_remove_description, checksums, is_temp, log); } -void DataPartStorageOnDisk::clearDirectory( +void DataPartStorageOnDiskBase::clearDirectory( const std::string & dir, - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + const CanRemoveDescription & can_remove_description, const MergeTreeDataPartChecksums & checksums, - const std::unordered_set & skip_directories, bool is_temp, - MergeTreeDataPartState state, - Poco::Logger * log, - bool is_projection) const + Poco::Logger * log) { auto disk = volume->getDisk(); + auto [can_remove_shared_data, names_not_to_remove] = can_remove_description; + names_not_to_remove = getActualFileNamesOnDisk(names_not_to_remove); /// It does not make sense to try fast path for incomplete temporary parts, because some files are probably absent. /// Sometimes we add something to checksums.files before actually writing checksums and columns on disk. /// Also sometimes we write checksums.txt and columns.txt in arbitrary order, so this check becomes complex... - bool is_temporary_part = is_temp || state == MergeTreeDataPartState::Temporary; - bool incomplete_temporary_part = is_temporary_part && (!disk->exists(fs::path(dir) / "checksums.txt") || !disk->exists(fs::path(dir) / "columns.txt")); + bool incomplete_temporary_part = is_temp && (!disk->exists(fs::path(dir) / "checksums.txt") || !disk->exists(fs::path(dir) / "columns.txt")); if (checksums.empty() || incomplete_temporary_part) { /// If the part is not completely written, we cannot use fast path by listing files. @@ -337,23 +667,21 @@ void DataPartStorageOnDisk::clearDirectory( try { + NameSet names_to_remove = {"checksums.txt", "columns.txt"}; + for (const auto & [file, _] : checksums.files) + if (!endsWith(file, ".proj")) + names_to_remove.emplace(file); + + names_to_remove = getActualFileNamesOnDisk(names_to_remove); + /// Remove each expected file in directory, then remove directory itself. RemoveBatchRequest request; - - for (const auto & [file, _] : checksums.files) - { - if (skip_directories.find(file) == skip_directories.end()) - request.emplace_back(fs::path(dir) / file); - } - - for (const auto & file : {"checksums.txt", "columns.txt"}) + for (const auto & file : names_to_remove) request.emplace_back(fs::path(dir) / file); request.emplace_back(fs::path(dir) / "default_compression_codec.txt", true); request.emplace_back(fs::path(dir) / "delete-on-destroy.txt", true); - - if (!is_projection) - request.emplace_back(fs::path(dir) / "txn_version.txt", true); + request.emplace_back(fs::path(dir) / "txn_version.txt", true); disk->removeSharedFiles(request, !can_remove_shared_data, names_not_to_remove); disk->removeDirectory(dir); @@ -367,330 +695,7 @@ void DataPartStorageOnDisk::clearDirectory( } } -std::optional DataPartStorageOnDisk::getRelativePathForPrefix(Poco::Logger * log, const String & prefix, bool detached, bool broken) const -{ - assert(!broken || detached); - String res; - - auto full_relative_path = fs::path(root_path); - if (detached) - full_relative_path /= "detached"; - - std::optional original_checksums_content; - std::optional original_files_list; - - for (int try_no = 0; try_no < 10; ++try_no) - { - if (prefix.empty()) - res = part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); - else if (prefix.ends_with("_")) - res = prefix + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); - else - res = prefix + "_" + part_dir + (try_no ? "_try" + DB::toString(try_no) : ""); - - if (!volume->getDisk()->exists(full_relative_path / res)) - return res; - - if (broken && looksLikeBrokenDetachedPartHasTheSameContent(res, original_checksums_content, original_files_list)) - { - LOG_WARNING(log, "Directory {} (to detach to) already exists, " - "but its content looks similar to content of the broken part which we are going to detach. " - "Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.", res); - return {}; - } - - LOG_WARNING(log, "Directory {} (to detach to) already exists. Will detach to directory with '_tryN' suffix.", res); - } - - return res; -} - -bool DataPartStorageOnDisk::looksLikeBrokenDetachedPartHasTheSameContent(const String & detached_part_path, - std::optional & original_checksums_content, - std::optional & original_files_list) const -{ - /// We cannot know for sure that content of detached part is the same, - /// but in most cases it's enough to compare checksums.txt and list of files. - - if (!exists("checksums.txt")) - return false; - - auto detached_full_path = fs::path(root_path) / "detached" / detached_part_path; - auto disk = volume->getDisk(); - if (!disk->exists(detached_full_path / "checksums.txt")) - return false; - - if (!original_checksums_content) - { - auto in = disk->readFile(detached_full_path / "checksums.txt", /* settings */ {}, /* read_hint */ {}, /* file_size */ {}); - original_checksums_content.emplace(); - readStringUntilEOF(*original_checksums_content, *in); - } - - if (original_checksums_content->empty()) - return false; - - auto part_full_path = fs::path(root_path) / part_dir; - String detached_checksums_content; - { - auto in = readFile("checksums.txt", /* settings */ {}, /* read_hint */ {}, /* file_size */ {}); - readStringUntilEOF(detached_checksums_content, *in); - } - - if (original_checksums_content != detached_checksums_content) - return false; - - if (!original_files_list) - { - original_files_list.emplace(); - disk->listFiles(part_full_path, *original_files_list); - std::sort(original_files_list->begin(), original_files_list->end()); - } - - Strings detached_files_list; - disk->listFiles(detached_full_path, detached_files_list); - std::sort(detached_files_list.begin(), detached_files_list.end()); - - return original_files_list == detached_files_list; -} - -std::string DataPartStorageOnDisk::getDiskName() const -{ - return volume->getDisk()->getName(); -} - -std::string DataPartStorageOnDisk::getDiskType() const -{ - return toString(volume->getDisk()->getDataSourceDescription().type); -} - -bool DataPartStorageOnDisk::isStoredOnRemoteDisk() const -{ - return volume->getDisk()->isRemote(); -} - -bool DataPartStorageOnDisk::supportZeroCopyReplication() const -{ - return volume->getDisk()->supportZeroCopyReplication(); -} - -bool DataPartStorageOnDisk::supportParallelWrite() const -{ - return volume->getDisk()->supportParallelWrite(); -} - -bool DataPartStorageOnDisk::isBroken() const -{ - return volume->getDisk()->isBroken(); -} - -void DataPartStorageOnDisk::syncRevision(UInt64 revision) const -{ - volume->getDisk()->syncRevision(revision); -} - -UInt64 DataPartStorageOnDisk::getRevision() const -{ - return volume->getDisk()->getRevision(); -} - -std::unordered_map DataPartStorageOnDisk::getSerializedMetadata(const std::vector & paths) const -{ - return volume->getDisk()->getSerializedMetadata(paths); -} - -std::string DataPartStorageOnDisk::getDiskPath() const -{ - return volume->getDisk()->getPath(); -} - -ReservationPtr DataPartStorageOnDisk::reserve(UInt64 bytes) const -{ - auto res = volume->reserve(bytes); - if (!res) - throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Cannot reserve {}, not enough space", ReadableSize(bytes)); - - return res; -} - -ReservationPtr DataPartStorageOnDisk::tryReserve(UInt64 bytes) const -{ - return volume->reserve(bytes); -} - -String DataPartStorageOnDisk::getUniqueId() const -{ - auto disk = volume->getDisk(); - if (!disk->supportZeroCopyReplication()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Disk {} doesn't support zero-copy replication", disk->getName()); - - return disk->getUniqueId(fs::path(getRelativePath()) / "checksums.txt"); -} - -void DataPartStorageOnDisk::backup( - const MergeTreeDataPartChecksums & checksums, - const NameSet & files_without_checksums, - const String & path_in_backup, - BackupEntries & backup_entries, - bool make_temporary_hard_links, - TemporaryFilesOnDisks * temp_dirs) const -{ - fs::path part_path_on_disk = fs::path{root_path} / part_dir; - fs::path part_path_in_backup = fs::path{path_in_backup} / part_dir; - - auto disk = volume->getDisk(); - - fs::path temp_part_dir; - std::shared_ptr temp_dir_owner; - if (make_temporary_hard_links) - { - assert(temp_dirs); - auto temp_dir_it = temp_dirs->find(disk); - if (temp_dir_it == temp_dirs->end()) - temp_dir_it = temp_dirs->emplace(disk, std::make_shared(disk, "tmp/")).first; - temp_dir_owner = temp_dir_it->second; - fs::path temp_dir = temp_dir_owner->getPath(); - temp_part_dir = temp_dir / part_path_in_backup.relative_path(); - disk->createDirectories(temp_part_dir); - } - - /// For example, - /// part_path_in_backup = /data/test/table/0_1_1_0 - /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0 - /// tmp_part_dir = tmp/1aaaaaa/data/test/table/0_1_1_0 - /// Or, for projections: - /// part_path_in_backup = /data/test/table/0_1_1_0/prjmax.proj - /// part_path_on_disk = store/f57/f5728353-44bb-4575-85e8-28deb893657a/0_1_1_0/prjmax.proj - /// tmp_part_dir = tmp/1aaaaaa/data/test/table/0_1_1_0/prjmax.proj - - for (const auto & [filepath, checksum] : checksums.files) - { - if (filepath.ends_with(".proj")) - continue; /// Skip *.proj files - they're actually directories and will be handled. - String filepath_on_disk = part_path_on_disk / filepath; - String filepath_in_backup = part_path_in_backup / filepath; - - if (make_temporary_hard_links) - { - String hardlink_filepath = temp_part_dir / filepath; - disk->createHardLink(filepath_on_disk, hardlink_filepath); - filepath_on_disk = hardlink_filepath; - } - - UInt128 file_hash{checksum.file_hash.first, checksum.file_hash.second}; - backup_entries.emplace_back( - filepath_in_backup, - std::make_unique(disk, filepath_on_disk, checksum.file_size, file_hash, temp_dir_owner)); - } - - for (const auto & filepath : files_without_checksums) - { - String filepath_on_disk = part_path_on_disk / filepath; - String filepath_in_backup = part_path_in_backup / filepath; - backup_entries.emplace_back(filepath_in_backup, std::make_unique(disk, filepath_on_disk)); - } -} - -MutableDataPartStoragePtr DataPartStorageOnDisk::freeze( - const std::string & to, - const std::string & dir_path, - bool make_source_readonly, - std::function save_metadata_callback, - bool copy_instead_of_hardlink, - const NameSet & files_to_copy_instead_of_hardlinks) const - -{ - auto disk = volume->getDisk(); - disk->createDirectories(to); - - localBackup(disk, getRelativePath(), fs::path(to) / dir_path, make_source_readonly, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks); - - if (save_metadata_callback) - save_metadata_callback(disk); - - disk->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt"); - disk->removeFileIfExists(fs::path(to) / dir_path / "txn_version.txt"); - - auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - return std::make_shared(single_disk_volume, to, dir_path); -} - -MutableDataPartStoragePtr DataPartStorageOnDisk::clonePart( - const std::string & to, - const std::string & dir_path, - const DiskPtr & disk, - Poco::Logger * log) const -{ - String path_to_clone = fs::path(to) / dir_path / ""; - - if (disk->exists(path_to_clone)) - { - LOG_WARNING(log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone)); - disk->removeRecursive(path_to_clone); - } - - disk->createDirectories(to); - volume->getDisk()->copy(getRelativePath(), disk, to); - volume->getDisk()->removeFileIfExists(fs::path(path_to_clone) / "delete-on-destroy.txt"); - - auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - return std::make_shared(single_disk_volume, to, dir_path); -} - -void DataPartStorageOnDisk::rename( - std::string new_root_path, - std::string new_part_dir, - Poco::Logger * log, - bool remove_new_dir_if_exists, - bool fsync_part_dir) -{ - if (new_root_path.ends_with('/')) - new_root_path.pop_back(); - if (new_part_dir.ends_with('/')) - new_part_dir.pop_back(); - - String to = fs::path(new_root_path) / new_part_dir / ""; - - if (volume->getDisk()->exists(to)) - { - if (remove_new_dir_if_exists) - { - Names files; - volume->getDisk()->listFiles(to, files); - - if (log) - LOG_WARNING(log, - "Part directory {} already exists and contains {} files. Removing it.", - fullPath(volume->getDisk(), to), files.size()); - - executeOperation([&](auto & disk) { disk.removeRecursive(to); }); - } - else - { - throw Exception( - ErrorCodes::DIRECTORY_ALREADY_EXISTS, - "Part directory {} already exists", - fullPath(volume->getDisk(), to)); - } - } - String from = getRelativePath(); - - /// Why? - executeOperation([&](auto & disk) - { - disk.setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); - disk.moveDirectory(from, to); - }); - - part_dir = new_part_dir; - root_path = new_root_path; - - SyncGuardPtr sync_guard; - if (fsync_part_dir) - sync_guard = volume->getDisk()->getDirectorySyncGuard(getRelativePath()); -} - -void DataPartStorageOnDisk::changeRootPath(const std::string & from_root, const std::string & to_root) +void DataPartStorageOnDiskBase::changeRootPath(const std::string & from_root, const std::string & to_root) { /// This is a very dumb implementation, here for root path like /// "some/current/path/to/part" and change like @@ -716,134 +721,34 @@ void DataPartStorageOnDisk::changeRootPath(const std::string & from_root, const root_path = to_root.substr(0, dst_size) + root_path.substr(prefix_size); } -SyncGuardPtr DataPartStorageOnDisk::getDirectorySyncGuard() const +SyncGuardPtr DataPartStorageOnDiskBase::getDirectorySyncGuard() const { return volume->getDisk()->getDirectorySyncGuard(fs::path(root_path) / part_dir); } -template -void DataPartStorageOnDisk::executeOperation(Op && op) -{ - if (transaction) - op(*transaction); - else - op(*volume->getDisk()); -} - -std::unique_ptr DataPartStorageOnDisk::writeFile( - const String & name, - size_t buf_size, - const WriteSettings & settings) -{ - return writeFile(name, buf_size, WriteMode::Rewrite, settings); -} - -std::unique_ptr DataPartStorageOnDisk::writeFile( - const String & name, - size_t buf_size, - WriteMode mode, - const WriteSettings & settings) -{ - if (transaction) - return transaction->writeFile(fs::path(root_path) / part_dir / name, buf_size, mode, settings, /* autocommit = */ false); - - return volume->getDisk()->writeFile(fs::path(root_path) / part_dir / name, buf_size, mode, settings); -} - -std::unique_ptr DataPartStorageOnDisk::writeTransactionFile(WriteMode mode) const +std::unique_ptr DataPartStorageOnDiskBase::writeTransactionFile(WriteMode mode) const { return volume->getDisk()->writeFile(fs::path(root_path) / part_dir / "txn_version.txt", 256, mode); } -void DataPartStorageOnDisk::createFile(const String & name) +void DataPartStorageOnDiskBase::removeRecursive() { - executeOperation([&](auto & disk) { disk.createFile(fs::path(root_path) / part_dir / name); }); + executeWriteOperation([&](auto & disk) { disk.removeRecursive(fs::path(root_path) / part_dir); }); } -void DataPartStorageOnDisk::moveFile(const String & from_name, const String & to_name) +void DataPartStorageOnDiskBase::removeSharedRecursive(bool keep_in_remote_fs) { - executeOperation([&](auto & disk) - { - auto relative_path = fs::path(root_path) / part_dir; - disk.moveFile(relative_path / from_name, relative_path / to_name); - }); + executeWriteOperation([&](auto & disk) { disk.removeSharedRecursive(fs::path(root_path) / part_dir, keep_in_remote_fs, {}); }); } -void DataPartStorageOnDisk::replaceFile(const String & from_name, const String & to_name) +void DataPartStorageOnDiskBase::createDirectories() { - executeOperation([&](auto & disk) - { - auto relative_path = fs::path(root_path) / part_dir; - disk.replaceFile(relative_path / from_name, relative_path / to_name); - }); + executeWriteOperation([&](auto & disk) { disk.createDirectories(fs::path(root_path) / part_dir); }); } -void DataPartStorageOnDisk::removeFile(const String & name) +bool DataPartStorageOnDiskBase::hasActiveTransaction() const { - executeOperation([&](auto & disk) { disk.removeFile(fs::path(root_path) / part_dir / name); }); -} - -void DataPartStorageOnDisk::removeFileIfExists(const String & name) -{ - executeOperation([&](auto & disk) { disk.removeFileIfExists(fs::path(root_path) / part_dir / name); }); -} - -void DataPartStorageOnDisk::removeRecursive() -{ - executeOperation([&](auto & disk) { disk.removeRecursive(fs::path(root_path) / part_dir); }); -} - -void DataPartStorageOnDisk::removeSharedRecursive(bool keep_in_remote_fs) -{ - executeOperation([&](auto & disk) { disk.removeSharedRecursive(fs::path(root_path) / part_dir, keep_in_remote_fs, {}); }); -} - -void DataPartStorageOnDisk::createHardLinkFrom(const IDataPartStorage & source, const std::string & from, const std::string & to) -{ - const auto * source_on_disk = typeid_cast(&source); - if (!source_on_disk) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cannot create hardlink from different storage. Expected DataPartStorageOnDisk, got {}", - typeid(source).name()); - - executeOperation([&](auto & disk) - { - disk.createHardLink( - fs::path(source_on_disk->getRelativePath()) / from, - fs::path(root_path) / part_dir / to); - }); -} - -void DataPartStorageOnDisk::createDirectories() -{ - executeOperation([&](auto & disk) { disk.createDirectories(fs::path(root_path) / part_dir); }); -} - -void DataPartStorageOnDisk::createProjection(const std::string & name) -{ - executeOperation([&](auto & disk) { disk.createDirectory(fs::path(root_path) / part_dir / name); }); -} - -void DataPartStorageOnDisk::beginTransaction() -{ - if (transaction) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Uncommitted {}transaction already exists", has_shared_transaction ? "shared " : ""); - - transaction = volume->getDisk()->createTransaction(); -} - -void DataPartStorageOnDisk::commitTransaction() -{ - if (!transaction) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no uncommitted transaction"); - - if (has_shared_transaction) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot commit shared transaction"); - - transaction->commit(); - transaction.reset(); + return transaction != nullptr; } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h similarity index 57% rename from src/Storages/MergeTree/DataPartStorageOnDisk.h rename to src/Storages/MergeTree/DataPartStorageOnDiskBase.h index 86b8784e2b1..7c408dcf381 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.h +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include #include #include @@ -10,48 +11,19 @@ namespace DB class IVolume; using VolumePtr = std::shared_ptr; - -class DataPartStorageOnDisk final : public IDataPartStorage +class DataPartStorageOnDiskBase : public IDataPartStorage { public: - DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_); + DataPartStorageOnDiskBase(VolumePtr volume_, std::string root_path_, std::string part_dir_); std::string getFullPath() const override; std::string getRelativePath() const override; - std::string getPartDirectory() const override { return part_dir; } + std::string getPartDirectory() const override; std::string getFullRootPath() const override; - MutableDataPartStoragePtr getProjection(const std::string & name, bool use_parent_transaction = true) override; // NOLINT - DataPartStoragePtr getProjection(const std::string & name) const override; - - bool exists() const override; - bool exists(const std::string & name) const override; - bool isDirectory(const std::string & name) const override; - Poco::Timestamp getLastModified() const override; - DataPartStorageIteratorPtr iterate() const override; - - size_t getFileSize(const std::string & file_name) const override; - UInt32 getRefCount(const std::string & file_name) const override; - UInt64 calculateTotalSizeOnDisk() const override; - std::unique_ptr readFile( - const std::string & name, - const ReadSettings & settings, - std::optional read_hint, - std::optional file_size) const override; - - void checkConsistency(const MergeTreeDataPartChecksums & checksums) const override; - - void remove( - CanRemoveCallback && can_remove_callback, - const MergeTreeDataPartChecksums & checksums, - std::list projections, - bool is_temp, - MergeTreeDataPartState state, - Poco::Logger * log) override; - /// Returns path to place detached part in or nullopt if we don't need to detach part (if it already exists and has the same content) std::optional getRelativePathForPrefix(Poco::Logger * log, const String & prefix, bool detached, bool broken) const override; @@ -69,11 +41,12 @@ public: bool isBroken() const override; void syncRevision(UInt64 revision) const override; UInt64 getRevision() const override; - std::unordered_map getSerializedMetadata(const std::vector & paths) const override; std::string getDiskPath() const override; ReservationPtr reserve(UInt64 bytes) const override; ReservationPtr tryReserve(UInt64 bytes) const override; - String getUniqueId() const override; + + ReplicatedFilesDescription getReplicatedFilesDescription(const NameSet & file_names) const override; + ReplicatedFilesDescription getReplicatedFilesDescriptionForRemoteDisk(const NameSet & file_names) const override; void backup( const MergeTreeDataPartChecksums & checksums, @@ -97,36 +70,6 @@ public: const DiskPtr & disk, Poco::Logger * log) const override; - void changeRootPath(const std::string & from_root, const std::string & to_root) override; - - void createDirectories() override; - void createProjection(const std::string & name) override; - - std::unique_ptr writeFile( - const String & name, - size_t buf_size, - const WriteSettings & settings) override; - std::unique_ptr writeFile( - const String & name, - size_t buf_size, - DB::WriteMode mode, - const WriteSettings & settings) override; - - std::unique_ptr writeTransactionFile(WriteMode mode) const override; - - void createFile(const String & name) override; - void moveFile(const String & from_name, const String & to_name) override; - void replaceFile(const String & from_name, const String & to_name) override; - - void removeFile(const String & name) override; - void removeFileIfExists(const String & name) override; - void removeRecursive() override; - void removeSharedRecursive(bool keep_in_remote_fs) override; - - SyncGuardPtr getDirectorySyncGuard() const override; - - void createHardLinkFrom(const IDataPartStorage & source, const std::string & from, const std::string & to) override; - void rename( std::string new_root_path, std::string new_part_dir, @@ -134,31 +77,58 @@ public: bool remove_new_dir_if_exists, bool fsync_part_dir) override; - void beginTransaction() override; - void commitTransaction() override; - bool hasActiveTransaction() const override { return transaction != nullptr; } + void remove( + CanRemoveCallback && can_remove_callback, + const MergeTreeDataPartChecksums & checksums, + std::list projections, + bool is_temp, + Poco::Logger * log) override; + + void changeRootPath(const std::string & from_root, const std::string & to_root) override; + void createDirectories() override; + + std::unique_ptr writeTransactionFile(WriteMode mode) const override; + + void removeRecursive() override; + void removeSharedRecursive(bool keep_in_remote_fs) override; + + SyncGuardPtr getDirectorySyncGuard() const override; + bool hasActiveTransaction() const override; + +protected: + DiskPtr getDisk() const; + + DataPartStorageOnDiskBase(VolumePtr volume_, std::string root_path_, std::string part_dir_, DiskTransactionPtr transaction_); + virtual MutableDataPartStoragePtr create(VolumePtr volume_, std::string root_path_, std::string part_dir_, bool initialize_) const = 0; -private: VolumePtr volume; std::string root_path; std::string part_dir; DiskTransactionPtr transaction; bool has_shared_transaction = false; - DataPartStorageOnDisk(VolumePtr volume_, std::string root_path_, std::string part_dir_, DiskTransactionPtr transaction_); - template - void executeOperation(Op && op); + void executeWriteOperation(Op && op) + { + if (transaction) + op(*transaction); + else + op(*volume->getDisk()); + } +private: void clearDirectory( const std::string & dir, - bool can_remove_shared_data, - const NameSet & names_not_to_remove, + const CanRemoveDescription & can_remove_description, const MergeTreeDataPartChecksums & checksums, - const std::unordered_set & skip_directories, bool is_temp, - MergeTreeDataPartState state, - Poco::Logger * log, - bool is_projection) const; + Poco::Logger * log); + + /// For names of expected data part files returns the actual names + /// of files in filesystem to which data of these files is written. + /// Actual file name may be the same as expected + /// or be the name of the file with packed data. + virtual NameSet getActualFileNamesOnDisk(const NameSet & file_names) const = 0; }; + } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp new file mode 100644 index 00000000000..5ad01774096 --- /dev/null +++ b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp @@ -0,0 +1,199 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +DataPartStorageOnDiskFull::DataPartStorageOnDiskFull(VolumePtr volume_, std::string root_path_, std::string part_dir_) + : DataPartStorageOnDiskBase(std::move(volume_), std::move(root_path_), std::move(part_dir_)) +{ +} + +DataPartStorageOnDiskFull::DataPartStorageOnDiskFull( + VolumePtr volume_, std::string root_path_, std::string part_dir_, DiskTransactionPtr transaction_) + : DataPartStorageOnDiskBase(std::move(volume_), std::move(root_path_), std::move(part_dir_), std::move(transaction_)) +{ +} + +MutableDataPartStoragePtr DataPartStorageOnDiskFull::create( + VolumePtr volume_, std::string root_path_, std::string part_dir_, bool /*initialize_*/) const +{ + return std::make_shared(std::move(volume_), std::move(root_path_), std::move(part_dir_)); +} + +MutableDataPartStoragePtr DataPartStorageOnDiskFull::getProjection(const std::string & name, bool use_parent_transaction) // NOLINT +{ + return std::shared_ptr(new DataPartStorageOnDiskFull(volume, std::string(fs::path(root_path) / part_dir), name, use_parent_transaction ? transaction : nullptr)); +} + +DataPartStoragePtr DataPartStorageOnDiskFull::getProjection(const std::string & name) const +{ + return std::make_shared(volume, std::string(fs::path(root_path) / part_dir), name); +} + +bool DataPartStorageOnDiskFull::exists() const +{ + return volume->getDisk()->exists(fs::path(root_path) / part_dir); +} + +bool DataPartStorageOnDiskFull::exists(const std::string & name) const +{ + return volume->getDisk()->exists(fs::path(root_path) / part_dir / name); +} + +bool DataPartStorageOnDiskFull::isDirectory(const std::string & name) const +{ + return volume->getDisk()->isDirectory(fs::path(root_path) / part_dir / name); +} + +class DataPartStorageIteratorOnDisk final : public IDataPartStorageIterator +{ +public: + DataPartStorageIteratorOnDisk(DiskPtr disk_, DirectoryIteratorPtr it_) + : disk(std::move(disk_)), it(std::move(it_)) + { + } + + void next() override { it->next(); } + bool isValid() const override { return it->isValid(); } + bool isFile() const override { return isValid() && disk->isFile(it->path()); } + std::string name() const override { return it->name(); } + std::string path() const override { return it->path(); } + +private: + DiskPtr disk; + DirectoryIteratorPtr it; +}; + +DataPartStorageIteratorPtr DataPartStorageOnDiskFull::iterate() const +{ + return std::make_unique( + volume->getDisk(), + volume->getDisk()->iterateDirectory(fs::path(root_path) / part_dir)); +} + +size_t DataPartStorageOnDiskFull::getFileSize(const String & file_name) const +{ + return volume->getDisk()->getFileSize(fs::path(root_path) / part_dir / file_name); +} + +UInt32 DataPartStorageOnDiskFull::getRefCount(const String & file_name) const +{ + return volume->getDisk()->getRefCount(fs::path(root_path) / part_dir / file_name); +} + +String DataPartStorageOnDiskFull::getUniqueId() const +{ + auto disk = volume->getDisk(); + if (!disk->supportZeroCopyReplication()) + throw Exception(fmt::format("Disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); + + return disk->getUniqueId(fs::path(getRelativePath()) / "checksums.txt"); +} + +std::unique_ptr DataPartStorageOnDiskFull::readFile( + const std::string & name, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const +{ + return volume->getDisk()->readFile(fs::path(root_path) / part_dir / name, settings, read_hint, file_size); +} + +std::unique_ptr DataPartStorageOnDiskFull::writeFile( + const String & name, + size_t buf_size, + WriteMode mode, + const WriteSettings & settings) +{ + if (transaction) + return transaction->writeFile(fs::path(root_path) / part_dir / name, buf_size, mode, settings, /* autocommit = */ false); + else + return volume->getDisk()->writeFile(fs::path(root_path) / part_dir / name, buf_size, mode, settings); +} + +void DataPartStorageOnDiskFull::createFile(const String & name) +{ + executeWriteOperation([&](auto & disk) { disk.createFile(fs::path(root_path) / part_dir / name); }); +} + +void DataPartStorageOnDiskFull::moveFile(const String & from_name, const String & to_name) +{ + executeWriteOperation([&](auto & disk) + { + auto relative_path = fs::path(root_path) / part_dir; + disk.moveFile(relative_path / from_name, relative_path / to_name); + }); +} + +void DataPartStorageOnDiskFull::replaceFile(const String & from_name, const String & to_name) +{ + executeWriteOperation([&](auto & disk) + { + auto relative_path = fs::path(root_path) / part_dir; + disk.replaceFile(relative_path / from_name, relative_path / to_name); + }); +} + +void DataPartStorageOnDiskFull::removeFile(const String & name) +{ + executeWriteOperation([&](auto & disk) { disk.removeFile(fs::path(root_path) / part_dir / name); }); +} + +void DataPartStorageOnDiskFull::removeFileIfExists(const String & name) +{ + executeWriteOperation([&](auto & disk) { disk.removeFileIfExists(fs::path(root_path) / part_dir / name); }); +} + +void DataPartStorageOnDiskFull::createHardLinkFrom(const IDataPartStorage & source, const std::string & from, const std::string & to) +{ + const auto * source_on_disk = typeid_cast(&source); + if (!source_on_disk) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot create hardlink from different storage. Expected DataPartStorageOnDiskFull, got {}", + typeid(source).name()); + + executeWriteOperation([&](auto & disk) + { + disk.createHardLink( + fs::path(source_on_disk->getRelativePath()) / from, + fs::path(root_path) / part_dir / to); + }); +} + +void DataPartStorageOnDiskFull::createProjection(const std::string & name) +{ + executeWriteOperation([&](auto & disk) { disk.createDirectory(fs::path(root_path) / part_dir / name); }); +} + +void DataPartStorageOnDiskFull::beginTransaction() +{ + if (transaction) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Uncommitted{}transaction already exists", has_shared_transaction ? " shared " : " "); + + transaction = volume->getDisk()->createTransaction(); +} + +void DataPartStorageOnDiskFull::commitTransaction() +{ + if (!transaction) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no uncommitted transaction"); + + if (has_shared_transaction) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot commit shared transaction"); + + transaction->commit(); + transaction.reset(); +} + +} diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskFull.h b/src/Storages/MergeTree/DataPartStorageOnDiskFull.h new file mode 100644 index 00000000000..8b38bfd7105 --- /dev/null +++ b/src/Storages/MergeTree/DataPartStorageOnDiskFull.h @@ -0,0 +1,62 @@ +#pragma once +#include "Storages/MergeTree/IDataPartStorage.h" +#include + +namespace DB +{ + +/// A storage for data part that stores files on filesystem as is. +class DataPartStorageOnDiskFull final : public DataPartStorageOnDiskBase +{ +public: + DataPartStorageOnDiskFull(VolumePtr volume_, std::string root_path_, std::string part_dir_); + MergeTreeDataPartStorageType getType() const override { return MergeTreeDataPartStorageType::Full; } + + MutableDataPartStoragePtr getProjection(const std::string & name, bool use_parent_transaction = true) override; // NOLINT + DataPartStoragePtr getProjection(const std::string & name) const override; + + bool exists() const override; + bool exists(const std::string & name) const override; + bool isDirectory(const std::string & name) const override; + + DataPartStorageIteratorPtr iterate() const override; + size_t getFileSize(const std::string & file_name) const override; + UInt32 getRefCount(const std::string & file_name) const override; + String getUniqueId() const override; + + std::unique_ptr readFile( + const std::string & name, + const ReadSettings & settings, + std::optional read_hint, + std::optional file_size) const override; + + void createProjection(const std::string & name) override; + + std::unique_ptr writeFile( + const String & name, + size_t buf_size, + WriteMode mode, + const WriteSettings & settings) override; + + void createFile(const String & name) override; + void moveFile(const String & from_name, const String & to_name) override; + void replaceFile(const String & from_name, const String & to_name) override; + + void removeFile(const String & name) override; + void removeFileIfExists(const String & name) override; + + void createHardLinkFrom(const IDataPartStorage & source, const std::string & from, const std::string & to) override; + + void beginTransaction() override; + void commitTransaction() override; + void precommitTransaction() override {} + bool hasActiveTransaction() const override { return transaction != nullptr; } + +private: + DataPartStorageOnDiskFull(VolumePtr volume_, std::string root_path_, std::string part_dir_, DiskTransactionPtr transaction_); + MutableDataPartStoragePtr create(VolumePtr volume_, std::string root_path_, std::string part_dir_, bool initialize_) const override; + + NameSet getActualFileNamesOnDisk(const NameSet & file_names) const override { return file_names; } +}; + +} diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f9cad146593..76f8b6b4e9c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -38,7 +38,6 @@ namespace DB namespace ErrorCodes { - extern const int DIRECTORY_ALREADY_EXISTS; extern const int NO_SUCH_DATA_PART; extern const int ABORTED; extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; @@ -67,7 +66,6 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; // Reserved for ALTER PRIMARY KEY // constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 8; - std::string getEndpointId(const std::string & node_id) { return "DataPartsExchange:" + node_id; @@ -181,6 +179,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write std::sregex_token_iterator()); bool send_projections = client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION; + if (send_projections) { const auto & projections = part->getProjectionParts(); @@ -197,7 +196,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write { /// Send metadata if the receiver's capability covers the source disk type. response.addCookie({"remote_fs_metadata", disk_type}); - sendPartFromDiskRemoteMeta(part, out, true, send_projections); + sendPartFromDisk(part, out, client_protocol_version, true, send_projections); return; } } @@ -205,7 +204,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write if (isInMemoryPart(part)) sendPartFromMemory(part, out, send_projections); else - sendPartFromDisk(part, out, client_protocol_version, send_projections); + sendPartFromDisk(part, out, client_protocol_version, false, send_projections); } catch (const NetException &) { @@ -261,30 +260,50 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( const MergeTreeData::DataPartPtr & part, WriteBuffer & out, int client_protocol_version, + bool from_remote_disk, bool send_projections) { - /// We'll take a list of files from the list of checksums. - MergeTreeData::DataPart::Checksums checksums = part->checksums; - /// Add files that are not in the checksum list. + NameSet files_to_replicate; auto file_names_without_checksums = part->getFileNamesWithoutChecksums(); - for (const auto & file_name : file_names_without_checksums) + + for (const auto & [name, _] : part->checksums.files) { - if (client_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION - && file_name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) + if (endsWith(name, ".proj")) continue; - checksums.files[file_name] = {}; + files_to_replicate.insert(name); + } + + for (const auto & name : file_names_without_checksums) + { + if (client_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION + && name == IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) + continue; + + files_to_replicate.insert(name); + } + + auto data_part_storage = part->getDataPartStoragePtr(); + IDataPartStorage::ReplicatedFilesDescription replicated_description; + + if (from_remote_disk) + { + replicated_description = data_part_storage->getReplicatedFilesDescriptionForRemoteDisk(files_to_replicate); + if (!part->isProjectionPart()) + writeStringBinary(replicated_description.unique_id, out); + } + else + { + replicated_description = data_part_storage->getReplicatedFilesDescription(files_to_replicate); } MergeTreeData::DataPart::Checksums data_checksums; for (const auto & [name, projection] : part->getProjectionParts()) { - // Get rid of projection files - checksums.files.erase(name + ".proj"); if (send_projections) { writeStringBinary(name, out); - MergeTreeData::DataPart::Checksums projection_checksum = sendPartFromDisk(projection, out, client_protocol_version, false); + MergeTreeData::DataPart::Checksums projection_checksum = sendPartFromDisk(projection, out, client_protocol_version, from_remote_disk, false); data_checksums.addFile(name + ".proj", projection_checksum.getTotalSizeOnDisk(), projection_checksum.getTotalChecksumUInt128()); } else if (part->checksums.has(name + ".proj")) @@ -295,29 +314,25 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( } } - writeBinary(checksums.files.size(), out); - for (const auto & it : checksums.files) + writeBinary(replicated_description.files.size(), out); + for (const auto & [file_name, desc] : replicated_description.files) { - String file_name = it.first; + writeStringBinary(file_name, out); + writeBinary(desc.file_size, out); - UInt64 size = part->getDataPartStorage().getFileSize(file_name); - - writeStringBinary(it.first, out); - writeBinary(size, out); - - auto file_in = part->getDataPartStorage().readFile(file_name, {}, std::nullopt, std::nullopt); + auto file_in = desc.input_buffer_getter(); HashingWriteBuffer hashing_out(out); copyDataWithThrottler(*file_in, hashing_out, blocker.getCounter(), data.getSendsThrottler()); if (blocker.isCancelled()) throw Exception(ErrorCodes::ABORTED, "Transferring part to replica was cancelled"); - if (hashing_out.count() != size) + if (hashing_out.count() != desc.file_size) throw Exception( ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}, expected {} got {}", std::string(fs::path(part->getDataPartStorage().getRelativePath()) / file_name), - hashing_out.count(), size); + desc.file_size, hashing_out.count()); writePODBinary(hashing_out.getHash(), out); @@ -325,90 +340,12 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( data_checksums.addFile(file_name, hashing_out.count(), hashing_out.getHash()); } - part->checksums.checkEqual(data_checksums, false); + if (!from_remote_disk && isFullPartStorage(part->getDataPartStorage())) + part->checksums.checkEqual(data_checksums, false); + return data_checksums; } -void Service::sendPartFromDiskRemoteMeta( - const MergeTreeData::DataPartPtr & part, - WriteBuffer & out, - bool send_part_id, - bool send_projections) -{ - auto data_part_storage = part->getDataPartStoragePtr(); - if (!data_part_storage->supportZeroCopyReplication()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Disk '{}' doesn't support zero-copy replication", data_part_storage->getDiskName()); - - /// We'll take a list of files from the list of checksums. - MergeTreeData::DataPart::Checksums checksums = part->checksums; - /// Add files that are not in the checksum list. - auto file_names_without_checksums = part->getFileNamesWithoutChecksums(); - for (const auto & file_name : file_names_without_checksums) - checksums.files[file_name] = {}; - - for (const auto & [name, projection] : part->getProjectionParts()) - { - // Get rid of projection files - checksums.files.erase(name + ".proj"); - } - - std::vector paths; - paths.reserve(checksums.files.size()); - for (const auto & it : checksums.files) - paths.push_back(fs::path(part->getDataPartStorage().getRelativePath()) / it.first); - - /// Serialized metadatadatas with zero ref counts. - auto metadatas = data_part_storage->getSerializedMetadata(paths); - - if (send_part_id) - { - String part_id = data_part_storage->getUniqueId(); - writeStringBinary(part_id, out); - } - - if (send_projections) - { - for (const auto & [name, projection] : part->getProjectionParts()) - { - writeStringBinary(name, out); - sendPartFromDiskRemoteMeta(projection, out, false, false); - } - } - - writeBinary(checksums.files.size(), out); - for (const auto & it : checksums.files) - { - const String & file_name = it.first; - String file_path_prefix = fs::path(part->getDataPartStorage().getRelativePath()) / file_name; - - /// Just some additional checks - String metadata_file_path = fs::path(data_part_storage->getDiskPath()) / file_path_prefix; - fs::path metadata(metadata_file_path); - if (!fs::exists(metadata)) - throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", file_name); - if (!fs::is_regular_file(metadata)) - throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", file_name); - - /// Actual metadata send - auto metadata_str = metadatas[file_path_prefix]; - UInt64 file_size = metadata_str.size(); - ReadBufferFromString buf(metadata_str); - - writeStringBinary(it.first, out); - writeBinary(file_size, out); - - HashingWriteBuffer hashing_out(out); - copyDataWithThrottler(buf, hashing_out, blocker.getCounter(), data.getSendsThrottler()); - if (blocker.isCancelled()) - throw Exception(ErrorCodes::ABORTED, "Transferring part to replica was cancelled"); - - if (hashing_out.count() != file_size) - throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file_path); - - writePODBinary(hashing_out.getHash(), out); - } -} - MergeTreeData::DataPartPtr Service::findPart(const String & name) { /// It is important to include PreActive and Outdated parts here because remote replicas cannot reliably @@ -602,9 +539,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); - String part_type = "Wide"; + using PartType = MergeTreeDataPartType; + PartType part_type = PartType::Wide; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE) - readStringBinary(part_type, *in); + { + String part_type_str; + readStringBinary(part_type_str, *in); + part_type.fromString(part_type_str); + } UUID part_uuid = UUIDHelpers::Nil; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) @@ -616,7 +558,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) readBinary(projections, *in); - MergeTreeData::DataPart::Checksums checksums; if (!remote_fs_metadata.empty()) { if (!try_zero_copy) @@ -626,20 +567,24 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( remote_fs_metadata, fmt::join(capability, ", ")); if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version); - if (part_type == "InMemory") + if (part_type == PartType::InMemory) throw Exception(ErrorCodes::INCORRECT_PART_TYPE, "Got 'remote_fs_metadata' cookie for in-memory part"); try { - return downloadPartToDiskRemoteMeta(part_name, replica_path, to_detached, tmp_prefix, disk, *in, projections, checksums, throttler); - } + auto output_buffer_getter = [](IDataPartStorage & part_storage, const auto & file_name, size_t file_size) + { + auto full_path = fs::path(part_storage.getFullPath()) / file_name; + return std::make_unique(full_path, std::min(DBMS_DEFAULT_BUFFER_SIZE, file_size)); + }; + return downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix, disk, true, *in, output_buffer_getter, projections, throttler, sync); + } catch (const Exception & e) { if (e.code() != ErrorCodes::S3_ERROR && e.code() != ErrorCodes::ZERO_COPY_REPLICATION_ERROR) throw; - #if USE_AWS_S3 if (const auto * s3_exception = dynamic_cast(&e)) { @@ -674,7 +619,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( } auto storage_id = data.getStorageID(); - String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(disk)) / part_name / ""; + String new_part_path = part_type == PartType::InMemory ? "memory" : fs::path(data.getFullPathOnDisk(disk)) / part_name / ""; auto entry = data.getContext()->getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), part_info.partition_id, part_name, new_part_path, @@ -682,11 +627,11 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( in->setNextCallback(ReplicatedFetchReadCallback(*entry)); - if (part_type == "InMemory") + if (part_type == PartType::InMemory) { auto volume = std::make_shared("volume_" + part_name, disk, 0); - auto data_part_storage = std::make_shared( + auto data_part_storage = std::make_shared( volume, data.getRelativeDataPath(), part_name); @@ -698,9 +643,15 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchSelectedPart( projections, false, throttler); } + auto output_buffer_getter = [](IDataPartStorage & part_storage, const String & file_name, size_t file_size) + { + return part_storage.writeFile(file_name, std::min(file_size, DBMS_DEFAULT_BUFFER_SIZE), {}); + }; + return downloadPartToDisk( part_name, replica_path, to_detached, tmp_prefix, - sync, disk, *in, projections, checksums, throttler); + disk, false, *in, output_buffer_getter, + projections, throttler, sync); } MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( @@ -763,67 +714,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( return new_data_part; } -void Fetcher::downloadBasePartOrProjectionPartToDiskRemoteMeta( - const String & replica_path, - const MutableDataPartStoragePtr & data_part_storage, - PooledReadWriteBufferFromHTTP & in, - MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) const -{ - size_t files; - readBinary(files, in); - - for (size_t i = 0; i < files; ++i) - { - String file_name; - UInt64 file_size; - - readStringBinary(file_name, in); - readBinary(file_size, in); - - String metadata_file = fs::path(data_part_storage->getFullPath()) / file_name; - - { - auto file_out = std::make_unique(metadata_file, DBMS_DEFAULT_BUFFER_SIZE, -1, 0666, nullptr, 0); - - HashingWriteBuffer hashing_out(*file_out); - - copyDataWithThrottler(in, hashing_out, file_size, blocker.getCounter(), throttler); - - if (blocker.isCancelled()) - { - /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, - /// performing a poll with a not very large timeout. - /// And now we check it only between read chunks (in the `copyData` function). - throw Exception(ErrorCodes::ABORTED, "Fetching of part was cancelled"); - } - - MergeTreeDataPartChecksum::uint128 expected_hash; - readPODBinary(expected_hash, in); - - if (expected_hash != hashing_out.getHash()) - { - throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, - "Checksum mismatch for file {} transferred from {}", - metadata_file, replica_path); - } - - if (file_name != "checksums.txt" && - file_name != "columns.txt" && - file_name != IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME) - checksums.addFile(file_name, file_size, expected_hash); - } - } -} - - void Fetcher::downloadBaseOrProjectionPartToDisk( const String & replica_path, const MutableDataPartStoragePtr & data_part_storage, - bool sync, PooledReadWriteBufferFromHTTP & in, + OutputBufferGetter output_buffer_getter, MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) const + ThrottlerPtr throttler, + bool sync) const { size_t files; readBinary(files, in); @@ -845,7 +743,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( "This may happen if we are trying to download part from malicious replica or logical error.", absolute_file_path, data_part_storage->getRelativePath()); - auto file_out = data_part_storage->writeFile(file_name, std::min(file_size, DBMS_DEFAULT_BUFFER_SIZE), {}); + auto file_out = output_buffer_getter(*data_part_storage, file_name, file_size); HashingWriteBuffer hashing_out(*file_out); copyDataWithThrottler(in, hashing_out, file_size, blocker.getCounter(), throttler); @@ -881,15 +779,32 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( const String & replica_path, bool to_detached, const String & tmp_prefix, - bool sync, DiskPtr disk, + bool to_remote_disk, PooledReadWriteBufferFromHTTP & in, + OutputBufferGetter output_buffer_getter, size_t projections, - MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) + ThrottlerPtr throttler, + bool sync) { - assert(!tmp_prefix.empty()); + String part_id; const auto data_settings = data.getSettings(); + MergeTreeData::DataPart::Checksums data_checksums; + + if (to_remote_disk) + { + readStringBinary(part_id, in); + + if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id)) + throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {} (with type {}).", part_name, part_id, disk->getName(), toString(disk->getDataSourceDescription().type)); + + LOG_DEBUG(log, "Downloading part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); + data.lockSharedDataTemporary(part_name, part_id, disk); + } + else + { + LOG_DEBUG(log, "Downloading part {} onto disk {}.", part_name, disk->getName()); + } /// We will remove directory if it's already exists. Make precautions. if (tmp_prefix.empty() //-V560 @@ -898,37 +813,34 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( || std::string::npos != part_name.find_first_of("/.")) throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: tmp_prefix and part_name cannot be empty or contain '.' or '/' characters."); - String part_dir = tmp_prefix + part_name; - String part_relative_path = data.getRelativeDataPath() + String(to_detached ? "detached/" : ""); + auto part_dir = tmp_prefix + part_name; + auto part_relative_path = data.getRelativeDataPath() + String(to_detached ? "detached/" : ""); + auto volume = std::make_shared("volume_" + part_name, disk); - auto volume = std::make_shared("volume_" + part_name, disk, 0); + /// Create temporary part storage to write sent files. + /// Actual part storage will be initialized later from metadata. + auto part_storage_for_loading = std::make_shared(volume, part_relative_path, part_dir); + part_storage_for_loading->beginTransaction(); - auto data_part_storage = std::make_shared( - volume, - part_relative_path, - part_dir); - - data_part_storage->beginTransaction(); - - if (data_part_storage->exists()) + if (part_storage_for_loading->exists()) { LOG_WARNING(log, "Directory {} already exists, probably result of a failed fetch. Will remove it before fetching part.", - data_part_storage->getFullPath()); + part_storage_for_loading->getFullPath()); /// Even if it's a temporary part it could be downloaded with zero copy replication and this function /// is executed as a callback. /// /// We don't control the amount of refs for temporary parts so we cannot decide can we remove blobs /// or not. So we are not doing it - bool keep_shared = disk->supportZeroCopyReplication() && data_settings->allow_remote_fs_zero_copy_replication; - data_part_storage->removeSharedRecursive(keep_shared); + bool keep_shared = part_storage_for_loading->supportZeroCopyReplication() && data_settings->allow_remote_fs_zero_copy_replication; + part_storage_for_loading->removeSharedRecursive(keep_shared); } - data_part_storage->createDirectories(); + part_storage_for_loading->createDirectories(); SyncGuardPtr sync_guard; if (data.getSettings()->fsync_part_directory) - sync_guard = data_part_storage->getDirectorySyncGuard(); + sync_guard = part_storage_for_loading->getDirectorySyncGuard(); CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; @@ -940,16 +852,18 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( readStringBinary(projection_name, in); MergeTreeData::DataPart::Checksums projection_checksum; - auto projection_part_storage = data_part_storage->getProjection(projection_name + ".proj"); + auto projection_part_storage = part_storage_for_loading->getProjection(projection_name + ".proj"); projection_part_storage->createDirectories(); + downloadBaseOrProjectionPartToDisk( - replica_path, projection_part_storage, sync, in, projection_checksum, throttler); - checksums.addFile( + replica_path, projection_part_storage, in, output_buffer_getter, projection_checksum, throttler, sync); + + data_checksums.addFile( projection_name + ".proj", projection_checksum.getTotalSizeOnDisk(), projection_checksum.getTotalChecksumUInt128()); } - // Download the base part - downloadBaseOrProjectionPartToDisk(replica_path, data_part_storage, sync, in, checksums, throttler); + downloadBaseOrProjectionPartToDisk( + replica_path, part_storage_for_loading, in, output_buffer_getter, data_checksums, throttler, sync); } catch (const Exception & e) { @@ -957,98 +871,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( /// part or fetch of any projection was stopped. if (e.code() == ErrorCodes::ABORTED) { - data_part_storage->removeRecursive(); - data_part_storage->commitTransaction(); - } - throw; - } - - assertEOF(in); - data_part_storage->commitTransaction(); - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, data_part_storage); - new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr); - new_data_part->is_temp = true; - new_data_part->modification_time = time(nullptr); - new_data_part->loadColumnsChecksumsIndexes(true, false); - new_data_part->checksums.checkEqual(checksums, false); - return new_data_part; -} - -MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( - const String & part_name, - const String & replica_path, - bool to_detached, - const String & tmp_prefix, - DiskPtr disk, - PooledReadWriteBufferFromHTTP & in, - size_t projections, - MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) -{ - String part_id; - readStringBinary(part_id, in); - - if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id)) - { - throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, - "Part {} unique id {} doesn't exist on {} (with type {}).", - part_name, part_id, disk->getName(), toString(disk->getDataSourceDescription().type)); - } - - LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", - part_name, part_id, disk->getName()); - - data.lockSharedDataTemporary(part_name, part_id, disk); - - assert(!tmp_prefix.empty()); - - String part_dir = tmp_prefix + part_name; - String part_relative_path = data.getRelativeDataPath() + String(to_detached ? "detached/" : ""); - - auto volume = std::make_shared("volume_" + part_name, disk); - - auto data_part_storage = std::make_shared( - volume, - part_relative_path, - part_dir); - - data_part_storage->beginTransaction(); - - if (data_part_storage->exists()) - throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", data_part_storage->getFullPath()); - - CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; - - data_part_storage->createDirectories(); - - try - { - for (size_t i = 0; i < projections; ++i) - { - String projection_name; - readStringBinary(projection_name, in); - MergeTreeData::DataPart::Checksums projection_checksum; - - auto projection_part_storage = data_part_storage->getProjection(projection_name + ".proj"); - projection_part_storage->createDirectories(); - downloadBasePartOrProjectionPartToDiskRemoteMeta( - replica_path, projection_part_storage, in, projection_checksum, throttler); - - checksums.addFile( - projection_name + ".proj", projection_checksum.getTotalSizeOnDisk(), projection_checksum.getTotalChecksumUInt128()); - } - - downloadBasePartOrProjectionPartToDiskRemoteMeta( - replica_path, data_part_storage, in, checksums, throttler); - } - catch (const Exception & e) - { - if (e.code() == ErrorCodes::ABORTED) - { - /// Remove the whole part directory if fetch of base - /// part or fetch of any projection was stopped. - data_part_storage->removeSharedRecursive(true); - data_part_storage->commitTransaction(); + part_storage_for_loading->removeSharedRecursive(true); + part_storage_for_loading->commitTransaction(); } throw; } @@ -1057,9 +881,11 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( MergeTreeData::MutableDataPartPtr new_data_part; try { - data_part_storage->commitTransaction(); + part_storage_for_loading->commitTransaction(); + + MergeTreeDataPartBuilder builder(data, part_name, volume, part_relative_path, part_dir); + new_data_part = builder.withPartFormatFromDisk().build(); - new_data_part = data.createPart(part_name, data_part_storage); new_data_part->version.setCreationTID(Tx::PrehistoricTID, nullptr); new_data_part->is_temp = true; new_data_part->modification_time = time(nullptr); @@ -1080,10 +906,17 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( throw; } - data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true, {}); - - LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", - part_name, part_id, disk->getName()); + if (to_remote_disk) + { + data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true, {}); + LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", part_name, part_id, disk->getName()); + } + else + { + if (isFullPartStorage(new_data_part->getDataPartStorage())) + new_data_part->checksums.checkEqual(data_checksums, false); + LOG_DEBUG(log, "Download of part {} onto disk {} finished.", part_name, disk->getName()); + } return new_data_part; } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 6c92fad4092..a322faee122 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -49,12 +49,7 @@ private: const MergeTreeData::DataPartPtr & part, WriteBuffer & out, int client_protocol_version, - bool send_projections); - - void sendPartFromDiskRemoteMeta( - const MergeTreeData::DataPartPtr & part, - WriteBuffer & out, - bool send_part_id, + bool from_remote_disk, bool send_projections); /// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish, @@ -93,32 +88,29 @@ public: ActionBlocker blocker; private: + using OutputBufferGetter = std::function(IDataPartStorage &, const String &, size_t)>; + void downloadBaseOrProjectionPartToDisk( const String & replica_path, const MutableDataPartStoragePtr & data_part_storage, - bool sync, PooledReadWriteBufferFromHTTP & in, + OutputBufferGetter output_buffer_getter, MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) const; - - void downloadBasePartOrProjectionPartToDiskRemoteMeta( - const String & replica_path, - const MutableDataPartStoragePtr & data_part_storage, - PooledReadWriteBufferFromHTTP & in, - MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler) const; + ThrottlerPtr throttler, + bool sync) const; MergeTreeData::MutableDataPartPtr downloadPartToDisk( const String & part_name, const String & replica_path, bool to_detached, const String & tmp_prefix_, - bool sync, DiskPtr disk, + bool to_remote_disk, PooledReadWriteBufferFromHTTP & in, + OutputBufferGetter output_buffer_getter, size_t projections, - MergeTreeData::DataPart::Checksums & checksums, - ThrottlerPtr throttler); + ThrottlerPtr throttler, + bool sync); MergeTreeData::MutableDataPartPtr downloadPartToMemory( MutableDataPartStoragePtr data_part_storage, diff --git a/src/Storages/MergeTree/FutureMergedMutatedPart.cpp b/src/Storages/MergeTree/FutureMergedMutatedPart.cpp index f7dc5f50568..959a0e50bca 100644 --- a/src/Storages/MergeTree/FutureMergedMutatedPart.cpp +++ b/src/Storages/MergeTree/FutureMergedMutatedPart.cpp @@ -16,20 +16,23 @@ void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_) size_t sum_rows = 0; size_t sum_bytes_uncompressed = 0; - MergeTreeDataPartType future_part_type = MergeTreeDataPartType::Unknown; + MergeTreeDataPartType future_part_type; + MergeTreeDataPartStorageType future_part_storage_type; for (const auto & part : parts_) { sum_rows += part->rows_count; sum_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; future_part_type = std::min(future_part_type, part->getType()); + future_part_storage_type = std::min(future_part_storage_type, part->getDataPartStorage().getType()); } - auto chosen_type = parts_.front()->storage.choosePartTypeOnDisk(sum_bytes_uncompressed, sum_rows); - future_part_type = std::min(future_part_type, chosen_type); - assign(std::move(parts_), future_part_type); + auto chosen_format = parts_.front()->storage.choosePartFormatOnDisk(sum_bytes_uncompressed, sum_rows); + future_part_type = std::min(future_part_type, chosen_format.part_type); + future_part_storage_type = std::min(future_part_storage_type, chosen_format.storage_type); + assign(std::move(parts_), {future_part_type, future_part_storage_type}); } -void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartType future_part_type) +void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartFormat future_part_format) { if (parts_.empty()) return; @@ -53,7 +56,7 @@ void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_, Merg max_mutation = std::max(max_mutation, part->info.mutation); } - type = future_part_type; + part_format = future_part_format; part_info.partition_id = parts.front()->info.partition_id; part_info.min_block = parts.front()->info.min_block; part_info.max_block = parts.back()->info.max_block; diff --git a/src/Storages/MergeTree/FutureMergedMutatedPart.h b/src/Storages/MergeTree/FutureMergedMutatedPart.h index 4447687c3d9..09fb7b01678 100644 --- a/src/Storages/MergeTree/FutureMergedMutatedPart.h +++ b/src/Storages/MergeTree/FutureMergedMutatedPart.h @@ -19,7 +19,7 @@ struct FutureMergedMutatedPart String name; UUID uuid = UUIDHelpers::Nil; String path; - MergeTreeDataPartType type; + MergeTreeDataPartFormat part_format; MergeTreePartInfo part_info; MergeTreeData::DataPartsVector parts; MergeType merge_type = MergeType::Regular; @@ -33,13 +33,13 @@ struct FutureMergedMutatedPart assign(std::move(parts_)); } - FutureMergedMutatedPart(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartType future_part_type) + FutureMergedMutatedPart(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartFormat future_part_format) { - assign(std::move(parts_), future_part_type); + assign(std::move(parts_), future_part_format); } void assign(MergeTreeData::DataPartsVector parts_); - void assign(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartType future_part_type); + void assign(MergeTreeData::DataPartsVector parts_, MergeTreeDataPartFormat future_part_format); void updatePath(const MergeTreeData & storage, const IReservation * reservation); }; diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 84ca3e733c6..f92784cb0da 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -1,10 +1,11 @@ #pragma once #include #include +#include #include #include #include -#include +#include #include #include #include @@ -22,7 +23,6 @@ struct CanRemoveDescription { bool can_remove_anything; NameSet files_not_to_remove; - }; using CanRemoveCallback = std::function; @@ -79,6 +79,8 @@ class IDataPartStorage : public boost::noncopyable public: virtual ~IDataPartStorage() = default; + virtual MergeTreeDataPartStorageType getType() const = 0; + /// Methods to get path components of a data part. virtual std::string getFullPath() const = 0; /// '/var/lib/clickhouse/data/database/table/moving/all_1_5_1' virtual std::string getRelativePath() const = 0; /// 'database/table/moving/all_1_5_1' @@ -117,8 +119,6 @@ public: std::optional read_hint, std::optional file_size) const = 0; - virtual void checkConsistency(const MergeTreeDataPartChecksums & checksums) const = 0; - struct ProjectionChecksums { const std::string & name; @@ -133,7 +133,6 @@ public: const MergeTreeDataPartChecksums & checksums, std::list projections, bool is_temp, - MergeTreeDataPartState state, Poco::Logger * log) = 0; /// Get a name like 'prefix_partdir_tryN' which does not exist in a root dir. @@ -157,7 +156,6 @@ public: virtual void syncRevision(UInt64 revision) const = 0; virtual UInt64 getRevision() const = 0; - virtual std::unordered_map getSerializedMetadata(const std::vector & paths) const = 0; /// Get a path for internal disk if relevant. It is used mainly for logging. virtual std::string getDiskPath() const = 0; @@ -172,6 +170,28 @@ public: /// Required for distinguish different copies of the same part on remote FS. virtual String getUniqueId() const = 0; + + /// Represents metadata which is required for fetching of part. + struct ReplicatedFilesDescription + { + using InputBufferGetter = std::function()>; + + struct ReplicatedFileDescription + { + InputBufferGetter input_buffer_getter; + size_t file_size; + }; + + std::map files; + + /// Unique string that is used to distinguish different + /// copies of the same part on remote disk + String unique_id; + }; + + virtual ReplicatedFilesDescription getReplicatedFilesDescription(const NameSet & file_names) const = 0; + virtual ReplicatedFilesDescription getReplicatedFilesDescriptionForRemoteDisk(const NameSet & file_names) const = 0; + /// Create a backup of a data part. /// This method adds a new entry to backup_entries. /// Also creates a new tmp_dir for internal disk (if disk is mentioned the first time). @@ -212,16 +232,20 @@ public: virtual void createDirectories() = 0; virtual void createProjection(const std::string & name) = 0; - virtual std::unique_ptr writeFile( - const String & name, - size_t buf_size, - const WriteSettings & settings) = 0; virtual std::unique_ptr writeFile( const String & name, size_t buf_size, WriteMode mode, const WriteSettings & settings) = 0; + std::unique_ptr writeFile( + const String & name, + size_t buf_size, + const WriteSettings & settings) + { + return writeFile(name, buf_size, WriteMode::Rewrite, settings); + } + /// A special const method to write transaction file. /// It's const, because file with transaction metadata /// can be modified after part creation. @@ -256,6 +280,9 @@ public: virtual void beginTransaction() = 0; /// Commits a transaction of mutable operations. virtual void commitTransaction() = 0; + /// Prepares transaction to commit. + /// It may be flush of buffered data or similar. + virtual void precommitTransaction() = 0; virtual bool hasActiveTransaction() const = 0; }; @@ -283,4 +310,9 @@ private: MutableDataPartStoragePtr storage; }; +inline bool isFullPartStorage(const IDataPartStorage & storage) +{ + return storage.getType() == MergeTreeDataPartStorageType::Full; +} + } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index e1427413f62..52d53ac1ddb 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -299,33 +299,6 @@ static void decrementTypeMetric(MergeTreeDataPartType type) } } -IMergeTreeDataPart::IMergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - Type part_type_, - const IMergeTreeDataPart * parent_part_) - : DataPartStorageHolder(data_part_storage_) - , storage(storage_) - , name(name_) - , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) - , index_granularity_info(storage_, part_type_) - , part_type(part_type_) - , parent_part(parent_part_) - , use_metadata_cache(storage.use_metadata_cache) -{ - if (parent_part) - state = MergeTreeDataPartState::Active; - - incrementStateMetric(state); - incrementTypeMetric(part_type); - - minmax_idx = std::make_shared(); - - initializeIndexGranularityInfo(); - initializePartMetadataManager(); -} - IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, @@ -700,18 +673,38 @@ void IMergeTreeDataPart::appendFilesOfColumnsChecksumsIndexes(Strings & files, b } } +MergeTreeDataPartBuilder IMergeTreeDataPart::getProjectionPartBuilder(const String & projection_name, bool is_temp_projection) +{ + auto projection_extension = is_temp_projection ? ".tmp_proj" : ".proj"; + auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection); + MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage); + return builder.withPartInfo({"all", 0, 0, 0}).withParentPart(this); +} + +void IMergeTreeDataPart::addProjectionPart( + const String & projection_name, + std::shared_ptr && projection_part) +{ + auto [it, inserted] = projection_parts.try_emplace(projection_name); + if (!inserted) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Projection with name {} already exists in part {}", + projection_name, getDataPartStorage().getRelativePath()); + + it->second = std::move(projection_part); +} + void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool check_consistency) { auto metadata_snapshot = storage.getInMemoryMetadataPtr(); for (const auto & projection : metadata_snapshot->projections) { - String path = /*getRelativePath() + */ projection.name + ".proj"; + auto path = projection.name + ".proj"; if (getDataPartStorage().exists(path)) { - auto projection_part_storage = getDataPartStorage().getProjection(projection.name + ".proj"); - auto part = storage.createPart(projection.name, {"all", 0, 0, 0}, projection_part_storage, this); + auto part = getProjectionPartBuilder(projection.name).withPartFormatFromDisk().build(); part->loadColumnsChecksumsIndexes(require_columns_checksums, check_consistency); - projection_parts.emplace(projection.name, std::move(part)); + addProjectionPart(projection.name, std::move(part)); } } } @@ -865,6 +858,8 @@ void IMergeTreeDataPart::writeMetadata(const String & filename, const WriteSetti auto & data_part_storage = getDataPartStorage(); auto tmp_filename = filename + ".tmp"; + data_part_storage.beginTransaction(); + try { { @@ -880,15 +875,20 @@ void IMergeTreeDataPart::writeMetadata(const String & filename, const WriteSetti try { if (data_part_storage.exists(tmp_filename)) + { data_part_storage.removeFile(tmp_filename); + data_part_storage.commitTransaction(); + } } catch (...) { - tryLogCurrentException("DataPartStorageOnDisk"); + tryLogCurrentException("DataPartStorageOnDiskFull"); } throw; } + + data_part_storage.commitTransaction(); } void IMergeTreeDataPart::writeChecksums(const MergeTreeDataPartChecksums & checksums_, const WriteSettings & settings) @@ -941,7 +941,7 @@ void IMergeTreeDataPart::writeVersionMetadata(const VersionMetadata & version_, } catch (...) { - tryLogCurrentException("DataPartStorageOnDisk"); + tryLogCurrentException("DataPartStorageOnDiskFull"); } throw; @@ -1629,9 +1629,9 @@ void IMergeTreeDataPart::initializePartMetadataManager() void IMergeTreeDataPart::initializeIndexGranularityInfo() { - auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(getDataPartStorage()); - if (mrk_ext) - index_granularity_info = MergeTreeIndexGranularityInfo(storage, MarkType{*mrk_ext}); + auto mrk_type = MergeTreeIndexGranularityInfo::getMarksTypeFromFilesystem(getDataPartStorage()); + if (mrk_type) + index_granularity_info = MergeTreeIndexGranularityInfo(storage, *mrk_type); else index_granularity_info = MergeTreeIndexGranularityInfo(storage, part_type); } @@ -1689,7 +1689,8 @@ void IMergeTreeDataPart::remove() projection_checksums.emplace_back(IDataPartStorage::ProjectionChecksums{.name = p_name, .checksums = projection_part->checksums}); } - getDataPartStorage().remove(std::move(can_remove_callback), checksums, projection_checksums, is_temp, getState(), storage.log); + bool is_temporary_part = is_temp || state == MergeTreeDataPartState::Temporary; + getDataPartStorage().remove(std::move(can_remove_callback), checksums, projection_checksums, is_temporary_part, storage.log); } std::optional IMergeTreeDataPart::getRelativePathForPrefix(const String & prefix, bool detached, bool broken) const @@ -1747,10 +1748,10 @@ DataPartStoragePtr IMergeTreeDataPart::makeCloneInDetached(const String & prefix return getDataPartStorage().freeze( storage.relative_data_path, *maybe_path_in_detached, - /*make_source_readonly*/ true, - {}, + /*make_source_readonly=*/ true, + /*save_metadata_callback=*/ {}, copy_instead_of_hardlink, - {}); + /*files_to_copy_instead_of_hardlinks=*/ {}); } MutableDataPartStoragePtr IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & directory_name) const @@ -1804,7 +1805,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const } } - getDataPartStorage().checkConsistency(checksums); + checksums.checkSizes(getDataPartStorage()); } else { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 68d5147362b..c77e4f75500 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -85,13 +86,6 @@ public: Type part_type_, const IMergeTreeDataPart * parent_part_); - IMergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - Type part_type_, - const IMergeTreeDataPart * parent_part_); - virtual MergeTreeReaderPtr getReader( const NamesAndTypesList & columns_, const StorageMetadataPtr & metadata_snapshot, @@ -141,6 +135,7 @@ public: void accumulateColumnSizes(ColumnToSize & /* column_to_size */) const; Type getType() const { return part_type; } + MergeTreeDataPartFormat getFormat() const { return {part_type, getDataPartStorage().getType()}; } String getTypeName() const { return getType().toString(); } @@ -359,15 +354,11 @@ public: const std::map> & getProjectionParts() const { return projection_parts; } - void addProjectionPart(const String & projection_name, std::shared_ptr && projection_part) - { - projection_parts.emplace(projection_name, std::move(projection_part)); - } + MergeTreeDataPartBuilder getProjectionPartBuilder(const String & projection_name, bool is_temp_projection = false); - bool hasProjection(const String & projection_name) const - { - return projection_parts.find(projection_name) != projection_parts.end(); - } + void addProjectionPart(const String & projection_name, std::shared_ptr && projection_part); + + bool hasProjection(const String & projection_name) const { return projection_parts.contains(projection_name); } void loadProjections(bool require_columns_checksums, bool check_consistency); @@ -604,6 +595,7 @@ using MergeTreeMutableDataPartPtr = std::shared_ptr; bool isCompactPart(const MergeTreeDataPartPtr & data_part); bool isWidePart(const MergeTreeDataPartPtr & data_part); bool isInMemoryPart(const MergeTreeDataPartPtr & data_part); + inline String getIndexExtension(bool is_compressed_primary_key) { return is_compressed_primary_key ? ".cidx" : ".idx"; } std::optional getIndexExtensionFromFilesystem(const IDataPartStorage & data_part_storage); bool isCompressedFromIndexExtension(const String & index_extension); diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index 8b3ee3f4325..1aa6082a29e 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -171,7 +171,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() StorageMetadataPtr metadata_snapshot = storage.getInMemoryMetadataPtr(); - auto future_merged_part = std::make_shared(parts, entry.new_part_type); + auto future_merged_part = std::make_shared(parts, entry.new_part_format); if (future_merged_part->name != entry.new_part_name) { throw Exception(ErrorCodes::BAD_DATA_PART_NAME, "Future merged part name {} differs from part name in log entry: {}", diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 35e1b82dae0..5874c257ad0 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -113,11 +113,12 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() if (isTTLMergeType(global_ctx->future_part->merge_type) && global_ctx->ttl_merges_blocker->isCancelled()) throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts with TTL"); - LOG_DEBUG(ctx->log, "Merging {} parts: from {} to {} into {}", + LOG_DEBUG(ctx->log, "Merging {} parts: from {} to {} into {} with storage {}", global_ctx->future_part->parts.size(), global_ctx->future_part->parts.front()->name, global_ctx->future_part->parts.back()->name, - global_ctx->future_part->type.toString()); + global_ctx->future_part->part_format.part_type.toString(), + global_ctx->future_part->part_format.storage_type.toString()); if (global_ctx->deduplicate) { @@ -128,31 +129,36 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() } ctx->disk = global_ctx->space_reservation->getDisk(); + auto local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; - String local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; - MutableDataPartStoragePtr data_part_storage; - + std::optional builder; if (global_ctx->parent_part) { - data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename); + auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename); + builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage); + builder->withParentPart(global_ctx->parent_part); } else { auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, ctx->disk, 0); - - data_part_storage = std::make_shared( - local_single_disk_volume, - global_ctx->data->relative_data_path, - local_tmp_part_basename); - - data_part_storage->beginTransaction(); + builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename)); + builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); } + builder->withPartInfo(global_ctx->future_part->part_info); + builder->withPartType(global_ctx->future_part->part_format.part_type); + + global_ctx->new_data_part = std::move(*builder).build(); + auto data_part_storage = global_ctx->new_data_part->getDataPartStoragePtr(); + if (data_part_storage->exists()) throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists", data_part_storage->getFullPath()); if (!global_ctx->parent_part) + { + data_part_storage->beginTransaction(); global_ctx->temporary_directory_lock = global_ctx->data->getTemporaryPartDirectoryHolder(local_tmp_part_basename); + } global_ctx->all_column_names = global_ctx->metadata_snapshot->getColumns().getNamesOfPhysical(); global_ctx->storage_columns = global_ctx->metadata_snapshot->getColumns().getAllPhysical(); @@ -171,13 +177,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() global_ctx->merging_columns, global_ctx->merging_column_names); - global_ctx->new_data_part = global_ctx->data->createPart( - global_ctx->future_part->name, - global_ctx->future_part->type, - global_ctx->future_part->part_info, - data_part_storage, - global_ctx->parent_part); - global_ctx->new_data_part->uuid = global_ctx->future_part->uuid; global_ctx->new_data_part->partition.assign(global_ctx->future_part->getPartition()); global_ctx->new_data_part->is_temp = global_ctx->parent_part == nullptr; @@ -699,9 +698,9 @@ bool MergeTask::MergeProjectionsStage::finalizeProjectionsAndWholeMerge() const if (global_ctx->chosen_merge_algorithm != MergeAlgorithm::Vertical) global_ctx->to->finalizePart(global_ctx->new_data_part, ctx->need_sync); else - global_ctx->to->finalizePart( - global_ctx->new_data_part, ctx->need_sync, &global_ctx->storage_columns, &global_ctx->checksums_gathered_columns); + global_ctx->to->finalizePart(global_ctx->new_data_part, ctx->need_sync, &global_ctx->storage_columns, &global_ctx->checksums_gathered_columns); + global_ctx->new_data_part->getDataPartStorage().precommitTransaction(); global_ctx->promise.set_value(global_ctx->new_data_part); return false; @@ -819,7 +818,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() global_ctx->horizontal_stage_progress = std::make_unique( ctx->column_sizes ? ctx->column_sizes->keyColumnsWeight() : 1.0); - for (const auto & part : global_ctx->future_part->parts) { Pipe pipe = createMergeTreeSequentialSource( @@ -957,7 +955,7 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm return MergeAlgorithm::Horizontal; for (const auto & part : global_ctx->future_part->parts) - if (!part->supportsVerticalMerge()) + if (!part->supportsVerticalMerge() || !isFullPartStorage(part->getDataPartStorage())) return MergeAlgorithm::Horizontal; bool is_supported_storage = diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 81f5783da4a..43e3ba54a6d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1,3 +1,4 @@ +#include "Storages/MergeTree/MergeTreeDataPartBuilder.h" #include #include @@ -44,7 +45,7 @@ #include #include #include -#include +#include #include #include #include @@ -147,7 +148,6 @@ namespace ErrorCodes extern const int BAD_DATA_PART_NAME; extern const int READONLY_SETTING; extern const int ABORTED; - extern const int UNKNOWN_PART_TYPE; extern const int UNKNOWN_DISK; extern const int NOT_ENOUGH_SPACE; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; @@ -1097,9 +1097,12 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( LoadPartResult res; auto single_disk_volume = std::make_shared("volume_" + part_name, part_disk_ptr, 0); - auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, part_name); + auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, part_name); - res.part = createPart(part_name, part_info, data_part_storage); + res.part = getDataPartBuilder(part_name, single_disk_volume, part_name) + .withPartInfo(part_info) + .withPartFormatFromDisk() + .build(); String part_path = fs::path(relative_data_path) / part_name; String marker_path = fs::path(part_path) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; @@ -1250,7 +1253,7 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart( LOG_TRACE(log, "Finished loading {} part {} on disk {}", magic_enum::enum_name(to_state), part_name, part_disk_ptr->getName()); return res; -}; +} std::vector MergeTreeData::loadDataPartsFromDisk( ThreadPool & pool, @@ -3105,72 +3108,39 @@ void MergeTreeData::checkMutationIsPossible(const MutationCommands & /*commands* /// Some validation will be added } -MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, size_t rows_count) const +MergeTreeDataPartFormat MergeTreeData::choosePartFormat(size_t bytes_uncompressed, size_t rows_count, bool only_on_disk) const { - const auto settings = getSettings(); - if (!canUsePolymorphicParts(*settings)) - return MergeTreeDataPartType::Wide; + using PartType = MergeTreeDataPartType; + using PartStorageType = MergeTreeDataPartStorageType; - if (bytes_uncompressed < settings->min_bytes_for_compact_part || rows_count < settings->min_rows_for_compact_part) - return MergeTreeDataPartType::InMemory; + const auto settings = getSettings(); + if (!canUsePolymorphicParts(*settings)) + return {PartType::Wide, PartStorageType::Full}; - if (bytes_uncompressed < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) - return MergeTreeDataPartType::Compact; - - return MergeTreeDataPartType::Wide; -} - -MergeTreeDataPartType MergeTreeData::choosePartTypeOnDisk(size_t bytes_uncompressed, size_t rows_count) const -{ - const auto settings = getSettings(); - if (!canUsePolymorphicParts(*settings)) - return MergeTreeDataPartType::Wide; - - if (bytes_uncompressed < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) - return MergeTreeDataPartType::Compact; - - return MergeTreeDataPartType::Wide; -} - - -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, - MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const -{ - if (type == MergeTreeDataPartType::Compact) - return std::make_shared(*this, name, part_info, data_part_storage, parent_part); - else if (type == MergeTreeDataPartType::Wide) - return std::make_shared(*this, name, part_info, data_part_storage, parent_part); - else if (type == MergeTreeDataPartType::InMemory) - return std::make_shared(*this, name, part_info, data_part_storage, parent_part); - else - throw Exception(ErrorCodes::UNKNOWN_PART_TYPE, "Unknown type of part {}", data_part_storage->getRelativePath()); -} - -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const -{ - return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), data_part_storage, parent_part); -} - -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const MergeTreePartInfo & part_info, - const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part) const -{ - MergeTreeDataPartType type; - auto mrk_ext = MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(*data_part_storage); - - if (mrk_ext) + auto satisfies = [&](const auto & min_bytes_for, const auto & min_rows_for) { - type = MarkType(*mrk_ext).part_type; - } - else - { - /// Didn't find any mark file, suppose that part is empty. - type = choosePartTypeOnDisk(0, 0); - } + return bytes_uncompressed < min_bytes_for || rows_count < min_rows_for; + }; - return createPart(name, type, part_info, data_part_storage, parent_part); + if (!only_on_disk && satisfies(settings->min_bytes_for_compact_part, settings->min_rows_for_compact_part)) + return {PartType::InMemory, PartStorageType::Full}; + + auto part_type = PartType::Wide; + if (satisfies(settings->min_bytes_for_wide_part, settings->min_rows_for_wide_part)) + part_type = PartType::Compact; + + return {part_type, PartStorageType::Full}; +} + +MergeTreeDataPartFormat MergeTreeData::choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const +{ + return choosePartFormat(bytes_uncompressed, rows_count, true); +} + +MergeTreeDataPartBuilder MergeTreeData::getDataPartBuilder( + const String & name, const VolumePtr & volume, const String & part_dir) const +{ + return MergeTreeDataPartBuilder(*this, name, volume, relative_data_path, part_dir); } void MergeTreeData::changeSettings( @@ -5008,8 +4978,9 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); - auto data_part_storage = std::make_shared(single_disk_volume, temp_part_dir.parent_path(), part_name); - auto part = createPart(part_name, part_info, data_part_storage); + MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, temp_part_dir.parent_path(), part_name); + builder.withPartFormatFromDisk(); + auto part = std::move(builder).build(); part->version.setCreationTID(Tx::PrehistoricTID, nullptr); part->loadColumnsChecksumsIndexes(false, true); @@ -5457,8 +5428,9 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const LOG_DEBUG(log, "Checking part {}", new_name); auto single_disk_volume = std::make_shared("volume_" + old_name, disk); - auto data_part_storage = std::make_shared(single_disk_volume, relative_data_path, source_dir + new_name); - MutableDataPartPtr part = createPart(old_name, data_part_storage); + auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir + new_name) + .withPartFormatFromDisk() + .build(); loadPartAndFixMetadataImpl(part); loaded_parts.push_back(part); @@ -5661,13 +5633,11 @@ bool MergeTreeData::isPartInTTLDestination(const TTLDescription & ttl, const IMe CompressionCodecPtr MergeTreeData::getCompressionCodecForPart(size_t part_size_compressed, const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t current_time) const { - auto metadata_snapshot = getInMemoryMetadataPtr(); const auto & recompression_ttl_entries = metadata_snapshot->getRecompressionTTLs(); auto best_ttl_entry = selectTTLDescriptionForTTLInfos(recompression_ttl_entries, ttl_infos.recompression_ttl, current_time, true); - if (best_ttl_entry) return CompressionCodecFactory::instance().get(best_ttl_entry->recompression_codec, {}); @@ -6909,9 +6879,13 @@ std::pair MergeTreeData::cloneAn src_flushed_tmp_dir_lock = src_part->storage.getTemporaryPartDirectoryHolder(tmp_src_part_file_name); auto flushed_part_storage = src_part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); - src_flushed_tmp_part = createPart(src_part->name, src_part->info, flushed_part_storage); - src_flushed_tmp_part->is_temp = true; + src_flushed_tmp_part = MergeTreeDataPartBuilder(*this, src_part->name, flushed_part_storage) + .withPartInfo(src_part->info) + .withPartFormatFromDisk() + .build(); + + src_flushed_tmp_part->is_temp = true; src_part_storage = flushed_part_storage; } @@ -6919,7 +6893,13 @@ std::pair MergeTreeData::cloneAn if (copy_instead_of_hardlink) with_copy = " (copying data)"; - auto dst_part_storage = src_part_storage->freeze(relative_data_path, tmp_dst_part_name, /* make_source_readonly */ false, {}, copy_instead_of_hardlink, files_to_copy_instead_of_hardlinks); + auto dst_part_storage = src_part_storage->freeze( + relative_data_path, + tmp_dst_part_name, + /*make_source_readonly=*/ false, + /*save_metadata_callback=*/ {}, + copy_instead_of_hardlink, + files_to_copy_instead_of_hardlinks); LOG_DEBUG(log, "Clone {} part {} to {}{}", src_flushed_tmp_part ? "flushed" : "", @@ -6927,7 +6907,9 @@ std::pair MergeTreeData::cloneAn std::string(fs::path(dst_part_storage->getFullRootPath()) / tmp_dst_part_name), with_copy); - auto dst_data_part = createPart(dst_part_name, dst_part_info, dst_part_storage); + auto dst_data_part = MergeTreeDataPartBuilder(*this, dst_part_name, dst_part_storage) + .withPartFormatFromDisk() + .build(); if (!copy_instead_of_hardlink && hardlinked_files) { @@ -7121,15 +7103,18 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( src_flushed_tmp_dir_lock = part->storage.getTemporaryPartDirectoryHolder("tmp_freeze" + part->name); auto flushed_part_storage = part_in_memory->flushToDisk(flushed_part_path, metadata_snapshot); - src_flushed_tmp_part = createPart(part->name, part->info, flushed_part_storage); - src_flushed_tmp_part->is_temp = true; + src_flushed_tmp_part = MergeTreeDataPartBuilder(*this, part->name, flushed_part_storage) + .withPartInfo(part->info) + .withPartFormatFromDisk() + .build(); + + src_flushed_tmp_part->is_temp = true; data_part_storage = flushed_part_storage; } auto callback = [this, &part, &backup_part_path](const DiskPtr & disk) { - // Store metadata for replicated table. // Do nothing for non-replicated. createAndStoreFreezeMetadata(disk, part, fs::path(backup_part_path) / part->getDataPartStorage().getPartDirectory()); @@ -7138,10 +7123,10 @@ PartitionCommandsResultInfo MergeTreeData::freezePartitionsByMatcher( auto new_storage = data_part_storage->freeze( backup_part_path, part->getDataPartStorage().getPartDirectory(), - /*make_source_readonly*/ true, + /*make_source_readonly=*/ true, callback, - /*copy_instead_of_hardlink*/ false, - {}); + /*copy_instead_of_hardlink=*/ false, + /*files_to_copy_instead_of_hardlinks=*/ {}); part->is_frozen.store(true, std::memory_order_relaxed); result.push_back(PartitionCommandResultInfo{ @@ -7501,6 +7486,11 @@ bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const return true; } +bool MergeTreeData::canUsePolymorphicParts() const +{ + return canUsePolymorphicParts(*getSettings(), nullptr); +} + bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, String * out_reason) const { if (!canUseAdaptiveGranularity()) @@ -7902,19 +7892,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( ReservationPtr reservation = reserveSpacePreferringTTLRules(metadata_snapshot, 0, move_ttl_infos, time(nullptr), 0, true); VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); - auto new_data_part_storage = std::make_shared( - data_part_volume, - getRelativeDataPath(), - EMPTY_PART_TMP_PREFIX + new_part_name); - - auto new_data_part = createPart( - new_part_name, - choosePartTypeOnDisk(0, block.rows()), - new_part_info, - new_data_part_storage - ); - - new_data_part->name = new_part_name; + auto new_data_part = getDataPartBuilder(new_part_name, data_part_volume, EMPTY_PART_TMP_PREFIX + new_part_name) + .withBytesAndRowsOnDisk(0, 0) + .withPartInfo(new_part_info) + .build(); if (settings->assign_part_uuids) new_data_part->uuid = UUIDHelpers::generateV4(); @@ -7927,6 +7908,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( new_data_part->minmax_idx = std::move(minmax_idx); new_data_part->is_temp = true; + auto new_data_part_storage = new_data_part->getDataPartStoragePtr(); + new_data_part_storage->beginTransaction(); + SyncGuardPtr sync_guard; if (new_data_part->isStoredOnDisk()) { @@ -7960,9 +7944,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createEmptyPart( out.write(block); /// Here is no projections as no data inside - out.finalizePart(new_data_part, sync_on_insert); + new_data_part_storage->precommitTransaction(); return new_data_part; } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 2e57d58ef41..7dcd0c40553 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -224,21 +225,9 @@ public: using OperationDataPartsLock = std::unique_lock; OperationDataPartsLock lockOperationsWithParts() const { return OperationDataPartsLock(operation_with_data_parts_mutex); } - MergeTreeDataPartType choosePartType(size_t bytes_uncompressed, size_t rows_count) const; - MergeTreeDataPartType choosePartTypeOnDisk(size_t bytes_uncompressed, size_t rows_count) const; - - /// After this method setColumns must be called - MutableDataPartPtr createPart(const String & name, - MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; - - /// Create part, that already exists on filesystem. - /// After this methods 'loadColumnsChecksumsIndexes' must be called. - MutableDataPartPtr createPart(const String & name, - const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; - - MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, - const MutableDataPartStoragePtr & data_part_storage, const IMergeTreeDataPart * parent_part = nullptr) const; + MergeTreeDataPartFormat choosePartFormat(size_t bytes_uncompressed, size_t rows_count, bool only_on_disk = false) const; + MergeTreeDataPartFormat choosePartFormatOnDisk(size_t bytes_uncompressed, size_t rows_count) const; + MergeTreeDataPartBuilder getDataPartBuilder(const String & name, const VolumePtr & volume, const String & part_dir) const; /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreActive parts (the parts are ready, but not yet in the active set). @@ -1054,6 +1043,7 @@ public: scope_guard getTemporaryPartDirectoryHolder(const String & part_dir_name) const; void waitForOutdatedPartsToBeLoaded() const; + bool canUsePolymorphicParts() const; protected: friend class IMergeTreeDataPart; diff --git a/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp new file mode 100644 index 00000000000..3aa68266a3e --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeDataPartBuilder.cpp @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int UNKNOWN_PART_TYPE; +} + +MergeTreeDataPartBuilder::MergeTreeDataPartBuilder( + const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_) + : data(data_) + , name(std::move(name_)) + , volume(std::move(volume_)) + , root_path(std::move(root_path_)) + , part_dir(std::move(part_dir_)) +{ +} + +MergeTreeDataPartBuilder::MergeTreeDataPartBuilder( + const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_) + : data(data_) + , name(std::move(name_)) + , part_storage(std::move(part_storage_)) +{ +} + +std::shared_ptr MergeTreeDataPartBuilder::build() +{ + using PartType = MergeTreeDataPartType; + using PartStorageType = MergeTreeDataPartStorageType; + + if (!part_type) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part {}, because part type is not set", name); + + if (!part_storage) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part {}, because part storage is not set", name); + + if (parent_part && data.format_version == MERGE_TREE_DATA_OLD_FORMAT_VERSION) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create projection part in MergeTree table created in old syntax"); + + auto part_storage_type = part_storage->getType(); + if (!data.canUsePolymorphicParts() && + (part_type != PartType::Wide || part_storage_type != PartStorageType::Full)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot create part with type {} and storage type {} because table does not support polymorphic parts", + part_type->toString(), part_storage_type.toString()); + } + + if (!part_info) + part_info = MergeTreePartInfo::fromPartName(name, data.format_version); + + switch (part_type->getValue()) + { + case PartType::Wide: + return std::make_shared(data, name, *part_info, part_storage, parent_part); + case PartType::Compact: + return std::make_shared(data, name, *part_info, part_storage, parent_part); + case PartType::InMemory: + return std::make_shared(data, name, *part_info, part_storage, parent_part); + default: + throw Exception(ErrorCodes::UNKNOWN_PART_TYPE, + "Unknown type of part {}", part_storage->getRelativePath()); + } +} + +MutableDataPartStoragePtr MergeTreeDataPartBuilder::getPartStorageByType( + MergeTreeDataPartStorageType storage_type_, + const VolumePtr & volume_, + const String & root_path_, + const String & part_dir_) +{ + if (!volume_) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create part storage, because volume is not specified"); + + using Type = MergeTreeDataPartStorageType; + switch (storage_type_.getValue()) + { + case Type::Full: + return std::make_shared(volume_, root_path_, part_dir_); + default: + throw Exception(ErrorCodes::UNKNOWN_PART_TYPE, + "Unknown type of storage for part {}", fs::path(root_path_) / part_dir_); + } +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartInfo(MergeTreePartInfo part_info_) +{ + part_info = std::move(part_info_); + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withParentPart(const IMergeTreeDataPart * parent_part_) +{ + if (parent_part_ && parent_part_->isProjectionPart()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Parent part cannot be projection"); + + parent_part = parent_part_; + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartType(MergeTreeDataPartType part_type_) +{ + part_type = part_type_; + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartStorageType(MergeTreeDataPartStorageType storage_type_) +{ + part_storage = getPartStorageByType(storage_type_, volume, root_path, part_dir); + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormat(MergeTreeDataPartFormat format_) +{ + withPartType(format_.part_type); + return part_storage ? *this : withPartStorageType(format_.storage_type); +} + +MergeTreeDataPartBuilder::PartStorageAndMarkType +MergeTreeDataPartBuilder::getPartStorageAndMarkType( + const VolumePtr & volume_, + const String & root_path_, + const String & part_dir_) +{ + auto disk = volume_->getDisk(); + auto part_relative_path = fs::path(root_path_) / part_dir_; + + for (auto it = disk->iterateDirectory(part_relative_path); it->isValid(); it->next()) + { + auto it_path = fs::path(it->name()); + auto ext = it_path.extension().string(); + + if (MarkType::isMarkFileExtension(ext)) + { + auto storage = getPartStorageByType(MergeTreeDataPartStorageType::Full, volume_, root_path_, part_dir_); + return {std::move(storage), MarkType(ext)}; + } + } + + return {}; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromDisk() +{ + if (part_storage) + return withPartFormatFromStorage(); + else + return withPartFormatFromVolume(); +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromVolume() +{ + assert(volume); + auto [storage, mark_type] = getPartStorageAndMarkType(volume, root_path, part_dir); + + if (!storage || !mark_type) + { + /// Didn't find any data or mark file, suppose that part is empty. + return withBytesAndRows(0, 0); + } + + part_storage = std::move(storage); + part_type = mark_type->part_type; + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withPartFormatFromStorage() +{ + assert(part_storage); + auto mark_type = MergeTreeIndexGranularityInfo::getMarksTypeFromFilesystem(*part_storage); + + if (!mark_type) + { + /// Didn't find any mark file, suppose that part is empty. + return withBytesAndRows(0, 0); + } + + part_type = mark_type->part_type; + return *this; +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withBytesAndRows(size_t bytes_uncompressed, size_t rows_count) +{ + return withPartFormat(data.choosePartFormat(bytes_uncompressed, rows_count)); +} + +MergeTreeDataPartBuilder & MergeTreeDataPartBuilder::withBytesAndRowsOnDisk(size_t bytes_uncompressed, size_t rows_count) +{ + return withPartFormat(data.choosePartFormatOnDisk(bytes_uncompressed, rows_count)); +} + +} diff --git a/src/Storages/MergeTree/MergeTreeDataPartBuilder.h b/src/Storages/MergeTree/MergeTreeDataPartBuilder.h new file mode 100644 index 00000000000..0f54ff0a631 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreeDataPartBuilder.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class IDataPartStorage; +class IMergeTreeDataPart; +class IVolume; +class IDisk; +class MergeTreeData; + +using MutableDataPartStoragePtr = std::shared_ptr; +using VolumePtr = std::shared_ptr; + +/// Class that helps to create a data part with different variations of arguments. +class MergeTreeDataPartBuilder +{ +public: + MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, VolumePtr volume_, String root_path_, String part_dir_); + MergeTreeDataPartBuilder(const MergeTreeData & data_, String name_, MutableDataPartStoragePtr part_storage_); + + std::shared_ptr build(); + + using Self = MergeTreeDataPartBuilder; + + Self & withPartInfo(MergeTreePartInfo part_info_); + Self & withParentPart(const IMergeTreeDataPart * parent_part_); + Self & withPartType(MergeTreeDataPartType part_type_); + Self & withPartStorageType(MergeTreeDataPartStorageType storage_type_); + Self & withPartFormat(MergeTreeDataPartFormat format_); + Self & withPartFormatFromDisk(); + Self & withBytesAndRows(size_t bytes_uncompressed, size_t rows_count); + Self & withBytesAndRowsOnDisk(size_t bytes_uncompressed, size_t rows_count); + + using PartStorageAndMarkType = std::pair>; + + static PartStorageAndMarkType getPartStorageAndMarkType( + const VolumePtr & volume_, + const String & root_path_, + const String & part_dir_); + +private: + Self & withPartFormatFromVolume(); + Self & withPartFormatFromStorage(); + + static MutableDataPartStoragePtr getPartStorageByType( + MergeTreeDataPartStorageType storage_type_, + const VolumePtr & volume_, + const String & root_path_, + const String & part_dir_); + + const MergeTreeData & data; + const String name; + const VolumePtr volume; + const String root_path; + const String part_dir; + + std::optional part_info; + std::optional part_type; + MutableDataPartStoragePtr part_storage; + const IMergeTreeDataPart * parent_part = nullptr; +}; + +} diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 4c7da39cc22..719a60b2f31 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -43,17 +44,20 @@ void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Checksum mismatch for file {} in data part", name); } -void MergeTreeDataPartChecksum::checkSize(const DiskPtr & disk, const String & path) const +void MergeTreeDataPartChecksum::checkSize(const IDataPartStorage & storage, const String & name) const { - if (!disk->exists(path)) - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "{} doesn't exist", fullPath(disk, path)); - if (disk->isDirectory(path)) - // This is a projection, no need to check its size. + if (!storage.exists(name)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "{} doesn't exist", fs::path(storage.getRelativePath()) / name); + + // This is a projection, no need to check its size. + if (storage.isDirectory(name)) return; - UInt64 size = disk->getFileSize(path); + + UInt64 size = storage.getFileSize(name); if (size != file_size) - throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "{} has unexpected size: {} instead of {}", - fullPath(disk, path), size, file_size); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, + "{} has unexpected size: {} instead of {}", + fs::path(storage.getRelativePath()) / name, size, file_size); } @@ -79,12 +83,12 @@ void MergeTreeDataPartChecksums::checkEqual(const MergeTreeDataPartChecksums & r } } -void MergeTreeDataPartChecksums::checkSizes(const DiskPtr & disk, const String & path) const +void MergeTreeDataPartChecksums::checkSizes(const IDataPartStorage & storage) const { for (const auto & it : files) { const String & name = it.first; - it.second.checkSize(disk, path + name); + it.second.checkSize(storage, name); } } diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h index 15acb88aa0f..db110043b74 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.h +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.h @@ -7,13 +7,13 @@ #include #include - class SipHash; - namespace DB { +class IDataPartStorage; + /// Checksum of one file. struct MergeTreeDataPartChecksum { @@ -33,7 +33,7 @@ struct MergeTreeDataPartChecksum uncompressed_size(uncompressed_size_), uncompressed_hash(uncompressed_hash_) {} void checkEqual(const MergeTreeDataPartChecksum & rhs, bool have_uncompressed, const String & name) const; - void checkSize(const DiskPtr & disk, const String & path) const; + void checkSize(const IDataPartStorage & storage, const String & name) const; }; @@ -54,10 +54,7 @@ struct MergeTreeDataPartChecksums bool has(const String & file_name) const { return files.find(file_name) != files.end(); } - bool empty() const - { - return files.empty(); - } + bool empty() const { return files.empty(); } /// Checks that the set of columns and their checksums are the same. If not, throws an exception. /// If have_uncompressed, for compressed files it compares the checksums of the decompressed data. @@ -67,7 +64,7 @@ struct MergeTreeDataPartChecksums static bool isBadChecksumsErrorCode(int code); /// Checks that the directory contains all the needed files of the correct size. Does not check the checksum. - void checkSizes(const DiskPtr & disk, const String & path) const; + void checkSizes(const IDataPartStorage & storage) const; /// Returns false if the checksum is too old. bool read(ReadBuffer & in); diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 2da1ea4aa0f..a5fb11244c7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -18,16 +18,6 @@ namespace ErrorCodes extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; } - -MergeTreeDataPartCompact::MergeTreeDataPartCompact( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::Compact, parent_part_) -{ -} - MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index e275c586cb9..ef553d290f7 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -28,12 +28,6 @@ public: const MutableDataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeDataPartCompact( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeReaderPtr getReader( const NamesAndTypesList & columns, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index ac56868894f..20049976acf 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -19,17 +19,6 @@ namespace ErrorCodes extern const int DIRECTORY_ALREADY_EXISTS; } - -MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::InMemory, parent_part_) -{ - default_codec = CompressionCodecFactory::instance().get("NONE", {}); -} - MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const MergeTreeData & storage_, const String & name_, @@ -76,17 +65,13 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & VolumePtr volume = storage.getStoragePolicy()->getVolume(0); VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); - auto new_data_part_storage = std::make_shared( - data_part_volume, - storage.getRelativeDataPath(), - new_relative_path); + auto new_data_part = storage.getDataPartBuilder(name, data_part_volume, new_relative_path) + .withPartFormat(storage.choosePartFormatOnDisk(block.bytes(), rows_count)) + .build(); + auto new_data_part_storage = new_data_part->getDataPartStoragePtr(); new_data_part_storage->beginTransaction(); - auto current_full_path = getDataPartStorage().getFullPath(); - auto new_type = storage.choosePartTypeOnDisk(block.bytes(), rows_count); - auto new_data_part = storage.createPart(name, new_type, info, new_data_part_storage); - new_data_part->uuid = uuid; new_data_part->setColumns(columns, {}); new_data_part->partition.value = partition.value; @@ -97,7 +82,7 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & throw Exception( ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Could not flush part {}. Part in {} already exists", - quoteString(current_full_path), + quoteString(getDataPartStorage().getFullPath()), new_data_part_storage->getFullPath()); } @@ -107,40 +92,42 @@ MutableDataPartStoragePtr MergeTreeDataPartInMemory::flushToDisk(const String & auto indices = MergeTreeIndexFactory::instance().getMany(metadata_snapshot->getSecondaryIndices()); MergedBlockOutputStream out(new_data_part, metadata_snapshot, columns, indices, compression_codec, NO_TRANSACTION_PTR); out.write(block); + const auto & projections = metadata_snapshot->getProjections(); for (const auto & [projection_name, projection] : projection_parts) { if (projections.has(projection_name)) { - auto projection_part_storage = new_data_part_storage->getProjection(projection_name + ".proj"); - if (projection_part_storage->exists()) + auto old_projection_part = asInMemoryPart(projection); + auto new_projection_part = new_data_part->getProjectionPartBuilder(projection_name) + .withPartFormat(storage.choosePartFormatOnDisk(old_projection_part->block.bytes(), rows_count)) + .build(); + + new_projection_part->is_temp = false; // clean up will be done on parent part + new_projection_part->setColumns(projection->getColumns(), {}); + + auto new_projection_part_storage = new_projection_part->getDataPartStoragePtr(); + if (new_projection_part_storage->exists()) { throw Exception( ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Could not flush projection part {}. Projection part in {} already exists", projection_name, - projection_part_storage->getFullPath()); + new_projection_part_storage->getFullPath()); } - auto projection_part = asInMemoryPart(projection); - auto projection_type = storage.choosePartTypeOnDisk(projection_part->block.bytes(), rows_count); - MergeTreePartInfo projection_info("all", 0, 0, 0); - auto projection_data_part - = storage.createPart(projection_name, projection_type, projection_info, projection_part_storage, parent_part); - projection_data_part->is_temp = false; // clean up will be done on parent part - projection_data_part->setColumns(projection->getColumns(), {}); - - projection_part_storage->createDirectories(); + new_projection_part_storage->createDirectories(); const auto & desc = projections.get(name); auto projection_compression_codec = storage.getContext()->chooseCompressionCodec(0, 0); auto projection_indices = MergeTreeIndexFactory::instance().getMany(desc.metadata->getSecondaryIndices()); MergedBlockOutputStream projection_out( - projection_data_part, desc.metadata, projection_part->columns, projection_indices, + new_projection_part, desc.metadata, + new_projection_part->getColumns(), projection_indices, projection_compression_codec, NO_TRANSACTION_PTR); - projection_out.write(projection_part->block); - projection_out.finalizePart(projection_data_part, false); - new_data_part->addProjectionPart(projection_name, std::move(projection_data_part)); + projection_out.write(old_projection_part->block); + projection_out.finalizePart(new_projection_part, false); + new_data_part->addProjectionPart(projection_name, std::move(new_projection_part)); } } diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h index acb1cd8c844..525c966476b 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.h +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.h @@ -17,12 +17,6 @@ public: const MutableDataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeDataPartInMemory( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeReaderPtr getReader( const NamesAndTypesList & columns, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.cpp b/src/Storages/MergeTree/MergeTreeDataPartType.cpp deleted file mode 100644 index ac277233d3c..00000000000 --- a/src/Storages/MergeTree/MergeTreeDataPartType.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int UNKNOWN_PART_TYPE; -} - -void MergeTreeDataPartType::fromString(const String & str) -{ - auto maybe_value = magic_enum::enum_cast(str); - if (!maybe_value || *maybe_value == Value::Unknown) - throw DB::Exception(ErrorCodes::UNKNOWN_PART_TYPE, "Unexpected string for part type: {}", str); - - value = *maybe_value; -} - -String MergeTreeDataPartType::toString() const -{ - return String(magic_enum::enum_name(value)); -} - -} diff --git a/src/Storages/MergeTree/MergeTreeDataPartType.h b/src/Storages/MergeTree/MergeTreeDataPartType.h index b4b1a56c161..bd2acb9ef65 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -1,10 +1,36 @@ #pragma once +#include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +template +requires std::is_enum_v +static E parseEnum(const String & str) +{ + auto value = magic_enum::enum_cast(str); + if (!value || *value == E::Unknown) + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, + "Unexpected string {} for enum {}", str, magic_enum::enum_type_name()); + + return *value; +} + +/// It's a bug in clang with three-way comparison operator +/// https://github.com/llvm/llvm-project/issues/55919 +#ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + /// Types of data part format. class MergeTreeDataPartType { @@ -24,36 +50,50 @@ public: Unknown, }; - MergeTreeDataPartType() : value(Unknown) {} + MergeTreeDataPartType() : value(Value::Unknown) {} MergeTreeDataPartType(Value value_) : value(value_) {} /// NOLINT - bool operator==(const MergeTreeDataPartType & other) const - { - return value == other.value; - } - - bool operator!=(const MergeTreeDataPartType & other) const - { - return !(*this == other); - } - - bool operator<(const MergeTreeDataPartType & other) const - { - return value < other.value; - } - - bool operator>(const MergeTreeDataPartType & other) const - { - return value > other.value; - } - - void fromString(const String & str); - String toString() const; + auto operator<=>(const MergeTreeDataPartType &) const = default; Value getValue() const { return value; } + String toString() const { return String(magic_enum::enum_name(value)); } + void fromString(const String & str) { value = parseEnum(str); } private: Value value; }; +/// Types of data part storage format. +class MergeTreeDataPartStorageType +{ +public: + enum Value + { + Full, + Unknown, + }; + + MergeTreeDataPartStorageType() : value(Value::Unknown) {} + MergeTreeDataPartStorageType(Value value_) : value(value_) {} /// NOLINT + + auto operator<=>(const MergeTreeDataPartStorageType &) const = default; + + Value getValue() const { return value; } + String toString() const { return String(magic_enum::enum_name(value)); } + void fromString(const String & str) { value = parseEnum(str); } + +private: + Value value; +}; + +#ifdef __clang__ + #pragma clang diagnostic pop +#endif + +struct MergeTreeDataPartFormat +{ + MergeTreeDataPartType part_type; + MergeTreeDataPartStorageType storage_type; +}; + } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index a31b88365c2..2d9fa14408d 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -17,16 +17,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } - -MergeTreeDataPartWide::MergeTreeDataPartWide( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_) - : IMergeTreeDataPart(storage_, name_, data_part_storage_, Type::Wide, parent_part_) -{ -} - MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 601bdff51a1..0d68334a623 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -23,12 +23,6 @@ public: const MutableDataPartStoragePtr & data_part_storage_, const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeDataPartWide( - MergeTreeData & storage_, - const String & name_, - const MutableDataPartStoragePtr & data_part_storage_, - const IMergeTreeDataPart * parent_part_ = nullptr); - MergeTreeReaderPtr getReader( const NamesAndTypesList & columns, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 692aed8a116..94c3651f1f9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -9,6 +9,13 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static CompressionCodecPtr getMarksCompressionCodec(const String & marks_compression_codec) +{ + ParserCodec codec_parser; + auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(marks_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + return CompressionCodecFactory::instance().get(ast, nullptr); +} + MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const MergeTreeMutableDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, @@ -38,7 +45,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( { marks_compressor = std::make_unique( *marks_file_hashing, - settings_.getMarksCompressionCodec(), + getMarksCompressionCodec(settings_.marks_compression_codec), settings_.marks_compress_block_size); marks_source_hashing = std::make_unique(*marks_compressor); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 125d7c12a1a..93b0abeca35 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -152,6 +152,10 @@ void MergeTreeDataWriter::TemporaryPart::finalize() { for (auto & stream : streams) stream.finalizer.finish(); + + part->getDataPartStorage().precommitTransaction(); + for (const auto & [_, projection] : part->getProjectionParts()) + projection->getDataPartStorage().precommitTransaction(); } std::vector scatterOffsetsBySelector(ChunkOffsetsPtr chunk_offsets, const IColumn::Selector & selector, size_t partition_num) @@ -392,6 +396,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( { part_dir = part_name; } + temp_part.temporary_directory_lock = data.getTemporaryPartDirectoryHolder(part_dir); /// If we need to calculate some columns to sort. @@ -443,19 +448,14 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( VolumePtr volume = data.getStoragePolicy()->getVolume(0); VolumePtr data_part_volume = createVolumeFromReservation(reservation, volume); - auto data_part_storage = std::make_shared( - data_part_volume, - data.relative_data_path, - part_dir); + auto new_data_part = data.getDataPartBuilder(part_name, data_part_volume, part_dir) + .withPartFormat(data.choosePartFormat(expected_size, block.rows())) + .withPartInfo(new_part_info) + .build(); + auto data_part_storage = new_data_part->getDataPartStoragePtr(); data_part_storage->beginTransaction(); - auto new_data_part = data.createPart( - part_name, - data.choosePartType(expected_size, block.rows()), - new_part_info, - data_part_storage); - if (data.storage_settings.get()->assign_part_uuids) new_data_part->uuid = UUIDHelpers::generateV4(); @@ -558,8 +558,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const ProjectionDescription & projection) { TemporaryPart temp_part; - const StorageMetadataPtr & metadata_snapshot = projection.metadata; - MergeTreePartInfo new_part_info("all", 0, 0, 0); + const auto & metadata_snapshot = projection.metadata; MergeTreeDataPartType part_type; if (parent_part->getType() == MergeTreeDataPartType::InMemory) @@ -572,21 +571,15 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( size_t expected_size = block.bytes(); // just check if there is enough space on parent volume data.reserveSpace(expected_size, parent_part->getDataPartStorage()); - part_type = data.choosePartTypeOnDisk(expected_size, block.rows()); + part_type = data.choosePartFormatOnDisk(expected_size, block.rows()).part_type; } - auto relative_path = part_name + (is_temp ? ".tmp_proj" : ".proj"); - auto projection_part_storage = parent_part->getDataPartStorage().getProjection(relative_path, !is_temp); + auto new_data_part = parent_part->getProjectionPartBuilder(part_name, is_temp).withPartType(part_type).build(); + auto projection_part_storage = new_data_part->getDataPartStoragePtr(); + if (is_temp) projection_part_storage->beginTransaction(); - auto new_data_part = data.createPart( - part_name, - part_type, - new_part_info, - projection_part_storage, - parent_part); - new_data_part->is_temp = is_temp; NamesAndTypesList columns = metadata_snapshot->getColumns().getAllPhysical().filter(block.getNames()); @@ -695,7 +688,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( IMergeTreeDataPart * parent_part, size_t block_num) { - String part_name = fmt::format("{}_{}", projection.name, block_num); + auto part_name = fmt::format("{}_{}", projection.name, block_num); return writeProjectionPartImpl( part_name, true /* is_temp */, diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 2020796f925..2f269663cd1 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -3,8 +3,6 @@ #include #include #include -#include -#include namespace DB @@ -59,13 +57,6 @@ struct MergeTreeWriterSettings { } - CompressionCodecPtr getMarksCompressionCodec() const - { - ParserCodec codec_parser; - auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(marks_compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - return CompressionCodecFactory::instance().get(ast, nullptr); - } - size_t min_compress_block_size; size_t max_compress_block_size; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index e1ce7d8075b..4e339964de3 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -89,13 +89,13 @@ std::string MarkType::getFileExtension() const } -std::optional MergeTreeIndexGranularityInfo::getMarksExtensionFromFilesystem(const IDataPartStorage & data_part_storage) +std::optional MergeTreeIndexGranularityInfo::getMarksTypeFromFilesystem(const IDataPartStorage & data_part_storage) { if (data_part_storage.exists()) for (auto it = data_part_storage.iterate(); it->isValid(); it->next()) if (it->isFile()) if (std::string ext = fs::path(it->name()).extension(); MarkType::isMarkFileExtension(ext)) - return ext; + return MarkType(ext); return {}; } @@ -112,8 +112,8 @@ MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const IDataPartStorage & data_part_storage) { - auto mrk_ext = getMarksExtensionFromFilesystem(data_part_storage); - if (mrk_ext && !MarkType(*mrk_ext).adaptive) + auto mrk_type = getMarksTypeFromFilesystem(data_part_storage); + if (mrk_type && !mrk_type->adaptive) { mark_type.adaptive = false; index_granularity_bytes = 0; diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index aed3081d3d0..4cb35ee64b1 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -57,7 +57,7 @@ public: size_t getMarkSizeInBytes(size_t columns_num = 1) const; - static std::optional getMarksExtensionFromFilesystem(const IDataPartStorage & data_part_storage); + static std::optional getMarksTypeFromFilesystem(const IDataPartStorage & data_part_storage); }; constexpr inline auto getNonAdaptiveMrkSizeWide() { return sizeof(UInt64) * 2; } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 2900e4aab7c..6f83a82e4e6 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -240,7 +240,8 @@ MergeTreeMutableDataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEn cloned_part_storage = part->makeCloneOnDisk(disk, MergeTreeData::MOVING_DIR_NAME); } - auto cloned_part = data->createPart(part->name, cloned_part_storage); + MergeTreeDataPartBuilder builder(*data, part->name, cloned_part_storage); + auto cloned_part = std::move(builder).withPartFormatFromDisk().build(); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getDataPartStorage().getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 9331c9723b5..fabf2acdad3 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -12,7 +12,7 @@ #include #include #include -#include "Storages/MergeTree/DataPartStorageOnDisk.h" +#include "Storages/MergeTree/DataPartStorageOnDiskFull.h" #include namespace DB @@ -176,16 +176,13 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore( else if (action_type == ActionType::ADD_PART) { auto single_disk_volume = std::make_shared("volume_" + part_name, disk, 0); - auto data_part_storage = std::make_shared(single_disk_volume, storage.getRelativeDataPath(), part_name); - part = storage.createPart( - part_name, - MergeTreeDataPartType::InMemory, - MergeTreePartInfo::fromPartName(part_name, storage.format_version), - data_part_storage); + part = storage.getDataPartBuilder(part_name, single_disk_volume, part_name) + .withPartType(MergeTreeDataPartType::InMemory) + .withPartStorageType(MergeTreeDataPartStorageType::Full) + .build(); part->uuid = metadata.part_uuid; - block = block_in.read(); if (storage.getActiveContainingPart(part->info, MergeTreeDataPartState::Active, parts_lock)) diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index b432841d5b0..ce2f485a1d8 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -106,7 +106,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() future_mutated_part->parts.push_back(source_part); future_mutated_part->part_info = new_part_info; future_mutated_part->updatePath(storage, reserved_space.get()); - future_mutated_part->type = source_part->getType(); + future_mutated_part->part_format = source_part->getFormat(); if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) { @@ -192,6 +192,10 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) { new_part = mutate_task->getFuture().get(); + auto & data_part_storage = new_part->getDataPartStorage(); + if (data_part_storage.hasActiveTransaction()) + data_part_storage.precommitTransaction(); + storage.renameTempPartAndReplace(new_part, *transaction_ptr); try diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 6ebf3e1fc22..f5e29d8cac5 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -60,7 +60,6 @@ void MutatePlainMergeTreeTask::prepare() bool MutatePlainMergeTreeTask::executeStep() { - /// Make out memory tracker a parent of current thread memory tracker MemoryTrackerThreadSwitcherPtr switcher; if (merge_list_entry) @@ -82,6 +81,9 @@ bool MutatePlainMergeTreeTask::executeStep() return true; new_part = mutate_task->getFuture().get(); + auto & data_part_storage = new_part->getDataPartStorage(); + if (data_part_storage.hasActiveTransaction()) + data_part_storage.precommitTransaction(); MergeTreeData::Transaction transaction(storage, merge_mutate_entry->txn.get()); /// FIXME Transactions: it's too optimistic, better to lock parts before starting transaction @@ -106,7 +108,7 @@ bool MutatePlainMergeTreeTask::executeStep() return false; } } - case State::NEED_FINISH : + case State::NEED_FINISH: { // Nothing to do state = State::SUCCESS; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 0a7affa4445..eb0abd79b8b 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -60,7 +60,7 @@ static void splitMutationCommands( { auto part_columns = part->getColumnsDescription(); - if (!isWidePart(part)) + if (!isWidePart(part) || !isFullPartStorage(part->getDataPartStorage())) { NameSet mutated_columns; for (const auto & command : commands) @@ -217,7 +217,7 @@ getColumnsForNewDataPart( /// In compact parts we read all columns, because they all stored in a /// single file - if (!isWidePart(source_part)) + if (!isWidePart(source_part) || !isFullPartStorage(source_part->getDataPartStorage())) return {updated_header.getNamesAndTypesList(), new_serialization_infos}; const auto & source_columns = source_part->getColumns(); @@ -692,7 +692,7 @@ void finalizeMutatedPart( new_data_part->index = source_part->index; new_data_part->minmax_idx = source_part->minmax_idx; new_data_part->modification_time = time(nullptr); - new_data_part->loadProjections(false, false); + /// All information about sizes is stored in checksums. /// It doesn't make sense to touch filesystem for sizes. new_data_part->setBytesOnDisk(new_data_part->checksums.getTotalSizeOnDisk()); @@ -1258,7 +1258,6 @@ public: case State::NEED_PREPARE: { prepare(); - state = State::NEED_EXECUTE; return true; } @@ -1487,7 +1486,6 @@ MutateTask::MutateTask( bool MutateTask::execute() { - switch (state) { case State::NEED_PREPARE: @@ -1577,7 +1575,6 @@ bool MutateTask::prepare() prefix = "tmp_clone_"; auto [part, lock] = ctx->data->cloneAndLoadDataPartOnSameDisk(ctx->source_part, prefix, ctx->future_part->part_info, ctx->metadata_snapshot, ctx->txn, &ctx->hardlinked_files, false, files_to_copy_instead_of_hardlinks); - part->getDataPartStorage().beginTransaction(); ctx->temporary_directory_lock = std::move(lock); @@ -1618,15 +1615,12 @@ bool MutateTask::prepare() String tmp_part_dir_name = prefix + ctx->future_part->name; ctx->temporary_directory_lock = ctx->data->getTemporaryPartDirectoryHolder(tmp_part_dir_name); - auto data_part_storage = std::make_shared( - single_disk_volume, - ctx->data->getRelativeDataPath(), - tmp_part_dir_name); + auto builder = ctx->data->getDataPartBuilder(ctx->future_part->name, single_disk_volume, tmp_part_dir_name); + builder.withPartFormat(ctx->future_part->part_format); + builder.withPartInfo(ctx->future_part->part_info); - data_part_storage->beginTransaction(); - - ctx->new_data_part = ctx->data->createPart( - ctx->future_part->name, ctx->future_part->type, ctx->future_part->part_info, data_part_storage); + ctx->new_data_part = std::move(builder).build(); + ctx->new_data_part->getDataPartStorage().beginTransaction(); ctx->new_data_part->uuid = ctx->future_part->uuid; ctx->new_data_part->is_temp = true; @@ -1654,7 +1648,7 @@ bool MutateTask::prepare() /// All columns from part are changed and may be some more that were missing before in part /// TODO We can materialize compact part without copying data - if (!isWidePart(ctx->source_part) + if (!isWidePart(ctx->source_part) || !isFullPartStorage(ctx->source_part->getDataPartStorage()) || (ctx->mutation_kind == MutationsInterpreter::MutationKind::MUTATE_OTHER && ctx->interpreter && ctx->interpreter->isAffectingAllColumns())) { task = std::make_unique(ctx); @@ -1710,8 +1704,8 @@ bool MutateTask::prepare() files_to_copy_instead_of_hardlinks.insert(IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK); auto [part, lock] = ctx->data->cloneAndLoadDataPartOnSameDisk(ctx->source_part, prefix, ctx->future_part->part_info, ctx->metadata_snapshot, ctx->txn, &ctx->hardlinked_files, false, files_to_copy_instead_of_hardlinks); - part->getDataPartStorage().beginTransaction(); + ctx->temporary_directory_lock = std::move(lock); promise.set_value(std::move(part)); return false; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index d7e3c3b1955..556751d4dbb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -165,8 +165,16 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const out << '\n'; - if (new_part_type != MergeTreeDataPartType::Wide && new_part_type != MergeTreeDataPartType::Unknown) - out << "part_type: " << new_part_type.toString() << "\n"; + using PartType = MergeTreeDataPartType; + using StorageType = MergeTreeDataPartStorageType; + + auto part_type = new_part_format.part_type; + if (part_type != PartType::Wide && part_type != PartType::Unknown) + out << "part_type: " << part_type.toString() << "\n"; + + auto storage_type = new_part_format.storage_type; + if (storage_type != StorageType::Full && storage_type != StorageType::Unknown) + out << "storage_type: " << storage_type.toString() << "\n"; if (quorum) out << "quorum: " << quorum << '\n'; @@ -342,13 +350,21 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) if (checkString("part_type: ", in)) { - String part_type_str; in >> type_str; - new_part_type.fromString(type_str); + new_part_format.part_type.fromString(type_str); in >> "\n"; } else - new_part_type = MergeTreeDataPartType::Wide; + new_part_format.part_type = MergeTreeDataPartType::Wide; + + if (checkString("storage_type: ", in)) + { + in >> type_str; + new_part_format.storage_type.fromString(type_str); + in >> "\n"; + } + else + new_part_format.storage_type = MergeTreeDataPartStorageType::Full; /// Optional field. if (!in.eof()) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index a8c72f3234c..7f504baaab3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -88,7 +88,7 @@ struct ReplicatedMergeTreeLogEntryData /// The name of resulting part for GET_PART and MERGE_PARTS /// Part range for DROP_RANGE and CLEAR_COLUMN String new_part_name; - MergeTreeDataPartType new_part_type; + MergeTreeDataPartFormat new_part_format; String block_id; /// For parts of level zero, the block identifier for deduplication (node name in /blocks/). mutable String actual_new_part_name; /// GET_PART could actually fetch a part covering 'new_part_name'. UUID new_part_uuid = UUIDHelpers::Nil; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index ba0a4e165a9..76b96899dac 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -748,9 +748,10 @@ std::vector ReplicatedMergeTreeSinkImpl::commitPart( log_entry.new_part_name = part->name; /// TODO maybe add UUID here as well? log_entry.quorum = getQuorumSize(replicas_num); + log_entry.new_part_format = part->getFormat(); + if constexpr (!async_insert) log_entry.block_id = block_id; - log_entry.new_part_type = part->getType(); ops.emplace_back(zkutil::makeCreateRequest( storage.zookeeper_path + "/log/log-", diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index b5631c6445f..60c5157d463 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1128,7 +1128,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( future_part->parts.push_back(part); future_part->part_info = new_part_info; future_part->name = part->getNewName(new_part_info); - future_part->type = part->getType(); + future_part->part_format = part->getFormat(); tagger = std::make_unique(future_part, MergeTreeDataMergerMutator::estimateNeededDiskSpace({part}), *this, metadata_snapshot, true); return std::make_shared(future_part, std::move(tagger), commands, txn); @@ -1988,8 +1988,8 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_ } catch (const Exception & ex) { - results.emplace_back(part->name, false, - "Check of part finished with error: '" + ex.message() + "'"); + tryLogCurrentException(log, __PRETTY_FUNCTION__); + results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'"); } } else diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 28a80607fa2..10f7615769f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -1441,11 +1441,10 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAndCommit(Transaction & transaction, - const DataPartPtr & part, std::optional hardlinked_files) + const MutableDataPartPtr & part, std::optional hardlinked_files) { auto zookeeper = getZooKeeper(); - while (true) { Coordination::Requests ops; @@ -1502,7 +1501,7 @@ String StorageReplicatedMergeTree::getChecksumsForZooKeeper(const MergeTreeDataP getSettings()->use_minimalistic_checksums_in_zookeeper); } -MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFoundValidPart(const LogEntry& entry) const +MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFoundValidPart(const LogEntry & entry) const { if (format_version != MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) return {}; @@ -1519,17 +1518,12 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo if (!part_info || part_info->partition_id != actual_part_info.partition_id) continue; - const String part_old_name = part_info->getPartNameV1(); + const auto part_old_name = part_info->getPartNameV1(); + const auto volume = std::make_shared("volume_" + part_old_name, disk); - const VolumePtr volume = std::make_shared("volume_" + part_old_name, disk); - - auto data_part_storage = std::make_shared( - volume, - fs::path(relative_data_path) / "detached", - part_old_name); - - /// actual_part_info is more recent than part_info so we use it - MergeTreeData::MutableDataPartPtr part = createPart(part_new_name, actual_part_info, data_part_storage); + auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path("detached") / part_old_name) + .withPartFormatFromDisk() + .build(); try { @@ -1544,7 +1538,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo if (entry.part_checksum == part->checksums.getTotalChecksumHex()) { - part->modification_time = data_part_storage->getLastModified().epochTime(); + part->modification_time = part->getDataPartStorage().getLastModified().epochTime(); return part; } } @@ -3217,7 +3211,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() future_merged_part->parts, future_merged_part->name, future_merged_part->uuid, - future_merged_part->type, + future_merged_part->part_format, deduplicate, deduplicate_by_columns, nullptr, @@ -3308,7 +3302,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c const DataPartsVector & parts, const String & merged_name, const UUID & merged_part_uuid, - const MergeTreeDataPartType & merged_part_type, + const MergeTreeDataPartFormat & merged_part_format, bool deduplicate, const Names & deduplicate_by_columns, ReplicatedMergeTreeLogEntryData * out_log_entry, @@ -3346,7 +3340,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c entry.source_replica = replica_name; entry.new_part_name = merged_name; entry.new_part_uuid = merged_part_uuid; - entry.new_part_type = merged_part_type; + entry.new_part_format = merged_part_format; entry.merge_type = merge_type; entry.deduplicate = deduplicate; entry.deduplicate_by_columns = deduplicate_by_columns; @@ -4867,9 +4861,12 @@ bool StorageReplicatedMergeTree::optimize( ReplicatedMergeTreeLogEntryData merge_entry; CreateMergeEntryResult create_result = createLogEntryToMergeParts( zookeeper, future_merged_part->parts, - future_merged_part->name, future_merged_part->uuid, future_merged_part->type, + future_merged_part->name, + future_merged_part->uuid, + future_merged_part->part_format, deduplicate, deduplicate_by_columns, - &merge_entry, can_merge.getVersion(), future_merged_part->merge_type); + &merge_entry, can_merge.getVersion(), + future_merged_part->merge_type); if (create_result == CreateMergeEntryResult::MissingPart) { @@ -7811,6 +7808,7 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, Context } catch (const Exception & ex) { + tryLogCurrentException(log, __PRETTY_FUNCTION__); results.emplace_back(part->name, false, "Check of part finished with error: '" + ex.message() + "'"); } } @@ -8570,7 +8568,6 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP LOG_INFO(log, "Going to replace lost part {} with empty part", lost_part_name); auto new_part_info = MergeTreePartInfo::fromPartName(lost_part_name, format_version); - auto metadata_snapshot = getInMemoryMetadataPtr(); MergeTreePartition partition; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 71f143aa8a4..568f9709aaa 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -528,7 +528,7 @@ private: String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const; /// Accepts a PreActive part, atomically checks its checksums with ones on other replicas and commit the part - DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part, std::optional hardlinked_files = {}); + DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const MutableDataPartPtr & part, std::optional hardlinked_files = {}); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; @@ -629,7 +629,7 @@ private: const DataPartsVector & parts, const String & merged_name, const UUID & merged_part_uuid, - const MergeTreeDataPartType & merged_part_type, + const MergeTreeDataPartFormat & merged_part_format, bool deduplicate, const Names & deduplicate_by_columns, ReplicatedMergeTreeLogEntryData * out_log_entry, diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp index 01c7b7d69e4..494f9c9c31f 100644 --- a/src/Storages/System/StorageSystemDetachedParts.cpp +++ b/src/Storages/System/StorageSystemDetachedParts.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include From ec1e0bde6bf1b2a9cba78c1285ad2d64d0924b32 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Mon, 23 Jan 2023 11:15:50 -0500 Subject: [PATCH 100/439] Update krb5 --- contrib/krb5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/krb5 b/contrib/krb5 index b89e20367b0..f8262a1b548 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit b89e20367b074bd02dd118a6534099b21e88b3c3 +Subproject commit f8262a1b548eb29d97e059260042036255d07f8d From d6d2414ef8d1166379aa0746eba2c85deb3257a8 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Mon, 23 Jan 2023 11:40:55 -0500 Subject: [PATCH 101/439] Remove aes.c and use the one krb5 --- contrib/krb5-cmake/CMakeLists.txt | 6 - contrib/krb5-cmake/aes.c | 302 ------------------------------ 2 files changed, 308 deletions(-) delete mode 100644 contrib/krb5-cmake/aes.c diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index 7e184d424aa..214d23bc2a9 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -578,12 +578,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin") list(APPEND ALL_SRCS "${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.c") endif() -if (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC) - list(REMOVE_ITEM ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/aes.c") - list(APPEND ALL_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/aes.c") -endif () - - target_sources(_krb5 PRIVATE ${ALL_SRCS} ) diff --git a/contrib/krb5-cmake/aes.c b/contrib/krb5-cmake/aes.c deleted file mode 100644 index c0c8c728bff..00000000000 --- a/contrib/krb5-cmake/aes.c +++ /dev/null @@ -1,302 +0,0 @@ -/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil -*- */ -/* lib/crypto/openssl/enc_provider/aes.c */ -/* - * Copyright (C) 2003, 2007, 2008, 2009 by the Massachusetts Institute of Technology. - * All rights reserved. - * - * Export of this software from the United States of America may - * require a specific license from the United States Government. - * It is the responsibility of any person or organization contemplating - * export to obtain such a license before exporting. - * - * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and - * distribute this software and its documentation for any purpose and - * without fee is hereby granted, provided that the above copyright - * notice appear in all copies and that both that copyright notice and - * this permission notice appear in supporting documentation, and that - * the name of M.I.T. not be used in advertising or publicity pertaining - * to distribution of the software without specific, written prior - * permission. Furthermore if you modify this software you must label - * your software as modified software and not distribute it in such a - * fashion that it might be confused with the original M.I.T. software. - * M.I.T. makes no representations about the suitability of - * this software for any purpose. It is provided "as is" without express - * or implied warranty. - */ - -#include "crypto_int.h" -#include -#include - -/* proto's */ -static krb5_error_code -cbc_enc(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data); -static krb5_error_code -cbc_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data); -static krb5_error_code -cts_encr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data, size_t dlen); -static krb5_error_code -cts_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data, size_t dlen); - -#define BLOCK_SIZE 16 -#define NUM_BITS 8 -#define IV_CTS_BUF_SIZE 16 /* 16 - hardcoded in CRYPTO_cts128_en/decrypt */ - -static const EVP_CIPHER * -map_mode(unsigned int len) -{ - if (len==16) - return EVP_aes_128_cbc(); - if (len==32) - return EVP_aes_256_cbc(); - else - return NULL; -} - -/* Encrypt one block using CBC. */ -static krb5_error_code -cbc_enc(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data) -{ - int ret, olen = BLOCK_SIZE; - unsigned char iblock[BLOCK_SIZE], oblock[BLOCK_SIZE]; - EVP_CIPHER_CTX *ctx; - struct iov_cursor cursor; - - ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) - return ENOMEM; - - ret = EVP_EncryptInit_ex(ctx, map_mode(key->keyblock.length), - NULL, key->keyblock.contents, (ivec) ? (unsigned char*)ivec->data : NULL); - if (ret == 0) { - EVP_CIPHER_CTX_free(ctx); - return KRB5_CRYPTO_INTERNAL; - } - - k5_iov_cursor_init(&cursor, data, num_data, BLOCK_SIZE, FALSE); - k5_iov_cursor_get(&cursor, iblock); - EVP_CIPHER_CTX_set_padding(ctx,0); - ret = EVP_EncryptUpdate(ctx, oblock, &olen, iblock, BLOCK_SIZE); - if (ret == 1) - k5_iov_cursor_put(&cursor, oblock); - EVP_CIPHER_CTX_free(ctx); - - zap(iblock, BLOCK_SIZE); - zap(oblock, BLOCK_SIZE); - return (ret == 1) ? 0 : KRB5_CRYPTO_INTERNAL; -} - -/* Decrypt one block using CBC. */ -static krb5_error_code -cbc_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data) -{ - int ret = 0, olen = BLOCK_SIZE; - unsigned char iblock[BLOCK_SIZE], oblock[BLOCK_SIZE]; - EVP_CIPHER_CTX *ctx; - struct iov_cursor cursor; - - ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) - return ENOMEM; - - ret = EVP_DecryptInit_ex(ctx, map_mode(key->keyblock.length), - NULL, key->keyblock.contents, (ivec) ? (unsigned char*)ivec->data : NULL); - if (ret == 0) { - EVP_CIPHER_CTX_free(ctx); - return KRB5_CRYPTO_INTERNAL; - } - - k5_iov_cursor_init(&cursor, data, num_data, BLOCK_SIZE, FALSE); - k5_iov_cursor_get(&cursor, iblock); - EVP_CIPHER_CTX_set_padding(ctx,0); - ret = EVP_DecryptUpdate(ctx, oblock, &olen, iblock, BLOCK_SIZE); - if (ret == 1) - k5_iov_cursor_put(&cursor, oblock); - EVP_CIPHER_CTX_free(ctx); - - zap(iblock, BLOCK_SIZE); - zap(oblock, BLOCK_SIZE); - return (ret == 1) ? 0 : KRB5_CRYPTO_INTERNAL; -} - -static krb5_error_code -cts_encr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data, size_t dlen) -{ - int ret = 0; - size_t size = 0; - unsigned char *oblock = NULL, *dbuf = NULL; - unsigned char iv_cts[IV_CTS_BUF_SIZE]; - struct iov_cursor cursor; - AES_KEY enck; - - memset(iv_cts,0,sizeof(iv_cts)); - if (ivec && ivec->data){ - if (ivec->length != sizeof(iv_cts)) - return KRB5_CRYPTO_INTERNAL; - memcpy(iv_cts, ivec->data,ivec->length); - } - - oblock = OPENSSL_malloc(dlen); - if (!oblock){ - return ENOMEM; - } - dbuf = OPENSSL_malloc(dlen); - if (!dbuf){ - OPENSSL_free(oblock); - return ENOMEM; - } - - k5_iov_cursor_init(&cursor, data, num_data, dlen, FALSE); - k5_iov_cursor_get(&cursor, dbuf); - - AES_set_encrypt_key(key->keyblock.contents, - NUM_BITS * key->keyblock.length, &enck); - - size = CRYPTO_cts128_encrypt((unsigned char *)dbuf, oblock, dlen, &enck, - iv_cts, AES_cbc_encrypt); - if (size <= 0) - ret = KRB5_CRYPTO_INTERNAL; - else - k5_iov_cursor_put(&cursor, oblock); - - if (!ret && ivec && ivec->data) - memcpy(ivec->data, iv_cts, sizeof(iv_cts)); - - zap(oblock, dlen); - zap(dbuf, dlen); - OPENSSL_free(oblock); - OPENSSL_free(dbuf); - - return ret; -} - -static krb5_error_code -cts_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data, - size_t num_data, size_t dlen) -{ - int ret = 0; - size_t size = 0; - unsigned char *oblock = NULL; - unsigned char *dbuf = NULL; - unsigned char iv_cts[IV_CTS_BUF_SIZE]; - struct iov_cursor cursor; - AES_KEY deck; - - memset(iv_cts,0,sizeof(iv_cts)); - if (ivec && ivec->data){ - if (ivec->length != sizeof(iv_cts)) - return KRB5_CRYPTO_INTERNAL; - memcpy(iv_cts, ivec->data,ivec->length); - } - - oblock = OPENSSL_malloc(dlen); - if (!oblock) - return ENOMEM; - dbuf = OPENSSL_malloc(dlen); - if (!dbuf){ - OPENSSL_free(oblock); - return ENOMEM; - } - - AES_set_decrypt_key(key->keyblock.contents, - NUM_BITS * key->keyblock.length, &deck); - - k5_iov_cursor_init(&cursor, data, num_data, dlen, FALSE); - k5_iov_cursor_get(&cursor, dbuf); - - size = CRYPTO_cts128_decrypt((unsigned char *)dbuf, oblock, - dlen, &deck, - iv_cts, AES_cbc_encrypt); - if (size <= 0) - ret = KRB5_CRYPTO_INTERNAL; - else - k5_iov_cursor_put(&cursor, oblock); - - if (!ret && ivec && ivec->data) - memcpy(ivec->data, iv_cts, sizeof(iv_cts)); - - zap(oblock, dlen); - zap(dbuf, dlen); - OPENSSL_free(oblock); - OPENSSL_free(dbuf); - - return ret; -} - -krb5_error_code -krb5int_aes_encrypt(krb5_key key, const krb5_data *ivec, - krb5_crypto_iov *data, size_t num_data) -{ - int ret = 0; - size_t input_length, nblocks; - - input_length = iov_total_length(data, num_data, FALSE); - nblocks = (input_length + BLOCK_SIZE - 1) / BLOCK_SIZE; - if (nblocks == 1) { - if (input_length != BLOCK_SIZE) - return KRB5_BAD_MSIZE; - ret = cbc_enc(key, ivec, data, num_data); - } else if (nblocks > 1) { - ret = cts_encr(key, ivec, data, num_data, input_length); - } - - return ret; -} - -krb5_error_code -krb5int_aes_decrypt(krb5_key key, const krb5_data *ivec, - krb5_crypto_iov *data, size_t num_data) -{ - int ret = 0; - size_t input_length, nblocks; - - input_length = iov_total_length(data, num_data, FALSE); - nblocks = (input_length + BLOCK_SIZE - 1) / BLOCK_SIZE; - if (nblocks == 1) { - if (input_length != BLOCK_SIZE) - return KRB5_BAD_MSIZE; - ret = cbc_decr(key, ivec, data, num_data); - } else if (nblocks > 1) { - ret = cts_decr(key, ivec, data, num_data, input_length); - } - - return ret; -} - -static krb5_error_code -krb5int_aes_init_state (const krb5_keyblock *key, krb5_keyusage usage, - krb5_data *state) -{ - state->length = 16; - state->data = (void *) malloc(16); - if (state->data == NULL) - return ENOMEM; - memset(state->data, 0, state->length); - return 0; -} -const struct krb5_enc_provider krb5int_enc_aes128 = { - 16, - 16, 16, - krb5int_aes_encrypt, - krb5int_aes_decrypt, - NULL, - krb5int_aes_init_state, - krb5int_default_free_state -}; - -const struct krb5_enc_provider krb5int_enc_aes256 = { - 16, - 32, 32, - krb5int_aes_encrypt, - krb5int_aes_decrypt, - NULL, - krb5int_aes_init_state, - krb5int_default_free_state -}; From 748c3ad0159f830fa6d8d81dad03a469bbe8be26 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Wed, 25 Jan 2023 13:35:31 -0500 Subject: [PATCH 102/439] Fix build error --- contrib/krb5-cmake/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index 214d23bc2a9..b2407e5b500 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -15,6 +15,10 @@ if(NOT AWK_PROGRAM) message(FATAL_ERROR "You need the awk program to build ClickHouse with krb5 enabled.") endif() +if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC)) + set(USE_BORINGSSL 1) +endif () + set(KRB5_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/krb5/src") set(KRB5_ET_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}/include_private") From 5df85ff2bbea34bf8d841dc50d1fde7b53b35ff4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 25 Jan 2023 19:09:24 +0000 Subject: [PATCH 103/439] Revert test to original version - the optimization shouldn't affect it --- tests/queries/0_stateless/02119_sumcount.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/queries/0_stateless/02119_sumcount.sql b/tests/queries/0_stateless/02119_sumcount.sql index 6e6409935d5..86625996f44 100644 --- a/tests/queries/0_stateless/02119_sumcount.sql +++ b/tests/queries/0_stateless/02119_sumcount.sql @@ -1,11 +1,3 @@ -set query_plan_remove_redundant_sorting=0; -- disable it for now since test with Float64 is failing with it --- while debugging I observe incorrect behavior which can affect the current test result --- but it's still unclear to the test is not failing w/o the optimization --- SELECT CAST('9007199254740992', 'Float64') + CAST('1', 'Float64') --- ┌─plus(CAST('9007199254740992', 'Float64'), CAST('1', 'Float64'))─┐ --- │ 9007199254740992 │ --- └─────────────────────────────────────────────────────────────────┘ - -- Integer types are added as integers SELECT toTypeName(sumCount(v)), sumCount(v) FROM ( From d0428a9b422858af2483e91a3110447583f0e215 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 25 Jan 2023 20:30:19 +0100 Subject: [PATCH 104/439] Fxi --- src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a77cce078ba..b304728f5c1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7967,6 +7967,7 @@ void StorageReplicatedMergeTree::createTableSharedID() const } else if (code == Coordination::Error::ZNONODE) /// table completely dropped, we can choose any id we want { + id = toString(UUIDHelpers::Nil); LOG_DEBUG(log, "Table was completely drop, we can use anything as ID (will use {})", id); } else if (code != Coordination::Error::ZOK) @@ -8113,7 +8114,7 @@ StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, co auto shared_id = getTableSharedID(); if (shared_id == toString(UUIDHelpers::Nil)) { - LOG_TRACE(log, "Part {} blobs can be removed, because table {} comletely dropped", part.name, getStorageID().getNameForLogs()); + LOG_TRACE(log, "Part {} blobs can be rem)oved, because table {} comletely dropped", part.name, getStorageID().getNameForLogs()); return std::make_pair(true, NameSet{}); } From 0f46c1ecc6f7c480df1a766d674ab75838625b7b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 25 Jan 2023 20:13:44 +0000 Subject: [PATCH 105/439] fix build --- src/Storages/MergeTree/DataPartsExchange.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index a322faee122..20c15039a2d 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -88,7 +88,7 @@ public: ActionBlocker blocker; private: - using OutputBufferGetter = std::function(IDataPartStorage &, const String &, size_t)>; + using OutputBufferGetter = std::function(IDataPartStorage &, const String &, size_t)>; void downloadBaseOrProjectionPartToDisk( const String & replica_path, From 3744fa2c63a2176be5c07cb2ca2d501cdd13b1ed Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 25 Jan 2023 21:16:42 +0100 Subject: [PATCH 106/439] format more messages --- src/Access/ContextAccess.cpp | 65 ++++++++++--------- src/Analyzer/Passes/QueryAnalysisPass.cpp | 9 +-- src/Common/Exception.cpp | 6 +- src/Common/Exception.h | 30 ++------- src/Common/LoggingFormatStringHelpers.cpp | 6 ++ src/Common/LoggingFormatStringHelpers.h | 36 +++++++++- src/Core/Block.cpp | 38 +++++------ src/DataTypes/DataTypeDecimalBase.h | 3 +- src/IO/parseDateTimeBestEffort.cpp | 61 ++++++++--------- src/Interpreters/executeQuery.cpp | 8 +-- src/Server/InterserverIOHTTPHandler.cpp | 4 +- src/Storages/AlterCommands.cpp | 24 +++---- .../MergeTree/MutatePlainMergeTreeTask.cpp | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 64 +++++++----------- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 10 +-- src/TableFunctions/TableFunctionRemote.h | 2 +- src/TableFunctions/TableFunctionS3Cluster.cpp | 2 +- tests/clickhouse-test | 14 ++-- 19 files changed, 194 insertions(+), 192 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index ed9f131860c..fbaacb2263b 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -484,13 +484,15 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg return true; }; - auto access_denied = [&](const String & error_msg, int error_code [[maybe_unused]]) + auto access_denied = [&](int error_code [[maybe_unused]], + FormatStringHelper fmt_string [[maybe_unused]], + FmtArgs && ...fmt_args [[maybe_unused]]) { if (trace_log) LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), (grant_option ? " WITH GRANT OPTION" : "")); if constexpr (throw_if_denied) - throw Exception::createDeprecated(getUserName() + ": " + error_msg, error_code); + throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward(fmt_args)...); return false; }; @@ -519,18 +521,16 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg { if (grant_option && acs->isGranted(flags, args...)) { - return access_denied( - "Not enough privileges. " + return access_denied(ErrorCodes::ACCESS_DENIED, + "{}: Not enough privileges. " "The required privileges have been granted, but without grant option. " - "To execute this query it's necessary to have grant " - + AccessRightsElement{flags, args...}.toStringWithoutOptions() + " WITH GRANT OPTION", - ErrorCodes::ACCESS_DENIED); + "To execute this query it's necessary to have grant {} WITH GRANT OPTION", + AccessRightsElement{flags, args...}.toStringWithoutOptions()); } - return access_denied( - "Not enough privileges. To execute this query it's necessary to have grant " - + AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""), - ErrorCodes::ACCESS_DENIED); + return access_denied(ErrorCodes::ACCESS_DENIED, + "{}: Not enough privileges. To execute this query it's necessary to have grant {}", + AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : "")); } struct PrecalculatedFlags @@ -557,32 +557,34 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg if (params.readonly) { if constexpr (grant_option) - return access_denied("Cannot change grants in readonly mode.", ErrorCodes::READONLY); + return access_denied(ErrorCodes::READONLY, "{}: Cannot change grants in readonly mode."); if ((flags & precalc.not_readonly_flags) || ((params.readonly == 1) && (flags & precalc.not_readonly_1_flags))) { if (params.interface == ClientInfo::Interface::HTTP && params.http_method == ClientInfo::HTTPMethod::GET) { - return access_denied( - "Cannot execute query in readonly mode. " - "For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries", - ErrorCodes::READONLY); + return access_denied(ErrorCodes::READONLY, + "{}: Cannot execute query in readonly mode. " + "For queries over HTTP, method GET implies readonly. " + "You should use method POST for modifying queries"); } else - return access_denied("Cannot execute query in readonly mode", ErrorCodes::READONLY); + return access_denied(ErrorCodes::READONLY, "{}: Cannot execute query in readonly mode"); } } if (!params.allow_ddl && !grant_option) { if (flags & precalc.ddl_flags) - return access_denied("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED); + return access_denied(ErrorCodes::QUERY_IS_PROHIBITED, + "Cannot execute query. DDL queries are prohibited for the user {}"); } if (!params.allow_introspection && !grant_option) { if (flags & precalc.introspection_flags) - return access_denied("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + return access_denied(ErrorCodes::FUNCTION_NOT_ALLOWED, "{}: Introspection functions are disabled, " + "because setting 'allow_introspection_functions' is set to 0"); } return access_granted(); @@ -679,11 +681,13 @@ void ContextAccess::checkGrantOption(const AccessRightsElements & elements) cons template bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const { - auto show_error = [this](const String & msg, int error_code [[maybe_unused]]) + auto show_error = [](int error_code [[maybe_unused]], + FormatStringHelper fmt_string [[maybe_unused]], + FmtArgs && ...fmt_args [[maybe_unused]]) { - UNUSED(this); if constexpr (throw_if_denied) - throw Exception::createDeprecated(getUserName() + ": " + msg, error_code); + throw Exception(error_code, std::move(fmt_string), std::forward(fmt_args)...); + return false; }; if (is_full_access) @@ -691,7 +695,7 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const if (user_was_dropped) { - show_error("User has been dropped", ErrorCodes::UNKNOWN_USER); + show_error(ErrorCodes::UNKNOWN_USER, "User has been dropped"); return false; } @@ -716,14 +720,15 @@ bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const role_name = "ID {" + toString(role_id) + "}"; if (info->enabled_roles.count(role_id)) - show_error("Not enough privileges. " - "Role " + backQuote(*role_name) + " is granted, but without ADMIN option. " - "To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option.", - ErrorCodes::ACCESS_DENIED); + show_error(ErrorCodes::ACCESS_DENIED, + "Not enough privileges. " + "Role {} is granted, but without ADMIN option. " + "To execute this query it's necessary to have the role {} granted with ADMIN option.", + backQuote(*role_name), backQuoteIfNeed(*role_name)); else - show_error("Not enough privileges. " - "To execute this query it's necessary to have the role " + backQuoteIfNeed(*role_name) + " granted with ADMIN option.", - ErrorCodes::ACCESS_DENIED); + show_error(ErrorCodes::ACCESS_DENIED, "Not enough privileges. " + "To execute this query it's necessary to have the role {} granted with ADMIN option.", + backQuoteIfNeed(*role_name)); } return false; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index fee040d973d..7d1f636754c 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4366,12 +4366,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi { if (!AggregateFunctionFactory::instance().isAggregateFunctionName(function_name)) { - std::string error_message = fmt::format("Aggregate function with name '{}' does not exists. In scope {}", - function_name, - scope.scope_node->formatASTForErrorMessage()); - - AggregateFunctionFactory::instance().appendHintsMessage(error_message, function_name); - throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, error_message); + throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, "Aggregate function with name '{}' does not exists. In scope {}{}", + function_name, scope.scope_node->formatASTForErrorMessage(), + getHintsErrorMessageSuffix(AggregateFunctionFactory::instance().getHints(function_name))); } if (!function_lambda_arguments_indexes.empty()) diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 637b6992ee1..a253260f69d 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -188,7 +188,7 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string { PreformattedMessage message = getCurrentExceptionMessageAndPattern(true); if (!start_of_message.empty()) - message.message = fmt::format("{}: {}", start_of_message, message.message); + message.text = fmt::format("{}: {}", start_of_message, message.text); LOG_ERROR(logger, message); } @@ -339,7 +339,7 @@ std::string getExtraExceptionInfo(const std::exception & e) std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/) { - return getCurrentExceptionMessageAndPattern(with_stacktrace, check_embedded_stacktrace, with_extra_info).message; + return getCurrentExceptionMessageAndPattern(with_stacktrace, check_embedded_stacktrace, with_extra_info).text; } PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/) @@ -481,7 +481,7 @@ void tryLogException(std::exception_ptr e, Poco::Logger * logger, const std::str std::string getExceptionMessage(const Exception & e, bool with_stacktrace, bool check_embedded_stacktrace) { - return getExceptionMessageAndPattern(e, with_stacktrace, check_embedded_stacktrace).message; + return getExceptionMessageAndPattern(e, with_stacktrace, check_embedded_stacktrace).text; } PreformattedMessage getExceptionMessageAndPattern(const Exception & e, bool with_stacktrace, bool check_embedded_stacktrace) diff --git a/src/Common/Exception.h b/src/Common/Exception.h index e548efce0d0..4fdfba9dde8 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -16,29 +16,6 @@ namespace Poco { class Logger; } -/// Extract format string from a string literal and constructs consteval fmt::format_string -template -struct FormatStringHelperImpl -{ - std::string_view message_format_string; - fmt::format_string fmt_str; - template - consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward(str)) - { - formatStringCheckArgsNumImpl(message_format_string, sizeof...(Args)); - } - template - FormatStringHelperImpl(fmt::basic_runtime && str) : message_format_string(), fmt_str(std::forward>(str)) {} - - PreformattedMessage format(Args && ...args) const - { - return PreformattedMessage{fmt::format(fmt_str, std::forward(args)...), message_format_string}; - } -}; - -template -using FormatStringHelper = FormatStringHelperImpl...>; - namespace DB { @@ -51,7 +28,12 @@ public: Exception() = default; - Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.message), code) + Exception(const PreformattedMessage & msg, int code): Exception(msg.text, code) + { + message_format_string = msg.format_string; + } + + Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.text), code) { message_format_string = msg.format_string; } diff --git a/src/Common/LoggingFormatStringHelpers.cpp b/src/Common/LoggingFormatStringHelpers.cpp index 9d1053335b9..186c2909382 100644 --- a/src/Common/LoggingFormatStringHelpers.cpp +++ b/src/Common/LoggingFormatStringHelpers.cpp @@ -4,3 +4,9 @@ { throw std::runtime_error(error); } + +template +PreformattedMessage FormatStringHelperImpl::format(Args && ...args) const +{ + return PreformattedMessage{fmt::format(fmt_str, std::forward(args)...), message_format_string}; +} diff --git a/src/Common/LoggingFormatStringHelpers.h b/src/Common/LoggingFormatStringHelpers.h index fd9eb43840e..b9415f51bee 100644 --- a/src/Common/LoggingFormatStringHelpers.h +++ b/src/Common/LoggingFormatStringHelpers.h @@ -2,14 +2,44 @@ #include #include +struct PreformattedMessage; +consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs); +template constexpr std::string_view tryGetStaticFormatString(T && x); + +/// Extract format string from a string literal and constructs consteval fmt::format_string +template +struct FormatStringHelperImpl +{ + std::string_view message_format_string; + fmt::format_string fmt_str; + template + consteval FormatStringHelperImpl(T && str) : message_format_string(tryGetStaticFormatString(str)), fmt_str(std::forward(str)) + { + formatStringCheckArgsNumImpl(message_format_string, sizeof...(Args)); + } + template + FormatStringHelperImpl(fmt::basic_runtime && str) : message_format_string(), fmt_str(std::forward>(str)) {} + + PreformattedMessage format(Args && ...args) const; +}; + +template +using FormatStringHelper = FormatStringHelperImpl...>; + /// Saves a format string for already formatted message struct PreformattedMessage { - std::string message; + std::string text; std::string_view format_string; - operator const std::string & () const { return message; } - operator std::string () && { return std::move(message); } + template + static PreformattedMessage create(FormatStringHelper fmt, Args &&... args) + { + return fmt.format(std::forward(args)...); + } + + operator const std::string & () const { return text; } + operator std::string () && { return std::move(text); } operator fmt::format_string<> () const { UNREACHABLE(); } }; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 93f25d6d0e2..948f305a701 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -29,11 +29,13 @@ namespace ErrorCodes extern const int AMBIGUOUS_COLUMN_NAME; } -template -static ReturnType onError(const std::string & message [[maybe_unused]], int code [[maybe_unused]]) +template +static ReturnType onError(int code [[maybe_unused]], + FormatStringHelper fmt_string [[maybe_unused]], + FmtArgs && ...fmt_args [[maybe_unused]]) { if constexpr (std::is_same_v) - throw Exception::createDeprecated(message, code); + throw Exception(error_code, std::move(fmt_string), std::forward(fmt_args)...); else return false; } @@ -44,13 +46,13 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con std::string_view context_description, bool allow_materialize, int code) { if (actual.name != expected.name) - return onError("Block structure mismatch in " + std::string(context_description) + " stream: different names of columns:\n" - + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + return onError(code, "Block structure mismatch in {} stream: different names of columns:\n{}\n{}", + context_description, actual.dumpStructure(), expected.dumpStructure()); if ((actual.type && !expected.type) || (!actual.type && expected.type) || (actual.type && expected.type && !actual.type->equals(*expected.type))) - return onError("Block structure mismatch in " + std::string(context_description) + " stream: different types:\n" - + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + return onError(code, "Block structure mismatch in {} stream: different types:\n{}\n{}", + context_description, actual.dumpStructure(), expected.dumpStructure()); if (!actual.column || !expected.column) return ReturnType(true); @@ -74,22 +76,18 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con if (actual_column_maybe_agg && expected_column_maybe_agg) { if (!actual_column_maybe_agg->getAggregateFunction()->haveSameStateRepresentation(*expected_column_maybe_agg->getAggregateFunction())) - return onError( - fmt::format( + return onError(code, "Block structure mismatch in {} stream: different columns:\n{}\n{}", context_description, actual.dumpStructure(), - expected.dumpStructure()), - code); + expected.dumpStructure()); } else if (actual_column->getName() != expected.column->getName()) - return onError( - fmt::format( + return onError(code, "Block structure mismatch in {} stream: different columns:\n{}\n{}", context_description, actual.dumpStructure(), - expected.dumpStructure()), - code); + expected.dumpStructure()); if (isColumnConst(*actual.column) && isColumnConst(*expected.column) && !actual.column->empty() && !expected.column->empty()) /// don't check values in empty columns @@ -98,14 +96,12 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con Field expected_value = assert_cast(*expected.column).getField(); if (actual_value != expected_value) - return onError( - fmt::format( + return onError(code, "Block structure mismatch in {} stream: different values of constants in column '{}': actual: {}, expected: {}", context_description, actual.name, applyVisitor(FieldVisitorToString(), actual_value), - applyVisitor(FieldVisitorToString(), expected_value)), - code); + applyVisitor(FieldVisitorToString(), expected_value)); } return ReturnType(true); @@ -117,8 +113,8 @@ static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, std: { size_t columns = rhs.columns(); if (lhs.columns() != columns) - return onError("Block structure mismatch in " + std::string(context_description) + " stream: different number of columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); + return onError(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch in {} stream: different number of columns:\n{}\n{}", + context_description, lhs.dumpStructure(), rhs.dumpStructure()); for (size_t i = 0; i < columns; ++i) { diff --git a/src/DataTypes/DataTypeDecimalBase.h b/src/DataTypes/DataTypeDecimalBase.h index a1580f246a2..0be345ba879 100644 --- a/src/DataTypes/DataTypeDecimalBase.h +++ b/src/DataTypes/DataTypeDecimalBase.h @@ -189,7 +189,8 @@ template