From 4ffe199d3947bd12d8a193aca42086f939596cb7 Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 23 Apr 2021 12:18:23 +0000 Subject: [PATCH 01/52] Implement table comments --- .../MySQL/DatabaseConnectionMySQL.cpp | 16 +++++++-- .../PostgreSQL/DatabasePostgreSQL.cpp | 9 +++-- src/Interpreters/DatabaseCatalog.cpp | 22 +++++------- src/Parsers/ASTCreateQuery.cpp | 11 ++++++ src/Parsers/ASTCreateQuery.h | 2 ++ src/Parsers/ParserCreateQuery.cpp | 13 +++++++ src/Storages/HDFS/StorageHDFS.cpp | 14 ++++---- src/Storages/HDFS/StorageHDFS.h | 4 ++- src/Storages/IStorage.h | 2 -- .../MergeTree/registerStorageMergeTree.cpp | 3 +- src/Storages/StorageBuffer.cpp | 10 ++++-- src/Storages/StorageBuffer.h | 1 + src/Storages/StorageDictionary.cpp | 10 +++--- src/Storages/StorageDictionary.h | 1 + src/Storages/StorageDistributed.cpp | 28 ++++++++++++--- src/Storages/StorageDistributed.h | 1 + src/Storages/StorageExternalDistributed.cpp | 14 ++++---- src/Storages/StorageExternalDistributed.h | 1 + src/Storages/StorageFactory.cpp | 10 +++++- src/Storages/StorageFactory.h | 1 + src/Storages/StorageFile.cpp | 2 ++ src/Storages/StorageFile.h | 1 + src/Storages/StorageGenerateRandom.cpp | 12 +++++-- src/Storages/StorageGenerateRandom.h | 9 +++-- src/Storages/StorageInMemoryMetadata.cpp | 6 ++++ src/Storages/StorageInMemoryMetadata.h | 4 +++ src/Storages/StorageJoin.cpp | 4 ++- src/Storages/StorageJoin.h | 4 ++- src/Storages/StorageLog.cpp | 12 +++++-- src/Storages/StorageLog.h | 1 + src/Storages/StorageMemory.cpp | 4 ++- src/Storages/StorageMemory.h | 1 + src/Storages/StorageMerge.cpp | 8 +++-- src/Storages/StorageMerge.h | 2 ++ src/Storages/StorageMongoDB.cpp | 7 ++-- src/Storages/StorageMongoDB.h | 3 +- src/Storages/StorageMySQL.cpp | 3 ++ src/Storages/StorageMySQL.h | 1 + src/Storages/StorageNull.cpp | 2 +- src/Storages/StorageNull.h | 4 ++- src/Storages/StoragePostgreSQL.cpp | 5 +-- src/Storages/StoragePostgreSQL.h | 1 + src/Storages/StorageS3.cpp | 6 ++-- src/Storages/StorageS3.h | 4 ++- src/Storages/StorageS3Cluster.cpp | 2 -- src/Storages/StorageS3Cluster.h | 14 ++++++-- src/Storages/StorageSet.cpp | 14 ++++---- src/Storages/StorageSet.h | 2 ++ src/Storages/StorageStripeLog.cpp | 12 +++++-- src/Storages/StorageStripeLog.h | 1 + src/Storages/StorageTinyLog.cpp | 12 +++++-- src/Storages/StorageTinyLog.h | 1 + src/Storages/StorageURL.cpp | 34 +++++++++--------- src/Storages/StorageURL.h | 19 +++++----- src/Storages/StorageView.cpp | 7 ++-- src/Storages/StorageView.h | 5 +-- src/Storages/StorageXDBC.cpp | 23 +++++++----- src/Storages/StorageXDBC.h | 1 + src/Storages/System/StorageSystemTables.cpp | 13 +++++-- src/TableFunctions/ITableFunctionXDBC.cpp | 3 +- .../TableFunctionDictionary.cpp | 3 +- src/TableFunctions/TableFunctionFile.cpp | 4 +-- .../TableFunctionGenerateRandom.cpp | 3 +- src/TableFunctions/TableFunctionHDFS.cpp | 1 + src/TableFunctions/TableFunctionMerge.cpp | 1 + src/TableFunctions/TableFunctionMySQL.cpp | 1 + src/TableFunctions/TableFunctionNull.cpp | 2 +- .../TableFunctionPostgreSQL.cpp | 10 ++++-- src/TableFunctions/TableFunctionRemote.cpp | 1 + src/TableFunctions/TableFunctionS3.cpp | 25 ++++++------- src/TableFunctions/TableFunctionS3Cluster.cpp | 18 +++++++--- src/TableFunctions/TableFunctionURL.cpp | 13 +++++-- src/TableFunctions/TableFunctionView.cpp | 2 +- .../0_stateless/01821_table_comment.reference | 3 ++ .../0_stateless/01821_table_comment.sql | 36 +++++++++++++++++++ 75 files changed, 408 insertions(+), 157 deletions(-) create mode 100644 tests/queries/0_stateless/01821_table_comment.reference create mode 100644 tests/queries/0_stateless/01821_table_comment.sql diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp index 9615aa85e05..2fd94370a5d 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp @@ -246,9 +246,19 @@ void DatabaseConnectionMySQL::fetchLatestTablesStructureIntoCache( local_tables_cache.erase(iterator); } - local_tables_cache[table_name] = std::make_pair(table_modification_time, StorageMySQL::create( - StorageID(database_name, table_name), std::move(mysql_pool), database_name_in_mysql, table_name, - false, "", ColumnsDescription{columns_name_and_type}, ConstraintsDescription{}, getContext())); + local_tables_cache[table_name] = std::make_pair( + table_modification_time, + StorageMySQL::create( + StorageID(database_name, table_name), + std::move(mysql_pool), + database_name_in_mysql, + table_name, + false, + "", + ColumnsDescription{columns_name_and_type}, + ConstraintsDescription{}, + String{}, + getContext())); } } diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index e12d7f975ce..ec47d7f81aa 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -170,8 +170,13 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr return StoragePtr{}; auto storage = StoragePostgreSQL::create( - StorageID(database_name, table_name), *connection_pool, table_name, - ColumnsDescription{*columns}, ConstraintsDescription{}, local_context); + StorageID(database_name, table_name), + *connection_pool, + table_name, + ColumnsDescription{*columns}, + ConstraintsDescription{}, + String{}, + local_context); if (cache_tables) cached_tables[table_name] = storage; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index d88b87a73d4..b9ba0bfcbdb 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -83,21 +83,17 @@ TemporaryTableHolder::TemporaryTableHolder( const ConstraintsDescription & constraints, const ASTPtr & query, bool create_for_global_subquery) - : TemporaryTableHolder - ( - context_, - [&](const StorageID & table_id) - { - auto storage = StorageMemory::create( - table_id, ColumnsDescription{columns}, ConstraintsDescription{constraints}); + : TemporaryTableHolder( + context_, + [&](const StorageID & table_id) { + auto storage = StorageMemory::create(table_id, ColumnsDescription{columns}, ConstraintsDescription{constraints}, String{}); - if (create_for_global_subquery) - storage->delayReadForGlobalSubqueries(); + if (create_for_global_subquery) + storage->delayReadForGlobalSubqueries(); - return storage; - }, - query - ) + return storage; + }, + query) { } diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 1192fcc6ebd..79e315f65f1 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -32,6 +32,12 @@ ASTPtr ASTStorage::clone() const if (settings) res->set(res->settings, settings->clone()); + if (comment) + { + res->comment = comment; + res->children.push_back(comment->clone()); + } + return res; } @@ -72,6 +78,11 @@ void ASTStorage::formatImpl(const FormatSettings & s, FormatState & state, Forma s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "SETTINGS " << (s.hilite ? hilite_none : ""); settings->formatImpl(s, state, frame); } + if (comment) + { + s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << "COMMENT " << (s.hilite ? hilite_none : ""); + comment->formatImpl(s, state, frame); + } } diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index d6d5c22240c..6fe486dbca4 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -25,6 +25,8 @@ public: IAST * ttl_table = nullptr; ASTSetQuery * settings = nullptr; + ASTPtr comment; + String getID(char) const override { return "Storage definition"; } ASTPtr clone() const override; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index bfd51b7633d..220c58f55ae 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -264,11 +264,13 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_sample_by("SAMPLE BY"); ParserKeyword s_ttl("TTL"); ParserKeyword s_settings("SETTINGS"); + ParserKeyword s_comment("COMMENT"); ParserIdentifierWithOptionalParameters ident_with_optional_params_p; ParserExpression expression_p; ParserSetQuery settings_p(/* parse_only_internals_ = */ true); ParserTTLExpressionList parser_ttl_list; + ParserStringLiteral string_literal_parser; ASTPtr engine; ASTPtr partition_by; @@ -277,6 +279,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr sample_by; ASTPtr ttl_table; ASTPtr settings; + ASTPtr comment_expression; if (!s_engine.ignore(pos, expected)) return false; @@ -334,6 +337,13 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } + if (s_comment.ignore(pos, expected)) + { + /// should be followed by a string literal + if (!string_literal_parser.parse(pos, comment_expression, expected)) + return false; + } + break; } @@ -347,6 +357,9 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) storage->set(storage->settings, settings); + storage->comment = comment_expression; + storage->children.push_back(std::move(comment_expression)); + node = storage; return true; } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 392495ff77f..728d49566d2 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -35,24 +35,23 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -StorageHDFS::StorageHDFS(const String & uri_, +StorageHDFS::StorageHDFS( + const String & uri_, const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const String & compression_method_ = "") - : IStorage(table_id_) - , WithContext(context_) - , uri(uri_) - , format_name(format_name_) - , compression_method(compression_method_) + : IStorage(table_id_), WithContext(context_), uri(uri_), format_name(format_name_), compression_method(compression_method_) { context_->getRemoteHostFilter().checkURL(Poco::URI(uri)); StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -335,7 +334,8 @@ void registerStorageHDFS(StorageFactory & factory) compression_method = engine_args[2]->as().value.safeGet(); } else compression_method = "auto"; - return StorageHDFS::create(url, args.table_id, format_name, args.columns, args.constraints, args.getContext(), compression_method); + return StorageHDFS::create( + url, args.table_id, format_name, args.columns, args.constraints, args.comment, args.getContext(), compression_method); }, { .source_access_type = AccessType::HDFS, diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 0dd57685354..8d6a5fdbd1e 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -33,11 +33,13 @@ public: NamesAndTypesList getVirtuals() const override; protected: - StorageHDFS(const String & uri_, + StorageHDFS( + const String & uri_, const StorageID & table_id_, const String & format_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const String & compression_method_); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index e48e9e49919..628085129e9 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -141,7 +141,6 @@ public: /// This is true for most storages that store data on disk. virtual bool prefersLargeBlocks() const { return true; } - /// Optional size information of each physical column. /// Currently it's only used by the MergeTree family for query optimizations. using ColumnSizeByName = std::unordered_map; @@ -165,7 +164,6 @@ public: metadata.set(std::make_unique(metadata_)); } - /// Return list of virtual columns (like _part, _table, etc). In the vast /// majority of cases virtual columns are static constant part of Storage /// class and don't depend on Storage object. But sometimes we have fake diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 862747abcb9..ff7d7396b4e 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -625,7 +625,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) String date_column_name; StorageInMemoryMetadata metadata; - metadata.columns = args.columns; + metadata.setColumns(args.columns); + metadata.setComment(args.comment); std::unique_ptr storage_settings; if (replicated) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index afe37d0bcbe..c93afee5ff2 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -102,6 +102,7 @@ StorageBuffer::StorageBuffer( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, size_t num_shards_, const Thresholds & min_thresholds_, @@ -111,7 +112,8 @@ StorageBuffer::StorageBuffer( bool allow_materialized_) : IStorage(table_id_) , WithContext(context_->getBufferContext()) - , num_shards(num_shards_), buffers(num_shards_) + , num_shards(num_shards_) + , buffers(num_shards_) , min_thresholds(min_thresholds_) , max_thresholds(max_thresholds_) , flush_thresholds(flush_thresholds_) @@ -123,6 +125,7 @@ StorageBuffer::StorageBuffer( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -1131,9 +1134,12 @@ void registerStorageBuffer(StorageFactory & factory) args.table_id, args.columns, args.constraints, + args.comment, args.getContext(), num_buckets, - min, max, flush, + min, + max, + flush, destination_id, static_cast(args.getLocalContext()->getSettingsRef().insert_allow_materialized_columns)); }, diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 1747c024a74..8844ff96c25 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -177,6 +177,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, size_t num_shards_, const Thresholds & min_thresholds_, diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index e2cab153092..dc0208fc30c 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -93,20 +93,20 @@ StorageDictionary::StorageDictionary( const StorageID & table_id_, const String & dictionary_name_, const ColumnsDescription & columns_, + const String & comment, Location location_) - : IStorage(table_id_) - , dictionary_name(dictionary_name_) - , location(location_) + : IStorage(table_id_), dictionary_name(dictionary_name_), location(location_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } StorageDictionary::StorageDictionary( const StorageID & table_id_, const String & dictionary_name_, const DictionaryStructure & dictionary_structure_, Location location_) - : StorageDictionary(table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_)}, location_) + : StorageDictionary(table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_)}, String{}, location_) { } @@ -158,7 +158,7 @@ void registerStorageDictionary(StorageFactory & factory) checkNamesAndTypesCompatibleWithDictionary(dictionary_name, args.columns, dictionary_structure); } - return StorageDictionary::create(args.table_id, dictionary_name, args.columns, StorageDictionary::Location::Custom); + return StorageDictionary::create(args.table_id, dictionary_name, args.columns, args.comment, StorageDictionary::Location::Custom); }); } diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h index 9e8564d4349..47a2b40cb5a 100644 --- a/src/Storages/StorageDictionary.h +++ b/src/Storages/StorageDictionary.h @@ -60,6 +60,7 @@ protected: const StorageID & table_id_, const String & dictionary_name_, const ColumnsDescription & columns_, + const String & comment, Location location_); StorageDictionary( diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 3d96796a79b..9f1e3b8e963 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -386,6 +386,7 @@ StorageDistributed::StorageDistributed( const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, const String & remote_database_, const String & remote_table_, const String & cluster_name_, @@ -411,6 +412,7 @@ StorageDistributed::StorageDistributed( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); if (sharding_key_) @@ -452,8 +454,21 @@ StorageDistributed::StorageDistributed( const DistributedSettings & distributed_settings_, bool attach, ClusterPtr owned_cluster_) - : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, - storage_policy_name_, relative_data_path_, distributed_settings_, attach, std::move(owned_cluster_)) + : StorageDistributed( + id_, + columns_, + constraints_, + String{}, + String{}, + String{}, + cluster_name_, + context_, + sharding_key_, + storage_policy_name_, + relative_data_path_, + distributed_settings_, + attach, + std::move(owned_cluster_)) { remote_table_function_ptr = std::move(remote_table_function_ptr_); } @@ -1251,8 +1266,13 @@ void registerStorageDistributed(StorageFactory & factory) } return StorageDistributed::create( - args.table_id, args.columns, args.constraints, - remote_database, remote_table, cluster_name, + args.table_id, + args.columns, + args.constraints, + args.comment, + remote_database, + remote_table, + cluster_name, args.getContext(), sharding_key, storage_policy, diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 886a8e032de..5c0c400ee25 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -123,6 +123,7 @@ private: const StorageID & id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, const String & remote_database_, const String & remote_table_, const String & cluster_name_, diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 3489de0161a..b7f07679c7e 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -36,12 +36,14 @@ StorageExternalDistributed::StorageExternalDistributed( const String & password, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context) : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; @@ -72,7 +74,9 @@ StorageExternalDistributed::StorageExternalDistributed( remote_table, /* replace_query = */ false, /* on_duplicate_clause = */ "", - columns_, constraints_, + columns_, + constraints_, + String{}, context); break; } @@ -90,12 +94,7 @@ StorageExternalDistributed::StorageExternalDistributed( context->getSettingsRef().postgresql_connection_pool_size, context->getSettingsRef().postgresql_connection_pool_wait_timeout); - shard = StoragePostgreSQL::create( - table_id_, - std::move(pool), - remote_table, - columns_, constraints_, - context); + shard = StoragePostgreSQL::create(table_id_, std::move(pool), remote_table, columns_, constraints_, String{}, context); break; } #endif @@ -177,6 +176,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) password, args.columns, args.constraints, + args.comment, args.getContext()); }, { diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index 71022f5eaa3..3a9eb5c821f 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -53,6 +53,7 @@ protected: const String & password, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_); private: diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 3a57c8ab4f6..8cba7df46d3 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -180,6 +180,13 @@ StoragePtr StorageFactory::get( } } + String comment; + + if (storage_def->comment) + { + comment = storage_def->comment->as().value.get(); + } + ASTs empty_engine_args; Arguments arguments { @@ -194,7 +201,8 @@ StoragePtr StorageFactory::get( .columns = columns, .constraints = constraints, .attach = query.attach, - .has_force_restore_data_flag = has_force_restore_data_flag + .has_force_restore_data_flag = has_force_restore_data_flag, + .comment = comment }; assert(arguments.getContext() == arguments.getContext()->getGlobalContext()); diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index 43f6a6d6f7d..8aa1273018b 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -45,6 +45,7 @@ public: const ConstraintsDescription & constraints; bool attach; bool has_force_restore_data_flag; + const String & comment; ContextPtr getContext() const; ContextPtr getLocalContext() const; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 14b91d29805..cfb4934dae0 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -221,6 +221,7 @@ StorageFile::StorageFile(CommonArguments args) storage_metadata.setColumns(args.columns); storage_metadata.setConstraints(args.constraints); + storage_metadata.setComment(args.comment); setInMemoryMetadata(storage_metadata); } @@ -687,6 +688,7 @@ void registerStorageFile(StorageFactory & factory) {}, factory_args.columns, factory_args.constraints, + factory_args.comment, }; ASTs & engine_args_ast = factory_args.engine_args; diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index a277dda7cc0..5be8378b53e 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -57,6 +57,7 @@ public: std::string compression_method; const ColumnsDescription & columns; const ConstraintsDescription & constraints; + const String & comment; }; NamesAndTypesList getVirtuals() const override; diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index bc158c38f37..1a1f96c3fae 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -385,8 +385,13 @@ private: } -StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, - UInt64 max_array_length_, UInt64 max_string_length_, std::optional random_seed_) +StorageGenerateRandom::StorageGenerateRandom( + const StorageID & table_id_, + const ColumnsDescription & columns_, + const String & comment, + UInt64 max_array_length_, + UInt64 max_string_length_, + std::optional random_seed_) : IStorage(table_id_), max_array_length(max_array_length_), max_string_length(max_string_length_) { static constexpr size_t MAX_ARRAY_SIZE = 1 << 30; @@ -402,6 +407,7 @@ StorageGenerateRandom::StorageGenerateRandom(const StorageID & table_id_, const random_seed = random_seed_ ? sipHash64(*random_seed_) : randomSeed(); StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -434,7 +440,7 @@ void registerStorageGenerateRandom(StorageFactory & factory) if (engine_args.size() == 3) max_array_length = engine_args[2]->as().value.safeGet(); - return StorageGenerateRandom::create(args.table_id, args.columns, max_array_length, max_string_length, random_seed); + return StorageGenerateRandom::create(args.table_id, args.columns, args.comment, max_array_length, max_string_length, random_seed); }); } diff --git a/src/Storages/StorageGenerateRandom.h b/src/Storages/StorageGenerateRandom.h index d9c2acb782b..4a4015ebc0d 100644 --- a/src/Storages/StorageGenerateRandom.h +++ b/src/Storages/StorageGenerateRandom.h @@ -30,8 +30,13 @@ private: UInt64 random_seed = 0; protected: - StorageGenerateRandom(const StorageID & table_id_, const ColumnsDescription & columns_, - UInt64 max_array_length, UInt64 max_string_length, std::optional random_seed); + StorageGenerateRandom( + const StorageID & table_id_, + const ColumnsDescription & columns_, + const String & comment, + UInt64 max_array_length, + UInt64 max_string_length, + std::optional random_seed); }; } diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 2f4a24a5c60..575099fbd21 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -35,6 +35,7 @@ StorageInMemoryMetadata::StorageInMemoryMetadata(const StorageInMemoryMetadata & , table_ttl(other.table_ttl) , settings_changes(other.settings_changes ? other.settings_changes->clone() : nullptr) , select(other.select) + , comment(other.comment) { } @@ -57,9 +58,14 @@ StorageInMemoryMetadata & StorageInMemoryMetadata::operator=(const StorageInMemo else settings_changes.reset(); select = other.select; + comment = other.comment; return *this; } +void StorageInMemoryMetadata::setComment(const String & comment_) +{ + comment = comment_; +} void StorageInMemoryMetadata::setColumns(ColumnsDescription columns_) { diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 00fb944c0b5..e3f02146181 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -43,6 +43,8 @@ struct StorageInMemoryMetadata /// SELECT QUERY. Supported for MaterializedView and View (have to support LiveView). SelectQueryDescription select; + String comment; + StorageInMemoryMetadata() = default; StorageInMemoryMetadata(const StorageInMemoryMetadata & other); @@ -52,6 +54,8 @@ struct StorageInMemoryMetadata /// structure from different threads. It should be used as MultiVersion /// object. See example in IStorage. + void setComment(const String & comment_); + /// Sets only real columns, possibly overwrites virtual ones. void setColumns(ColumnsDescription columns_); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 983b9213a35..4cd5eb1da71 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -46,9 +46,10 @@ StorageJoin::StorageJoin( ASTTableJoin::Strictness strictness_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool overwrite_, bool persistent_) - : StorageSetOrJoinBase{disk_, relative_path_, table_id_, columns_, constraints_, persistent_} + : StorageSetOrJoinBase{disk_, relative_path_, table_id_, columns_, constraints_, comment, persistent_} , key_names(key_names_) , use_nulls(use_nulls_) , limits(limits_) @@ -263,6 +264,7 @@ void registerStorageJoin(StorageFactory & factory) strictness, args.columns, args.constraints, + args.comment, join_any_take_last_row, persistent); }; diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index 4baac53c69c..b89780a0f75 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -81,9 +81,11 @@ protected: const Names & key_names_, bool use_nulls_, SizeLimits limits_, - ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_, + ASTTableJoin::Kind kind_, + ASTTableJoin::Strictness strictness_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool overwrite, bool persistent_); }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 8ed68e0b44d..663deddfd1f 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -465,6 +465,7 @@ StorageLog::StorageLog( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_) : IStorage(table_id_) @@ -476,6 +477,7 @@ StorageLog::StorageLog( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); if (relative_path_.empty()) @@ -734,8 +736,14 @@ void registerStorageLog(StorageFactory & factory) DiskPtr disk = args.getContext()->getDisk(disk_name); return StorageLog::create( - disk, args.relative_data_path, args.table_id, args.columns, args.constraints, - args.attach, args.getContext()->getSettings().max_compress_block_size); + disk, + args.relative_data_path, + args.table_id, + args.columns, + args.constraints, + args.comment, + args.attach, + args.getContext()->getSettings().max_compress_block_size); }, features); } diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 4fbaf53529f..20103f51479 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -57,6 +57,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 4cae7367606..289a17366bb 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -164,12 +164,14 @@ StorageMemory::StorageMemory( const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_, + const String & comment, bool compress_) : IStorage(table_id_), data(std::make_unique()), compress(compress_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(std::move(columns_description_)); storage_metadata.setConstraints(std::move(constraints_)); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -349,7 +351,7 @@ void registerStorageMemory(StorageFactory & factory) if (has_settings) settings.loadFromQuery(*args.storage_def); - return StorageMemory::create(args.table_id, args.columns, args.constraints, settings.compress); + return StorageMemory::create(args.table_id, args.columns, args.constraints, args.comment, settings.compress); }, { .supports_settings = true, diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 1118474deee..e623f29e2e6 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -115,6 +115,7 @@ protected: const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_, + const String & comment, bool compress_ = false); }; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 6ad7b0bce6e..91af4cb46b0 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -102,6 +102,7 @@ TreeRewriterResult modifySelect(ASTSelectQuery & select, const TreeRewriterResul StorageMerge::StorageMerge( const StorageID & table_id_, const ColumnsDescription & columns_, + const String & comment, const String & source_database_, const Strings & source_tables_, ContextPtr context_) @@ -112,12 +113,14 @@ StorageMerge::StorageMerge( { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } StorageMerge::StorageMerge( const StorageID & table_id_, const ColumnsDescription & columns_, + const String & comment, const String & source_database_, const String & source_table_regexp_, ContextPtr context_) @@ -128,6 +131,7 @@ StorageMerge::StorageMerge( { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -617,9 +621,7 @@ void registerStorageMerge(StorageFactory & factory) String source_database = engine_args[0]->as().value.safeGet(); String table_name_regexp = engine_args[1]->as().value.safeGet(); - return StorageMerge::create( - args.table_id, args.columns, - source_database, table_name_regexp, args.getContext()); + return StorageMerge::create(args.table_id, args.columns, args.comment, source_database, table_name_regexp, args.getContext()); }); } diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index ff016952686..d1269d9e5ed 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -70,6 +70,7 @@ protected: StorageMerge( const StorageID & table_id_, const ColumnsDescription & columns_, + const String & comment, const String & source_database_, const Strings & source_tables_, ContextPtr context_); @@ -77,6 +78,7 @@ protected: StorageMerge( const StorageID & table_id_, const ColumnsDescription & columns_, + const String & comment, const String & source_database_, const String & source_table_regexp_, ContextPtr context_); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 2b0200f3643..f5f604c6f41 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -34,7 +34,8 @@ StorageMongoDB::StorageMongoDB( const std::string & username_, const std::string & password_, const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_) + const ConstraintsDescription & constraints_, + const String & comment) : IStorage(table_id_) , host(host_) , port(port_) @@ -46,6 +47,7 @@ StorageMongoDB::StorageMongoDB( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -125,7 +127,8 @@ void registerStorageMongoDB(StorageFactory & factory) username, password, args.columns, - args.constraints); + args.constraints, + args.comment); }, { .source_access_type = AccessType::MONGO, diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 5e96d1543a2..e5f60f59c1a 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -27,7 +27,8 @@ public: const std::string & username_, const std::string & password_, const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_); + const ConstraintsDescription & constraints_, + const String & comment); std::string getName() const override { return "MongoDB"; } diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index c635b0e939c..fa4bb29adee 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -49,6 +49,7 @@ StorageMySQL::StorageMySQL( const std::string & on_duplicate_clause_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_) : IStorage(table_id_) , WithContext(context_->getGlobalContext()) @@ -61,6 +62,7 @@ StorageMySQL::StorageMySQL( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -268,6 +270,7 @@ void registerStorageMySQL(StorageFactory & factory) on_duplicate_clause, args.columns, args.constraints, + args.comment, args.getContext()); }, { diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index a68c06c1abe..a7aca48197e 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -32,6 +32,7 @@ public: const std::string & on_duplicate_clause_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_); std::string getName() const override { return "MySQL"; } diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 46f88bbc7ac..5eb27949f21 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -29,7 +29,7 @@ void registerStorageNull(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageNull::create(args.table_id, args.columns, args.constraints); + return StorageNull::create(args.table_id, args.columns, args.constraints, args.comment); }, { .supports_parallel_insert = true, diff --git a/src/Storages/StorageNull.h b/src/Storages/StorageNull.h index 7fe65eb25dc..85fae2cc470 100644 --- a/src/Storages/StorageNull.h +++ b/src/Storages/StorageNull.h @@ -57,12 +57,14 @@ public: private: protected: - StorageNull(const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_) + StorageNull( + const StorageID & table_id_, ColumnsDescription columns_description_, ConstraintsDescription constraints_, const String & comment) : IStorage(table_id_) { StorageInMemoryMetadata metadata_; metadata_.setColumns(columns_description_); metadata_.setConstraints(constraints_); + metadata_.setComment(comment); setInMemoryMetadata(metadata_); } }; diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index a99568c0036..ac301502aba 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -46,6 +46,7 @@ StoragePostgreSQL::StoragePostgreSQL( const String & remote_table_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const String & remote_table_schema_) : IStorage(table_id_) @@ -57,6 +58,7 @@ StoragePostgreSQL::StoragePostgreSQL( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -329,8 +331,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) args.getContext()->getSettingsRef().postgresql_connection_pool_wait_timeout); return StoragePostgreSQL::create( - args.table_id, pool, remote_table, - args.columns, args.constraints, args.getContext(), remote_table_schema); + args.table_id, pool, remote_table, args.columns, args.constraints, args.comment, args.getContext(), remote_table_schema); }, { .source_access_type = AccessType::POSTGRES, diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index e4ab59f7a06..5dbd857307d 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -26,6 +26,7 @@ public: const String & remote_table_name_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const std::string & remote_table_schema_ = ""); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a5cbd004d55..f908eac63d8 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -317,6 +317,7 @@ StorageS3::StorageS3( UInt64 max_connections_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const String & compression_method_, bool distributed_processing_) @@ -333,6 +334,7 @@ StorageS3::StorageS3( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); updateClientAndAuthSettings(context_, client_auth); } @@ -501,9 +503,9 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory) max_connections, args.columns, args.constraints, + args.comment, args.getContext(), - compression_method - ); + compression_method); }, { .source_access_type = AccessType::S3, diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 1e1d76fa6e3..14eee9a1aa3 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -95,7 +95,8 @@ private: class StorageS3 : public ext::shared_ptr_helper, public IStorage, WithContext { public: - StorageS3(const S3::URI & uri, + StorageS3( + const S3::URI & uri, const String & access_key_id, const String & secret_access_key, const StorageID & table_id_, @@ -105,6 +106,7 @@ public: UInt64 max_connections_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, ContextPtr context_, const String & compression_method_ = "", bool distributed_processing_ = false); diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 8afc0e44023..5b924892feb 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -51,8 +51,6 @@ namespace DB { - - StorageS3Cluster::StorageS3Cluster( const String & filename_, const String & access_key_id_, diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index c98840d62fc..e30948cd49a 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -42,9 +42,17 @@ public: protected: StorageS3Cluster( - const String & filename_, const String & access_key_id_, const String & secret_access_key_, const StorageID & table_id_, - String cluster_name_, const String & format_name_, UInt64 max_connections_, const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, ContextPtr context_, const String & compression_method_); + const String & filename_, + const String & access_key_id_, + const String & secret_access_key_, + const StorageID & table_id_, + String cluster_name_, + const String & format_name_, + UInt64 max_connections_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + ContextPtr context_, + const String & compression_method_); private: /// Connections from initiator to other nodes diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 34bbfed874f..f585a5747b8 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -112,14 +112,14 @@ StorageSetOrJoinBase::StorageSetOrJoinBase( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool persistent_) - : IStorage(table_id_), - disk(disk_), - persistent(persistent_) + : IStorage(table_id_), disk(disk_), persistent(persistent_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); @@ -136,9 +136,10 @@ StorageSet::StorageSet( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool persistent_) - : StorageSetOrJoinBase{disk_, relative_path_, table_id_, columns_, constraints_, persistent_}, - set(std::make_shared(SizeLimits(), false, true)) + : StorageSetOrJoinBase{disk_, relative_path_, table_id_, columns_, constraints_, comment, persistent_} + , set(std::make_shared(SizeLimits(), false, true)) { Block header = getInMemoryMetadataPtr()->getSampleBlock(); @@ -247,7 +248,8 @@ void registerStorageSet(StorageFactory & factory) set_settings.loadFromQuery(*args.storage_def); DiskPtr disk = args.getContext()->getDisk(set_settings.disk); - return StorageSet::create(disk, args.relative_data_path, args.table_id, args.columns, args.constraints, set_settings.persistent); + return StorageSet::create( + disk, args.relative_data_path, args.table_id, args.columns, args.constraints, args.comment, set_settings.persistent); }, StorageFactory::StorageFeatures{ .supports_settings = true, }); } diff --git a/src/Storages/StorageSet.h b/src/Storages/StorageSet.h index b87dcf21a23..5dd497527a9 100644 --- a/src/Storages/StorageSet.h +++ b/src/Storages/StorageSet.h @@ -35,6 +35,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool persistent_); DiskPtr disk; @@ -91,6 +92,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool persistent_); }; diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index d845dfb71f2..36b10dfd2bb 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -258,6 +258,7 @@ StorageStripeLog::StorageStripeLog( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_) : IStorage(table_id_) @@ -270,6 +271,7 @@ StorageStripeLog::StorageStripeLog( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); if (relative_path_.empty()) @@ -410,8 +412,14 @@ void registerStorageStripeLog(StorageFactory & factory) DiskPtr disk = args.getContext()->getDisk(disk_name); return StorageStripeLog::create( - disk, args.relative_data_path, args.table_id, args.columns, args.constraints, - args.attach, args.getContext()->getSettings().max_compress_block_size); + disk, + args.relative_data_path, + args.table_id, + args.columns, + args.constraints, + args.comment, + args.attach, + args.getContext()->getSettings().max_compress_block_size); }, features); } diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h index 7fad94870dc..21dbbd4bae8 100644 --- a/src/Storages/StorageStripeLog.h +++ b/src/Storages/StorageStripeLog.h @@ -52,6 +52,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_); diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 41c2961e929..689b1307f4d 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -386,6 +386,7 @@ StorageTinyLog::StorageTinyLog( const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_) : IStorage(table_id_) @@ -398,6 +399,7 @@ StorageTinyLog::StorageTinyLog( StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); if (relative_path_.empty()) @@ -551,8 +553,14 @@ void registerStorageTinyLog(StorageFactory & factory) DiskPtr disk = args.getContext()->getDisk(disk_name); return StorageTinyLog::create( - disk, args.relative_data_path, args.table_id, args.columns, args.constraints, - args.attach, args.getContext()->getSettings().max_compress_block_size); + disk, + args.relative_data_path, + args.table_id, + args.columns, + args.constraints, + args.comment, + args.attach, + args.getContext()->getSettings().max_compress_block_size); }, features); } diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 01652169b62..72cf1ce7e2e 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -52,6 +52,7 @@ protected: const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, bool attach, size_t max_compress_block_size_); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 8b6d7839de0..cd09886abc2 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -39,16 +39,14 @@ IStorageURLBase::IStorageURLBase( const std::optional & format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, const String & compression_method_) - : IStorage(table_id_) - , uri(uri_) - , compression_method(compression_method_) - , format_name(format_name_) - , format_settings(format_settings_) + : IStorage(table_id_), uri(uri_), compression_method(compression_method_), format_name(format_name_), format_settings(format_settings_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); + storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } @@ -242,16 +240,17 @@ BlockOutputStreamPtr IStorageURLBase::write(const ASTPtr & /*query*/, const Stor chooseCompressionMethod(uri.toString(), compression_method)); } -StorageURL::StorageURL(const Poco::URI & uri_, - const StorageID & table_id_, - const String & format_name_, - const std::optional & format_settings_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - ContextPtr context_, - const String & compression_method_) - : IStorageURLBase(uri_, context_, table_id_, format_name_, - format_settings_, columns_, constraints_, compression_method_) +StorageURL::StorageURL( + const Poco::URI & uri_, + const StorageID & table_id_, + const String & format_name_, + const std::optional & format_settings_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + const String & comment, + ContextPtr context_, + const String & compression_method_) + : IStorageURLBase(uri_, context_, table_id_, format_name_, format_settings_, columns_, constraints_, comment, compression_method_) { context_->getRemoteHostFilter().checkURL(uri); } @@ -321,7 +320,10 @@ void registerStorageURL(StorageFactory & factory) args.table_id, format_name, format_settings, - args.columns, args.constraints, args.getContext(), + args.columns, + args.constraints, + args.comment, + args.getContext(), compression_method); }, { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 6fc1a6006ec..46acedb80b9 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -42,6 +42,7 @@ protected: const std::optional & format_settings_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, + const String & comment, const String & compression_method_); Poco::URI uri; @@ -106,14 +107,16 @@ class StorageURL final : public ext::shared_ptr_helper, public IStor { friend struct ext::shared_ptr_helper; public: - StorageURL(const Poco::URI & uri_, - const StorageID & table_id_, - const String & format_name_, - const std::optional & format_settings_, - const ColumnsDescription & columns_, - const ConstraintsDescription & constraints_, - ContextPtr context_, - const String & compression_method_); + StorageURL( + const Poco::URI & uri_, + const StorageID & table_id_, + const String & format_name_, + const std::optional & format_settings_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + const String & comment, + ContextPtr context_, + const String & compression_method_); String getName() const override { diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 75bd4b2967f..a83e0f62c29 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -31,13 +31,12 @@ namespace ErrorCodes StorageView::StorageView( - const StorageID & table_id_, - const ASTCreateQuery & query, - const ColumnsDescription & columns_) + const StorageID & table_id_, const ASTCreateQuery & query, const ColumnsDescription & columns_, const String & comment) : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); + storage_metadata.setComment(comment); if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); @@ -173,7 +172,7 @@ void registerStorageView(StorageFactory & factory) if (args.query.storage) throw Exception("Specifying ENGINE is not allowed for a View", ErrorCodes::INCORRECT_QUERY); - return StorageView::create(args.table_id, args.query, args.columns); + return StorageView::create(args.table_id, args.query, args.columns, args.comment); }); } diff --git a/src/Storages/StorageView.h b/src/Storages/StorageView.h index fa11472218d..b5b8ae2dd8e 100644 --- a/src/Storages/StorageView.h +++ b/src/Storages/StorageView.h @@ -49,10 +49,7 @@ public: static ASTPtr restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name); protected: - StorageView( - const StorageID & table_id_, - const ASTCreateQuery & query, - const ColumnsDescription & columns_); + StorageView(const StorageID & table_id_, const ASTCreateQuery & query, const ColumnsDescription & columns_, const String & comment); }; } diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index f94696c716b..88d54669889 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -30,17 +30,20 @@ StorageXDBC::StorageXDBC( const std::string & remote_database_name_, const std::string & remote_table_name_, const ColumnsDescription & columns_, + const String & comment, ContextPtr context_, const BridgeHelperPtr bridge_helper_) /// Please add support for constraints as soon as StorageODBC or JDBC will support insertion. - : IStorageURLBase(Poco::URI(), - context_, - table_id_, - IXDBCBridgeHelper::DEFAULT_FORMAT, - getFormatSettings(context_), - columns_, - ConstraintsDescription{}, - "" /* CompressionMethod */) + : IStorageURLBase( + Poco::URI(), + context_, + table_id_, + IXDBCBridgeHelper::DEFAULT_FORMAT, + getFormatSettings(context_), + columns_, + ConstraintsDescription{}, + comment, + "" /* CompressionMethod */) , bridge_helper(bridge_helper_) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) @@ -167,10 +170,12 @@ namespace BridgeHelperPtr bridge_helper = std::make_shared>(args.getContext(), args.getContext()->getSettingsRef().http_receive_timeout.value, engine_args[0]->as().value.safeGet()); - return std::make_shared(args.table_id, + return std::make_shared( + args.table_id, engine_args[1]->as().value.safeGet(), engine_args[2]->as().value.safeGet(), args.columns, + args.comment, args.getContext(), bridge_helper); diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index 064912fda92..db0b506546d 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -29,6 +29,7 @@ public: const std::string & remote_database_name, const std::string & remote_table_name, const ColumnsDescription & columns_, + const String & comment, ContextPtr context_, BridgeHelperPtr bridge_helper_); diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 9602339f381..97c88313b71 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -34,8 +34,7 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) : IStorage(table_id_) { StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(ColumnsDescription( - { + storage_metadata.setColumns(ColumnsDescription({ {"database", std::make_shared()}, {"name", std::make_shared()}, {"uuid", std::make_shared()}, @@ -57,6 +56,7 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) {"total_bytes", std::make_shared(std::make_shared())}, {"lifetime_rows", std::make_shared(std::make_shared())}, {"lifetime_bytes", std::make_shared(std::make_shared())}, + {"comment", std::make_shared()}, })); setInMemoryMetadata(storage_metadata); } @@ -244,6 +244,9 @@ protected: // lifetime_bytes if (columns_mask[src_index++]) res_columns[res_index++]->insertDefault(); + + if (columns_mask[src_index++]) + res_columns[res_index++]->insertDefault(); } } @@ -478,6 +481,12 @@ protected: else res_columns[res_index++]->insertDefault(); } + + if (columns_mask[src_index++]) + { + assert(metadata_snapshot != nullptr); + res_columns[res_index++]->insert(metadata_snapshot->comment); + } } } diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 51431a1e3a6..9aa099c1333 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -95,7 +95,8 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & /*ast_function*/, Cont { startBridgeIfNot(context); auto columns = getActualTableStructure(context); - auto result = std::make_shared(StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, columns, context, helper); + auto result = std::make_shared( + StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, columns, String{}, context, helper); result->startup(); return result; } diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp index 46d3183bba9..f299dabd1fa 100644 --- a/src/TableFunctions/TableFunctionDictionary.cpp +++ b/src/TableFunctions/TableFunctionDictionary.cpp @@ -52,7 +52,8 @@ StoragePtr TableFunctionDictionary::executeImpl( { StorageID dict_id(getDatabaseName(), table_name); auto dictionary_table_structure = getActualTableStructure(context); - return StorageDictionary::create(dict_id, dictionary_name, std::move(dictionary_table_structure), StorageDictionary::Location::Custom); + return StorageDictionary::create( + dict_id, dictionary_name, std::move(dictionary_table_structure), String{}, StorageDictionary::Location::Custom); } void registerTableFunctionDictionary(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 6ecb5606d56..f732e160751 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -17,8 +17,7 @@ StoragePtr TableFunctionFile::getStorage(const String & source, { // For `file` table function, we are going to use format settings from the // query context. - StorageFile::CommonArguments args - { + StorageFile::CommonArguments args{ WithContext(global_context), StorageID(getDatabaseName(), table_name), format_, @@ -26,6 +25,7 @@ StoragePtr TableFunctionFile::getStorage(const String & source, compression_method_, columns, ConstraintsDescription{}, + String{}, }; return StorageFile::create(source, global_context->getUserFilesPath(), args); diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index b19be7bd7a3..ff9630eabd8 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -82,7 +82,8 @@ ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(ContextP StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); - auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); + auto res = StorageGenerateRandom::create( + StorageID(getDatabaseName(), table_name), columns, String{}, max_array_length, max_string_length, random_seed); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionHDFS.cpp b/src/TableFunctions/TableFunctionHDFS.cpp index 714c6ea1f59..245674b0e06 100644 --- a/src/TableFunctions/TableFunctionHDFS.cpp +++ b/src/TableFunctions/TableFunctionHDFS.cpp @@ -19,6 +19,7 @@ StoragePtr TableFunctionHDFS::getStorage( format_, columns, ConstraintsDescription{}, + String{}, global_context, compression_method_); } diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 6d10b0d04b6..051aa38398f 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -109,6 +109,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & /*ast_function*/, Cont auto res = StorageMerge::create( StorageID(getDatabaseName(), table_name), getActualTableStructure(context), + String{}, source_database, getSourceTables(context), context); diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index c7dda873a00..5fbd1c2b8d6 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -101,6 +101,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & /*ast_function*/, Cont on_duplicate_clause, columns, ConstraintsDescription{}, + String{}, context); pool.reset(); diff --git a/src/TableFunctions/TableFunctionNull.cpp b/src/TableFunctions/TableFunctionNull.cpp index 334d7c3dcbd..63e496ecd0c 100644 --- a/src/TableFunctions/TableFunctionNull.cpp +++ b/src/TableFunctions/TableFunctionNull.cpp @@ -38,7 +38,7 @@ ColumnsDescription TableFunctionNull::getActualTableStructure(ContextPtr context StoragePtr TableFunctionNull::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); - auto res = StorageNull::create(StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription()); + auto res = StorageNull::create(StorageID(getDatabaseName(), table_name), columns, ConstraintsDescription(), String{}); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 6e7ba1825fc..11de6780d1e 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -30,8 +30,14 @@ StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/, { auto columns = getActualTableStructure(context); auto result = std::make_shared( - StorageID(getDatabaseName(), table_name), *connection_pool, remote_table_name, - columns, ConstraintsDescription{}, context, remote_table_schema); + StorageID(getDatabaseName(), table_name), + *connection_pool, + remote_table_name, + columns, + ConstraintsDescription{}, + String{}, + context, + remote_table_schema); result->startup(); return result; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index ab2458b64f4..8539705a2d2 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -240,6 +240,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con StorageID(getDatabaseName(), table_name), cached_columns, ConstraintsDescription{}, + String{}, remote_table_id.database_name, remote_table_id.table_name, String{}, diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 2da597f49ff..488b4489577 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -88,18 +88,19 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context UInt64 max_connections = context->getSettingsRef().s3_max_connections; StoragePtr storage = StorageS3::create( - s3_uri, - access_key_id, - secret_access_key, - StorageID(getDatabaseName(), table_name), - format, - min_upload_part_size, - max_single_part_upload_size, - max_connections, - getActualTableStructure(context), - ConstraintsDescription{}, - context, - compression_method); + s3_uri, + access_key_id, + secret_access_key, + StorageID(getDatabaseName(), table_name), + format, + min_upload_part_size, + max_single_part_upload_size, + max_connections, + getActualTableStructure(context), + ConstraintsDescription{}, + String{}, + context, + compression_method); storage->startup(); diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 26ef07ef97f..bf6de500dc3 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -113,10 +113,20 @@ StoragePtr TableFunctionS3Cluster::executeImpl( UInt64 max_single_part_upload_size = context->getSettingsRef().s3_max_single_part_upload_size; UInt64 max_connections = context->getSettingsRef().s3_max_connections; storage = StorageS3::create( - s3_uri, access_key_id, secret_access_key, StorageID(getDatabaseName(), table_name), - format, min_upload_part_size, max_single_part_upload_size, max_connections, - getActualTableStructure(context), ConstraintsDescription{}, - context, compression_method, /*distributed_processing=*/true); + s3_uri, + access_key_id, + secret_access_key, + StorageID(getDatabaseName(), table_name), + format, + min_upload_part_size, + max_single_part_upload_size, + max_connections, + getActualTableStructure(context), + ConstraintsDescription{}, + String{}, + context, + compression_method, + /*distributed_processing=*/true); } else { diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index a77b9140508..795b2b77e1c 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -16,9 +16,16 @@ StoragePtr TableFunctionURL::getStorage( const std::string & table_name, const String & compression_method_) const { Poco::URI uri(source); - return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), - format_, std::nullopt /*format settings*/, columns, - ConstraintsDescription{}, global_context, compression_method_); + return StorageURL::create( + uri, + StorageID(getDatabaseName(), table_name), + format_, + std::nullopt /*format settings*/, + columns, + ConstraintsDescription{}, + String{}, + global_context, + compression_method_); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 3f51e0bbc95..8ca061c43d5 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -42,7 +42,7 @@ StoragePtr TableFunctionView::executeImpl( const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); - auto res = StorageView::create(StorageID(getDatabaseName(), table_name), create, columns); + auto res = StorageView::create(StorageID(getDatabaseName(), table_name), create, columns, String{}); res->startup(); return res; } diff --git a/tests/queries/0_stateless/01821_table_comment.reference b/tests/queries/0_stateless/01821_table_comment.reference new file mode 100644 index 00000000000..8365f394cfb --- /dev/null +++ b/tests/queries/0_stateless/01821_table_comment.reference @@ -0,0 +1,3 @@ +default t1 this is a temtorary table +default t2 this is a MergeTree table +default t3 this is a Log table diff --git a/tests/queries/0_stateless/01821_table_comment.sql b/tests/queries/0_stateless/01821_table_comment.sql new file mode 100644 index 00000000000..4b6bd4a5998 --- /dev/null +++ b/tests/queries/0_stateless/01821_table_comment.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; + +CREATE TABLE t1 +( + `n` Int8 +) +ENGINE = Memory +COMMENT 'this is a temtorary table'; + +CREATE TABLE t2 +( + `n` Int8 +) +ENGINE = MergeTree +ORDER BY n +COMMENT 'this is a MergeTree table'; + +CREATE TABLE t3 +( + `n` Int8 +) +ENGINE = Log +COMMENT 'this is a Log table'; + +SELECT + database, + name, + comment +FROM system.tables +WHERE name IN ('t1', 't2', 't3'); + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; From 3916870100746f14fa15d1fbad85d497fe70045f Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 23 Apr 2021 12:25:09 +0000 Subject: [PATCH 02/52] fix fix fix --- src/Parsers/ASTCreateQuery.cpp | 3 +-- src/Parsers/ParserCreateQuery.cpp | 1 - src/Storages/IStorage.h | 2 ++ src/Storages/StorageFactory.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 79e315f65f1..124e7f445b0 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -34,8 +34,7 @@ ASTPtr ASTStorage::clone() const if (comment) { - res->comment = comment; - res->children.push_back(comment->clone()); + res->comment = comment->clone(); } return res; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 220c58f55ae..9b3f4a92162 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -358,7 +358,6 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) storage->set(storage->settings, settings); storage->comment = comment_expression; - storage->children.push_back(std::move(comment_expression)); node = storage; return true; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 628085129e9..e48e9e49919 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -141,6 +141,7 @@ public: /// This is true for most storages that store data on disk. virtual bool prefersLargeBlocks() const { return true; } + /// Optional size information of each physical column. /// Currently it's only used by the MergeTree family for query optimizations. using ColumnSizeByName = std::unordered_map; @@ -164,6 +165,7 @@ public: metadata.set(std::make_unique(metadata_)); } + /// Return list of virtual columns (like _part, _table, etc). In the vast /// majority of cases virtual columns are static constant part of Storage /// class and don't depend on Storage object. But sometimes we have fake diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 8cba7df46d3..cbfa3c6078f 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -182,7 +182,7 @@ StoragePtr StorageFactory::get( String comment; - if (storage_def->comment) + if (storage_def && storage_def->comment) { comment = storage_def->comment->as().value.get(); } From 3df8423b6ee45fe614b84fec81d7f066702e59d3 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 25 Apr 2021 06:51:21 +0000 Subject: [PATCH 03/52] fix tests build --- src/Storages/tests/gtest_storage_log.cpp | 2 +- .../tests/gtest_transform_query_for_external_database.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 41c1b6ac75a..6e903b63955 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -33,7 +33,7 @@ DB::StoragePtr createStorage(DB::DiskPtr & disk) names_and_types.emplace_back("a", std::make_shared()); StoragePtr table = StorageLog::create( - disk, "table/", StorageID("test", "test"), ColumnsDescription{names_and_types}, ConstraintsDescription{}, false, 1048576); + disk, "table/", StorageID("test", "test"), ColumnsDescription{names_and_types}, ConstraintsDescription{}, String{}, false, 1048576); table->startup(); diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index d774fd144cf..6735206862d 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -85,7 +85,8 @@ private: const auto & db_name = tab.table.database; database->attachTable( table_name, - StorageMemory::create(StorageID(db_name, table_name), ColumnsDescription{getColumns()}, ConstraintsDescription{})); + StorageMemory::create( + StorageID(db_name, table_name), ColumnsDescription{getColumns()}, ConstraintsDescription{}, String{})); } DatabaseCatalog::instance().attachDatabase(database->getDatabaseName(), database); context->setCurrentDatabase("test"); From 79a3800679a2113d7e224588ac802d138bef27de Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 25 Apr 2021 07:58:40 +0000 Subject: [PATCH 04/52] fix test --- tests/queries/0_stateless/01821_table_comment.reference | 6 +++--- tests/queries/0_stateless/01821_table_comment.sql | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01821_table_comment.reference b/tests/queries/0_stateless/01821_table_comment.reference index 8365f394cfb..7cdd5372102 100644 --- a/tests/queries/0_stateless/01821_table_comment.reference +++ b/tests/queries/0_stateless/01821_table_comment.reference @@ -1,3 +1,3 @@ -default t1 this is a temtorary table -default t2 this is a MergeTree table -default t3 this is a Log table +t1 this is a temtorary table +t2 this is a MergeTree table +t3 this is a Log table diff --git a/tests/queries/0_stateless/01821_table_comment.sql b/tests/queries/0_stateless/01821_table_comment.sql index 4b6bd4a5998..fe058d23997 100644 --- a/tests/queries/0_stateless/01821_table_comment.sql +++ b/tests/queries/0_stateless/01821_table_comment.sql @@ -25,11 +25,10 @@ ENGINE = Log COMMENT 'this is a Log table'; SELECT - database, name, comment FROM system.tables -WHERE name IN ('t1', 't2', 't3'); +WHERE name IN ('t1', 't2', 't3') order by name; DROP TABLE t1; DROP TABLE t2; From e694af921d9306b6f2486744815ef8bfa486917c Mon Sep 17 00:00:00 2001 From: adevyatova Date: Sun, 25 Apr 2021 20:04:42 +0000 Subject: [PATCH 05/52] add settings desc --- .../settings/merge-tree-settings.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 77b68715ba9..f36d46c4d3d 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -115,6 +115,39 @@ Default value: 604800 (1 week). Similar to [replicated_deduplication_window](#replicated-deduplication-window), `replicated_deduplication_window_seconds` specifies how long to store hash sums of blocks for insert deduplication. Hash sums older than `replicated_deduplication_window_seconds` are removed from Zookeeper, even if they are less than ` replicated_deduplication_window`. +## replicated_fetches_http_connection_timeout + +HTTP connection timeout (in seconds) for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly. + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + +## replicated_fetches_http_send_timeout + +HTTP send timeout (in seconds) for part fetch requests. Inherited from default profile `http_send_timeout` if not set explicitly. + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + +## replicated_fetches_http_receive_timeout + +HTTP receive timeout (in seconds) for fetch part requests. Inherited from default profile `http_receive_timeout` if not set explicitly. + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + ## old_parts_lifetime {#old-parts-lifetime} The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots. From 2347a1f6f3676432b242bb0f0c6dce7b1fbc220f Mon Sep 17 00:00:00 2001 From: feng lv Date: Mon, 3 May 2021 16:19:48 +0000 Subject: [PATCH 06/52] update test --- tests/queries/0_stateless/01821_table_comment.reference | 1 + tests/queries/0_stateless/01821_table_comment.sql | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/01821_table_comment.reference b/tests/queries/0_stateless/01821_table_comment.reference index 7cdd5372102..05acabae3d4 100644 --- a/tests/queries/0_stateless/01821_table_comment.reference +++ b/tests/queries/0_stateless/01821_table_comment.reference @@ -1,3 +1,4 @@ t1 this is a temtorary table t2 this is a MergeTree table t3 this is a Log table +CREATE TABLE default.t1\n(\n `n` Int8\n)\nENGINE = Memory\nCOMMENT \'this is a temtorary table\' diff --git a/tests/queries/0_stateless/01821_table_comment.sql b/tests/queries/0_stateless/01821_table_comment.sql index fe058d23997..c09f121459c 100644 --- a/tests/queries/0_stateless/01821_table_comment.sql +++ b/tests/queries/0_stateless/01821_table_comment.sql @@ -30,6 +30,8 @@ SELECT FROM system.tables WHERE name IN ('t1', 't2', 't3') order by name; +SHOW CREATE TABLE t1; + DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; From ec8ca5bf5e00d08ae9668c298dd2e31f886245db Mon Sep 17 00:00:00 2001 From: Chao Ma Date: Wed, 21 Apr 2021 10:32:33 +0800 Subject: [PATCH 07/52] Fix kafka storage does not work with parquet format messages --- .../Formats/Impl/ParquetBlockInputFormat.cpp | 6 ++- tests/integration/test_storage_kafka/test.py | 40 ++++--------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 162185e75b8..df264406cfe 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_) : IInputFormat(std::move(header_), in_) { - prepareReader(); } Chunk ParquetBlockInputFormat::generate() @@ -41,6 +40,9 @@ Chunk ParquetBlockInputFormat::generate() Chunk res; const Block & header = getPort().getHeader(); + if (!file_reader) + prepareReader(); + if (row_group_current >= row_group_total) return res; @@ -62,7 +64,7 @@ void ParquetBlockInputFormat::resetParser() file_reader.reset(); column_indices.clear(); - prepareReader(); + row_group_current = 0; } void ParquetBlockInputFormat::prepareReader() diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 8bb36cfa8fd..f705788e04e 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -574,39 +574,13 @@ def test_kafka_formats(kafka_cluster): ], 'extra_settings': ", kafka_schema='test:TestRecordStruct'" }, - - # 'Parquet' : { - # not working at all with Kafka: DB::Exception: IOError: Invalid Parquet file size is 0 bytes - # /contrib/libcxx/include/exception:129: std::exception::capture() @ 0x15c33fe8 in /usr/bin/clickhouse - # /contrib/libcxx/include/exception:109: std::exception::exception() @ 0x15c33fb5 in /usr/bin/clickhouse - # /contrib/poco/Foundation/src/Exception.cpp:27: Poco::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int) @ 0x21877833 in /usr/bin/clickhouse - # /src/Common/Exception.cpp:37: DB::Exception::Exception(std::__1::basic_string, std::__1::allocator > const&, int) @ 0x15c2d2a3 in /usr/bin/clickhouse - # /src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp:70: DB::ParquetBlockInputFormat::prepareReader() @ 0x1df2b0c2 in /usr/bin/clickhouse - # /src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp:36: DB::ParquetBlockInputFormat::ParquetBlockInputFormat(DB::ReadBuffer&, DB::Block) @ 0x1df2af8b in /usr/bin/clickhouse - # /contrib/libcxx/include/memory:2214: std::__1::__compressed_pair_elem::__compressed_pair_elem(std::__1::piecewise_construct_t, std::__1::tuple, std::__1::__tuple_indices<0ul, 1ul>) @ 0x1df2dc88 in /usr/bin/clickhouse - # /contrib/libcxx/include/memory:2299: std::__1::__compressed_pair, DB::ParquetBlockInputFormat>::__compressed_pair&, DB::ReadBuffer&, DB::Block const&>(std::__1::piecewise_construct_t, std::__1::tuple&>, std::__1::tuple) @ 0x1df2d9c8 in /usr/bin/clickhouse - # /contrib/libcxx/include/memory:3569: std::__1::__shared_ptr_emplace >::__shared_ptr_emplace(std::__1::allocator, DB::ReadBuffer&, DB::Block const&) @ 0x1df2d687 in /usr/bin/clickhouse - # /contrib/libcxx/include/memory:4400: std::__1::enable_if::value), std::__1::shared_ptr >::type std::__1::make_shared(DB::ReadBuffer&, DB::Block const&) @ 0x1df2d455 in /usr/bin/clickhouse - # /src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp:95: DB::registerInputFormatProcessorParquet(DB::FormatFactory&)::$_0::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1df2cec7 in /usr/bin/clickhouse - # /contrib/libcxx/include/type_traits:3519: decltype(std::__1::forward(fp)(std::__1::forward(fp0), std::__1::forward(fp0), std::__1::forward(fp0), std::__1::forward(fp0))) std::__1::__invoke(DB::registerInputFormatProcessorParquet(DB::FormatFactory&)::$_0&, DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1df2ce6a in /usr/bin/clickhouse - # /contrib/libcxx/include/__functional_base:317: std::__1::shared_ptr std::__1::__invoke_void_return_wrapper >::__call(DB::registerInputFormatProcessorParquet(DB::FormatFactory&)::$_0&, DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1df2cd7d in /usr/bin/clickhouse - # /contrib/libcxx/include/functional:1540: std::__1::__function::__alloc_func, std::__1::shared_ptr (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1df2ccda in /usr/bin/clickhouse - # /contrib/libcxx/include/functional:1714: std::__1::__function::__func, std::__1::shared_ptr (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) @ 0x1df2bdec in /usr/bin/clickhouse - # /contrib/libcxx/include/functional:1867: std::__1::__function::__value_func (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1dd14dbd in /usr/bin/clickhouse - # /contrib/libcxx/include/functional:2473: std::__1::function (DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&)>::operator()(DB::ReadBuffer&, DB::Block const&, DB::RowInputFormatParams const&, DB::FormatSettings const&) const @ 0x1dd07035 in /usr/bin/clickhouse - # /src/Formats/FormatFactory.cpp:258: DB::FormatFactory::getInputFormat(std::__1::basic_string, std::__1::allocator > const&, DB::ReadBuffer&, DB::Block const&, DB::Context const&, unsigned long, std::__1::function) const @ 0x1dd04007 in /usr/bin/clickhouse - # /src/Storages/Kafka/KafkaBlockInputStream.cpp:76: DB::KafkaBlockInputStream::readImpl() @ 0x1d8f6559 in /usr/bin/clickhouse - # /src/DataStreams/IBlockInputStream.cpp:60: DB::IBlockInputStream::read() @ 0x1c9c92fd in /usr/bin/clickhouse - # /src/DataStreams/copyData.cpp:26: void DB::copyDataImpl*)::$_0&, void (&)(DB::Block const&)>(DB::IBlockInputStream&, DB::IBlockOutputStream&, DB::copyData(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::__1::atomic*)::$_0&, void (&)(DB::Block const&)) @ 0x1c9ea01c in /usr/bin/clickhouse - # /src/DataStreams/copyData.cpp:63: DB::copyData(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::__1::atomic*) @ 0x1c9e9fc7 in /usr/bin/clickhouse - # /src/Storages/Kafka/StorageKafka.cpp:565: DB::StorageKafka::streamToViews() @ 0x1d8cc3fa in /usr/bin/clickhouse - # # 'data_sample' : [ - # # b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', - # # b'\x50\x41\x52\x31\x15\x04\x15\xf0\x01\x15\x90\x01\x4c\x15\x1e\x15\x04\x12\x00\x00\x78\x04\x01\x00\x09\x01\x00\x02\x09\x07\x04\x00\x03\x0d\x08\x00\x04\x0d\x08\x00\x05\x0d\x08\x00\x06\x0d\x08\x00\x07\x0d\x08\x00\x08\x0d\x08\x00\x09\x0d\x08\x00\x0a\x0d\x08\x00\x0b\x0d\x08\x00\x0c\x0d\x08\x00\x0d\x0d\x08\x3c\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x14\x15\x18\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x24\x04\x05\x10\x32\x54\x76\x98\xba\xdc\x0e\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x1e\x19\x1c\x19\x5c\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\xa6\x06\x16\x1e\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc5\x01\x00\x00\x50\x41\x52\x31', - # # b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', - # # '' - # # ], - # }, + 'Parquet' : { + 'data_sample': [ + b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', + b'\x50\x41\x52\x31\x15\x04\x15\xf0\x01\x15\x90\x01\x4c\x15\x1e\x15\x04\x12\x00\x00\x78\x04\x01\x00\x09\x01\x00\x02\x09\x07\x04\x00\x03\x0d\x08\x00\x04\x0d\x08\x00\x05\x0d\x08\x00\x06\x0d\x08\x00\x07\x0d\x08\x00\x08\x0d\x08\x00\x09\x0d\x08\x00\x0a\x0d\x08\x00\x0b\x0d\x08\x00\x0c\x0d\x08\x00\x0d\x0d\x08\x3c\x0e\x00\x00\x00\x00\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x14\x15\x18\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0a\x24\x04\x05\x10\x32\x54\x76\x98\xba\xdc\x0e\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x1e\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x1e\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x1e\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x1e\x19\x1c\x19\x5c\x26\xca\x02\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x1e\x16\x9e\x03\x16\xc2\x02\x26\xb8\x01\x26\x08\x1c\x18\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x0f\x00\x00\x00\x00\x00\x00\x00\x18\x08\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xd8\x04\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\x8c\x04\x26\xe4\x03\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xb2\x06\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x1e\x16\x68\x16\x70\x26\xee\x05\x26\xc2\x05\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x9a\x08\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x1e\x16\x84\x01\x16\x8c\x01\x26\xb6\x07\x26\x8e\x07\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\x8e\x0a\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x1e\x16\x6c\x16\x74\x26\xc2\x09\x26\x9a\x09\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\xa6\x06\x16\x1e\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc5\x01\x00\x00\x50\x41\x52\x31', + b'\x50\x41\x52\x31\x15\x04\x15\x10\x15\x14\x4c\x15\x02\x15\x04\x12\x00\x00\x08\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x15\x04\x15\x0c\x15\x10\x4c\x15\x02\x15\x04\x12\x00\x00\x06\x14\x02\x00\x00\x00\x41\x4d\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x03\x08\x01\x02\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x00\x00\x00\x3f\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x03\x08\x01\x02\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x15\x04\x15\x08\x15\x0c\x4c\x15\x02\x15\x04\x12\x00\x00\x04\x0c\x01\x00\x00\x00\x15\x00\x15\x06\x15\x0a\x2c\x15\x02\x15\x04\x15\x06\x15\x06\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x03\x08\x01\x02\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x15\x02\x19\x6c\x35\x00\x18\x06\x73\x63\x68\x65\x6d\x61\x15\x0a\x00\x15\x04\x25\x00\x18\x02\x69\x64\x00\x15\x02\x25\x00\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x25\x18\x4c\xac\x13\x10\x12\x00\x00\x00\x15\x0c\x25\x00\x18\x04\x76\x61\x6c\x31\x25\x00\x4c\x1c\x00\x00\x00\x15\x08\x25\x00\x18\x04\x76\x61\x6c\x32\x00\x15\x02\x25\x00\x18\x04\x76\x61\x6c\x33\x25\x16\x4c\xac\x13\x08\x12\x00\x00\x00\x16\x02\x19\x1c\x19\x5c\x26\xbc\x01\x1c\x15\x04\x19\x35\x04\x00\x06\x19\x18\x02\x69\x64\x15\x02\x16\x02\x16\xac\x01\x16\xb4\x01\x26\x38\x26\x08\x1c\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x16\x00\x28\x08\x00\x00\x00\x00\x00\x00\x00\x00\x18\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x26\xc8\x03\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x07\x62\x6c\x6f\x63\x6b\x4e\x6f\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xfc\x02\x26\xd4\x02\x1c\x36\x00\x28\x04\x00\x00\x00\x00\x18\x04\x00\x00\x00\x00\x00\x00\x00\x26\xa2\x05\x1c\x15\x0c\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x31\x15\x02\x16\x02\x16\x68\x16\x70\x26\xde\x04\x26\xb2\x04\x1c\x36\x00\x28\x02\x41\x4d\x18\x02\x41\x4d\x00\x00\x00\x26\x8a\x07\x1c\x15\x08\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x32\x15\x02\x16\x02\x16\x84\x01\x16\x8c\x01\x26\xa6\x06\x26\xfe\x05\x1c\x18\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x16\x00\x28\x04\x00\x00\x00\x3f\x18\x04\x00\x00\x00\x3f\x00\x00\x00\x26\xfe\x08\x1c\x15\x02\x19\x35\x04\x00\x06\x19\x18\x04\x76\x61\x6c\x33\x15\x02\x16\x02\x16\x6c\x16\x74\x26\xb2\x08\x26\x8a\x08\x1c\x36\x00\x28\x04\x01\x00\x00\x00\x18\x04\x01\x00\x00\x00\x00\x00\x00\x16\x98\x05\x16\x02\x00\x28\x22\x70\x61\x72\x71\x75\x65\x74\x2d\x63\x70\x70\x20\x76\x65\x72\x73\x69\x6f\x6e\x20\x31\x2e\x35\x2e\x31\x2d\x53\x4e\x41\x50\x53\x48\x4f\x54\x19\x5c\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x1c\x00\x00\x00\xc4\x01\x00\x00\x50\x41\x52\x31', + ], + }, 'AvroConfluent': { 'data_sample': [ avro_confluent_message(cluster.schema_registry_client, From dd619e1e37fef85a44d7646fe89b2cac1bddb341 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Thu, 13 May 2021 16:03:32 +0200 Subject: [PATCH 08/52] Try to stabilize flapping test --- tests/integration/test_storage_kafka/test.py | 59 ++++++++++---------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index f705788e04e..9e035b4a98a 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -2457,8 +2457,8 @@ def test_kafka_csv_with_thread_per_consumer(kafka_cluster): CREATE TABLE test.kafka (key UInt64, value UInt64) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', - kafka_topic_list = 'csv', - kafka_group_name = 'csv', + kafka_topic_list = 'csv_with_thread_per_consumer', + kafka_group_name = 'csv_with_thread_per_consumer', kafka_format = 'CSV', kafka_row_delimiter = '\\n', kafka_num_consumers = 4, @@ -2468,7 +2468,7 @@ def test_kafka_csv_with_thread_per_consumer(kafka_cluster): messages = [] for i in range(50): messages.append('{i}, {i}'.format(i=i)) - kafka_produce('csv', messages) + kafka_produce('csv_with_thread_per_consumer', messages) result = '' while True: @@ -2490,8 +2490,8 @@ def test_kafka_engine_put_errors_to_stream(kafka_cluster): CREATE TABLE test.kafka (i Int64, s String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', - kafka_topic_list = 'json', - kafka_group_name = 'json', + kafka_topic_list = 'kafka_engine_put_errors_to_stream', + kafka_group_name = 'kafka_engine_put_errors_to_stream', kafka_format = 'JSONEachRow', kafka_max_block_size = 128, kafka_handle_error_mode = 'stream'; @@ -2519,17 +2519,11 @@ def test_kafka_engine_put_errors_to_stream(kafka_cluster): # Unexpected json content for table test.kafka. messages.append(json.dumps({'i': 'n_' + random_string(4), 's': random_string(8)})) - kafka_produce('json', messages) + kafka_produce('kafka_engine_put_errors_to_stream', messages) + instance.wait_for_log_line("Committed offset 128") - while True: - total_rows = instance.query('SELECT count() FROM test.kafka_data', ignore_error=True) - if total_rows == '64\n': - break - - while True: - total_error_rows = instance.query('SELECT count() FROM test.kafka_errors', ignore_error=True) - if total_error_rows == '64\n': - break + assert TSV(instance.query('SELECT count() FROM test.kafka_data')) == TSV('64') + assert TSV(instance.query('SELECT count() FROM test.kafka_errors')) == TSV('64') instance.query(''' DROP TABLE test.kafka; @@ -2545,8 +2539,15 @@ def gen_malformed_json(): def gen_message_with_jsons(jsons = 10, malformed = 0): s = io.StringIO() + + # we don't care on which position error will be added + # (we skip whole broken message), but we need to be + # sure that at least one error will be added, + # otherwise test will fail. + error_pos = random.randint(0,jsons-1) + for i in range (jsons): - if malformed and random.randint(0,1) == 1: + if malformed and i == error_pos: s.write(gen_malformed_json()) else: s.write(gen_normal_json()) @@ -2562,8 +2563,8 @@ def test_kafka_engine_put_errors_to_stream_with_random_malformed_json(kafka_clus CREATE TABLE test.kafka (i Int64, s String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', - kafka_topic_list = 'json', - kafka_group_name = 'json', + kafka_topic_list = 'kafka_engine_put_errors_to_stream_with_random_malformed_json', + kafka_group_name = 'kafka_engine_put_errors_to_stream_with_random_malformed_json', kafka_format = 'JSONEachRow', kafka_max_block_size = 100, kafka_poll_max_batch_size = 1, @@ -2591,17 +2592,13 @@ def test_kafka_engine_put_errors_to_stream_with_random_malformed_json(kafka_clus else: messages.append(gen_message_with_jsons(10, 0)) - kafka_produce('json', messages) + kafka_produce('kafka_engine_put_errors_to_stream_with_random_malformed_json', messages) - while True: - total_rows = instance.query('SELECT count() FROM test.kafka_data', ignore_error=True) - if total_rows == '640\n': - break - - while True: - total_error_rows = instance.query('SELECT count() FROM test.kafka_errors', ignore_error=True) - if total_error_rows == '64\n': - break + instance.wait_for_log_line("Committed offset 128") + # 64 good messages, each containing 10 rows + assert TSV(instance.query('SELECT count() FROM test.kafka_data')) == TSV('640') + # 64 bad messages, each containing some broken row + assert TSV(instance.query('SELECT count() FROM test.kafka_errors')) == TSV('64') instance.query(''' DROP TABLE test.kafka; @@ -2847,12 +2844,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster): 15 0 AM 0.5 1 {topic_name} 0 {offset_1} 0 0 AM 0.5 1 {topic_name} 0 {offset_2} '''.format(topic_name=topic_name, offset_0=offsets[0], offset_1=offsets[1], offset_2=offsets[2]) - print(('Checking result\n {result} \n expected \n {expected}\n'.format(result=str(result), expected=str(expected)))) + # print(('Checking result\n {result} \n expected \n {expected}\n'.format(result=str(result), expected=str(expected)))) assert TSV(result) == TSV(expected), 'Proper result for format: {}'.format(format_name) errors_result = instance.query('SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow'.format(format_name=format_name)) errors_expected = format_opts['expected'] - print(errors_result.strip()) - print(errors_expected.strip()) + # print(errors_result.strip()) + # print(errors_expected.strip()) assert errors_result.strip() == errors_expected.strip(), 'Proper errors for format: {}'.format(format_name) if __name__ == '__main__': From 6100245f7511c6df45909b8c6250e85bcea453d8 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Thu, 13 May 2021 17:45:26 +0000 Subject: [PATCH 09/52] update --- .../settings/merge-tree-settings.md | 12 +++---- docs/en/operations/settings/settings.md | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index f36d46c4d3d..76e58e10563 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -115,9 +115,9 @@ Default value: 604800 (1 week). Similar to [replicated_deduplication_window](#replicated-deduplication-window), `replicated_deduplication_window_seconds` specifies how long to store hash sums of blocks for insert deduplication. Hash sums older than `replicated_deduplication_window_seconds` are removed from Zookeeper, even if they are less than ` replicated_deduplication_window`. -## replicated_fetches_http_connection_timeout +## replicated_fetches_http_connection_timeout {#replicated_fetches_http_connection_timeout} -HTTP connection timeout (in seconds) for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly. +HTTP connection timeout (in seconds) for part fetch requests. Inherited from default profile [http_connection_timeout](./settings.md#http_connection_timeout) if not set explicitly. Possible values: @@ -126,9 +126,9 @@ Possible values: Default value: 0. -## replicated_fetches_http_send_timeout +## replicated_fetches_http_send_timeout {#replicated_fetches_http_send_timeout} -HTTP send timeout (in seconds) for part fetch requests. Inherited from default profile `http_send_timeout` if not set explicitly. +HTTP send timeout (in seconds) for part fetch requests. Inherited from default profile [http_send_timeout](./settings.md#http_send_timeout) if not set explicitly. Possible values: @@ -137,9 +137,9 @@ Possible values: Default value: 0. -## replicated_fetches_http_receive_timeout +## replicated_fetches_http_receive_timeout {#replicated_fetches_http_receive_timeout} -HTTP receive timeout (in seconds) for fetch part requests. Inherited from default profile `http_receive_timeout` if not set explicitly. +HTTP receive timeout (in seconds) for fetch part requests. Inherited from default profile [http_receive_timeout](./settings.md#http_receive_timeout) if not set explicitly. Possible values: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a5c3902f8f2..e78c9a5949f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2822,4 +2822,37 @@ Sets the interval in seconds after which periodically refreshed [live view](../. Default value: `60`. +## http_connection_timeout {#http_connection_timeout} + +HTTP connection timeout (in seconds). + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + +## http_send_timeout {#replicated_fetches_http_send_timeout} + +HTTP send timeout (in seconds). + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + +## http_receive_timeout {#http_receive_timeout} + +HTTP receive timeout (in seconds). + +Possible values: + +- Any positive integer. +- 0 - Disabled. + +Default value: 0. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From 4ee1896fafa64c3330ad2499aeb0311d816d220e Mon Sep 17 00:00:00 2001 From: adevyatova Date: Thu, 13 May 2021 18:08:38 +0000 Subject: [PATCH 10/52] add ru docs --- .../settings/merge-tree-settings.md | 33 +++++++++++++++++++ docs/ru/operations/settings/settings.md | 32 ++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index f9093d379e3..c610828fc37 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -149,6 +149,39 @@ Eсли суммарное число активных кусков во все Стандартное значение Linux dirty_expire_centisecs - 30 секунд (максимальное время, которое записанные данные хранятся только в оперативной памяти), но при больших нагрузках на дисковую систему, данные могут быть записаны намного позже. Экспериментально было найдено время - 480 секунд, за которое гарантированно новый кусок будет записан на диск. +## replicated_fetches_http_connection_timeout {#replicated_fetches_http_connection_timeout} + +Тайм-аут HTTP-соединения (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. + +## replicated_fetches_http_send_timeout {#replicated_fetches_http_send_timeout} + +Тайм-аут отправки HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. + +## replicated_fetches_http_receive_timeout {#replicated_fetches_http_receive_timeout} + +Тайм-аут приёма HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. + ## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool} Максимальный суммарный размер кусков (в байтах) в одном слиянии, при наличии свободных ресурсов в фоновом пуле. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 467d27dad32..26513350b63 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2857,5 +2857,37 @@ SELECT * FROM test LIMIT 10 OFFSET 100; │ 109 │ └─────┘ ``` +## http_connection_timeout {#http_connection_timeout} + +Тайм-аут HTTP-соединения (в секундах). + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. + +## http_send_timeout {#replicated_fetches_http_send_timeout} + +Тайм-аут отправки HTTP (в секундах). + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. + +## http_receive_timeout {#http_receive_timeout} + +Тайм-аут приема HTTP (в секундах). + +Возможные значения: + +- 0 - тайм-аут не задан. +- Любое положительное целое число. + +Значение по умолчанию: `0`. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) From 2ef60f8eaa62803036278f013ef7a744ed2f7ba4 Mon Sep 17 00:00:00 2001 From: adevyatova Date: Thu, 13 May 2021 18:10:39 +0000 Subject: [PATCH 11/52] minor fixes --- docs/ru/operations/settings/merge-tree-settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index c610828fc37..a89140800eb 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -151,7 +151,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_connection_timeout {#replicated_fetches_http_connection_timeout} -Тайм-аут HTTP-соединения (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. +Тайм-аут HTTP-соединения (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: @@ -162,7 +162,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_send_timeout {#replicated_fetches_http_send_timeout} -Тайм-аут отправки HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. +Тайм-аут отправки HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: @@ -173,7 +173,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_receive_timeout {#replicated_fetches_http_receive_timeout} -Тайм-аут приёма HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout) если не задан явно. +Тайм-аут приёма HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: From 67e43937694352cdee6e306e705fd7e6ffc36b9f Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 11:32:41 +0300 Subject: [PATCH 12/52] If table was not active set readonly mode --- .../ReplicatedMergeTreeRestartingThread.cpp | 44 ++++++++++++----- .../ReplicatedMergeTreeRestartingThread.h | 5 +- .../test.py | 47 ++++++++++++++++--- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index ca6ea3103d1..b43770f0923 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -57,7 +57,7 @@ void ReplicatedMergeTreeRestartingThread::run() try { - if (first_time || storage.getZooKeeper()->expired()) + if (first_time || readonly_mode_was_set || storage.getZooKeeper()->expired()) { startup_completed = false; @@ -67,15 +67,15 @@ void ReplicatedMergeTreeRestartingThread::run() } else { - LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); - - bool old_val = false; - if (storage.is_readonly.compare_exchange_strong(old_val, true)) + if (!readonly_mode_was_set) { - incr_readonly = true; - CurrentMetrics::add(CurrentMetrics::ReadonlyReplica); + LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); + setReadonly(); + } + else + { + LOG_WARNING(log, "Table was in readonly mode. Will try to activate it."); } - partialShutdown(); } @@ -98,8 +98,14 @@ void ReplicatedMergeTreeRestartingThread::run() if (!need_stop && !tryStartup()) { + /// We couldn't startup replication. Table must be readonly. + /// Otherwise it can have partially initialized queue and other + /// strange parts of state. + setReadonly(); + if (first_time) storage.startup_event.set(); + task->scheduleAfter(retry_period_ms); return; } @@ -116,7 +122,7 @@ void ReplicatedMergeTreeRestartingThread::run() bool old_val = true; if (storage.is_readonly.compare_exchange_strong(old_val, false)) { - incr_readonly = false; + readonly_mode_was_set = false; CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); } @@ -125,6 +131,8 @@ void ReplicatedMergeTreeRestartingThread::run() } catch (...) { + /// We couldn't activate table let's set it into readonly mode + setReadonly(); storage.startup_event.set(); tryLogCurrentException(log, __PRETTY_FUNCTION__); } @@ -184,7 +192,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() } catch (const Coordination::Exception & e) { - LOG_ERROR(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); + LOG_ERROR(log, "Couldn't start replication (table will be in readonly mode): {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } catch (const Exception & e) @@ -192,7 +200,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() if (e.code() != ErrorCodes::REPLICA_IS_ALREADY_ACTIVE) throw; - LOG_ERROR(log, "Couldn't start replication: {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); + LOG_ERROR(log, "Couldn't start replication (table will be in readonly mode): {}. {}", e.what(), DB::getCurrentExceptionMessage(true)); return false; } } @@ -356,14 +364,24 @@ void ReplicatedMergeTreeRestartingThread::shutdown() LOG_TRACE(log, "Restarting thread finished"); /// For detach table query, we should reset the ReadonlyReplica metric. - if (incr_readonly) + if (readonly_mode_was_set) { CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); - incr_readonly = false; + readonly_mode_was_set = false; } /// Stop other tasks. partialShutdown(); } +void ReplicatedMergeTreeRestartingThread::setReadonly() +{ + bool old_val = false; + if (storage.is_readonly.compare_exchange_strong(old_val, true)) + { + readonly_mode_was_set = true; + CurrentMetrics::add(CurrentMetrics::ReadonlyReplica); + } +} + } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index 824ed73c171..cb10d628349 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -37,7 +37,7 @@ private: std::atomic need_stop {false}; // We need it besides `storage.is_readonly`, because `shutdown()` may be called many times, that way `storage.is_readonly` will not change. - bool incr_readonly = false; + bool readonly_mode_was_set = false; /// The random data we wrote into `/replicas/me/is_active`. String active_node_identifier; @@ -62,6 +62,9 @@ private: void updateQuorumIfWeHavePart(); void partialShutdown(); + + /// Set readonly mode for table + void setReadonly(); }; diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 79e5dece174..2549dc0a8cf 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -1,4 +1,5 @@ import pytest +import time from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry @@ -34,12 +35,32 @@ def test_mutate_and_upgrade(start_cluster): node1.query("ALTER TABLE mt DELETE WHERE id = 2", settings={"mutations_sync": "2"}) node2.query("SYSTEM SYNC REPLICA mt", timeout=5) - node1.restart_with_latest_version() - node2.restart_with_latest_version() + node1.restart_with_latest_version(signal=9) + node2.restart_with_latest_version(signal=9) - node2.query("INSERT INTO mt VALUES ('2020-02-13', 3);") + exception = None + # After hard restart table can be in readonly mode + for _ in range(40): + try: + node2.query("INSERT INTO mt VALUES ('2020-02-13', 3);") + break + except Exception as ex: + print("Cannot insert into node2 with error {}", ex) + time.sleep(0.5) + exception = ex + else: + raise exception - node1.query("SYSTEM SYNC REPLICA mt", timeout=5) + for _ in range(40): + try: + node1.query("SYSTEM SYNC REPLICA mt", timeout=5) + break + except Exception as ex: + print("Cannot sync node1 with error {}", ex) + time.sleep(0.5) + exception = ex + else: + raise exception assert node1.query("SELECT COUNT() FROM mt") == "2\n" assert node2.query("SELECT COUNT() FROM mt") == "2\n" @@ -73,12 +94,24 @@ def test_upgrade_while_mutation(start_cluster): node3.query("INSERT INTO mt1 select '2020-02-13', number from numbers(100000)") - node3.query("SYSTEM STOP MERGES") + node3.query("SYSTEM STOP MERGES mt1") node3.query("ALTER TABLE mt1 DELETE WHERE id % 2 == 0") - node3.restart_with_latest_version() + node3.restart_with_latest_version(signal=9) + + # After hard restart table can be in readonly mode + exception = None + for _ in range(40): + try: + node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) + break + except Exception as ex: + print("Cannot alter node3 with error {}", ex) + time.sleep(0.5) + exception = ex + else: + raise exception # will delete nothing, but previous async mutation will finish with this query - node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) assert_eq_with_retry(node3, "SELECT COUNT() from mt1", "50000\n") From d4c6a5a05e4e62a5ddb9aa9d2c610a4efcba96c5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 11:38:53 +0300 Subject: [PATCH 13/52] Better logging --- .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index b43770f0923..6b7fb3bf17f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -67,12 +67,12 @@ void ReplicatedMergeTreeRestartingThread::run() } else { - if (!readonly_mode_was_set) + if (storage.getZooKeeper()->expired()) { LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); setReadonly(); } - else + else if (readonly_mode_was_set) { LOG_WARNING(log, "Table was in readonly mode. Will try to activate it."); } From 340cd2027445927f87f30bff6c9186f839f58bd4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 11:55:51 +0300 Subject: [PATCH 14/52] Fix flaky TTL replicated test --- tests/integration/test_ttl_replicated/test.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 67614b88029..c18831e5d9d 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -392,11 +392,33 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): time.sleep(5) # Wait for TTL - node_right.query("OPTIMIZE TABLE test_ttl_delete FINAL") + # after restart table can be in readonly mode + exception = None + for _ in range(40): + try: + node_right.query("OPTIMIZE TABLE test_ttl_delete FINAL") + break + except Exception as ex: + print("Cannot optimaze table on node", node_right.name, "exception", ex) + time.sleep(0.5) + exception = ex + else: + raise ex + node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") - node_left.query("SYSTEM SYNC REPLICA test_ttl_delete", timeout=20) + for _ in range(40): + try: + node_left.query("SYSTEM SYNC REPLICA test_ttl_delete", timeout=20) + break + except Exception as ex: + print("Cannot sync replica table on node", node_left.name, "exception", ex) + time.sleep(0.5) + exception = ex + else: + raise ex + node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) From 8de016d33aac87e22c2bb6941806fcf8577cce55 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 12 May 2021 23:16:55 +0800 Subject: [PATCH 15/52] Fix empty key projection query analysis --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- tests/queries/0_stateless/01710_projections.reference | 1 + tests/queries/0_stateless/01710_projections.sql | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 22fe540222e..802227ade98 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4041,7 +4041,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( auto required_columns = candidate.before_aggregation->foldActionsByProjection(keys, projection.sample_block_for_keys); // std::cerr << fmt::format("before_aggregation = \n{}", candidate.before_aggregation->dumpDAG()) << std::endl; // std::cerr << fmt::format("aggregate_required_columns = \n{}", fmt::join(required_columns, ", ")) << std::endl; - if (required_columns.empty()) + if (required_columns.empty() && !keys.empty()) continue; if (analysis_result.optimize_aggregation_in_order) diff --git a/tests/queries/0_stateless/01710_projections.reference b/tests/queries/0_stateless/01710_projections.reference index 9f30d82e23e..578f7523830 100644 --- a/tests/queries/0_stateless/01710_projections.reference +++ b/tests/queries/0_stateless/01710_projections.reference @@ -3,3 +3,4 @@ 2020-10-24 00:00:00 1.3619605237696326 0.16794469697335793 0.7637956767025532 0.8899329799574005 0.6227685185389797 0.30795997278638165 0.7637956767025532 2020-10-24 00:00:00 19 -1.9455094931672063 0.7759802460082872 0.6 0 2020-10-24 00:00:00 852 894 +999 diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index 49635b9a2c7..cac134ff216 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -39,3 +39,10 @@ select toStartOfMinute(datetime) dt_m, domain, sum(retry_count) / sum(duration), select toStartOfHour(toStartOfMinute(datetime)) dt_h, uniqHLL12(x_id), uniqHLL12(y_id) from projection_test group by dt_h order by dt_h; drop table if exists projection_test; + +drop table if exists projection_without_key; +create table projection_without_key (key UInt32, PROJECTION x (SELECT max(key))) engine MergeTree order by key; +insert into projection_without_key select number from numbers(1000); +set force_optimize_projection = 1, allow_experimental_projection_optimization = 1; +select max(key) from projection_without_key; +drop table projection_without_key; From a113acc40c570aad2ea9200ef0cd279881a43094 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 14 May 2021 22:26:09 +0800 Subject: [PATCH 16/52] Fix empty key --- src/Interpreters/Aggregator.cpp | 9 ++++++--- src/Processors/Transforms/AggregatingTransform.cpp | 7 ++++++- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 6 +++++- tests/queries/0_stateless/01710_projections.reference | 1 + tests/queries/0_stateless/01710_projections.sql | 3 +++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index db047c0fc54..64b20a54955 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1894,9 +1894,12 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( res = place; } - /// Adding Values - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->merge(res + offsets_of_aggregate_states[i], (*aggregate_columns[i])[0], result.aggregates_pool); + if (block.rows() > 0) + { + /// Adding Values + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_functions[i]->merge(res + offsets_of_aggregate_states[i], (*aggregate_columns[i])[0], result.aggregates_pool); + } /// Early release memory. block.clear(); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 0724742e275..90e0e7cb066 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -546,7 +546,12 @@ void AggregatingTransform::initGenerate() /// If there was no data, and we aggregate without keys, and we must return single row with the result of empty aggregation. /// To do this, we pass a block with zero rows to aggregate. if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) - params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); + { + if (params->only_merge) + params->aggregator.mergeBlock(getInputs().front().getHeader(), variants, no_more_keys); + else + params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); + } double elapsed_seconds = watch.elapsedSeconds(); size_t rows = variants.sizeWithoutOverflowRow(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 786536891cd..e22b9461b26 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -161,7 +161,11 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( query_info.merge_tree_data_select_cache.get()); } - LOG_DEBUG(log, "Choose projection {}", query_info.projection->desc->name); + LOG_DEBUG( + log, + "Choose {} projection {}", + ProjectionDescription::typeToString(query_info.projection->desc->type), + query_info.projection->desc->name); if (query_info.projection->merge_tree_data_select_base_cache->sum_marks + query_info.projection->merge_tree_data_select_projection_cache->sum_marks diff --git a/tests/queries/0_stateless/01710_projections.reference b/tests/queries/0_stateless/01710_projections.reference index 578f7523830..1e4f659c639 100644 --- a/tests/queries/0_stateless/01710_projections.reference +++ b/tests/queries/0_stateless/01710_projections.reference @@ -3,4 +3,5 @@ 2020-10-24 00:00:00 1.3619605237696326 0.16794469697335793 0.7637956767025532 0.8899329799574005 0.6227685185389797 0.30795997278638165 0.7637956767025532 2020-10-24 00:00:00 19 -1.9455094931672063 0.7759802460082872 0.6 0 2020-10-24 00:00:00 852 894 +2 -1 999 diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index cac134ff216..c929bfaa273 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -38,6 +38,9 @@ select toStartOfMinute(datetime) dt_m, domain, sum(retry_count) / sum(duration), select toStartOfHour(toStartOfMinute(datetime)) dt_h, uniqHLL12(x_id), uniqHLL12(y_id) from projection_test group by dt_h order by dt_h; +-- found by fuzzer +SELECT 2, -1 FROM projection_test PREWHERE domain_alias = 1. WHERE domain = NULL GROUP BY -9223372036854775808 ORDER BY countIf(first_time = 0) / count(-2147483649) DESC NULLS LAST, 1048576 DESC NULLS LAST; + drop table if exists projection_test; drop table if exists projection_without_key; From 8c82e3f1b919ca2e68acd4a55c0304e3d4a933fd Mon Sep 17 00:00:00 2001 From: feng lv Date: Thu, 13 May 2021 06:22:05 +0000 Subject: [PATCH 17/52] fix style fix --- src/Interpreters/DatabaseCatalog.cpp | 3 ++- src/Parsers/ASTCreateQuery.cpp | 4 +--- src/Parsers/ASTCreateQuery.h | 2 +- src/Parsers/ParserCreateQuery.cpp | 2 +- src/Storages/StorageFactory.cpp | 7 +++---- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 6335e26d116..09166103155 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -86,7 +86,8 @@ TemporaryTableHolder::TemporaryTableHolder( bool create_for_global_subquery) : TemporaryTableHolder( context_, - [&](const StorageID & table_id) { + [&](const StorageID & table_id) + { auto storage = StorageMemory::create(table_id, ColumnsDescription{columns}, ConstraintsDescription{constraints}, String{}); if (create_for_global_subquery) diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 7eef5c25aec..d6d424beb3a 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -33,9 +33,7 @@ ASTPtr ASTStorage::clone() const res->set(res->settings, settings->clone()); if (comment) - { - res->comment = comment->clone(); - } + res->set(res->comment, comment->clone()); return res; } diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index 00cc05ad438..c7be67d9b78 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -23,9 +23,9 @@ public: IAST * order_by = nullptr; IAST * sample_by = nullptr; IAST * ttl_table = nullptr; + IAST * comment = nullptr; ASTSetQuery * settings = nullptr; - ASTPtr comment; String getID(char) const override { return "Storage definition"; } diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index d6424ebb772..4bbe9cc0848 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -408,7 +408,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) storage->set(storage->settings, settings); - storage->comment = comment_expression; + storage->set(storage->comment, comment_expression); node = storage; return true; diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 1365647db9f..4e3baee5a03 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -199,8 +199,7 @@ StoragePtr StorageFactory::get( } ASTs empty_engine_args; - Arguments arguments - { + Arguments arguments{ .engine_name = name, .engine_args = has_engine_args ? storage_def->engine->arguments->children : empty_engine_args, .storage_def = storage_def, @@ -213,8 +212,8 @@ StoragePtr StorageFactory::get( .constraints = constraints, .attach = query.attach, .has_force_restore_data_flag = has_force_restore_data_flag, - .comment = comment - }; + .comment = comment}; + assert(arguments.getContext() == arguments.getContext()->getGlobalContext()); auto res = storages.at(name).creator_fn(arguments); From 20ef8e8bf23ef026c0dd7245f4f38746d114884f Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:39:29 +0300 Subject: [PATCH 18/52] Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/merge-tree-settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index a89140800eb..bc71caf1bda 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -162,7 +162,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_send_timeout {#replicated_fetches_http_send_timeout} -Тайм-аут отправки HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. +Тайм-аут (в секундах) для отправки HTTP-запросов на скачивание кусков. Наследуется из профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: From 73c7552adde4d68894c7c9918a2be3663a8d67e8 Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:39:38 +0300 Subject: [PATCH 19/52] Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/merge-tree-settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index bc71caf1bda..2ee92e5cd56 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -151,7 +151,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_connection_timeout {#replicated_fetches_http_connection_timeout} -Тайм-аут HTTP-соединения (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. +Тайм-аут HTTP-соединения (в секундах) для запросов на скачивание кусков. Наследуется из профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: From aea177ea42095b1a75ab6583bdd4cfe5af2df797 Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:39:46 +0300 Subject: [PATCH 20/52] Update docs/ru/operations/settings/settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 26513350b63..b77ba99488e 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2859,7 +2859,7 @@ SELECT * FROM test LIMIT 10 OFFSET 100; ``` ## http_connection_timeout {#http_connection_timeout} -Тайм-аут HTTP-соединения (в секундах). +Тайм-аут для HTTP-соединения (в секундах). Возможные значения: From a0c0723c3db58e758ee58273bac76217332e07a2 Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:39:58 +0300 Subject: [PATCH 21/52] Update docs/ru/operations/settings/settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index b77ba99488e..3ee19670061 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2881,7 +2881,7 @@ SELECT * FROM test LIMIT 10 OFFSET 100; ## http_receive_timeout {#http_receive_timeout} -Тайм-аут приема HTTP (в секундах). +Тайм-аут для получения HTTP-запроса (в секундах). Возможные значения: From 2b05303d7d810678805f26edfabd0cd3bf600a9b Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:40:08 +0300 Subject: [PATCH 22/52] Update docs/ru/operations/settings/settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 3ee19670061..c32cf1d8158 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2870,7 +2870,7 @@ SELECT * FROM test LIMIT 10 OFFSET 100; ## http_send_timeout {#replicated_fetches_http_send_timeout} -Тайм-аут отправки HTTP (в секундах). +Тайм-аут для отправки HTTP-запросов (в секундах). Возможные значения: From b9f332127001a08c66079503cc47c085ac85aafe Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Sat, 15 May 2021 06:40:18 +0300 Subject: [PATCH 23/52] Update docs/ru/operations/settings/merge-tree-settings.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/operations/settings/merge-tree-settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 2ee92e5cd56..be154bbd19a 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -173,7 +173,7 @@ Eсли суммарное число активных кусков во все ## replicated_fetches_http_receive_timeout {#replicated_fetches_http_receive_timeout} -Тайм-аут приёма HTTP (в секундах) для запросов на скачивание кусков. Наследуется от профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. +Тайм-аут (в секундах) для получения HTTP-запросов на скачивание кусков. Наследуется из профиля по умолчанию [http_connection_timeout](./settings.md#http_connection_timeout), если не задан явно. Возможные значения: From 1d73cd64064ea67101ad5c5800cabe60c50fa4c0 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sat, 15 May 2021 09:05:27 +0300 Subject: [PATCH 24/52] Fix PVS check --- src/Storages/StorageFactory.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 4e3baee5a03..bb9d13ddbda 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -192,8 +192,7 @@ StoragePtr StorageFactory::get( } String comment; - - if (storage_def && storage_def->comment) + if (storage_def->comment) { comment = storage_def->comment->as().value.get(); } From fe6a956499b3f2a077cc759b20c27ae58a2db652 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 15 May 2021 14:15:51 +0800 Subject: [PATCH 25/52] bump simdjson to fix crash when alloc fails --- contrib/simdjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/simdjson b/contrib/simdjson index 95b4870e20b..8df32cea335 160000 --- a/contrib/simdjson +++ b/contrib/simdjson @@ -1 +1 @@ -Subproject commit 95b4870e20be5f97d9dcf63b23b1c6f520c366c1 +Subproject commit 8df32cea3359cb30120795da6020b3b73da01d38 From 9e65f376cc5403c1c5ea1b915a06e59a3f092019 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sat, 15 May 2021 10:05:31 +0300 Subject: [PATCH 26/52] Fix PVS check --- src/Storages/StorageFactory.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index bb9d13ddbda..4baaf2e8ff6 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -65,7 +65,7 @@ StoragePtr StorageFactory::get( const ConstraintsDescription & constraints, bool has_force_restore_data_flag) const { - String name; + String name, comment; ASTStorage * storage_def = query.storage; bool has_engine_args = false; @@ -145,6 +145,9 @@ StoragePtr StorageFactory::get( else throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); } + + if (storage_def->comment) + comment = storage_def->comment->as().value.get(); auto check_feature = [&](String feature_description, FeatureMatcherFn feature_matcher_fn) { @@ -191,12 +194,6 @@ StoragePtr StorageFactory::get( } } - String comment; - if (storage_def->comment) - { - comment = storage_def->comment->as().value.get(); - } - ASTs empty_engine_args; Arguments arguments{ .engine_name = name, From 6bcc99e5e6ba549b2dce30abcec3f862c3200171 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sat, 15 May 2021 10:56:46 +0300 Subject: [PATCH 27/52] Whitespace --- src/Storages/StorageFactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 4baaf2e8ff6..836d8bae7f3 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -145,7 +145,7 @@ StoragePtr StorageFactory::get( else throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); } - + if (storage_def->comment) comment = storage_def->comment->as().value.get(); From 33e9f1bcf12c0a5f9fa838a7117104f2f15cb6cb Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 15 May 2021 15:33:01 +0300 Subject: [PATCH 28/52] Better tests --- tests/integration/helpers/test_tools.py | 13 ++++++ .../test_polymorphic_parts/test.py | 19 +++++---- tests/integration/test_ttl_replicated/test.py | 26 ++---------- .../test.py | 40 ++----------------- 4 files changed, 31 insertions(+), 67 deletions(-) diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 5fedadd3380..93478c4dd49 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -80,3 +80,16 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti time.sleep(sleep_time) else: raise AssertionError("'{}' not found in logs".format(substring)) + +def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, settings={}): + exception = None + for _ in range(retry_count): + try: + instance.query(query, timeout=30, settings=settings) + break + except Exception as ex: + exception = ex + print("Failed to execute query '", query, "' on instance", instance.name, "will retry") + time.sleep(sleep_time) + else: + raise exception diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index dc16bab0ca4..9fe3ef77da8 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -7,7 +7,7 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry cluster = ClickHouseCluster(__file__) @@ -408,8 +408,9 @@ def test_in_memory_wal_rotate(start_cluster): def test_in_memory_deduplication(start_cluster): for i in range(3): - node9.query("INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") - node10.query("INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") + # table can be in readonly node + exec_query_with_retry(node9, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") + exec_query_with_retry(node10, "INSERT INTO deduplication_table (date, id, s) VALUES (toDate('2020-03-03'), 1, 'foo')") node9.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20) node10.query("SYSTEM SYNC REPLICA deduplication_table", timeout=20) @@ -430,10 +431,10 @@ def test_in_memory_alters(start_cluster): node9.restart_clickhouse(kill=True) expected = "1\tab\t0\n2\tcd\t0\n" - assert node9.query("SELECT id, s, col1 FROM alters_table") == expected + assert node9.query("SELECT id, s, col1 FROM alters_table ORDER BY id") == expected check_parts_type(1) - - node9.query("INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)") + # After hard restart table can be in readonly mode + exec_query_with_retry(node9, "INSERT INTO alters_table (date, id, col1) VALUES (toDate('2020-10-10'), 3, 100)") node9.query("ALTER TABLE alters_table MODIFY COLUMN col1 String") node9.query("ALTER TABLE alters_table DROP COLUMN s") node9.restart_clickhouse(kill=True) @@ -442,8 +443,10 @@ def test_in_memory_alters(start_cluster): with pytest.raises(Exception): node9.query("SELECT id, s, col1 FROM alters_table") - expected = expected = "1\t0_foo\n2\t0_foo\n3\t100_foo\n" - assert node9.query("SELECT id, col1 || '_foo' FROM alters_table") + # Values of col1 was not materialized as integers, so they have + # default string values after alter + expected = "1\t_foo\n2\t_foo\n3\t100_foo\n" + assert node9.query("SELECT id, col1 || '_foo' FROM alters_table ORDER BY id") == expected def test_polymorphic_parts_index(start_cluster): diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index c18831e5d9d..f32edc36a71 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -3,7 +3,7 @@ import time import helpers.client as client import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, exec_query_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance('node1', with_zookeeper=True) @@ -393,31 +393,11 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): time.sleep(5) # Wait for TTL # after restart table can be in readonly mode - exception = None - for _ in range(40): - try: - node_right.query("OPTIMIZE TABLE test_ttl_delete FINAL") - break - except Exception as ex: - print("Cannot optimaze table on node", node_right.name, "exception", ex) - time.sleep(0.5) - exception = ex - else: - raise ex + exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL") node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") - - for _ in range(40): - try: - node_left.query("SYSTEM SYNC REPLICA test_ttl_delete", timeout=20) - break - except Exception as ex: - print("Cannot sync replica table on node", node_left.name, "exception", ex) - time.sleep(0.5) - exception = ex - else: - raise ex + exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 2549dc0a8cf..1ef65512959 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -2,7 +2,7 @@ import pytest import time from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, exec_query_with_retry cluster = ClickHouseCluster(__file__) @@ -38,29 +38,9 @@ def test_mutate_and_upgrade(start_cluster): node1.restart_with_latest_version(signal=9) node2.restart_with_latest_version(signal=9) - exception = None # After hard restart table can be in readonly mode - for _ in range(40): - try: - node2.query("INSERT INTO mt VALUES ('2020-02-13', 3);") - break - except Exception as ex: - print("Cannot insert into node2 with error {}", ex) - time.sleep(0.5) - exception = ex - else: - raise exception - - for _ in range(40): - try: - node1.query("SYSTEM SYNC REPLICA mt", timeout=5) - break - except Exception as ex: - print("Cannot sync node1 with error {}", ex) - time.sleep(0.5) - exception = ex - else: - raise exception + exec_query_with_retry(node2, "INSERT INTO mt VALUES ('2020-02-13', 3)") + exec_query_with_retry(node1, "SYSTEM SYNC REPLICA mt") assert node1.query("SELECT COUNT() FROM mt") == "2\n" assert node2.query("SELECT COUNT() FROM mt") == "2\n" @@ -99,19 +79,7 @@ def test_upgrade_while_mutation(start_cluster): node3.restart_with_latest_version(signal=9) - # After hard restart table can be in readonly mode - exception = None - for _ in range(40): - try: - node3.query("ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) - break - except Exception as ex: - print("Cannot alter node3 with error {}", ex) - time.sleep(0.5) - exception = ex - else: - raise exception - + exec_query_with_retry(node3, "ALTER TABLE mt1 DELETE WHERE id > 100000", settings={"mutations_sync": "2"}) # will delete nothing, but previous async mutation will finish with this query assert_eq_with_retry(node3, "SELECT COUNT() from mt1", "50000\n") From 5b668fc3512d0102551f3585bac85651cf1c08ff Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 15 May 2021 11:14:40 +0300 Subject: [PATCH 29/52] Remove trailing whitespace for Processed client message --- programs/client/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index aae2e0f1c59..ccf92ebc419 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2278,7 +2278,7 @@ private: size_t elapsed_ns = watch.elapsed(); if (elapsed_ns) std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " - << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) "; + << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; else std::cout << ". "; } From 1cdeafe4731c9472c34816e8b7dcfffc43ce7336 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 15 May 2021 11:09:17 +0300 Subject: [PATCH 30/52] Remove superfluous trailing whitespace in Markdown format --- src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp | 5 ++++- tests/queries/0_stateless/01231_markdown_format.reference | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp index ee5d4193a45..ebf0384445c 100644 --- a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp @@ -17,7 +17,10 @@ void MarkdownRowOutputFormat::writePrefix() for (size_t i = 0; i < columns; ++i) { writeEscapedString(header.safeGetByPosition(i).name, out); - writeCString(" | ", out); + if (i == (columns - 1)) + writeCString(" |", out); + else + writeCString(" | ", out); } writeCString("\n|", out); String left_alignment = ":-|"; diff --git a/tests/queries/0_stateless/01231_markdown_format.reference b/tests/queries/0_stateless/01231_markdown_format.reference index 65838bfede7..55cf44381ad 100644 --- a/tests/queries/0_stateless/01231_markdown_format.reference +++ b/tests/queries/0_stateless/01231_markdown_format.reference @@ -1,4 +1,4 @@ -| id | name | array | nullable | low_cardinality | decimal | +| id | name | array | nullable | low_cardinality | decimal | |-:|:-|:-|:-|:-|-:| | 1 | name1 | [1,2,3] | Some long string | name1 | 1.110000 | | 2 | name2 | [4,5,60000] | \N | Another long string | 222.222222 | From 947f28d43038dc2740ed1e963138e767c61a91ca Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 15 May 2021 20:33:15 +0300 Subject: [PATCH 31/52] IFunction refactoring --- src/CMakeLists.txt | 4 +- src/Common/ya.make | 1 - src/Functions/CMakeLists.txt | 4 +- src/Functions/CustomWeekTransforms.h | 2 +- src/Functions/DateTimeTransforms.h | 2 +- src/Functions/FunctionBinaryArithmetic.h | 14 +- src/Functions/FunctionBitTestMany.h | 2 +- src/Functions/FunctionCustomWeekToSomething.h | 2 +- .../FunctionDateOrDateTimeAddInterval.h | 2 +- .../FunctionDateOrDateTimeToSomething.h | 2 +- src/Functions/FunctionFQDN.cpp | 2 +- src/Functions/FunctionFactory.cpp | 11 +- src/Functions/FunctionFactory.h | 19 +- src/Functions/FunctionHelpers.cpp | 73 ++- src/Functions/FunctionHelpers.h | 13 + src/Functions/FunctionIfBase.h | 2 +- src/Functions/FunctionJoinGet.cpp | 6 +- src/Functions/FunctionJoinGet.h | 18 +- src/Functions/FunctionMathBinaryFloat64.h | 2 +- src/Functions/FunctionMathConstFloat64.h | 2 +- src/Functions/FunctionMathUnary.h | 2 +- src/Functions/FunctionNumericPredicate.h | 2 +- src/Functions/FunctionStartsEndsWith.h | 2 +- src/Functions/FunctionStringOrArrayToT.h | 2 +- src/Functions/FunctionStringToString.h | 2 +- src/Functions/FunctionUnaryArithmetic.h | 2 +- src/Functions/FunctionUnixTimestamp64.h | 2 +- src/Functions/FunctionsBitmap.h | 2 +- src/Functions/FunctionsCoding.h | 2 +- src/Functions/FunctionsComparison.h | 5 +- src/Functions/FunctionsConsistentHashing.h | 2 +- src/Functions/FunctionsConversion.h | 44 +- src/Functions/FunctionsEmbeddedDictionaries.h | 2 +- src/Functions/FunctionsExternalDictionaries.h | 2 +- src/Functions/FunctionsExternalModels.h | 2 +- src/Functions/FunctionsHashing.h | 2 +- src/Functions/FunctionsJSON.h | 2 +- src/Functions/FunctionsLogical.h | 2 +- src/Functions/FunctionsMiscellaneous.h | 26 +- .../FunctionsMultiStringFuzzySearch.h | 2 +- src/Functions/FunctionsMultiStringPosition.h | 2 +- src/Functions/FunctionsMultiStringSearch.h | 2 +- src/Functions/FunctionsRandom.h | 2 +- src/Functions/FunctionsRound.h | 2 +- src/Functions/FunctionsStringArray.h | 2 +- src/Functions/FunctionsStringHash.h | 2 +- src/Functions/FunctionsStringSearch.h | 2 +- src/Functions/FunctionsStringSearchToString.h | 2 +- src/Functions/FunctionsStringSimilarity.h | 2 +- src/Functions/FunctionsVisitParam.h | 2 +- src/Functions/IFunction.cpp | 578 +++++------------- src/Functions/IFunction.h | 172 ++++-- src/Functions/IFunctionAdaptors.h | 179 +----- src/Functions/IFunctionImpl.h | 307 ---------- src/Functions/IFunctionOld.cpp | 91 +++ src/Functions/IFunctionOld.h | 129 ++++ src/Functions/LeastGreatestGeneric.h | 14 +- src/Functions/PerformanceAdaptors.h | 2 +- src/Functions/URL/port.cpp | 2 +- src/Functions/addressToLine.cpp | 2 +- src/Functions/addressToSymbol.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/Functions/array/FunctionArrayMapped.h | 2 +- src/Functions/array/array.cpp | 2 +- src/Functions/array/arrayConcat.cpp | 2 +- src/Functions/array/arrayDistinct.cpp | 2 +- src/Functions/array/arrayElement.cpp | 2 +- src/Functions/array/arrayEnumerate.cpp | 2 +- src/Functions/array/arrayEnumerateExtended.h | 2 +- src/Functions/array/arrayEnumerateRanked.h | 2 +- src/Functions/array/arrayFlatten.cpp | 2 +- src/Functions/array/arrayIndex.h | 2 +- src/Functions/array/arrayIntersect.cpp | 2 +- src/Functions/array/arrayJoin.cpp | 2 +- src/Functions/array/arrayPop.h | 2 +- src/Functions/array/arrayPush.h | 2 +- src/Functions/array/arrayReduce.cpp | 2 +- src/Functions/array/arrayReduceInRanges.cpp | 2 +- src/Functions/array/arrayResize.cpp | 2 +- src/Functions/array/arrayReverse.cpp | 2 +- src/Functions/array/arraySlice.cpp | 2 +- src/Functions/array/arrayUniq.cpp | 2 +- src/Functions/array/arrayWithConstant.cpp | 2 +- src/Functions/array/emptyArray.cpp | 2 +- src/Functions/array/emptyArrayToSingle.cpp | 2 +- src/Functions/array/hasAllAny.h | 2 +- src/Functions/array/range.cpp | 2 +- src/Functions/assumeNotNull.cpp | 2 +- src/Functions/bar.cpp | 2 +- src/Functions/bitmaskToList.cpp | 2 +- src/Functions/blockNumber.cpp | 2 +- src/Functions/blockSerializedSize.cpp | 2 +- src/Functions/blockSize.cpp | 2 +- src/Functions/buildId.cpp | 2 +- src/Functions/byteSize.cpp | 2 +- src/Functions/coalesce.cpp | 2 +- src/Functions/concat.cpp | 14 +- src/Functions/connectionId.cpp | 2 +- src/Functions/countDigits.cpp | 2 +- src/Functions/countMatches.h | 2 +- src/Functions/currentDatabase.cpp | 2 +- src/Functions/currentUser.cpp | 2 +- src/Functions/dateDiff.cpp | 2 +- src/Functions/defaultValueOfArgumentType.cpp | 2 +- src/Functions/defaultValueOfTypeName.cpp | 2 +- src/Functions/dumpColumnStructure.cpp | 2 +- src/Functions/errorCodeToName.cpp | 2 +- src/Functions/evalMLMethod.cpp | 2 +- src/Functions/extractAllGroups.h | 2 +- src/Functions/extractTextFromHTML.cpp | 2 +- src/Functions/filesystem.cpp | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/formatDateTime.cpp | 2 +- src/Functions/formatReadable.h | 2 +- src/Functions/formatReadableTimeDelta.cpp | 2 +- src/Functions/formatRow.cpp | 12 +- src/Functions/formatString.cpp | 2 +- src/Functions/fromModifiedJulianDay.cpp | 21 +- src/Functions/fromUnixTimestamp64Micro.cpp | 2 +- src/Functions/fromUnixTimestamp64Milli.cpp | 2 +- src/Functions/fromUnixTimestamp64Nano.cpp | 2 +- src/Functions/fuzzBits.cpp | 2 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/geohashesInBox.cpp | 2 +- src/Functions/getMacro.cpp | 2 +- src/Functions/getScalar.cpp | 2 +- src/Functions/getSetting.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 2 +- src/Functions/globalVariable.cpp | 2 +- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/hasColumnInTable.cpp | 2 +- src/Functions/hostName.cpp | 2 +- src/Functions/identity.cpp | 2 +- src/Functions/if.cpp | 2 +- src/Functions/ifNotFinite.cpp | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/ignore.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/indexHint.cpp | 2 +- src/Functions/initializeAggregation.cpp | 2 +- src/Functions/isConstant.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isIPAddressContainedIn.cpp | 2 +- src/Functions/isNotNull.cpp | 2 +- src/Functions/isNull.cpp | 2 +- src/Functions/isZeroOrNull.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Functions/lowCardinalityIndices.cpp | 2 +- src/Functions/lowCardinalityKeys.cpp | 2 +- src/Functions/map.cpp | 2 +- src/Functions/materialize.h | 2 +- src/Functions/neighbor.cpp | 2 +- src/Functions/now.cpp | 18 +- src/Functions/now64.cpp | 16 +- src/Functions/nullIf.cpp | 2 +- src/Functions/partitionId.cpp | 2 +- src/Functions/pointInEllipses.cpp | 2 +- src/Functions/randConstant.cpp | 16 +- src/Functions/randomFixedString.cpp | 2 +- src/Functions/randomPrintableASCII.cpp | 2 +- src/Functions/randomString.cpp | 2 +- src/Functions/randomStringUTF8.cpp | 2 +- src/Functions/repeat.cpp | 2 +- src/Functions/replicate.cpp | 2 +- src/Functions/replicate.h | 2 +- src/Functions/reverse.cpp | 12 +- src/Functions/rowNumberInAllBlocks.cpp | 2 +- src/Functions/rowNumberInBlock.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/Functions/runningConcurrency.cpp | 20 +- src/Functions/runningDifference.h | 2 +- src/Functions/sleep.h | 2 +- src/Functions/substring.cpp | 2 +- src/Functions/subtractDays.cpp | 2 +- src/Functions/subtractHours.cpp | 2 +- src/Functions/subtractMinutes.cpp | 2 +- src/Functions/subtractMonths.cpp | 2 +- src/Functions/subtractQuarters.cpp | 2 +- src/Functions/subtractSeconds.cpp | 2 +- src/Functions/subtractWeeks.cpp | 2 +- src/Functions/subtractYears.cpp | 2 +- src/Functions/throwIf.cpp | 2 +- src/Functions/tid.cpp | 2 +- src/Functions/timeSlot.cpp | 2 +- src/Functions/timeSlots.cpp | 2 +- src/Functions/timezone.cpp | 2 +- src/Functions/timezoneOf.cpp | 18 +- src/Functions/toColumnTypeName.cpp | 2 +- src/Functions/toCustomWeek.cpp | 2 +- src/Functions/toFixedString.h | 2 +- src/Functions/toLowCardinality.cpp | 2 +- src/Functions/toModifiedJulianDay.cpp | 18 +- src/Functions/toNullable.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toTimezone.cpp | 2 +- src/Functions/toTypeName.cpp | 18 +- src/Functions/toUnixTimestamp64Micro.cpp | 2 +- src/Functions/toUnixTimestamp64Milli.cpp | 2 +- src/Functions/toUnixTimestamp64Nano.cpp | 2 +- src/Functions/today.cpp | 18 +- src/Functions/transform.cpp | 5 +- src/Functions/tuple.cpp | 2 +- src/Functions/tupleElement.cpp | 2 +- src/Functions/uptime.cpp | 2 +- src/Functions/version.cpp | 2 +- src/Functions/visibleWidth.cpp | 2 +- src/Functions/ya.make | 1 + src/Functions/yesterday.cpp | 18 +- src/Interpreters/ActionsDAG.cpp | 29 +- src/Interpreters/ActionsVisitor.cpp | 5 +- src/Interpreters/ExpressionJIT.cpp | 13 +- src/Interpreters/ExpressionJIT.h | 2 +- src/Interpreters/JIT/CompileDAG.h | 2 +- src/Interpreters/JIT/compileFunction.cpp | 4 +- src/Interpreters/JIT/compileFunction.h | 4 +- src/Interpreters/castColumn.cpp | 3 +- src/Storages/MergeTree/KeyCondition.cpp | 3 +- 217 files changed, 1022 insertions(+), 1322 deletions(-) delete mode 100644 src/Functions/IFunctionImpl.h create mode 100644 src/Functions/IFunctionOld.cpp create mode 100644 src/Functions/IFunctionOld.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 292adc0e124..d22a69c211c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -106,8 +106,8 @@ endif() list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) -list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) -list (APPEND dbms_headers Functions/IFunctionImpl.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) +list (APPEND dbms_sources Functions/IFunction.cpp Functions/IFunctionOld.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) +list (APPEND dbms_headers Functions/IFunctionOld.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) list (APPEND dbms_sources AggregateFunctions/AggregateFunctionFactory.cpp diff --git a/src/Common/ya.make b/src/Common/ya.make index dde1e6ae013..f12b17827f7 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -18,7 +18,6 @@ PEERDIR( contrib/libs/openssl contrib/libs/poco/NetSSL_OpenSSL contrib/libs/re2 - contrib/libs/cxxsupp/libcxxabi-parts contrib/restricted/dragonbox ) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 7e9e953eabe..ba876e02424 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -5,8 +5,8 @@ add_subdirectory(divide) include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions .) -list(REMOVE_ITEM clickhouse_functions_sources IFunctionImpl.cpp FunctionFactory.cpp FunctionHelpers.cpp) -list(REMOVE_ITEM clickhouse_functions_headers IFunctionImpl.h FunctionFactory.h FunctionHelpers.h) +list(REMOVE_ITEM clickhouse_functions_sources IFunctionOld.cpp FunctionFactory.cpp FunctionHelpers.cpp) +list(REMOVE_ITEM clickhouse_functions_headers IFunctionOld.h FunctionFactory.h FunctionHelpers.h) add_library(clickhouse_functions ${clickhouse_functions_sources}) diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index 98b7c38f266..f07f2777cec 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 86cac4f4222..aa8f52b335e 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 508c598b0ed..0d90eece3e2 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -23,7 +23,7 @@ #include #include #include "Core/DecimalFunctions.h" -#include "IFunctionImpl.h" +#include "IFunctionOld.h" #include "FunctionHelpers.h" #include "IsOperation.h" #include "DivisionUtils.h" @@ -1532,11 +1532,11 @@ private: }; template