From 0fa4c37e855036acfad7c0a541ca32911690bc76 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 22 Sep 2020 20:47:07 +0300 Subject: [PATCH 001/291] fix not equi join on issue --- src/Interpreters/CollectJoinOnKeysVisitor.cpp | 6 ++++++ tests/queries/0_stateless/01478_not_equi-join_on.reference | 0 tests/queries/0_stateless/01478_not_equi-join_on.sql | 7 +++++++ 3 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/01478_not_equi-join_on.reference create mode 100644 tests/queries/0_stateless/01478_not_equi-join_on.sql diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.cpp b/src/Interpreters/CollectJoinOnKeysVisitor.cpp index e0fce4854d2..943db234c7e 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.cpp +++ b/src/Interpreters/CollectJoinOnKeysVisitor.cpp @@ -127,6 +127,12 @@ std::pair CollectJoinOnKeysMatcher::getTableNumbers(const ASTPtr getIdentifiers(left_ast, left_identifiers); getIdentifiers(right_ast, right_identifiers); + if (left_identifiers.empty() || right_identifiers.empty()) + { + throw Exception("Not equi-join ON expression: " + queryToString(expr) + ". No columns in one of equality side.", + ErrorCodes::INVALID_JOIN_ON_EXPRESSION); + } + size_t left_idents_table = getTableForIdentifiers(left_identifiers, data); size_t right_idents_table = getTableForIdentifiers(right_identifiers, data); diff --git a/tests/queries/0_stateless/01478_not_equi-join_on.reference b/tests/queries/0_stateless/01478_not_equi-join_on.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01478_not_equi-join_on.sql b/tests/queries/0_stateless/01478_not_equi-join_on.sql new file mode 100644 index 00000000000..b52af5fcb46 --- /dev/null +++ b/tests/queries/0_stateless/01478_not_equi-join_on.sql @@ -0,0 +1,7 @@ +SELECT * FROM (SELECT NULL AS a, 1 AS b) AS foo +RIGHT JOIN (SELECT 1024 AS b) AS bar +ON 1 = foo.b; -- { serverError 403 } + +SELECT * FROM (SELECT NULL AS a, 1 AS b) AS foo +RIGHT JOIN (SELECT 1024 AS b) AS bar +ON 1 = bar.b; -- { serverError 403 } From e349769524ed3cdab6883782f1922700ddaf6146 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 22 Sep 2020 22:22:47 +0300 Subject: [PATCH 002/291] update tests --- .../0_stateless/00878_join_unexpected_results.reference | 4 ---- .../queries/0_stateless/00878_join_unexpected_results.sql | 8 ++++---- .../01081_PartialSortingTransform_full_column.sql | 4 ++-- .../0_stateless/01101_literal_column_clash.reference | 4 ---- tests/queries/0_stateless/01101_literal_column_clash.sql | 5 ++--- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/00878_join_unexpected_results.reference b/tests/queries/0_stateless/00878_join_unexpected_results.reference index 65fcbc257ca..a389cb47a96 100644 --- a/tests/queries/0_stateless/00878_join_unexpected_results.reference +++ b/tests/queries/0_stateless/00878_join_unexpected_results.reference @@ -23,8 +23,6 @@ join_use_nulls = 1 - \N \N - -1 1 \N \N -2 2 \N \N - 1 1 1 1 2 2 \N \N @@ -51,8 +49,6 @@ join_use_nulls = 0 - - - -1 1 0 0 -2 2 0 0 - 1 1 1 1 2 2 0 0 diff --git a/tests/queries/0_stateless/00878_join_unexpected_results.sql b/tests/queries/0_stateless/00878_join_unexpected_results.sql index 6f6cd6e6479..0aef5208b26 100644 --- a/tests/queries/0_stateless/00878_join_unexpected_results.sql +++ b/tests/queries/0_stateless/00878_join_unexpected_results.sql @@ -30,11 +30,11 @@ select * from t left outer join s on (t.a=s.a and t.b=s.b) where s.a is null; select '-'; select s.* from t left outer join s on (t.a=s.a and t.b=s.b) where s.a is null; select '-'; -select t.*, s.* from t left join s on (s.a=t.a and t.b=s.b and t.a=toInt64(2)) order by t.a; +select t.*, s.* from t left join s on (s.a=t.a and t.b=s.b and t.a=toInt64(2)) order by t.a; -- {serverError 403 } select '-'; select t.*, s.* from t left join s on (s.a=t.a) order by t.a; select '-'; -select t.*, s.* from t left join s on (t.b=toInt64(2) and s.a=t.a) where s.b=2; +select t.*, s.* from t left join s on (t.b=toInt64(2) and s.a=t.a) where s.b=2; -- {serverError 403 } select 'join_use_nulls = 0'; set join_use_nulls = 0; @@ -58,11 +58,11 @@ select '-'; select '-'; -- select s.* from t left outer join s on (t.a=s.a and t.b=s.b) where s.a is null; -- TODO select '-'; -select t.*, s.* from t left join s on (s.a=t.a and t.b=s.b and t.a=toInt64(2)) order by t.a; +select t.*, s.* from t left join s on (s.a=t.a and t.b=s.b and t.a=toInt64(2)) order by t.a; -- {serverError 403 } select '-'; select t.*, s.* from t left join s on (s.a=t.a) order by t.a; select '-'; -select t.*, s.* from t left join s on (t.b=toInt64(2) and s.a=t.a) where s.b=2; +select t.*, s.* from t left join s on (t.b=toInt64(2) and s.a=t.a) where s.b=2; -- {serverError 403 } drop table t; drop table s; diff --git a/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql b/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql index 146107d6f3d..cfa37b06477 100644 --- a/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql +++ b/tests/queries/0_stateless/01081_PartialSortingTransform_full_column.sql @@ -2,7 +2,7 @@ drop table if exists test_01081; create table test_01081 (key Int) engine=MergeTree() order by key; insert into test_01081 select * from system.numbers limit 10; -select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system.one as rhs on rhs.dummy = 1 order by 1; +select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system.one as rhs on rhs.dummy = 1 order by 1; -- { serverError 403 } -- With multiple blocks triggers: -- @@ -13,6 +13,6 @@ select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system -- With experimental_use_processors=1 (default at the time of writing). insert into test_01081 select * from system.numbers limit 10; -select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system.one as rhs on rhs.dummy = 1 order by 1; +select 1 from remote('127.{1,2}', currentDatabase(), test_01081) lhs join system.one as rhs on rhs.dummy = 1 order by 1; -- { serverError 403 } drop table if exists test_01081; diff --git a/tests/queries/0_stateless/01101_literal_column_clash.reference b/tests/queries/0_stateless/01101_literal_column_clash.reference index 47458541f79..e63f96416af 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.reference +++ b/tests/queries/0_stateless/01101_literal_column_clash.reference @@ -8,7 +8,3 @@ xyzabc 2 1 0 0 3 \N 1 2 \N 0 \N 1 0 \N 3 -1 -1 -1 hello\0\0\0 -1 hello\0\0\0 diff --git a/tests/queries/0_stateless/01101_literal_column_clash.sql b/tests/queries/0_stateless/01101_literal_column_clash.sql index 2c665d3833b..f96aad4bacf 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.sql +++ b/tests/queries/0_stateless/01101_literal_column_clash.sql @@ -17,7 +17,6 @@ select null, isConstant(null), * from (select 2 x, null) a right join (select 3 -- other cases with joins and constants -select cast(1, 'UInt8') from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); - -select isConstant('UInt8'), toFixedString('hello', toUInt8(substring('UInt8', 5, 1))) from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); +select cast(1, 'UInt8') from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); -- { serverError 403 } +select isConstant('UInt8'), toFixedString('hello', toUInt8(substring('UInt8', 5, 1))) from (select arrayJoin([1, 2]) as a) t1 left join (select 1 as b) t2 on b = ignore('UInt8'); -- { serverError 403 } From 1127232446a1133a1137a6fcd62661eb9eaa6673 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Wed, 23 Sep 2020 00:03:20 +0300 Subject: [PATCH 003/291] one more test fix --- .../queries/0_stateless/01018_Distributed__shard_num.reference | 2 -- tests/queries/0_stateless/01018_Distributed__shard_num.sql | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.reference b/tests/queries/0_stateless/01018_Distributed__shard_num.reference index 679383f969e..f4b92011759 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.reference +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.reference @@ -29,8 +29,6 @@ remote(Distributed) JOIN system.clusters 1 10 localhost ::1 9000 1 20 localhost ::1 9000 -1 10 localhost ::1 9000 -1 20 localhost ::1 9000 dist_3 100 foo foo 100 foo diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.sql b/tests/queries/0_stateless/01018_Distributed__shard_num.sql index ddb8505a415..f6d5f23eca8 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.sql +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.sql @@ -48,11 +48,12 @@ FROM (SELECT *, _shard_num FROM dist_1) a JOIN system.clusters b ON a._shard_num = b.shard_num WHERE b.cluster = 'test_cluster_two_shards_localhost'; + SELECT _shard_num, key, b.host_name, b.host_address, b.port FROM dist_1 a JOIN system.clusters b ON _shard_num = b.shard_num -WHERE b.cluster = 'test_cluster_two_shards_localhost'; +WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 403 } -- rewrite does not work with aliases, hence Missing columns (47) SELECT a._shard_num, key FROM dist_1 a; -- { serverError 47; } From 47d150b11639ec2fbfb5c691b38c172da344d6da Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 25 Sep 2020 02:29:16 +0300 Subject: [PATCH 004/291] Revert "Revert "Avoid deadlocks in Log/TinyLog"" --- src/Common/FileChecker.cpp | 5 ++ src/Common/FileChecker.h | 4 +- src/Storages/StorageFile.cpp | 40 ++++++--- src/Storages/StorageFile.h | 2 +- src/Storages/StorageLog.cpp | 60 +++++++++---- src/Storages/StorageLog.h | 4 +- src/Storages/StorageStripeLog.cpp | 45 +++++++--- src/Storages/StorageStripeLog.h | 2 +- src/Storages/StorageTinyLog.cpp | 87 +++++++++++++------ src/Storages/StorageTinyLog.h | 4 +- .../0_stateless/01499_log_deadlock.reference | 3 + .../0_stateless/01499_log_deadlock.sql | 26 ++++++ ...2_long_log_tinylog_deadlock_race.reference | 6 ++ .../01502_long_log_tinylog_deadlock_race.sh | 85 ++++++++++++++++++ .../01505_log_distributed_deadlock.reference | 0 .../01505_log_distributed_deadlock.sql | 12 +++ .../queries/0_stateless/arcadia_skip_list.txt | 1 + 17 files changed, 307 insertions(+), 79 deletions(-) create mode 100644 tests/queries/0_stateless/01499_log_deadlock.reference create mode 100644 tests/queries/0_stateless/01499_log_deadlock.sql create mode 100644 tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference create mode 100755 tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh create mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.reference create mode 100644 tests/queries/0_stateless/01505_log_distributed_deadlock.sql diff --git a/src/Common/FileChecker.cpp b/src/Common/FileChecker.cpp index 6cbec3bda77..b306c3af990 100644 --- a/src/Common/FileChecker.cpp +++ b/src/Common/FileChecker.cpp @@ -41,6 +41,11 @@ void FileChecker::setEmpty(const String & full_file_path) map[fileName(full_file_path)] = 0; } +FileChecker::Map FileChecker::getFileSizes() const +{ + return map; +} + CheckResults FileChecker::check() const { // Read the files again every time you call `check` - so as not to violate the constancy. diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h index 015d4cadb07..59e7331952e 100644 --- a/src/Common/FileChecker.h +++ b/src/Common/FileChecker.h @@ -27,10 +27,12 @@ public: /// The purpose of this function is to rollback a group of unfinished writes. void repair(); -private: /// File name -> size. using Map = std::map; + Map getFileSizes() const; + +private: void initialize(); void updateImpl(const String & file_path); void load(Map & local_map, const String & path) const; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index cc47047dc78..7b094f9bc06 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -52,6 +52,7 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int INCORRECT_FILE_NAME; extern const int FILE_DOESNT_EXIST; + extern const int TIMEOUT_EXCEEDED; } namespace @@ -199,6 +200,17 @@ StorageFile::StorageFile(CommonArguments args) setInMemoryMetadata(storage_metadata); } + +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + class StorageFileSource : public SourceWithProgress { public: @@ -245,7 +257,9 @@ public: { if (storage->use_table_fd) { - unique_lock = std::unique_lock(storage->rwlock); + unique_lock = std::unique_lock(storage->rwlock, getLockTimeout(context)); + if (!unique_lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); /// We could use common ReadBuffer and WriteBuffer in storage to leverage cache /// and add ability to seek unseekable files, but cache sync isn't supported. @@ -264,7 +278,9 @@ public: } else { - shared_lock = std::shared_lock(storage->rwlock); + shared_lock = std::shared_lock(storage->rwlock, getLockTimeout(context)); + if (!shared_lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } } @@ -373,8 +389,8 @@ private: bool finished_generate = false; - std::shared_lock shared_lock; - std::unique_lock unique_lock; + std::shared_lock shared_lock; + std::unique_lock unique_lock; }; @@ -417,7 +433,7 @@ Pipe StorageFile::read( for (size_t i = 0; i < num_streams; ++i) pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, metadata_snapshot->getColumns().getDefaults())); return Pipe::unitePipes(std::move(pipes)); } @@ -429,12 +445,16 @@ public: explicit StorageFileBlockOutputStream( StorageFile & storage_, const StorageMetadataPtr & metadata_snapshot_, + std::unique_lock && lock_, const CompressionMethod compression_method, const Context & context) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + std::unique_ptr naked_buffer = nullptr; if (storage.use_table_fd) { @@ -488,7 +508,7 @@ public: private: StorageFile & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; std::unique_ptr write_buf; BlockOutputStreamPtr writer; bool prefix_written{false}; @@ -506,7 +526,7 @@ BlockOutputStreamPtr StorageFile::write( if (!paths.empty()) path = paths[0]; - return std::make_shared(*this, metadata_snapshot, + return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}, chooseCompressionMethod(path, compression_method), context); } @@ -529,8 +549,6 @@ void StorageFile::rename(const String & new_path_to_table_data, const StorageID if (path_new == paths[0]) return; - std::unique_lock lock(rwlock); - Poco::File(Poco::Path(path_new).parent()).createDirectories(); Poco::File(paths[0]).renameTo(path_new); @@ -547,8 +565,6 @@ void StorageFile::truncate( if (paths.size() != 1) throw Exception("Can't truncate table '" + getStorageID().getNameForLogs() + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); - std::unique_lock lock(rwlock); - if (use_table_fd) { if (0 != ::ftruncate(table_fd, 0)) diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index ea70dcd5311..babc56e3a11 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -89,7 +89,7 @@ private: std::atomic table_fd_was_used{false}; /// To detect repeating reads from stdin off_t table_fd_init_offset = -1; /// Initial position of fd, used for repeating reads - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log = &Poco::Logger::get("StorageFile"); }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index e437bfb05f1..2fbce21655c 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -39,6 +39,7 @@ namespace DB namespace ErrorCodes { + extern const int TIMEOUT_EXCEEDED; extern const int LOGICAL_ERROR; extern const int DUPLICATE_COLUMN; extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT; @@ -50,7 +51,6 @@ namespace ErrorCodes class LogSource final : public SourceWithProgress { public: - static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -116,13 +116,16 @@ private: class LogBlockOutputStream final : public IBlockOutputStream { public: - explicit LogBlockOutputStream(StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_) + explicit LogBlockOutputStream( + StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) , marks_stream( storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~LogBlockOutputStream() override @@ -149,7 +152,7 @@ public: private: StorageLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -507,9 +510,11 @@ void StorageLog::addFiles(const String & column_name, const IDataType & type) } -void StorageLog::loadMarks() +void StorageLog::loadMarks(std::chrono::seconds lock_timeout) { - std::unique_lock lock(rwlock); + std::unique_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); if (loaded_marks) return; @@ -552,8 +557,6 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -569,8 +572,6 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { - std::shared_lock lock(rwlock); - files.clear(); file_count = 0; loaded_marks = false; @@ -610,6 +611,17 @@ const StorageLog::Marks & StorageLog::getMarksWithRealRowCount(const StorageMeta return it->second.marks; } + +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -620,11 +632,15 @@ Pipe StorageLog::read( unsigned num_streams) { metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); - loadMarks(); + + auto lock_timeout = getLockTimeout(context); + loadMarks(lock_timeout); NamesAndTypesList all_columns = Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)); - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); Pipes pipes; @@ -653,18 +669,28 @@ Pipe StorageLog::read( max_read_buffer_size)); } + /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe::unitePipes(std::move(pipes)); } -BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - loadMarks(); - return std::make_shared(*this, metadata_snapshot); + auto lock_timeout = getLockTimeout(context); + loadMarks(lock_timeout); + + std::unique_lock lock(rwlock, lock_timeout); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + return std::make_shared(*this, metadata_snapshot, std::move(lock)); } -CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 49fc9a576c5..3553426b9e6 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -83,7 +83,7 @@ private: DiskPtr disk; String table_path; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Files files; @@ -104,7 +104,7 @@ private: /// Read marks files if they are not already read. /// It is done lazily, so that with a large number of tables, the server starts quickly. /// You can not call with a write locked `rwlock`. - void loadMarks(); + void loadMarks(std::chrono::seconds lock_timeout); /** For normal columns, the number of rows in the block is specified in the marks. * For array columns and nested structures, there are more than one group of marks that correspond to different files diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c4344cf6f1f..8ff8035c128 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -47,13 +47,13 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int INCORRECT_FILE_NAME; + extern const int TIMEOUT_EXCEEDED; } class StripeLogSource final : public SourceWithProgress { public: - static Block getHeader( StorageStripeLog & storage, const StorageMetadataPtr & metadata_snapshot, @@ -157,10 +157,11 @@ private: class StripeLogBlockOutputStream final : public IBlockOutputStream { public: - explicit StripeLogBlockOutputStream(StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_) + explicit StripeLogBlockOutputStream( + StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) - , lock(storage.rwlock) + , lock(std::move(lock_)) , data_out_file(storage.table_path + "data.bin") , data_out_compressed(storage.disk->writeFile(data_out_file, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append)) , data_out(std::make_unique( @@ -170,6 +171,8 @@ public: , index_out(std::make_unique(*index_out_compressed)) , block_out(*data_out, 0, metadata_snapshot->getSampleBlock(), false, index_out.get(), storage.disk->getFileSize(data_out_file)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~StripeLogBlockOutputStream() override @@ -223,7 +226,7 @@ public: private: StorageStripeLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; String data_out_file; std::unique_ptr data_out_compressed; @@ -286,8 +289,6 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -297,6 +298,16 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora } +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageStripeLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -306,7 +317,9 @@ Pipe StorageStripeLog::read( const size_t /*max_block_size*/, unsigned num_streams) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); @@ -345,24 +358,28 @@ Pipe StorageStripeLog::read( } -BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageStripeLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - return std::make_shared(*this, metadata_snapshot); + std::unique_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + return std::make_shared(*this, metadata_snapshot, std::move(lock)); } -CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageStripeLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } void StorageStripeLog::truncate(const ASTPtr &, const StorageMetadataPtr &, const Context &, TableExclusiveLockHolder &) { - std::shared_lock lock(rwlock); - disk->clearDirectory(table_path); - file_checker = FileChecker{disk, table_path + "sizes.json"}; } diff --git a/src/Storages/StorageStripeLog.h b/src/Storages/StorageStripeLog.h index f88120a932e..ca3bfe4ff75 100644 --- a/src/Storages/StorageStripeLog.h +++ b/src/Storages/StorageStripeLog.h @@ -67,7 +67,7 @@ private: size_t max_compress_block_size; FileChecker file_checker; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; }; diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 0bdcab8abf4..4d646c7451e 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -46,6 +47,7 @@ namespace DB namespace ErrorCodes { + extern const int TIMEOUT_EXCEEDED; extern const int DUPLICATE_COLUMN; extern const int INCORRECT_FILE_NAME; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -55,7 +57,6 @@ namespace ErrorCodes class TinyLogSource final : public SourceWithProgress { public: - static Block getHeader(const NamesAndTypesList & columns) { Block res; @@ -66,10 +67,17 @@ public: return Nested::flatten(res); } - TinyLogSource(size_t block_size_, const NamesAndTypesList & columns_, StorageTinyLog & storage_, size_t max_read_buffer_size_) + TinyLogSource( + size_t block_size_, + const NamesAndTypesList & columns_, + StorageTinyLog & storage_, + size_t max_read_buffer_size_, + FileChecker::Map file_sizes_) : SourceWithProgress(getHeader(columns_)) - , block_size(block_size_), columns(columns_), storage(storage_), lock(storage_.rwlock) - , max_read_buffer_size(max_read_buffer_size_) {} + , block_size(block_size_), columns(columns_), storage(storage_) + , max_read_buffer_size(max_read_buffer_size_), file_sizes(std::move(file_sizes_)) + { + } String getName() const override { return "TinyLog"; } @@ -80,19 +88,21 @@ private: size_t block_size; NamesAndTypesList columns; StorageTinyLog & storage; - std::shared_lock lock; bool is_finished = false; size_t max_read_buffer_size; + FileChecker::Map file_sizes; struct Stream { - Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_) + Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_, size_t file_size) : plain(disk->readFile(data_path, std::min(max_read_buffer_size_, disk->getFileSize(data_path)))), + limited(std::make_unique(*plain, file_size, false)), compressed(*plain) { } std::unique_ptr plain; + std::unique_ptr limited; CompressedReadBuffer compressed; }; @@ -110,9 +120,14 @@ private: class TinyLogBlockOutputStream final : public IBlockOutputStream { public: - explicit TinyLogBlockOutputStream(StorageTinyLog & storage_, const StorageMetadataPtr & metadata_snapshot_) - : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(storage_.rwlock) + explicit TinyLogBlockOutputStream( + StorageTinyLog & storage_, + const StorageMetadataPtr & metadata_snapshot_, + std::unique_lock && lock_) + : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(std::move(lock_)) { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); } ~TinyLogBlockOutputStream() override @@ -140,7 +155,7 @@ public: private: StorageTinyLog & storage; StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; + std::unique_lock lock; bool done = false; struct Stream @@ -231,13 +246,17 @@ void TinyLogSource::readData(const String & name, const IDataType & type, IColum String stream_name = IDataType::getFileNameForStream(name, path); if (!streams.count(stream_name)) - streams[stream_name] = std::make_unique(storage.disk, storage.files[stream_name].data_file_path, max_read_buffer_size); + { + String file_path = storage.files[stream_name].data_file_path; + streams[stream_name] = std::make_unique( + storage.disk, file_path, max_read_buffer_size, file_sizes[fileName(file_path)]); + } return &streams[stream_name]->compressed; }; if (deserialize_states.count(name) == 0) - type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); + type.deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]); type.deserializeBinaryBulkWithMultipleStreams(column, limit, settings, deserialize_states[name]); } @@ -410,8 +429,6 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage { assert(table_path != new_path_to_table_data); { - std::unique_lock lock(rwlock); - disk->moveDirectory(table_path, new_path_to_table_data); table_path = new_path_to_table_data; @@ -424,6 +441,16 @@ void StorageTinyLog::rename(const String & new_path_to_table_data, const Storage } +static std::chrono::seconds getLockTimeout(const Context & context) +{ + const Settings & settings = context.getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + + Pipe StorageTinyLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -437,28 +464,40 @@ Pipe StorageTinyLog::read( // When reading, we lock the entire storage, because we only have one file // per column and can't modify it concurrently. + const Settings & settings = context.getSettingsRef(); + + std::shared_lock lock{rwlock, getLockTimeout(context)}; + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data. return Pipe(std::make_shared( - max_block_size, Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, + Nested::collect(metadata_snapshot->getColumns().getAllPhysical().addTypes(column_names)), + *this, + settings.max_read_buffer_size, + file_checker.getFileSizes())); } -BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { - return std::make_shared(*this, metadata_snapshot); + return std::make_shared(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)}); } -CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & /* context */) +CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, const Context & context) { - std::shared_lock lock(rwlock); + std::shared_lock lock(rwlock, getLockTimeout(context)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + return file_checker.check(); } void StorageTinyLog::truncate( const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) { - std::unique_lock lock(rwlock); - disk->clearDirectory(table_path); files.clear(); @@ -468,14 +507,6 @@ void StorageTinyLog::truncate( addFiles(column.name, *column.type); } -void StorageTinyLog::drop() -{ - std::unique_lock lock(rwlock); - if (disk->exists(table_path)) - disk->removeRecursive(table_path); - files.clear(); -} - void registerStorageTinyLog(StorageFactory & factory) { diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index dc6ff101503..95b7d9f2941 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -43,8 +43,6 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) override; - void drop() override; - protected: StorageTinyLog( DiskPtr disk_, @@ -70,7 +68,7 @@ private: Files files; FileChecker file_checker; - mutable std::shared_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; diff --git a/tests/queries/0_stateless/01499_log_deadlock.reference b/tests/queries/0_stateless/01499_log_deadlock.reference new file mode 100644 index 00000000000..166be640db5 --- /dev/null +++ b/tests/queries/0_stateless/01499_log_deadlock.reference @@ -0,0 +1,3 @@ +6 +6 +6 diff --git a/tests/queries/0_stateless/01499_log_deadlock.sql b/tests/queries/0_stateless/01499_log_deadlock.sql new file mode 100644 index 00000000000..e98b37f2455 --- /dev/null +++ b/tests/queries/0_stateless/01499_log_deadlock.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (x UInt8) ENGINE = TinyLog; + +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; + +DROP TABLE t; + + +CREATE TABLE t (x UInt8) ENGINE = Log; + +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; + +DROP TABLE t; + + +CREATE TABLE t (x UInt8) ENGINE = StripeLog; + +INSERT INTO t VALUES (1), (2), (3); +INSERT INTO t SELECT * FROM t; +SELECT count() FROM t; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference new file mode 100644 index 00000000000..4bf85ae79f3 --- /dev/null +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.reference @@ -0,0 +1,6 @@ +Testing TinyLog +Done TinyLog +Testing StripeLog +Done StripeLog +Testing Log +Done Log diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh new file mode 100755 index 00000000000..29c5f868617 --- /dev/null +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +export -f thread_create +export -f thread_drop +export -f thread_rename +export -f thread_select +export -f thread_insert +export -f thread_insert_select + + +# Do randomized queries and expect nothing extraordinary happens. + +function test_with_engine { + echo "Testing $1" + + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & + + wait + echo "Done $1" +} + +test_with_engine TinyLog +test_with_engine StripeLog +test_with_engine Log diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.reference b/tests/queries/0_stateless/01505_log_distributed_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01505_log_distributed_deadlock.sql b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql new file mode 100644 index 00000000000..2b0b2b97188 --- /dev/null +++ b/tests/queries/0_stateless/01505_log_distributed_deadlock.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS t_local; +DROP TABLE IF EXISTS t_dist; + +create table t_local(a int) engine Log; +create table t_dist (a int) engine Distributed(test_shard_localhost, currentDatabase(), 't_local', cityHash64(a)); + +set insert_distributed_sync = 1; + +insert into t_dist values (1); + +DROP TABLE t_local; +DROP TABLE t_dist; diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 69391ca9fd4..6d1c6444d1b 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -145,3 +145,4 @@ 01461_query_start_time_microseconds 01455_shard_leaf_max_rows_bytes_to_read 01505_distributed_local_type_conversion_enum +01505_log_distributed_deadlock From 91e80598cd1ff50cb7af1c32ae4fb47b53a0816d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 04:54:35 +0300 Subject: [PATCH 005/291] Add "harmful" library --- base/glibc-compatibility/musl/lgamma.c | 8 --- base/glibc-compatibility/musl/lgammal.c | 9 --- base/harmful/CMakeLists.txt | 3 + base/harmful/README.md | 1 + base/harmful/harmful.c | 89 +++++++++++++++++++++++++ cmake/linux/default_libs.cmake | 1 + 6 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 base/harmful/CMakeLists.txt create mode 100644 base/harmful/README.md create mode 100644 base/harmful/harmful.c diff --git a/base/glibc-compatibility/musl/lgamma.c b/base/glibc-compatibility/musl/lgamma.c index fbbe1fedc92..fb9d105d0fa 100644 --- a/base/glibc-compatibility/musl/lgamma.c +++ b/base/glibc-compatibility/musl/lgamma.c @@ -258,11 +258,3 @@ double lgamma_r(double x, int *signgamp) r = nadj - r; return r; } - - -int signgam; - -double lgamma(double x) -{ - return lgamma_r(x, &signgam); -} diff --git a/base/glibc-compatibility/musl/lgammal.c b/base/glibc-compatibility/musl/lgammal.c index 3b5d94c5051..a8208a6d9c0 100644 --- a/base/glibc-compatibility/musl/lgammal.c +++ b/base/glibc-compatibility/musl/lgammal.c @@ -328,12 +328,3 @@ long double lgammal_r(long double x, int *sg) return lgamma_r(x, sg); } #endif - - -int signgam_lgammal; - -long double lgammal(long double x) -{ - return lgammal_r(x, &signgam_lgammal); -} - diff --git a/base/harmful/CMakeLists.txt b/base/harmful/CMakeLists.txt new file mode 100644 index 00000000000..f4ae3ba82b9 --- /dev/null +++ b/base/harmful/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(harmful harmful.c) +target_link_libraries(global-libs INTERFACE harmful) +install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib) diff --git a/base/harmful/README.md b/base/harmful/README.md new file mode 100644 index 00000000000..7e56c1f62e4 --- /dev/null +++ b/base/harmful/README.md @@ -0,0 +1 @@ +A library that traps whenever harmful functions from libc are called. diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c new file mode 100644 index 00000000000..893d451e0c6 --- /dev/null +++ b/base/harmful/harmful.c @@ -0,0 +1,89 @@ +#pragma GCC diagnostic ignored "-Wincompatible-library-redeclaration" + +#define TRAP(func) void func() { __builtin_trap(); } + +/// Trap all non thread-safe functions: +/// nm -D /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} | grep -P '_r@?$' | awk '{ print $3 }' | sed -r -e 's/_r//' | grep -vP '^_' + +TRAP(asctime) +TRAP(ctime) +TRAP(drand48) +TRAP(ecvt) +TRAP(erand48) +TRAP(ether_aton) +TRAP(ether_ntoa) +TRAP(fcvt) +TRAP(fgetgrent) +TRAP(fgetpwent) +TRAP(fgetsgent) +TRAP(fgetspent) +TRAP(getaliasbyname) +TRAP(getaliasent) +TRAP(getdate) +TRAP(getgrent) +TRAP(getgrgid) +TRAP(getgrnam) +TRAP(gethostbyaddr) +TRAP(gethostbyname2) +TRAP(gethostbyname) +TRAP(gethostent) +TRAP(getlogin) +TRAP(getmntent) +TRAP(getnetbyaddr) +TRAP(getnetbyname) +TRAP(getnetent) +TRAP(getnetgrent) +TRAP(getprotobyname) +TRAP(getprotobynumber) +TRAP(getprotoent) +TRAP(getpwent) +TRAP(getpwnam) +TRAP(getpwuid) +TRAP(getrpcbyname) +TRAP(getrpcbynumber) +TRAP(getrpcent) +TRAP(getservbyname) +TRAP(getservbyport) +TRAP(getservent) +TRAP(getsgent) +TRAP(getsgnam) +TRAP(getspent) +TRAP(getspnam) +TRAP(getutent) +TRAP(getutid) +TRAP(getutline) +TRAP(gmtime) +TRAP(hcreate) +TRAP(hdestroy) +TRAP(hsearch) +TRAP(initstate) +TRAP(jrand48) +TRAP(lcong48) +TRAP(localtime) +TRAP(lrand48) +TRAP(mrand48) +TRAP(nrand48) +TRAP(ptsname) +TRAP(qecvt) +TRAP(qfcvt) +TRAP(random) +TRAP(rand) +TRAP(seed48) +TRAP(setstate) +TRAP(sgetsgent) +TRAP(sgetspent) +TRAP(srand48) +TRAP(srandom) +TRAP(strerror) +TRAP(strtok) +TRAP(tmpnam) +TRAP(ttyname) +TRAP(twalk) +TRAP(lgammaf128) +TRAP(lgammaf32) +TRAP(lgammaf32x) +TRAP(lgammaf64) +TRAP(lgammaf64x) +TRAP(lgammaf) +TRAP(lgammal) +TRAP(lgamma) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index 2a325b4d095..c4c441634f4 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -43,6 +43,7 @@ find_package(Threads REQUIRED) if (NOT OS_ANDROID) # Our compatibility layer doesn't build under Android, many errors in musl. add_subdirectory(base/glibc-compatibility) + add_subdirectory(base/harmful) endif () include (cmake/find/unwind.cmake) From 53ef792fe507cec3caa575424a79ee75770ef933 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 06:12:04 +0300 Subject: [PATCH 006/291] Fix gcc build --- base/harmful/harmful.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 893d451e0c6..cd53da9e9ac 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -1,4 +1,8 @@ -#pragma GCC diagnostic ignored "-Wincompatible-library-redeclaration" +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wincompatible-library-redeclaration" +#else + #pragma GCC diagnostic ignored "-Wbuiltin-declaration-mismatch" +#endif #define TRAP(func) void func() { __builtin_trap(); } From 2a6c1463a1c1ca39a593c56edb9a66468220cde7 Mon Sep 17 00:00:00 2001 From: roverxu Date: Fri, 4 Dec 2020 11:42:58 +0800 Subject: [PATCH 007/291] Add metrics for part number in MergeTree --- src/Common/CurrentMetrics.cpp | 3 ++ src/Storages/MergeTree/MergeTreeData.cpp | 50 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index c48e76e1d98..14fdf74b5e0 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -5,6 +5,9 @@ #define APPLY_FOR_METRICS(M) \ M(Query, "Number of executing queries") \ M(Merge, "Number of executing background merges") \ + M(Parts, "Total number of data parts") \ + M(PartsActive, "Number of active data parts") \ + M(PartsInactive, "Number of inactive data parts") \ M(PartMutation, "Number of mutations (ALTER DELETE/UPDATE)") \ M(ReplicatedFetch, "Number of data parts being fetched from replica") \ M(ReplicatedSend, "Number of data parts being sent to replicas") \ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b213bb7b6f9..af3e323763e 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -72,6 +72,9 @@ namespace CurrentMetrics { extern const Metric DelayedInserts; extern const Metric BackgroundMovePoolTask; + extern const Metric Parts; + extern const Metric PartsActive; + extern const Metric PartsInactive; } @@ -877,6 +880,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) std::lock_guard loading_lock(mutex); if (!data_parts_indexes.insert(part).second) throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); + + CurrentMetrics::add(CurrentMetrics::Parts); + CurrentMetrics::add(CurrentMetrics::PartsActive); }); } @@ -893,6 +899,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (!data_parts_indexes.insert(part).second) throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); + + CurrentMetrics::add(CurrentMetrics::Parts); + CurrentMetrics::add(CurrentMetrics::PartsActive); } if (has_non_adaptive_parts && has_adaptive_parts && !settings->enable_mixed_granularity_parts) @@ -924,6 +933,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { (*it)->remove_time.store((*it)->modification_time, std::memory_order_relaxed); modifyPartState(it, DataPartState::Outdated); + + CurrentMetrics::sub(CurrentMetrics::PartsActive); + CurrentMetrics::add(CurrentMetrics::PartsInactive); }; (*prev_jt)->assertState({DataPartState::Committed}); @@ -1091,6 +1103,9 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa (*it)->assertState({DataPartState::Deleting}); data_parts_indexes.erase(it); + + CurrentMetrics::sub(CurrentMetrics::Parts); + CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } @@ -1265,10 +1280,15 @@ void MergeTreeData::dropAllData() LOG_TRACE(log, "dropAllData: removing data from memory."); DataPartsVector all_parts(data_parts_by_info.begin(), data_parts_by_info.end()); + DataPartsVector committed_parts = getDataPartsVector({DataPartState::Committed}); data_parts_indexes.clear(); column_sizes.clear(); + CurrentMetrics::sub(CurrentMetrics::Parts, all_parts.size()); + CurrentMetrics::sub(CurrentMetrics::PartsActive, committed_parts.size()); + CurrentMetrics::sub(CurrentMetrics::PartsInactive, all_parts.size() - committed_parts.size()); + /// Tables in atomic databases have UUID and stored in persistent locations. /// No need to drop caches (that are keyed by filesystem path) because collision is not possible. if (!getStorageID().hasUUID()) @@ -1981,6 +2001,9 @@ bool MergeTreeData::renameTempPartAndReplace( auto part_it = data_parts_indexes.insert(part).first; + CurrentMetrics::add(CurrentMetrics::Parts); + CurrentMetrics::add(CurrentMetrics::PartsInactive); + if (out_transaction) { out_transaction->precommitted_parts.insert(part); @@ -1993,10 +2016,16 @@ bool MergeTreeData::renameTempPartAndReplace( covered_part->remove_time.store(current_time, std::memory_order_relaxed); modifyPartState(covered_part, DataPartState::Outdated); removePartContributionToColumnSizes(covered_part); + + CurrentMetrics::sub(CurrentMetrics::PartsActive); + CurrentMetrics::add(CurrentMetrics::PartsInactive); } modifyPartState(part_it, DataPartState::Committed); addPartContributionToColumnSizes(part); + + CurrentMetrics::add(CurrentMetrics::PartsActive); + CurrentMetrics::sub(CurrentMetrics::PartsInactive); } auto part_in_memory = asInMemoryPart(part); @@ -2037,7 +2066,11 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect for (const DataPartPtr & part : remove) { if (part->state == IMergeTreeDataPart::State::Committed) + { removePartContributionToColumnSizes(part); + CurrentMetrics::sub(CurrentMetrics::PartsActive); + CurrentMetrics::add(CurrentMetrics::PartsInactive); + } if (part->state == IMergeTreeDataPart::State::Committed || clear_without_timeout) part->remove_time.store(remove_time, std::memory_order_relaxed); @@ -2063,6 +2096,9 @@ void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(con modifyPartState(part, IMergeTreeDataPart::State::Temporary); /// Erase immediately data_parts_indexes.erase(it_part); + + CurrentMetrics::sub(CurrentMetrics::Parts); + CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } @@ -2150,13 +2186,21 @@ restore_covered) DataPartPtr part = *it_part; if (part->state == DataPartState::Committed) + { removePartContributionToColumnSizes(part); + CurrentMetrics::sub(CurrentMetrics::PartsActive); + CurrentMetrics::add(CurrentMetrics::PartsInactive); + } + modifyPartState(it_part, DataPartState::Deleting); part->renameToDetached(prefix); data_parts_indexes.erase(it_part); + CurrentMetrics::sub(CurrentMetrics::Parts); + CurrentMetrics::sub(CurrentMetrics::PartsInactive); + if (restore_covered && part->info.level == 0) { LOG_WARNING(log, "Will not recover parts covered by zero-level part {}", part->name); @@ -3385,10 +3429,16 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: covered_part->remove_time.store(current_time, std::memory_order_relaxed); data.modifyPartState(covered_part, DataPartState::Outdated); data.removePartContributionToColumnSizes(covered_part); + + CurrentMetrics::sub(CurrentMetrics::PartsActive); + CurrentMetrics::add(CurrentMetrics::PartsInactive); } data.modifyPartState(part, DataPartState::Committed); data.addPartContributionToColumnSizes(part); + + CurrentMetrics::add(CurrentMetrics::PartsActive); + CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } } From 5914e6e451700d5e9b0d17ae57406f920cf2ea87 Mon Sep 17 00:00:00 2001 From: spongedc Date: Mon, 7 Dec 2020 16:04:29 +0800 Subject: [PATCH 008/291] Refine documents for system log table, and fix a link error in config.xml --- docs/en/operations/system-tables/index.md | 28 ++++++++++++++++++++++- docs/zh/operations/system-tables/index.md | 28 ++++++++++++++++++++++- programs/server/config.xml | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 7a9e386d419..e660b418a76 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,33 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), [crash_log](../../operations/system-tables/crash_log.md#system.crash_log) and [text_log](../../operations/system-tables/crash_log.md#system.text_log) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. + +System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: + +- `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`. +- `table`: table to insert data. +- `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression. +- `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) expression. +- `flush_interval_milliseconds`: interval of flushing data to disk. +- `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit. + +An example: + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` By default, table growth is unlimited. To control a size of a table, you can use [TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables. diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index fcf6741761b..2dc1e2712e5 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -22,7 +22,33 @@ toc_title: "\u7CFB\u7EDF\u8868" 大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。 -与其他系统表不同，系统表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log) 由 [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 表引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 +与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), [crash_log](../../operations/system-tables/crash_log.md#system.crash_log) and [text_log](../../operations/system-tables/crash_log.md#system.text_log) 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 + +用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下: + +- `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。 +- `table`: 系统日志表名。 +- `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。 +- `ttl`: 指定系统日志表[TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl)选项。 +- `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。 +- `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。 + +一个配置定义的例子如下： + +``` + + + system + query_log
+ toYYYYMM(event_date) + event_date + INTERVAL 30 DAY DELETE + + 7500 + + +``` 默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 diff --git a/programs/server/config.xml b/programs/server/config.xml index dde3702a44b..e062c8d7b80 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -581,7 +581,7 @@ system query_log
From e476dcdba78a1f24dd4522b6ef042bbe4756b1b8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 4 Jan 2021 20:58:26 +0300 Subject: [PATCH 150/291] Update replication.md --- .../table-engines/mergetree-family/replication.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 04a239e0b06..ef34c8d3804 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -38,15 +38,15 @@ Example of setting the addresses of the ZooKeeper cluster: ``` xml - + example1 2181 - + example2 2181 - + example3 2181 @@ -61,21 +61,21 @@ Example of setting the addresses of the auxiliary ZooKeeper cluster: ``` xml - + example_2_1 2181 - + example_2_2 2181 - + example_2_3 2181 - + example_3_1 2181 From 063360511ac09e3eb40474daa59026f940f90f73 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 4 Jan 2021 21:01:17 +0300 Subject: [PATCH 151/291] Remove obsolete "incl" from /etc/metrika.xml; correct links in config --- programs/server/config.xml | 36 ++++++++++++++++++++++++++++-------- programs/server/embedded.xml | 2 +- programs/server/users.xml | 2 +- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 2bdcbd47ccf..a4c13be493e 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -204,7 +204,7 @@ /etc/clickhouse-server/server.key /etc/clickhouse-server/dhparam.pem @@ -432,7 +432,7 @@ - + - + - + @@ -810,8 +830,8 @@ - + --> diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml index 1ac568682f9..a66f57d1eb7 100644 --- a/programs/server/embedded.xml +++ b/programs/server/embedded.xml @@ -20,7 +20,7 @@ - + ::/0 diff --git a/programs/server/users.xml b/programs/server/users.xml index 838b46e6938..3223d855651 100644 --- a/programs/server/users.xml +++ b/programs/server/users.xml @@ -77,7 +77,7 @@ Strongly recommended that regexp is ends with $ All results of DNS requests are cached till server restart. --> - + ::/0 From 12a0ef907ece6cbda587209ace70c58fee9b0b1a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 4 Jan 2021 21:07:52 +0300 Subject: [PATCH 152/291] Slightly better --- src/IO/WriteHelpers.h | 4 ++-- src/IO/WriteIntText.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 58fdaeab6f0..624a6c3496a 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -730,7 +730,7 @@ static const char digits100[201] = template inline void writeDateText(const LocalDate & date, WriteBuffer & buf) { - if (buf.position() && buf.position() + 10 <= buf.buffer().end()) + if (reinterpret_cast(buf.position()) + 10 <= reinterpret_cast(buf.buffer().end())) { memcpy(buf.position(), &digits100[date.year() / 100 * 2], 2); buf.position() += 2; @@ -767,7 +767,7 @@ inline void writeDateText(DayNum date, WriteBuffer & buf) template inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) { - if (buf.position() && buf.position() + 19 <= buf.buffer().end()) + if (reinterpret_cast(buf.position()) + 19 <= reinterpret_cast(buf.buffer().end())) { memcpy(buf.position(), &digits100[datetime.year() / 100 * 2], 2); buf.position() += 2; diff --git a/src/IO/WriteIntText.h b/src/IO/WriteIntText.h index 2dca2af8c40..93444e7df73 100644 --- a/src/IO/WriteIntText.h +++ b/src/IO/WriteIntText.h @@ -24,7 +24,7 @@ namespace detail template void writeIntText(T x, WriteBuffer & buf) { - if (likely(buf.position() && buf.position() + WRITE_HELPERS_MAX_INT_WIDTH < buf.buffer().end())) + if (likely(reinterpret_cast(buf.position()) + WRITE_HELPERS_MAX_INT_WIDTH < reinterpret_cast(buf.buffer().end()))) buf.position() = itoa(x, buf.position()); else detail::writeUIntTextFallback(x, buf); From fc9c440d3ecfe3cb4489b7dcf6269995ff7f3374 Mon Sep 17 00:00:00 2001 From: fastio Date: Tue, 5 Jan 2021 04:42:57 +0800 Subject: [PATCH 153/291] Fix the unexpected behaviour of show tables when antlr parser enabled (#18431) * Fix the unexpected behaviour of show tables when antlr parser enabled * fix compile error --- src/Interpreters/executeQuery.cpp | 2 +- src/Parsers/New/ParseTreeVisitor.cpp | 15 ++++++++------- src/Parsers/New/ParseTreeVisitor.h | 2 ++ src/Parsers/New/parseQuery.cpp | 9 ++++----- src/Parsers/New/parseQuery.h | 7 +++---- utils/syntax-analyzer/main.cpp | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ae35e009bce..5928da156f3 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -351,7 +351,7 @@ static std::tuple executeQueryImpl( #if !defined(ARCADIA_BUILD) if (settings.use_antlr_parser) { - ast = parseQuery(begin, end, max_query_size, settings.max_parser_depth); + ast = parseQuery(begin, end, max_query_size, settings.max_parser_depth, context.getCurrentDatabase()); } else { diff --git a/src/Parsers/New/ParseTreeVisitor.cpp b/src/Parsers/New/ParseTreeVisitor.cpp index 2179e44d78b..a7c7a2758eb 100644 --- a/src/Parsers/New/ParseTreeVisitor.cpp +++ b/src/Parsers/New/ParseTreeVisitor.cpp @@ -34,7 +34,6 @@ // Include last, because antlr-runtime undefines EOF macros, which is required in boost multiprecision numbers. #include - namespace DB { @@ -114,15 +113,17 @@ antlrcpp::Any ParseTreeVisitor::visitShowTablesStmt(ClickHouseParser::ShowTables auto and_args = PtrTo(new ColumnExprList{ColumnExpr::createLiteral(Literal::createNumber("1"))}); + auto current_database = ColumnExpr::createLiteral(Literal::createString(current_database_name)); if (ctx->databaseIdentifier()) { - auto database = std::make_shared(nullptr, std::make_shared("database")); - auto args = PtrTo(new ColumnExprList{ - ColumnExpr::createIdentifier(database), - ColumnExpr::createLiteral(Literal::createString(visit(ctx->databaseIdentifier()).as>()->getName())) - }); - and_args->push(ColumnExpr::createFunction(std::make_shared("equals"), nullptr, args)); + current_database = ColumnExpr::createLiteral(Literal::createString(visit(ctx->databaseIdentifier()).as>()->getName())); } + auto database = std::make_shared(nullptr, std::make_shared("database")); + auto equals_args = PtrTo(new ColumnExprList{ + ColumnExpr::createIdentifier(database), + current_database + }); + and_args->push(ColumnExpr::createFunction(std::make_shared("equals"), nullptr, equals_args)); if (ctx->LIKE()) { diff --git a/src/Parsers/New/ParseTreeVisitor.h b/src/Parsers/New/ParseTreeVisitor.h index 4f6d93f0327..8301cb5c5b8 100644 --- a/src/Parsers/New/ParseTreeVisitor.h +++ b/src/Parsers/New/ParseTreeVisitor.h @@ -7,7 +7,9 @@ namespace DB { class ParseTreeVisitor : public ClickHouseParserVisitor { + const String & current_database_name; public: + explicit ParseTreeVisitor(const String & database_name) : ClickHouseParserVisitor(), current_database_name(database_name) {} virtual ~ParseTreeVisitor() override = default; // Top-level statements diff --git a/src/Parsers/New/parseQuery.cpp b/src/Parsers/New/parseQuery.cpp index af334717392..c66772385ca 100644 --- a/src/Parsers/New/parseQuery.cpp +++ b/src/Parsers/New/parseQuery.cpp @@ -12,7 +12,6 @@ #include - namespace DB { @@ -20,7 +19,7 @@ using namespace antlr4; using namespace AST; // For testing only -PtrTo parseQuery(const String & query) +PtrTo parseQuery(const String & query, const String & current_database) { ANTLRInputStream input(query); ClickHouseLexer lexer(&input); @@ -34,12 +33,12 @@ PtrTo parseQuery(const String & query) lexer.addErrorListener(&lexer_error_listener); parser.addErrorListener(&parser_error_listener); - ParseTreeVisitor visitor; + ParseTreeVisitor visitor { current_database }; return visitor.visit(parser.queryStmt()); } -ASTPtr parseQuery(const char * begin, const char * end, size_t, size_t) +ASTPtr parseQuery(const char * begin, const char * end, size_t, size_t, const String & current_database) { // TODO: do not ignore |max_parser_depth|. @@ -60,7 +59,7 @@ ASTPtr parseQuery(const char * begin, const char * end, size_t, size_t) lexer.addErrorListener(&lexer_error_listener); parser.addErrorListener(&parser_error_listener); - ParseTreeVisitor visitor; + ParseTreeVisitor visitor { current_database }; PtrTo new_ast = visitor.visit(parser.queryStmt()); auto old_ast = new_ast->convertToOld(); diff --git a/src/Parsers/New/parseQuery.h b/src/Parsers/New/parseQuery.h index ae86dc32eda..8d9c8efd337 100644 --- a/src/Parsers/New/parseQuery.h +++ b/src/Parsers/New/parseQuery.h @@ -2,14 +2,13 @@ #include #include - +#include namespace DB { // Compatibility interface - -AST::PtrTo parseQuery(const std::string & query); -ASTPtr parseQuery(const char * begin, const char * end, size_t max_query_size, size_t max_parser_depth); +AST::PtrTo parseQuery(const std::string & query, const String & current_database); +ASTPtr parseQuery(const char * begin, const char * end, size_t max_query_size, size_t max_parser_depth, const String & current_database); } diff --git a/utils/syntax-analyzer/main.cpp b/utils/syntax-analyzer/main.cpp index 7f1d2c979ec..cf264160407 100644 --- a/utils/syntax-analyzer/main.cpp +++ b/utils/syntax-analyzer/main.cpp @@ -36,7 +36,7 @@ int main(int argc, const char **) ASTPtr orig_ast = parseQuery(parser, q, 10000000, 10000); std::cout << std::endl << "New AST:" << std::endl; - auto new_ast = parseQuery(q); + auto new_ast = parseQuery(q, ""); new_ast->dump(); auto old_ast = new_ast->convertToOld(); From 858c2a474aacb16bd7513b14df651563477b420d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 00:28:57 +0300 Subject: [PATCH 154/291] Do not throw from Parser --- src/Parsers/ParserCreateQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 8e6040169c9..552ea9df320 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -228,7 +228,8 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E { if (primary_key) { - throw Exception("Multiple primary keys are not allowed.", ErrorCodes::BAD_ARGUMENTS); + /// Multiple primary keys are not allowed. + return false; } primary_key = elem; } From a15eb69f049fe0ab78f21236468fc64248a1afa1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 00:56:00 +0300 Subject: [PATCH 155/291] Remove pink screen with confusing questions about Kerberos --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 809b5bb5d2a..9b34e982698 100644 --- a/debian/control +++ b/debian/control @@ -40,7 +40,7 @@ Description: Common files for ClickHouse Package: clickhouse-server Architecture: all Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-common-static (= ${binary:Version}), adduser -Recommends: libcap2-bin, krb5-user +Recommends: libcap2-bin Replaces: clickhouse-server-common, clickhouse-server-base Provides: clickhouse-server-common Description: Server binary for ClickHouse From 039edbb72434a1f9ba93936a8337457f8e1699de Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 01:31:34 +0300 Subject: [PATCH 156/291] Don't allow conversion between UUID and numeric types --- src/Functions/FunctionsConversion.h | 4 ++ .../queries/0_stateless/00735_conditional.sql | 54 ------------------- .../0_stateless/01634_uuid_fuzz.reference | 0 tests/queries/0_stateless/01634_uuid_fuzz.sql | 1 + 4 files changed, 5 insertions(+), 54 deletions(-) create mode 100644 tests/queries/0_stateless/01634_uuid_fuzz.reference create mode 100644 tests/queries/0_stateless/01634_uuid_fuzz.sql diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 6669a369a9d..91a67799115 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -192,6 +192,10 @@ struct ConvertImpl else throw Exception("Unexpected UInt128 to big int conversion", ErrorCodes::NOT_IMPLEMENTED); } + else if constexpr (std::is_same_v != std::is_same_v) + { + throw Exception("Conversion between numeric types and UUID is not supported", ErrorCodes::NOT_IMPLEMENTED); + } else { if constexpr (IsDataTypeDecimal || IsDataTypeDecimal) diff --git a/tests/queries/0_stateless/00735_conditional.sql b/tests/queries/0_stateless/00735_conditional.sql index 0c272ac69f8..04439f4062e 100644 --- a/tests/queries/0_stateless/00735_conditional.sql +++ b/tests/queries/0_stateless/00735_conditional.sql @@ -14,7 +14,6 @@ SELECT toInt8(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt8(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -31,7 +30,6 @@ SELECT toInt16(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt16(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -48,7 +46,6 @@ SELECT toInt32(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt32(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -65,7 +62,6 @@ SELECT toInt64(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toInt64(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -82,7 +78,6 @@ SELECT toUInt8(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x) SELECT toUInt8(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt8(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt8(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt8(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt8(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -99,7 +94,6 @@ SELECT toUInt16(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt16(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt16(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt16(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt16(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt16(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -116,7 +110,6 @@ SELECT toUInt32(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt32(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt32(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt32(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt32(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt32(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -133,7 +126,6 @@ SELECT toUInt64(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x SELECT toUInt64(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toUInt64(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUInt64(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toUInt64(0) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt64(0) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toUInt64(0) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -150,7 +142,6 @@ SELECT toDate(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), SELECT toDate(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toDate('2000-01-01') AS x, toDateTime('2000-01-01 00:00:01', 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toDate(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT toDate(0) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -167,28 +158,10 @@ SELECT toDateTime(0, 'Europe/Moscow') AS x, toFloat32(1) AS y, ((x > y) ? x : y) SELECT toDateTime(0, 'Europe/Moscow') AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toDateTime('2000-01-01 00:00:00', 'Europe/Moscow') AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT toDateTime(0, 'Europe/Moscow') AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toDateTime(0, 'Europe/Moscow') AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT toDateTime(0, 'Europe/Moscow') AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT toUUID(0) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT toUUID(0) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT toUUID(0) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } - SELECT 'column vs value'; SELECT materialize(toInt8(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); @@ -203,7 +176,6 @@ SELECT materialize(toInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt8(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -220,7 +192,6 @@ SELECT materialize(toInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt16(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -237,7 +208,6 @@ SELECT materialize(toInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt32(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -254,7 +224,6 @@ SELECT materialize(toInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toInt64(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -271,7 +240,6 @@ SELECT materialize(toUInt8(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt8(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt8(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt8(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt8(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt8(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -288,7 +256,6 @@ SELECT materialize(toUInt16(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt16(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt16(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt16(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt16(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt16(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -305,7 +272,6 @@ SELECT materialize(toUInt32(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt32(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toUInt32(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt32(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt32(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt32(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -322,7 +288,6 @@ SELECT materialize(toUInt64(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, SELECT materialize(toUInt64(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUInt64(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toUInt64(0)) AS x, toDecimal32(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toDecimal64(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toUInt64(0)) AS x, toDecimal128(1, 0) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -339,7 +304,6 @@ SELECT materialize(toDate(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, t SELECT materialize(toDate(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toDate('2000-01-01')) AS x, toDateTime('2000-01-01 00:00:01', 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toDate(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } SELECT materialize(toDate(0)) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } @@ -356,24 +320,6 @@ SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toFloat32(1) AS y, ((x SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Europe/Moscow')) AS x, toDate('2000-01-02') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } SELECT materialize(toDateTime(0, 'Europe/Moscow')) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } - -SELECT materialize(toUUID(0)) AS x, toInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toUInt8(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toUInt16(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toUInt32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toUInt64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toFloat32(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toFloat64(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toDate(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 43 } -SELECT materialize(toUUID(0)) AS x, toDateTime(1, 'Europe/Moscow') AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toUUID(1) AS y, ((x > y) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -SELECT materialize(toUUID(0)) AS x, toDecimal32(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toDecimal64(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } -SELECT materialize(toUUID(0)) AS x, toDecimal128(1, 0) AS y, ((x = 0) ? x : y) AS z, toTypeName(x), toTypeName(y), toTypeName(z); -- { serverError 386 } diff --git a/tests/queries/0_stateless/01634_uuid_fuzz.reference b/tests/queries/0_stateless/01634_uuid_fuzz.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01634_uuid_fuzz.sql b/tests/queries/0_stateless/01634_uuid_fuzz.sql new file mode 100644 index 00000000000..62ca209f6f3 --- /dev/null +++ b/tests/queries/0_stateless/01634_uuid_fuzz.sql @@ -0,0 +1 @@ +SELECT toUUID(-1.1); -- { serverError 48 } From 3bbdacc66b7d86669f278a8f036e2edd19879565 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 01:35:54 +0300 Subject: [PATCH 157/291] Update test --- .../0_stateless/01516_create_table_primary_key.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/queries/0_stateless/01516_create_table_primary_key.sql b/tests/queries/0_stateless/01516_create_table_primary_key.sql index 0611612f89f..c0e9ce4a34d 100644 --- a/tests/queries/0_stateless/01516_create_table_primary_key.sql +++ b/tests/queries/0_stateless/01516_create_table_primary_key.sql @@ -18,10 +18,6 @@ ATTACH TABLE primary_key_test(v Int32) ENGINE=ReplacingMergeTree ORDER BY v PRIM SELECT * FROM primary_key_test FINAL; DROP TABLE primary_key_test; -CREATE TABLE primary_key_test(v Int32, PRIMARY KEY(v), PRIMARY KEY(v)) ENGINE=ReplacingMergeTree ORDER BY v; -- { clientError 36; } - -CREATE TABLE primary_key_test(v Int32, PRIMARY KEY(v)) ENGINE=ReplacingMergeTree ORDER BY v PRIMARY KEY(v); -- { clientError 36; } - CREATE TABLE primary_key_test(v1 Int32, v2 Int32, PRIMARY KEY(v1, v2)) ENGINE=ReplacingMergeTree ORDER BY (v1, v2); INSERT INTO primary_key_test VALUES (1, 1), (1, 1), (1, 1); DETACH TABLE primary_key_test; @@ -36,10 +32,6 @@ ATTACH TABLE primary_key_test(v1 Int32, v2 Int32) ENGINE=ReplacingMergeTree ORDE SELECT * FROM primary_key_test FINAL; DROP TABLE primary_key_test; -CREATE TABLE primary_key_test(v1 Int32, v2 Int32, PRIMARY KEY(v1, v2), PRIMARY KEY(v1, v2)) ENGINE=ReplacingMergeTree ORDER BY (v1, v2); -- { clientError 36; } - -CREATE TABLE primary_key_test(v1 Int32, v2 Int32, PRIMARY KEY(v1, v2)) ENGINE=ReplacingMergeTree ORDER BY (v1, v2) PRIMARY KEY(v1, v2); -- { clientError 36; } - CREATE TABLE primary_key_test(v1 Int64, v2 Int32, v3 String, PRIMARY KEY(v1, gcd(v1, v2))) ENGINE=ReplacingMergeTree ORDER BY v1; -- { serverError 36; } CREATE TABLE primary_key_test(v1 Int64, v2 Int32, v3 String, PRIMARY KEY(v1, gcd(v1, v2))) ENGINE=ReplacingMergeTree ORDER BY (v1, gcd(v1, v2)); From 535889003987145bc366f874e486540daa24ec53 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 02:49:31 +0300 Subject: [PATCH 158/291] Remove bad code in HashJoin --- src/Interpreters/HashJoin.cpp | 22 +++++++++++++++++++--- src/Interpreters/HashJoin.h | 8 ++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 8f74a7be493..7c472a27d88 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -357,10 +357,14 @@ void HashJoin::init(Type type_) joinDispatch(kind, strictness, data->maps, [&](auto, auto, auto & map) { map.create(data->type); }); } -size_t HashJoin::getTotalRowCount() const +bool HashJoin::overDictionary() const { - std::shared_lock lock(data->rwlock); - return getTotalRowCountLocked(); + return data->type == Type::DICT; +} + +bool HashJoin::empty() const +{ + return data->type == Type::EMPTY; } size_t HashJoin::getTotalByteCount() const @@ -369,6 +373,18 @@ size_t HashJoin::getTotalByteCount() const return getTotalByteCountLocked(); } +size_t HashJoin::getTotalRowCount() const +{ + std::shared_lock lock(data->rwlock); + return getTotalRowCountLocked(); +} + +bool HashJoin::alwaysReturnsEmptySet() const +{ + std::shared_lock lock(data->rwlock); + return isInnerOrRight(getKind()) && data->empty && !overDictionary(); +} + size_t HashJoin::getTotalRowCountLocked() const { size_t res = 0; diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 37bc9a9d345..c14ad4e5a99 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -150,9 +150,6 @@ class HashJoin : public IJoin public: HashJoin(std::shared_ptr table_join_, const Block & right_sample_block, bool any_take_last_row_ = false); - bool empty() const { return data->type == Type::EMPTY; } - bool overDictionary() const { return data->type == Type::DICT; } - /** Add block of data from right hand of JOIN to the map. * Returns false, if some limit was exceeded and you should not insert more data. */ @@ -188,7 +185,7 @@ public: /// Sum size in bytes of all buffers, used for JOIN maps and for all memory pools. size_t getTotalByteCount() const final; - bool alwaysReturnsEmptySet() const final { return isInnerOrRight(getKind()) && data->empty && !overDictionary(); } + bool alwaysReturnsEmptySet() const final; ASTTableJoin::Kind getKind() const { return kind; } ASTTableJoin::Strictness getStrictness() const { return strictness; } @@ -397,6 +394,9 @@ private: /// Call with already locked rwlock. size_t getTotalRowCountLocked() const; size_t getTotalByteCountLocked() const; + + bool empty() const; + bool overDictionary() const; }; } From 06405531a1b5118ef20112dfd7044599cc720aa2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 02:58:16 +0300 Subject: [PATCH 159/291] Update tests --- .../0_stateless/00306_insert_values_and_expressions.sql | 2 +- tests/queries/0_stateless/00735_conditional.reference | 2 -- .../queries/0_stateless/01322_cast_keep_nullable.reference | 1 - tests/queries/0_stateless/01322_cast_keep_nullable.sql | 1 - .../01554_bloom_filter_index_big_integer_uuid.reference | 6 +++--- .../01554_bloom_filter_index_big_integer_uuid.sql | 6 +++--- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/00306_insert_values_and_expressions.sql b/tests/queries/0_stateless/00306_insert_values_and_expressions.sql index 10a1415f287..01a66282bb8 100644 --- a/tests/queries/0_stateless/00306_insert_values_and_expressions.sql +++ b/tests/queries/0_stateless/00306_insert_values_and_expressions.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS insert; CREATE TABLE insert (i UInt64, s String, u UUID, d Date, t DateTime, a Array(UInt32)) ENGINE = Memory; -INSERT INTO insert VALUES (1, 'Hello', 'ab41bdd6-5cd4-11e7-907b-a6006ad3dba0', '2016-01-01', '2016-01-02 03:04:05', [1, 2, 3]), (1 + 1, concat('Hello', ', world'), toUUID(0), toDate('2016-01-01') + 1, toStartOfMinute(toDateTime('2016-01-02 03:04:05')), [[0,1],[2]][1]), (round(pi()), concat('hello', ', world!'), toUUID(toString('ab41bdd6-5cd4-11e7-907b-a6006ad3dba0')), toDate(toDateTime('2016-01-03 03:04:05')), toStartOfHour(toDateTime('2016-01-02 03:04:05')), []), (4, 'World', 'ab41bdd6-5cd4-11e7-907b-a6006ad3dba0', '2016-01-04', '2016-12-11 10:09:08', [3,2,1]); +INSERT INTO insert VALUES (1, 'Hello', 'ab41bdd6-5cd4-11e7-907b-a6006ad3dba0', '2016-01-01', '2016-01-02 03:04:05', [1, 2, 3]), (1 + 1, concat('Hello', ', world'), toUUID('00000000-0000-0000-0000-000000000000'), toDate('2016-01-01') + 1, toStartOfMinute(toDateTime('2016-01-02 03:04:05')), [[0,1],[2]][1]), (round(pi()), concat('hello', ', world!'), toUUID(toString('ab41bdd6-5cd4-11e7-907b-a6006ad3dba0')), toDate(toDateTime('2016-01-03 03:04:05')), toStartOfHour(toDateTime('2016-01-02 03:04:05')), []), (4, 'World', 'ab41bdd6-5cd4-11e7-907b-a6006ad3dba0', '2016-01-04', '2016-12-11 10:09:08', [3,2,1]); SELECT * FROM insert ORDER BY i; DROP TABLE insert; diff --git a/tests/queries/0_stateless/00735_conditional.reference b/tests/queries/0_stateless/00735_conditional.reference index b37fce430ec..6bee974769d 100644 --- a/tests/queries/0_stateless/00735_conditional.reference +++ b/tests/queries/0_stateless/00735_conditional.reference @@ -95,7 +95,6 @@ value vs value 2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime 2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime 1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') -00000000-0000-0000-0000-000000000000 00000000-0000-0001-0000-000000000000 00000000-0000-0001-0000-000000000000 UUID UUID UUID column vs value 0 1 1 Int8 Int8 Int8 0 1 1 Int8 Int16 Int16 @@ -169,4 +168,3 @@ column vs value 2000-01-01 2000-01-01 00:00:01 2000-01-01 00:00:01 Date DateTime(\'Europe/Moscow\') DateTime 2000-01-01 00:00:00 2000-01-02 2000-01-02 00:00:00 DateTime(\'Europe/Moscow\') Date DateTime 1970-01-01 03:00:00 1970-01-01 03:00:01 1970-01-01 03:00:01 DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') DateTime(\'Europe/Moscow\') -00000000-0000-0000-0000-000000000000 00000000-0000-0001-0000-000000000000 00000000-0000-0001-0000-000000000000 UUID UUID UUID diff --git a/tests/queries/0_stateless/01322_cast_keep_nullable.reference b/tests/queries/0_stateless/01322_cast_keep_nullable.reference index cfe80a7c1b1..8ad99a10170 100644 --- a/tests/queries/0_stateless/01322_cast_keep_nullable.reference +++ b/tests/queries/0_stateless/01322_cast_keep_nullable.reference @@ -4,7 +4,6 @@ 1 Nullable(Int32) 2 Nullable(Float32) 2 Nullable(UInt8) -00000000-0000-0002-0000-000000000000 Nullable(UUID) 3 Nullable(Int32) \N Nullable(Int32) 42 Nullable(Int32) diff --git a/tests/queries/0_stateless/01322_cast_keep_nullable.sql b/tests/queries/0_stateless/01322_cast_keep_nullable.sql index ef5f5747a7d..2f6fd80c35f 100644 --- a/tests/queries/0_stateless/01322_cast_keep_nullable.sql +++ b/tests/queries/0_stateless/01322_cast_keep_nullable.sql @@ -10,7 +10,6 @@ SELECT CAST(toNullable(toInt8(1)) AS Int32) as x, toTypeName(x); SELECT CAST(toNullable(toFloat32(2)), 'Float32') as x, toTypeName(x); SELECT CAST(toNullable(toFloat32(2)), 'UInt8') as x, toTypeName(x); -SELECT CAST(toNullable(toFloat32(2)), 'UUID') as x, toTypeName(x); SELECT CAST(if(1 = 1, toNullable(toInt8(3)), NULL) AS Int32) as x, toTypeName(x); SELECT CAST(if(1 = 0, toNullable(toInt8(3)), NULL) AS Int32) as x, toTypeName(x); diff --git a/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.reference b/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.reference index 39beae69d65..f3b92461df7 100644 --- a/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.reference +++ b/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.reference @@ -7,6 +7,6 @@ 1 1 2 -00000000-0000-0001-0000-000000000000 -00000000-0000-0001-0000-000000000000 -00000000-0000-0002-0000-000000000000 +00000000-0000-0000-0000-000000000001 +00000000-0000-0000-0000-000000000001 +00000000-0000-0000-0000-000000000002 diff --git a/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.sql b/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.sql index 265f931c556..3472f41092d 100644 --- a/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.sql +++ b/tests/queries/0_stateless/01554_bloom_filter_index_big_integer_uuid.sql @@ -19,7 +19,7 @@ SELECT x FROM 01154_test WHERE x IN (1, 2); DROP TABLE 01154_test; CREATE TABLE 01154_test (x UUID, INDEX ix_x x TYPE bloom_filter(0.01) GRANULARITY 1) ENGINE = MergeTree() ORDER BY x SETTINGS index_granularity=8192; -INSERT INTO 01154_test VALUES (toUUID(1)), (toUUID(2)), (toUUID(3)); -SELECT x FROM 01154_test WHERE x = toUUID(1); -SELECT x FROM 01154_test WHERE x IN (toUUID(1), toUUID(2)); +INSERT INTO 01154_test VALUES (toUUID('00000000-0000-0000-0000-000000000001')), (toUUID('00000000-0000-0000-0000-000000000002')), (toUUID('00000000-0000-0000-0000-000000000003')); +SELECT x FROM 01154_test WHERE x = toUUID('00000000-0000-0000-0000-000000000001'); +SELECT x FROM 01154_test WHERE x IN (toUUID('00000000-0000-0000-0000-000000000001'), toUUID('00000000-0000-0000-0000-000000000002')); DROP TABLE 01154_test; From 6ecf4fe921934aa79fe77f54d25f1295cce2bf1b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 03:02:40 +0300 Subject: [PATCH 160/291] More robust stateful test --- docker/test/stateful/run.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 91c040ce6ad..f2fcefd604f 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -37,7 +37,15 @@ chmod 777 -R /var/lib/clickhouse clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary" clickhouse-client --query "CREATE DATABASE test" -service clickhouse-server restart && sleep 5 + +service clickhouse-server restart + +# Wait for server to start accepting connections +for _ in {1..120}; do + clickhouse-client --query "SELECT 1" && break + sleep 1 +done + clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "SHOW TABLES FROM test" clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" From 21467cf97a1b40522ca6de1c02307fedd8f243c4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 03:44:14 +0300 Subject: [PATCH 161/291] Added a test from #15641 --- .../0_stateless/01635_nullable_fuzz.reference | 0 .../0_stateless/01635_nullable_fuzz.sql | 21 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/01635_nullable_fuzz.reference create mode 100644 tests/queries/0_stateless/01635_nullable_fuzz.sql diff --git a/tests/queries/0_stateless/01635_nullable_fuzz.reference b/tests/queries/0_stateless/01635_nullable_fuzz.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01635_nullable_fuzz.sql b/tests/queries/0_stateless/01635_nullable_fuzz.sql new file mode 100644 index 00000000000..c134578b221 --- /dev/null +++ b/tests/queries/0_stateless/01635_nullable_fuzz.sql @@ -0,0 +1,21 @@ +SELECT + 'Nul\0able\0String)Nul\0\0ble(String)Nul\0able(String)Nul\0able(String)', + NULL AND 2, + '', + number, + NULL AS k +FROM +( + SELECT + materialize(NULL) OR materialize(-9223372036854775808), + number + FROM system.numbers + LIMIT 1000000 +) +ORDER BY + k ASC, + number ASC, + k ASC +LIMIT 1023, 1023 +SETTINGS max_bytes_before_external_sort = 1000000 +FORMAT Null; From f6dff13db8545967475f52416d01946d0d660269 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 03:56:56 +0300 Subject: [PATCH 162/291] Add a test for #14974 --- .../01636_nullable_fuzz2.reference | 6 ++++ .../0_stateless/01636_nullable_fuzz2.sql | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/queries/0_stateless/01636_nullable_fuzz2.reference create mode 100644 tests/queries/0_stateless/01636_nullable_fuzz2.sql diff --git a/tests/queries/0_stateless/01636_nullable_fuzz2.reference b/tests/queries/0_stateless/01636_nullable_fuzz2.reference new file mode 100644 index 00000000000..b26b82b1cb9 --- /dev/null +++ b/tests/queries/0_stateless/01636_nullable_fuzz2.reference @@ -0,0 +1,6 @@ +2020-07-09 +2020-07-10 +2020-07-11 +2020-07-12 + +1970-01-01 diff --git a/tests/queries/0_stateless/01636_nullable_fuzz2.sql b/tests/queries/0_stateless/01636_nullable_fuzz2.sql new file mode 100644 index 00000000000..a40da51c38c --- /dev/null +++ b/tests/queries/0_stateless/01636_nullable_fuzz2.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS open_events_tmp; +DROP TABLE IF EXISTS tracking_events_tmp; + +CREATE TABLE open_events_tmp (`APIKey` UInt32, `EventDate` Date) ENGINE = MergeTree PARTITION BY toMonday(EventDate) ORDER BY (APIKey, EventDate); +CREATE TABLE tracking_events_tmp (`APIKey` UInt32, `EventDate` Date) ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (APIKey, EventDate); + +insert into open_events_tmp select 2, '2020-07-10' from numbers(32); +insert into open_events_tmp select 2, '2020-07-11' from numbers(31); +insert into open_events_tmp select 2, '2020-07-12' from numbers(30); + +insert into tracking_events_tmp select 2, '2020-07-09' from numbers(1555); +insert into tracking_events_tmp select 2, '2020-07-10' from numbers(1881); +insert into tracking_events_tmp select 2, '2020-07-11' from numbers(1623); + +SELECT EventDate +FROM +( + SELECT EventDate + FROM tracking_events_tmp AS t1 + WHERE (EventDate >= toDate('2020-07-09')) AND (EventDate <= toDate('2020-07-11')) AND (APIKey = 2) + GROUP BY EventDate +) +FULL OUTER JOIN +( + SELECT EventDate + FROM remote('127.0.0.{1,3}', currentDatabase(), open_events_tmp) AS t2 + WHERE (EventDate <= toDate('2020-07-12')) AND (APIKey = 2) + GROUP BY EventDate + WITH TOTALS +) USING (EventDate) +ORDER BY EventDate ASC +SETTINGS totals_mode = 'after_having_auto', group_by_overflow_mode = 'any', max_rows_to_group_by = 10000000, joined_subquery_requires_alias = 0; + +DROP TABLE open_events_tmp; +DROP TABLE tracking_events_tmp; From c2c1d97da26412f57b252c077a1879ae26fd9652 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 03:59:09 +0300 Subject: [PATCH 163/291] Arcadia does not support distributed queries --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index c1f7c14f58a..ec5acc3e4d2 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -179,3 +179,4 @@ 01558_ttest_scipy 01561_mann_whitney_scipy 01601_custom_tld +01636_nullable_fuzz2 From ab6798addd9de3b26e0eb801f051b87404f4617c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 04:03:38 +0300 Subject: [PATCH 164/291] One more test --- tests/queries/0_stateless/01637_nullable_fuzz3.reference | 2 ++ tests/queries/0_stateless/01637_nullable_fuzz3.sql | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/01637_nullable_fuzz3.reference create mode 100644 tests/queries/0_stateless/01637_nullable_fuzz3.sql diff --git a/tests/queries/0_stateless/01637_nullable_fuzz3.reference b/tests/queries/0_stateless/01637_nullable_fuzz3.reference new file mode 100644 index 00000000000..d9cf16b9d2a --- /dev/null +++ b/tests/queries/0_stateless/01637_nullable_fuzz3.reference @@ -0,0 +1,2 @@ + +0 diff --git a/tests/queries/0_stateless/01637_nullable_fuzz3.sql b/tests/queries/0_stateless/01637_nullable_fuzz3.sql new file mode 100644 index 00000000000..21bf8999eae --- /dev/null +++ b/tests/queries/0_stateless/01637_nullable_fuzz3.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (`item_id` UInt64, `price_sold` Float32, `date` Date) ENGINE = MergeTree ORDER BY item_id; +SELECT item_id FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) AS l FULL OUTER JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) AS r USING (item_id); +DROP TABLE t; From a1ce6c41659eb6f6a530b737fb913e0fc054adb7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 04:26:29 +0300 Subject: [PATCH 165/291] Fix errors --- src/AggregateFunctions/AggregateFunctionSumMap.h | 12 +++++++----- .../0_stateless/01422_map_skip_null.reference | 7 ++----- tests/queries/0_stateless/01422_map_skip_null.sql | 6 +++--- .../0_stateless/01621_summap_check_types.reference | 1 + .../queries/0_stateless/01621_summap_check_types.sql | 5 +++++ 5 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 tests/queries/0_stateless/01621_summap_check_types.reference create mode 100644 tests/queries/0_stateless/01621_summap_check_types.sql diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 8be25194dae..6c8155f967c 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -96,6 +96,11 @@ public: if constexpr (overflow) { + if (value_type->onlyNull()) + throw Exception{ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Cannot calculate {} of type {}", + getName(), value_type->getName()}; + // Overflow, meaning that the returned type is the same as // the input type. Nulls are skipped. result_type = removeNullable(value_type); @@ -190,11 +195,8 @@ public: { // Create a value array for this key Array new_values; - new_values.resize(values_types.size()); - for (size_t k = 0; k < new_values.size(); ++k) - { - new_values[k] = (k == col) ? value : values_types[k]->getDefault(); - } + new_values.resize(size); + new_values[col] = value; if constexpr (IsDecimalNumber) { diff --git a/tests/queries/0_stateless/01422_map_skip_null.reference b/tests/queries/0_stateless/01422_map_skip_null.reference index 7211e0ac75d..83456126278 100644 --- a/tests/queries/0_stateless/01422_map_skip_null.reference +++ b/tests/queries/0_stateless/01422_map_skip_null.reference @@ -1,7 +1,4 @@ -([],[]) -([],[]) -([],[]) -([2],[11]) -([2],[22]) +([1,2],[0,11]) +([1,2],[0,22]) ([2],[33]) ([2],[33]) diff --git a/tests/queries/0_stateless/01422_map_skip_null.sql b/tests/queries/0_stateless/01422_map_skip_null.sql index 9af46758289..683757a473b 100644 --- a/tests/queries/0_stateless/01422_map_skip_null.sql +++ b/tests/queries/0_stateless/01422_map_skip_null.sql @@ -1,7 +1,7 @@ -select minMap(arrayJoin([([1], [null]), ([1], [null])])); -select maxMap(arrayJoin([([1], [null]), ([1], [null])])); +select minMap(arrayJoin([([1], [null]), ([1], [null])])); -- { serverError 43 } +select maxMap(arrayJoin([([1], [null]), ([1], [null])])); -- { serverError 43 } select sumMap(arrayJoin([([1], [null]), ([1], [null])])); -- { serverError 43 } -select sumMapWithOverflow(arrayJoin([([1], [null]), ([1], [null])])); +select sumMapWithOverflow(arrayJoin([([1], [null]), ([1], [null])])); -- { serverError 43 } select minMap(arrayJoin([([1, 2], [null, 11]), ([1, 2], [null, 22])])); select maxMap(arrayJoin([([1, 2], [null, 11]), ([1, 2], [null, 22])])); diff --git a/tests/queries/0_stateless/01621_summap_check_types.reference b/tests/queries/0_stateless/01621_summap_check_types.reference new file mode 100644 index 00000000000..1aeb897d4d0 --- /dev/null +++ b/tests/queries/0_stateless/01621_summap_check_types.reference @@ -0,0 +1 @@ +([1,2],[1,2],[1,0]) diff --git a/tests/queries/0_stateless/01621_summap_check_types.sql b/tests/queries/0_stateless/01621_summap_check_types.sql new file mode 100644 index 00000000000..a950f3ea094 --- /dev/null +++ b/tests/queries/0_stateless/01621_summap_check_types.sql @@ -0,0 +1,5 @@ +select initializeAggregation('sumMap', [1, 2], [1, 2], [1, null]); + +CREATE TEMPORARY TABLE sum_map_overflow (events Array(UInt8), counts Array(UInt8)); +INSERT INTO sum_map_overflow VALUES ([1], [255]), ([1], [2]); +SELECT [NULL], sumMapWithOverflow(events, [NULL], [[(NULL)]], counts) FROM sum_map_overflow; -- { serverError 43 } From 5368b20a719740f7572d5df3b5cd58b6523a6c3a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 04:49:15 +0300 Subject: [PATCH 166/291] Fix error --- src/Common/FileChecker.h | 1 + src/Storages/StorageLog.cpp | 188 +++++++++++++++--------------- src/Storages/StorageStripeLog.cpp | 13 ++- src/Storages/StorageTinyLog.cpp | 14 ++- 4 files changed, 116 insertions(+), 100 deletions(-) diff --git a/src/Common/FileChecker.h b/src/Common/FileChecker.h index b540112b5c4..73e4470f231 100644 --- a/src/Common/FileChecker.h +++ b/src/Common/FileChecker.h @@ -18,6 +18,7 @@ public: void update(const String & full_file_path); void setEmpty(const String & full_file_path); void save() const; + bool empty() const { return map.empty(); } /// Check the files whose parameters are specified in sizes.json CheckResults check() const; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index f9e6685b908..06e9bb8a2d6 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -113,93 +113,6 @@ private: }; -class LogBlockOutputStream final : public IBlockOutputStream -{ -public: - explicit LogBlockOutputStream( - StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) - : storage(storage_) - , metadata_snapshot(metadata_snapshot_) - , lock(std::move(lock_)) - , marks_stream( - storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite)) - { - if (!lock) - throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); - } - - ~LogBlockOutputStream() override - { - try - { - if (!done) - { - /// Rollback partial writes. - streams.clear(); - storage.file_checker.repair(); - } - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - - Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } - void write(const Block & block) override; - void writeSuffix() override; - -private: - StorageLog & storage; - StorageMetadataPtr metadata_snapshot; - std::unique_lock lock; - bool done = false; - - struct Stream - { - Stream(const DiskPtr & disk, const String & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : - plain(disk->writeFile(data_path, max_compress_block_size, WriteMode::Append)), - compressed(*plain, std::move(codec), max_compress_block_size), - plain_offset(disk->getFileSize(data_path)) - { - } - - std::unique_ptr plain; - CompressedWriteBuffer compressed; - - size_t plain_offset; /// How many bytes were in the file at the time the LogBlockOutputStream was created. - - void finalize() - { - compressed.next(); - plain->next(); - } - }; - - using Mark = StorageLog::Mark; - using MarksForColumns = std::vector>; - - using FileStreams = std::map; - FileStreams streams; - - using WrittenStreams = std::set; - - std::unique_ptr marks_stream; /// Declared below `lock` to make the file open when rwlock is captured. - - using SerializeState = IDataType::SerializeBinaryBulkStatePtr; - using SerializeStates = std::map; - SerializeStates serialize_states; - - IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenStreams & written_streams); - - void writeData(const String & name, const IDataType & type, const IColumn & column, - MarksForColumns & out_marks, - WrittenStreams & written_streams); - - void writeMarks(MarksForColumns && marks); -}; - - Chunk LogSource::generate() { Block res; @@ -207,7 +120,7 @@ Chunk LogSource::generate() if (rows_read == rows_limit) return {}; - if (storage.disk->isDirectoryEmpty(storage.table_path)) + if (storage.file_checker.empty()) return {}; /// How many rows to read for the next block. @@ -284,6 +197,101 @@ void LogSource::readData(const String & name, const IDataType & type, IColumn & } +class LogBlockOutputStream final : public IBlockOutputStream +{ +public: + explicit LogBlockOutputStream( + StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_, std::unique_lock && lock_) + : storage(storage_) + , metadata_snapshot(metadata_snapshot_) + , lock(std::move(lock_)) + , marks_stream( + storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite)) + { + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + /// If there were no files, add info to rollback in case of error. + if (storage.file_checker.empty()) + { + for (const auto & file : storage.files) + storage.file_checker.setEmpty(file.second.data_file_path); + storage.file_checker.save(); + } + } + + ~LogBlockOutputStream() override + { + try + { + if (!done) + { + /// Rollback partial writes. + streams.clear(); + storage.file_checker.repair(); + } + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } + void write(const Block & block) override; + void writeSuffix() override; + +private: + StorageLog & storage; + StorageMetadataPtr metadata_snapshot; + std::unique_lock lock; + bool done = false; + + struct Stream + { + Stream(const DiskPtr & disk, const String & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : + plain(disk->writeFile(data_path, max_compress_block_size, WriteMode::Append)), + compressed(*plain, std::move(codec), max_compress_block_size), + plain_offset(disk->getFileSize(data_path)) + { + } + + std::unique_ptr plain; + CompressedWriteBuffer compressed; + + size_t plain_offset; /// How many bytes were in the file at the time the LogBlockOutputStream was created. + + void finalize() + { + compressed.next(); + plain->next(); + } + }; + + using Mark = StorageLog::Mark; + using MarksForColumns = std::vector>; + + using FileStreams = std::map; + FileStreams streams; + + using WrittenStreams = std::set; + + std::unique_ptr marks_stream; /// Declared below `lock` to make the file open when rwlock is captured. + + using SerializeState = IDataType::SerializeBinaryBulkStatePtr; + using SerializeStates = std::map; + SerializeStates serialize_states; + + IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenStreams & written_streams); + + void writeData(const String & name, const IDataType & type, const IColumn & column, + MarksForColumns & out_marks, + WrittenStreams & written_streams); + + void writeMarks(MarksForColumns && marks); +}; + + void LogBlockOutputStream::write(const Block & block) { metadata_snapshot->check(block, true); @@ -477,10 +485,6 @@ StorageLog::StorageLog( addFiles(column.name, *column.type); marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME; - - if (!attach) - for (const auto & file : files) - file_checker.setEmpty(file.second.data_file_path); } diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 0bf70d4e837..db4fbff78cd 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -98,6 +98,9 @@ public: protected: Chunk generate() override { + if (storage.file_checker.empty()) + return {}; + Block res; start(); @@ -170,6 +173,13 @@ public: { if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + if (storage.file_checker.empty()) + { + storage.file_checker.setEmpty(storage.table_path + "data.bin"); + storage.file_checker.setEmpty(storage.table_path + "index.mrk"); + storage.file_checker.save(); + } } ~StripeLogBlockOutputStream() override @@ -264,9 +274,6 @@ StorageStripeLog::StorageStripeLog( { /// create directories if they do not exist disk->createDirectories(table_path); - - file_checker.setEmpty(table_path + "data.bin"); - file_checker.setEmpty(table_path + "index.mrk"); } else { diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 4f3dcec4ddc..fe8a25ba13b 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -121,7 +121,7 @@ Chunk TinyLogSource::generate() { Block res; - if (is_finished || (!streams.empty() && streams.begin()->second->compressed.eof())) + if (is_finished || file_sizes.empty() || (!streams.empty() && streams.begin()->second->compressed.eof())) { /** Close the files (before destroying the object). * When many sources are created, but simultaneously reading only a few of them, @@ -196,6 +196,14 @@ public: { if (!lock) throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + /// If there were no files, add info to rollback in case of error. + if (storage.file_checker.empty()) + { + for (const auto & file : storage.files) + storage.file_checker.setEmpty(file.second.data_file_path); + storage.file_checker.save(); + } } ~TinyLogBlockOutputStream() override @@ -393,10 +401,6 @@ StorageTinyLog::StorageTinyLog( for (const auto & col : storage_metadata.getColumns().getAllPhysical()) addFiles(col.name, *col.type); - - if (!attach) - for (const auto & file : files) - file_checker.setEmpty(file.second.data_file_path); } From 0b65605c34a95ae3a21493ba77bf19fd909b6414 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 05:30:24 +0300 Subject: [PATCH 167/291] Mark some TestFlows as flaky --- tests/testflows/ldap/authentication/regression.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index ff004a998ca..b18e5a93b67 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -23,7 +23,9 @@ xfails = { "connection protocols/starttls with custom port": [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], "connection protocols/tls cipher suite": - [(Fail, "can't get it to work")] + [(Fail, "can't get it to work")], + "external user directory/user authentications/valid verification cooldown value ldap unavailable": + [(Fail, "flaky, ask Vitaly Zakaznikov, Telegram @vzakaznikov")] } @TestFeature From dab4719aac3077e41a7759798355c690580941e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 06:22:06 +0300 Subject: [PATCH 168/291] Remove some headers --- src/DataStreams/PushingToViewsBlockOutputStream.cpp | 3 +++ src/DataStreams/PushingToViewsBlockOutputStream.h | 8 +++++--- src/Interpreters/InterpreterInsertQuery.cpp | 1 + src/Processors/Merges/MergingSortedTransform.cpp | 1 - src/Storages/StorageMerge.cpp | 1 - 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 1252dd7f4de..35607bcb182 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include namespace DB diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index 18dc3ffd800..b97b3fe2061 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -1,14 +1,16 @@ #pragma once -#include #include -#include -#include +#include +#include +#include + namespace DB { class ReplicatedMergeTreeBlockOutputStream; +class Context; /** Writes data to the specified table and to all dependent materialized views. diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 2a193dc7249..742c9f6736f 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Merges/MergingSortedTransform.cpp b/src/Processors/Merges/MergingSortedTransform.cpp index e680304ccee..ec1bdc59683 100644 --- a/src/Processors/Merges/MergingSortedTransform.cpp +++ b/src/Processors/Merges/MergingSortedTransform.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 18534d36e71..bc156b22abd 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include From 38b3d2844345ca844b829220f75cf407123450bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jan 2021 06:26:53 +0300 Subject: [PATCH 169/291] Mark another flaky test --- tests/testflows/ldap/authentication/regression.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index b18e5a93b67..50677d78cb8 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -25,6 +25,8 @@ xfails = { "connection protocols/tls cipher suite": [(Fail, "can't get it to work")], "external user directory/user authentications/valid verification cooldown value ldap unavailable": + [(Fail, "flaky, ask Vitaly Zakaznikov, Telegram @vzakaznikov")], + "user authentications/rbac=True/verification cooldown/verification cooldown performance": [(Fail, "flaky, ask Vitaly Zakaznikov, Telegram @vzakaznikov")] } From 99335563d122ab3801ce15631f1f3c275d9180f4 Mon Sep 17 00:00:00 2001 From: spongedc Date: Tue, 5 Jan 2021 13:03:19 +0800 Subject: [PATCH 170/291] [For #18707] MySQL compatibility: support DIV and MOD operators --- src/Parsers/ExpressionListParsers.cpp | 2 + .../0_stateless/00977_int_div.reference | 40 +++++++++++++++++++ tests/queries/0_stateless/00977_int_div.sql | 6 +++ .../0_stateless/01412_mod_float.reference | 4 ++ tests/queries/0_stateless/01412_mod_float.sql | 4 ++ 5 files changed, 56 insertions(+) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index f50cf71f54d..afe85f069c7 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -18,6 +18,8 @@ const char * ParserMultiplicativeExpression::operators[] = "*", "multiply", "/", "divide", "%", "modulo", + "MOD", "modulo", + "DIV", "intDiv", nullptr }; diff --git a/tests/queries/0_stateless/00977_int_div.reference b/tests/queries/0_stateless/00977_int_div.reference index 39e135d8eed..e5882f4aaba 100644 --- a/tests/queries/0_stateless/00977_int_div.reference +++ b/tests/queries/0_stateless/00977_int_div.reference @@ -81,3 +81,43 @@ -1 -1 -1 +-1 +0 +0 +0 +0 +0 +0 +0 +0 +0 +-1 +-2 +-3 +-4 +-5 +-6 +-7 +-8 +-9 +-10 +-1 +-2 +-3 +-4 +-5 +-6 +-7 +-8 +-9 +-10 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 +-1 diff --git a/tests/queries/0_stateless/00977_int_div.sql b/tests/queries/0_stateless/00977_int_div.sql index 08085020ca5..4184475e3a0 100644 --- a/tests/queries/0_stateless/00977_int_div.sql +++ b/tests/queries/0_stateless/00977_int_div.sql @@ -23,3 +23,9 @@ SELECT intDiv(toInt64(number), -1) FROM numbers(1, 10); SELECT intDivOrZero(toInt64(number), -1) FROM numbers(1, 10); SELECT intDiv(number, -number) FROM numbers(1, 10); SELECT intDivOrZero(number, -number) FROM numbers(1, 10); + +SELECT -1 DIV number FROM numbers(1, 10); +SELECT toInt32(number) DIV -1 FROM numbers(1, 10); +SELECT toInt64(number) DIV -1 FROM numbers(1, 10); +SELECT number DIV -number FROM numbers(1, 10); +SELECT -1 DIV 0; -- { serverError 153 } diff --git a/tests/queries/0_stateless/01412_mod_float.reference b/tests/queries/0_stateless/01412_mod_float.reference index 77e3431f7a9..94f7f6d3af7 100644 --- a/tests/queries/0_stateless/01412_mod_float.reference +++ b/tests/queries/0_stateless/01412_mod_float.reference @@ -1,3 +1,7 @@ 1 -1 1 -1 0.125 -0.125 0.125 -0.125 1 -1 1 -1 +1 -1 1 -1 +0.125 -0.125 0.125 -0.125 +nan +nan diff --git a/tests/queries/0_stateless/01412_mod_float.sql b/tests/queries/0_stateless/01412_mod_float.sql index 147cb88ff24..f2a5f2fcee6 100644 --- a/tests/queries/0_stateless/01412_mod_float.sql +++ b/tests/queries/0_stateless/01412_mod_float.sql @@ -1,3 +1,7 @@ WITH 8.5 AS a, 2.5 AS b SELECT a % b, -a % b, a % -b, -a % -b; WITH 10.125 AS a, 2.5 AS b SELECT a % b, -a % b, a % -b, -a % -b; WITH 8.5 AS a, 2.5 AS b SELECT mod(a, b), MOD(-a, b), modulo(a, -b), moduloOrZero(-a, -b); +WITH 8.5 AS a, 2.5 AS b SELECT a MOD b, -a MOD b, a MOD -b, -a MOD -b; +WITH 10.125 AS a, 2.5 AS b SELECT a MOD b, -a MOD b, a MOD -b, -a MOD -b; +SELECT 3.5 % 0; +SELECT 3.5 MOD 0; From 6c50f537a014ce744da22738d54bcd3999e60caa Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 5 Jan 2021 15:13:10 +0300 Subject: [PATCH 171/291] Add a regression test for marks corruption In #17120 (that was reverted in #17918 already) marks adjustemnt was introduced, but it may lead to corruption of marks files, that may cause two things: 1) Incorrect marks files, which lead to reading more rows than there are (can be reproduced with `max_threads`>1 or/and `PREWHERE`, and `optimize_move_to_prewhere`) 2) Can't adjust last granule because it has X rows, but try to subtract Y rows (#9260) And 1) is pretty hard to diagnosis, since it does not throw any error, it just may return wrong result. Fortunately both problems can be fixed with `OPTIMIZE TABLE ... [ FINAL]`. Cc: @alesapin Refs: #17943 Refs: #18223 --- ...1640_marks_corruption_regression.reference | 4 ++ .../01640_marks_corruption_regression.sql | 55 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/queries/0_stateless/01640_marks_corruption_regression.reference create mode 100644 tests/queries/0_stateless/01640_marks_corruption_regression.sql diff --git a/tests/queries/0_stateless/01640_marks_corruption_regression.reference b/tests/queries/0_stateless/01640_marks_corruption_regression.reference new file mode 100644 index 00000000000..a71b69fef89 --- /dev/null +++ b/tests/queries/0_stateless/01640_marks_corruption_regression.reference @@ -0,0 +1,4 @@ +"marks",7 +"optimize_trivial_count_query",16384 +"max_threads=1",16384 +"max_threads=100",16384 diff --git a/tests/queries/0_stateless/01640_marks_corruption_regression.sql b/tests/queries/0_stateless/01640_marks_corruption_regression.sql new file mode 100644 index 00000000000..63c180e518f --- /dev/null +++ b/tests/queries/0_stateless/01640_marks_corruption_regression.sql @@ -0,0 +1,55 @@ +DROP TABLE IF EXISTS adaptive_table; + +CREATE TABLE adaptive_table( + key UInt64, + value String +) ENGINE MergeTree() +ORDER BY key +SETTINGS + index_granularity_bytes=1048576, + min_bytes_for_wide_part=0, + old_parts_lifetime=0, + index_granularity=8192 +; + +-- This triggers adjustment of the granules that was introduced in PR#17120 +INSERT INTO adaptive_table SELECT number, randomPrintableASCII(if(number BETWEEN 8192-30 AND 8192, 102400, 1)) FROM system.numbers LIMIT 16384; +-- This creates the following marks: +-- +-- $ check-marks /path/to/db/adaptive_table/all_*/key.{mrk2,bin} +-- Mark 0, points to 0, 0, has rows after 8192, decompressed size 72808. + + + From 4f726784c92ae43389725d0981f96c58498078c3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Jan 2021 04:56:10 +0300 Subject: [PATCH 195/291] Fix shellcheck --- docker/test/fuzzer/run-fuzzer.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2cf97d683ac..0488cdce155 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -81,9 +81,9 @@ function fuzz echo Server started fuzzer_exit_code=0 - # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. - # They are all alphanumeric. - # shellcheck disable=SC2012 + # SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric. + # SC2046: Quote this to prevent word splitting. Actually I need word splitting. + # shellcheck disable=SC2012,SC2046 ./clickhouse-client --query-fuzzer-runs=1000 --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \ > >(tail -n 10000 > fuzzer.log) \ 2>&1 \ From 3f5b94fe3bfea38e628d9df16a697bc478dbb53e Mon Sep 17 00:00:00 2001 From: booknouse <34814088+booknouse@users.noreply.github.com> Date: Wed, 6 Jan 2021 15:12:28 +0800 Subject: [PATCH 196/291] Update style.md Update translation by machine, and improve document more human-readable. --- docs/zh/development/style.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md index 36e4acb6a24..8f104e3a7d8 100644 --- a/docs/zh/development/style.md +++ b/docs/zh/development/style.md @@ -267,7 +267,7 @@ void executeQuery( **9.** 多行注释的开头和结尾不得有空行（关闭多行注释的行除外）。 -**10.** 要注释掉代码，请使用基本注释，而不是«记录»注释。 +**10.** 要注释掉代码，请使用基本注释，而不是“文档”注释。 **11.** 在提交之前删除代码的无效注释部分。 @@ -335,7 +335,7 @@ template struct ExtractDomain ``` -**7.** 对于抽象类型（接口），用 `I` 前缀。 +**7.** 对于抽象类（接口），用 `I` 前缀。 ``` cpp class IBlockInputStream @@ -349,7 +349,7 @@ class IBlockInputStream bool info_successfully_loaded = false; ``` -**9.** `define` 和全局常量的名称使用带下划线的 `ALL_CAPS`。 +**9.** `define` 和全局常量的名称使用全大写带下划线的形式，如 `ALL_CAPS`。 ``` cpp #define MAX_SRC_TABLE_NAMES_TO_STORE 1000 @@ -357,14 +357,14 @@ bool info_successfully_loaded = false; **10.** 文件名应使用与其内容相同的样式。 -如果文件包含单个类，则以与该类名称相同的方式命名该文件。 +如果文件包含单个类，则以与该类名称相同的方式命名该文件（CamelCase）。 -如果文件包含单个函数，则以与函数名称相同的方式命名文件。 +如果文件包含单个函数，则以与函数名称相同的方式命名文件（camelCase）。 **11.** 如果名称包含缩写，则： - 对于变量名，缩写应使用小写字母 `mysql_connection`（不是 `mySQL_connection` ）。 -- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection` 。 +- 对于类和函数的名称，请将大写字母保留在缩写 `MySQLConnection`（不是 `MySqlConnection`）。 **12.** 仅用于初始化类成员的构造方法参数的命名方式应与类成员相同，但最后使用下划线。 @@ -411,7 +411,7 @@ enum class CompressionMethod 如果缩短版本是常用的，则可以接受不完整的单词。 -如果注释中旁边包含全名，您也可以使用缩写。 +如果旁边有注释包含全名，您也可以使用缩写。 **17.** C++ 源码文件名称必须为 `.cpp` 拓展名。 头文件必须为 `.h` 拓展名。 @@ -441,7 +441,7 @@ enum class CompressionMethod 在离线数据处理应用程序中，通常可以接受不捕获异常。 -在处理用户请求的服务器中，通常足以捕获连接处理程序顶层的异常。 +在处理用户请求的服务器中，捕获连接处理程序顶层的异常通常就足够了。 在线程函数中，你应该在 `join` 之后捕获并保留所有异常以在主线程中重新抛出它们。 @@ -548,7 +548,7 @@ Fork不用于并行化。 **10.** 常量。 -使用 const 引用，指向常量的指针，`const_iterator`和 const 指针。 +使用 const 引用、指针，指向常量、`const_iterator`和 const 方法。 将 `const` 视为默认值，仅在必要时使用非 `const`。 @@ -560,7 +560,7 @@ Fork不用于并行化。 **12.** 数值类型。 -使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32`， 以及 `Int64`， `size_t`， `ssize_t` 还有 `ptrdiff_t`。 +使用 `UInt8`， `UInt16`， `UInt32`， `UInt64`， `Int8`， `Int16`， `Int32` 和 `Int64`，同样还有 `size_t`， `ssize_t` 和 `ptrdiff_t`。 不要使用这些类型：`signed / unsigned long`，`long long`，`short`，`signed / unsigned char`，`char`。 @@ -732,11 +732,11 @@ CPU指令集是我们服务器中支持的最小集合。 目前，它是SSE 4.2 **8.** 尽可能经常地进行提交，即使代码只是部分准备好了。 -目的明确的功能，使用分支。 +为了这种目的可以创建分支。 -如果 `master` 分支中的代码尚不可构建，请在 `push` 之前将其从构建中排除。您需要在几天内完成或删除它。 +如果您的代码在 `master` 分支中尚不可构建，在 `push` 之前需要将其从构建中排除。您需要在几天内完成或删除它。 -**9.** 对于不重要的更改，请使用分支并在服务器上发布它们。 +**9.** 对于非一般的更改，请使用分支并在服务器上发布它们。 **10.** 未使用的代码将从 repo 中删除。 From 0260953a4785ac1daa6b3c17d6dc555ce59f8caf Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 6 Jan 2021 17:18:48 +0800 Subject: [PATCH 197/291] better --- src/Core/Settings.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 5 +++-- .../01632_max_partitions_to_read.reference | 12 ++++++------ .../0_stateless/01632_max_partitions_to_read.sql | 10 +++++----- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ba8f6563ec9..e2aa4e2f2d8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -141,7 +141,6 @@ class IColumn; \ M(Bool, force_index_by_date, 0, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \ M(Bool, force_primary_key, 0, "Throw an exception if there is primary key in a table, and it is not used.", 0) \ - M(Bool, force_max_partition_limit, 0, "Throw an exception if max_partitions_to_read is violated.", 0) \ M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \ \ M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \ @@ -354,6 +353,7 @@ class IColumn; M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ \ M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \ + M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ \ diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ada216b38b9..75760e145d3 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -707,8 +707,9 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( if (parts_with_ranges.empty()) return std::make_unique(); - auto max_partitions_to_read = data.getSettings()->max_partitions_to_read; - if (settings.force_max_partition_limit && max_partitions_to_read) + auto max_partitions_to_read + = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data.getSettings()->max_partitions_to_read; + if (max_partitions_to_read) { std::set partitions; for (auto & part_with_ranges : parts_with_ranges) diff --git a/tests/queries/0_stateless/01632_max_partitions_to_read.reference b/tests/queries/0_stateless/01632_max_partitions_to_read.reference index e24dfbd1231..ea2526e1301 100644 --- a/tests/queries/0_stateless/01632_max_partitions_to_read.reference +++ b/tests/queries/0_stateless/01632_max_partitions_to_read.reference @@ -1,6 +1,6 @@ -2021-01-03 1 2 -2021-01-04 3 4 -2021-01-03 1 2 -2021-01-04 3 4 -2021-01-03 1 2 -2021-01-04 3 4 +2021-01-01 1 2 +2021-01-02 4 5 +2021-01-01 1 2 +2021-01-02 4 5 +2021-01-01 1 2 +2021-01-02 4 5 diff --git a/tests/queries/0_stateless/01632_max_partitions_to_read.sql b/tests/queries/0_stateless/01632_max_partitions_to_read.sql index 8e0d2ba0ed6..b91405569bc 100644 --- a/tests/queries/0_stateless/01632_max_partitions_to_read.sql +++ b/tests/queries/0_stateless/01632_max_partitions_to_read.sql @@ -2,16 +2,16 @@ drop table if exists p; create table p(d Date, i int, j int) engine MergeTree partition by d order by i settings max_partitions_to_read = 1; -insert into p values (yesterday(), 1, 2), (today(), 3, 4); +insert into p values ('2021-01-01', 1, 2), ('2021-01-02', 4, 5); -select * from p order by i; -- default no limit +select * from p order by i; -- { serverError 565 } -select * from p order by i settings force_max_partition_limit = 0; +select * from p order by i settings max_partitions_to_read = 2; -select * from p order by i settings force_max_partition_limit = 1; -- { serverError 565 } +select * from p order by i settings max_partitions_to_read = 0; -- unlimited alter table p modify setting max_partitions_to_read = 2; -select * from p order by i settings force_max_partition_limit = 1; +select * from p order by i; drop table if exists p; From b2e4c0ee718b7840cb646c4ca0891f0e5e65799b Mon Sep 17 00:00:00 2001 From: weeds085490 <917923572@qq.com> Date: Wed, 6 Jan 2021 19:29:51 +0800 Subject: [PATCH 198/291] refine test --- .../01600_count_of_parts_metrics.reference | 1 + .../01600_count_of_parts_metrics.sh | 45 ++++++++++++++++ .../01600_count_of_parts_metrics.sql | 54 ------------------- 3 files changed, 46 insertions(+), 54 deletions(-) create mode 100755 tests/queries/0_stateless/01600_count_of_parts_metrics.sh delete mode 100644 tests/queries/0_stateless/01600_count_of_parts_metrics.sql diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.reference b/tests/queries/0_stateless/01600_count_of_parts_metrics.reference index e8183f05f5d..98fb6a68656 100644 --- a/tests/queries/0_stateless/01600_count_of_parts_metrics.reference +++ b/tests/queries/0_stateless/01600_count_of_parts_metrics.reference @@ -1,3 +1,4 @@ 1 1 1 +1 diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.sh b/tests/queries/0_stateless/01600_count_of_parts_metrics.sh new file mode 100755 index 00000000000..f55ee58ad15 --- /dev/null +++ b/tests/queries/0_stateless/01600_count_of_parts_metrics.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +verify_sql="SELECT COUNT(1) +FROM +(SELECT + SUM(IF(metric = 'Parts', value, 0)) AS Parts, + SUM(IF(metric = 'PartsActive', value, 0)) AS PartsActive, + SUM(IF(metric = 'PartsInactive', value, 0)) AS PartsInactive +FROM system.metrics) as a INNER JOIN +(SELECT + toInt64(SUM(1)) AS Parts, + toInt64(SUM(IF(active = 1, 1, 0))) AS PartsActive, + toInt64(SUM(IF(active = 0, 1, 0))) AS PartsInactive +FROM system.parts +) as b USING (Parts,PartsActive,PartsInactive)" + +verify(){ +for i in $(seq 1 10) +do +result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" ) +if [ "$result" = "1" ];then + echo 1 + return +fi +done +echo 0 +} + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_table" 2>/dev/null +$CLICKHOUSE_CLIENT --query="CREATE TABLE test_table(data Date) ENGINE = MergeTree PARTITION BY toYear(data) ORDER BY data;" 2>/dev/null + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_table VALUES ('1992-01-01')" 2>/dev/null +verify + +$CLICKHOUSE_CLIENT --query="INSERT INTO test_table VALUES ('1992-01-02')" 2>/dev/null +verify + +$CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE test_table FINAL" 2>/dev/null +verify + +$CLICKHOUSE_CLIENT --query="DROP TABLE test_table" 2>/dev/null +verify diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.sql b/tests/queries/0_stateless/01600_count_of_parts_metrics.sql deleted file mode 100644 index 31c8f75f3dc..00000000000 --- a/tests/queries/0_stateless/01600_count_of_parts_metrics.sql +++ /dev/null @@ -1,54 +0,0 @@ - -DROP TABLE IF EXISTS test_table; -CREATE TABLE test_table(data Date) ENGINE = MergeTree PARTITION BY toYear(data) ORDER BY data; - -INSERT INTO test_table VALUES ('1992-01-01'); - -SELECT COUNT(1) -FROM -(SELECT - SUM(IF(metric = 'Parts', value, 0)) AS Parts, - SUM(IF(metric = 'PartsActive', value, 0)) AS PartsActive, - SUM(IF(metric = 'PartsInactive', value, 0)) AS PartsInactive -FROM system.metrics) as a INNER JOIN -(SELECT - toInt64(SUM(1)) AS Parts, - toInt64(SUM(IF(active = 1, 1, 0))) AS PartsActive, - toInt64(SUM(IF(active = 0, 1, 0))) AS PartsInactive -FROM system.parts -) as b USING (Parts,PartsActive,PartsInactive); - -INSERT INTO test_table VALUES ('1992-01-02'); - -SELECT COUNT(1) -FROM -(SELECT - SUM(IF(metric = 'Parts', value, 0)) AS Parts, - SUM(IF(metric = 'PartsActive', value, 0)) AS PartsActive, - SUM(IF(metric = 'PartsInactive', value, 0)) AS PartsInactive -FROM system.metrics) as a INNER JOIN -(SELECT - toInt64(SUM(1)) AS Parts, - toInt64(SUM(IF(active = 1, 1, 0))) AS PartsActive, - toInt64(SUM(IF(active = 0, 1, 0))) AS PartsInactive -FROM system.parts -) as b USING (Parts,PartsActive,PartsInactive); - -OPTIMIZE TABLE test_table FINAL; - - -SELECT COUNT(1) -FROM -(SELECT - SUM(IF(metric = 'Parts', value, 0)) AS Parts, - SUM(IF(metric = 'PartsActive', value, 0)) AS PartsActive, - SUM(IF(metric = 'PartsInactive', value, 0)) AS PartsInactive -FROM system.metrics) as a INNER JOIN -(SELECT - toInt64(SUM(1)) AS Parts, - toInt64(SUM(IF(active = 1, 1, 0))) AS PartsActive, - toInt64(SUM(IF(active = 0, 1, 0))) AS PartsInactive -FROM system.parts -) as b USING (Parts,PartsActive,PartsInactive); - -DROP TABLE test_table; From a32f6dcf1b914e4e769902bf3b357595d66b0d10 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 6 Jan 2021 20:05:19 +0800 Subject: [PATCH 199/291] Apply alter settings on cluster to all replicas --- src/Interpreters/DDLWorker.cpp | 7 +++ src/Parsers/ASTAlterQuery.cpp | 16 ++++++ src/Parsers/ASTAlterQuery.h | 2 + .../__init__.py | 0 .../configs/config.d/clusters.xml | 17 ++++++ .../configs/config.d/distributed_ddl.xml | 5 ++ .../test_alter_settings_on_cluster/test.py | 53 +++++++++++++++++++ 7 files changed, 100 insertions(+) create mode 100644 tests/integration/test_alter_settings_on_cluster/__init__.py create mode 100644 tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml create mode 100644 tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml create mode 100644 tests/integration/test_alter_settings_on_cluster/test.py diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index cce62b1a6c4..e0b5591a708 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -847,6 +847,13 @@ bool DDLWorker::taskShouldBeExecutedOnLeader(const ASTPtr ast_ddl, const Storage if (!ast_ddl->as() && !ast_ddl->as() && !ast_ddl->as()) return false; + if (auto * alter = ast_ddl->as()) + { + // Setting alters should be executed on all replicas + if (alter->isSettingsAlter()) + return false; + } + return storage->supportsReplication(); } diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index b997164684b..8a44dcc7c3b 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -344,6 +344,22 @@ void ASTAlterCommand::formatImpl( throw Exception("Unexpected type of ALTER", ErrorCodes::UNEXPECTED_AST_STRUCTURE); } +bool ASTAlterQuery::isSettingsAlter() const +{ + if (command_list) + { + if (command_list->children.empty()) + return false; + for (const auto & child : command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::MODIFY_SETTING) + return false; + } + return true; + } + return false; +} /** Get the text that identifies this element. */ String ASTAlterQuery::getID(char delim) const diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 91c80867738..f53a987905e 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -187,6 +187,8 @@ public: ASTExpressionList * command_list = nullptr; + bool isSettingsAlter() const; + String getID(char) const override; ASTPtr clone() const override; diff --git a/tests/integration/test_alter_settings_on_cluster/__init__.py b/tests/integration/test_alter_settings_on_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml b/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml new file mode 100644 index 00000000000..26c9caa63db --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/configs/config.d/clusters.xml @@ -0,0 +1,17 @@ + + + + + true + + ch1 + 9000 + + + ch2 + 9000 + + + + + diff --git a/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml b/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml new file mode 100644 index 00000000000..6a88929c8ac --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/configs/config.d/distributed_ddl.xml @@ -0,0 +1,5 @@ + + + /clickhouse/task_queue/ddl + + diff --git a/tests/integration/test_alter_settings_on_cluster/test.py b/tests/integration/test_alter_settings_on_cluster/test.py new file mode 100644 index 00000000000..d56ef4461aa --- /dev/null +++ b/tests/integration/test_alter_settings_on_cluster/test.py @@ -0,0 +1,53 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) +ch2 = cluster.add_instance( + "ch2", + main_configs=[ + "configs/config.d/clusters.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + ch1.query("CREATE DATABASE test_default_database ON CLUSTER 'cluster';") + yield cluster + + finally: + cluster.shutdown() + + +def test_default_database_on_cluster(started_cluster): + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r1') ORDER BY tuple();", + ) + + ch2.query( + database="test_default_database", + sql="CREATE TABLE test_local_table (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_local_table', 'r2') ORDER BY tuple();", + ) + + ch1.query( + database="test_default_database", + sql="ALTER TABLE test_local_table ON CLUSTER 'cluster' MODIFY SETTING old_parts_lifetime = 100;", + ) + + for node in [ch1, ch2]: + assert node.query("SHOW CREATE test_local_table FORMAT TSV").endswith( + "old_parts_lifetime = 100\n" + ) From 38bc0f8348e1578c75336912f026e68bf3ff553b Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 6 Jan 2021 08:16:01 -0500 Subject: [PATCH 200/291] Updating `verification cooldown performance` and `valid verification cooldown value ldap unavailable` tests to fix fails due to timing issues caused by slow test executions. --- .../external_user_directory/tests/authentications.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 8229947adf7..ced54e95115 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -874,7 +874,7 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, "enable_tls": "no", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", - "verification_cooldown": "2" + "verification_cooldown": "300" }} self.context.ldap_node = self.context.cluster.node(server) @@ -898,11 +898,6 @@ def valid_verification_cooldown_value_ldap_unavailable(self, server, rbac=False, with Then("when I try to login again with the server offline it should work"): login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with And("when I sleep for 2 seconds and try to log in, it should fail"): - time.sleep(2) - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=error_exitcode, message=error_message) - finally: with Finally("I start the ldap server back up"): self.context.ldap_node.start() @@ -957,7 +952,7 @@ def repeat_requests(self, server, iterations, vcd_value, rbac=False): RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_VerificationCooldown_Performance("1.0") ) def verification_cooldown_performance(self, server, rbac=False, iterations=5000): - """Check that login performance is better when the verification cooldown + """Check login performance when the verification cooldown parameter is set to a positive value when comparing to the case when the verification cooldown parameter is turned off. """ @@ -973,9 +968,6 @@ def verification_cooldown_performance(self, server, rbac=False, iterations=5000) no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) - with Then("The performance with verification cooldown parameter set is better than the performance with no verification cooldown parameter."): - assert no_vcd_time > vcd_time, error() - with And("Log the performance improvement as a percentage."): metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) From 85003c8b5c2a7039999a636b891419dc1c97aeb6 Mon Sep 17 00:00:00 2001 From: Bertrand Junqua Date: Wed, 6 Jan 2021 16:36:51 +0100 Subject: [PATCH 201/291] Fix typo in array functions' documentation --- docs/en/sql-reference/functions/array-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index c4b7156ad95..ee4449d46fa 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1308,7 +1308,7 @@ Note that the `arraySum` is a [higher-order function](../../sql-reference/functi ## arrayAvg(\[func,\] arr1, …) {#array-avg} -Returns the sum of the `func` values. If the function is omitted, it just returns the average of the array elements. +Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements. Note that the `arrayAvg` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. From 22562273612b65e73abb0b027ac4f393d52b24ab Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Wed, 6 Jan 2021 11:21:31 +0800 Subject: [PATCH 202/291] SQL compatibility: provide POSITION(needle IN haystack) syntax. --- src/Parsers/ExpressionElementParsers.cpp | 16 ++++++++++++++++ .../00233_position_function_family.reference | 13 +++++++++++++ .../00233_position_function_family.sql | 15 +++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 7c82c4aca1e..8960803d936 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -343,6 +343,22 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) throw Exception("Argument of function toDate is unquoted: toDate(" + contents_str + "), must be: toDate('" + contents_str + "')" , ErrorCodes::SYNTAX_ERROR); } + else if (Poco::toLower(getIdentifierName(identifier)) == "position") + { + /// POSITION(needle IN haystack) is equivalent to function position(haystack, needle) + const auto & list = expr_list_args->as(); + if (list.children.size() == 1) + { + const auto & in_func = list.children[0]->as(); + if (in_func.name == "in") + { + // switch the two arguments + const auto & arg_list = in_func.arguments->as(); + expr_list_args->children = {arg_list.children[1], arg_list.children[0]}; + } + + } + } /// The parametric aggregate function has two lists (parameters and arguments) in parentheses. Example: quantile(0.9)(x). if (allow_function_parameters && pos->type == TokenType::OpeningRoundBracket) diff --git a/tests/queries/0_stateless/00233_position_function_family.reference b/tests/queries/0_stateless/00233_position_function_family.reference index 1523094261f..97f5276f200 100644 --- a/tests/queries/0_stateless/00233_position_function_family.reference +++ b/tests/queries/0_stateless/00233_position_function_family.reference @@ -23796,3 +23796,16 @@ 1 1 1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/00233_position_function_family.sql b/tests/queries/0_stateless/00233_position_function_family.sql index 8e443f5cd10..21e688c8f35 100644 --- a/tests/queries/0_stateless/00233_position_function_family.sql +++ b/tests/queries/0_stateless/00233_position_function_family.sql @@ -13,6 +13,14 @@ select 1 = position('a', '', 1); select 2 = position('a', '', 2); select 0 = position('a', '', 3); +select 1 = position('' in ''); +select 1 = position('' in 'abc'); +select 0 = position('abc' in ''); +select 1 = position('abc' in 'abc'); +select 2 = position('bc' in 'abc'); +select 3 = position('c' in 'abc'); +select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); + select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', '', number)) from numbers(10); select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), '', number)) from numbers(10); select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', materialize(''), number)) from numbers(10); @@ -54,6 +62,13 @@ select 0 = position('abcabc', 'b', 6); select 2 = position('abcabc', 'bca', 0); select 0 = position('abcabc', 'bca', 3); +select 1 = position('' in ''); +select 1 = position('' in 'абв'); +select 0 = position('абв' in ''); +select 1 = position('абв' in 'абв'); +select 3 = position('бв' in 'абв'); +select 5 = position('в' in 'абв'); + select 1 = position(materialize(''), ''); select 1 = position(materialize('абв'), ''); select 0 = position(materialize(''), 'абв'); From 4950377fbf7f3d36a5ebcb9f5ee99de32fc52b30 Mon Sep 17 00:00:00 2001 From: jianmei zhang Date: Wed, 6 Jan 2021 15:42:23 +0800 Subject: [PATCH 203/291] code changes according to reviewer's comments --- src/Parsers/ExpressionElementParsers.cpp | 20 +++++++++++-------- .../00233_position_function_family.reference | 13 ------------ .../00233_position_function_family.sql | 15 -------------- ...sition_function_sql_comparibilty.reference | 13 ++++++++++++ ...233_position_function_sql_comparibilty.sql | 16 +++++++++++++++ 5 files changed, 41 insertions(+), 36 deletions(-) create mode 100644 tests/queries/0_stateless/00233_position_function_sql_comparibilty.reference create mode 100644 tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 8960803d936..ff7c4d2d53e 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -346,17 +346,21 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else if (Poco::toLower(getIdentifierName(identifier)) == "position") { /// POSITION(needle IN haystack) is equivalent to function position(haystack, needle) - const auto & list = expr_list_args->as(); - if (list.children.size() == 1) + if (const auto * list = expr_list_args->as()) { - const auto & in_func = list.children[0]->as(); - if (in_func.name == "in") + if (list->children.size() == 1) { - // switch the two arguments - const auto & arg_list = in_func.arguments->as(); - expr_list_args->children = {arg_list.children[1], arg_list.children[0]}; + if (const auto * in_func = list->children[0]->as()) + { + if (in_func->name == "in") + { + // switch the two arguments + const auto & arg_list = in_func->arguments->as(); + if (arg_list.children.size() == 2) + expr_list_args->children = {arg_list.children[1], arg_list.children[0]}; + } + } } - } } diff --git a/tests/queries/0_stateless/00233_position_function_family.reference b/tests/queries/0_stateless/00233_position_function_family.reference index 97f5276f200..1523094261f 100644 --- a/tests/queries/0_stateless/00233_position_function_family.reference +++ b/tests/queries/0_stateless/00233_position_function_family.reference @@ -23796,16 +23796,3 @@ 1 1 1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/00233_position_function_family.sql b/tests/queries/0_stateless/00233_position_function_family.sql index 21e688c8f35..8e443f5cd10 100644 --- a/tests/queries/0_stateless/00233_position_function_family.sql +++ b/tests/queries/0_stateless/00233_position_function_family.sql @@ -13,14 +13,6 @@ select 1 = position('a', '', 1); select 2 = position('a', '', 2); select 0 = position('a', '', 3); -select 1 = position('' in ''); -select 1 = position('' in 'abc'); -select 0 = position('abc' in ''); -select 1 = position('abc' in 'abc'); -select 2 = position('bc' in 'abc'); -select 3 = position('c' in 'abc'); -select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); - select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', '', number)) from numbers(10); select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), '', number)) from numbers(10); select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', materialize(''), number)) from numbers(10); @@ -62,13 +54,6 @@ select 0 = position('abcabc', 'b', 6); select 2 = position('abcabc', 'bca', 0); select 0 = position('abcabc', 'bca', 3); -select 1 = position('' in ''); -select 1 = position('' in 'абв'); -select 0 = position('абв' in ''); -select 1 = position('абв' in 'абв'); -select 3 = position('бв' in 'абв'); -select 5 = position('в' in 'абв'); - select 1 = position(materialize(''), ''); select 1 = position(materialize('абв'), ''); select 0 = position(materialize(''), 'абв'); diff --git a/tests/queries/0_stateless/00233_position_function_sql_comparibilty.reference b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.reference new file mode 100644 index 00000000000..71c9a23879f --- /dev/null +++ b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.reference @@ -0,0 +1,13 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql new file mode 100644 index 00000000000..53b08de296a --- /dev/null +++ b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql @@ -0,0 +1,16 @@ +SET send_logs_level = 'fatal'; +select 1 = position('' in ''); +select 1 = position('' in 'abc'); +select 0 = position('abc' in ''); +select 1 = position('abc' in 'abc'); +select 2 = position('bc' in 'abc'); +select 3 = position('c' in 'abc'); + +select 1 = position('' in ''); +select 1 = position('' in 'абв'); +select 0 = position('абв' in ''); +select 1 = position('абв' in 'абв'); +select 3 = position('бв' in 'абв'); +select 5 = position('в' in 'абв'); + +select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); \ No newline at end of file From 8ab40d2747435a42c58f5fae8869a4c4491ecc2e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 6 Jan 2021 21:06:34 +0300 Subject: [PATCH 204/291] Newline --- .../0_stateless/00233_position_function_sql_comparibilty.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql index 53b08de296a..ae9409cd0c0 100644 --- a/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql +++ b/tests/queries/0_stateless/00233_position_function_sql_comparibilty.sql @@ -13,4 +13,4 @@ select 1 = position('абв' in 'абв'); select 3 = position('бв' in 'абв'); select 5 = position('в' in 'абв'); -select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); \ No newline at end of file +select 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); From 5b9876765229f744ddb5f401861a419bb6baa331 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Jan 2021 21:29:45 +0300 Subject: [PATCH 205/291] Update MergeTreeSettings.h --- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 98bce556fa7..97c6114a0be 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -110,7 +110,7 @@ struct Settings; M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \ M(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \ - M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited.", 0) \ + M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ From 6261d4135cab3cb0c368dad1293ba3cd8c1b153e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 6 Jan 2021 22:23:49 +0300 Subject: [PATCH 206/291] Add sanity checks for Sim/Min hash arguments Fixes: #18799 Fixes: #18524 --- src/Functions/FunctionsStringHash.h | 13 ++++++++++--- tests/queries/0_stateless/01016_simhash_minhash.sql | 4 ++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h index c09abc33319..907625ebaf5 100644 --- a/src/Functions/FunctionsStringHash.h +++ b/src/Functions/FunctionsStringHash.h @@ -18,6 +18,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int ARGUMENT_OUT_OF_BOUND; } // FunctionStringHash @@ -30,6 +31,8 @@ public: static constexpr auto name = Name::name; static constexpr size_t default_shingle_size = 3; static constexpr size_t default_num_hashes = 6; + static constexpr size_t max_shingle_size = 1000; + static constexpr size_t max_num_hashes = 1000; static FunctionPtr create(const Context &) { return std::make_shared(); } @@ -100,10 +103,14 @@ public: } if (shingle_size == 0) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument (shingle size) of function {} cannot be zero", getName()); - + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Second argument (shingle size) of function {} cannot be zero", getName()); if (num_hashes == 0) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Third argument (num hashes) of function {} cannot be zero", getName()); + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Third argument (num hashes) of function {} cannot be zero", getName()); + + if (shingle_size > max_shingle_size) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Second argument (shingle size) of function {} cannot be greater then {}", getName(), max_shingle_size); + if (num_hashes > max_num_hashes) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Third argument (num hashes) of function {} cannot be greater then {}", getName(), max_num_hashes); auto type = std::make_shared(); if constexpr (is_simhash) diff --git a/tests/queries/0_stateless/01016_simhash_minhash.sql b/tests/queries/0_stateless/01016_simhash_minhash.sql index 225bdd2bdc6..01af9451381 100644 --- a/tests/queries/0_stateless/01016_simhash_minhash.sql +++ b/tests/queries/0_stateless/01016_simhash_minhash.sql @@ -108,4 +108,8 @@ SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinH SELECT 'wordShingleMinHashCaseInsensitiveUTF8'; SELECT arrayStringConcat(groupArray(s), '\n:::::::\n'), count(), wordShingleMinHashCaseInsensitiveUTF8(s, 2, 3) as h FROM defaults GROUP BY h; +SELECT wordShingleSimHash('foobar', 9223372036854775807); -- { serverError 69 } +SELECT wordShingleSimHash('foobar', 1001); -- { serverError 69 } +SELECT wordShingleSimHash('foobar', 0); -- { serverError 69 } + DROP TABLE defaults; From 8e0cbea7387249d303d782a371b59e419d429295 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Jan 2021 22:49:48 +0300 Subject: [PATCH 207/291] Update brown-benchmark.md --- docs/en/getting-started/example-datasets/brown-benchmark.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/getting-started/example-datasets/brown-benchmark.md b/docs/en/getting-started/example-datasets/brown-benchmark.md index 80dbd0c1b6a..24b8f9e8eb0 100644 --- a/docs/en/getting-started/example-datasets/brown-benchmark.md +++ b/docs/en/getting-started/example-datasets/brown-benchmark.md @@ -410,3 +410,5 @@ GROUP BY yr, ORDER BY yr, mo; ``` + +The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==) From 058cdcc62c2e617a0198310ca54b4cb44e55f0ae Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Jan 2021 22:50:08 +0300 Subject: [PATCH 208/291] Update brown-benchmark.md --- docs/en/getting-started/example-datasets/brown-benchmark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/example-datasets/brown-benchmark.md b/docs/en/getting-started/example-datasets/brown-benchmark.md index 24b8f9e8eb0..b5ca23eddb9 100644 --- a/docs/en/getting-started/example-datasets/brown-benchmark.md +++ b/docs/en/getting-started/example-datasets/brown-benchmark.md @@ -411,4 +411,4 @@ ORDER BY yr, mo; ``` -The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==) +The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==). From af0a359ceab7dc48ce8257dfb0a8b087290753b9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Jan 2021 22:53:23 +0300 Subject: [PATCH 209/291] Update metrica.md --- docs/en/getting-started/example-datasets/metrica.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index cdbb9b56eeb..159c99b15a0 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -71,4 +71,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" [ClickHouse tutorial](../../getting-started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. -Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there). +Additional examples of queries to these tables can be found among [stateful tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/1_stateful) of ClickHouse (they are named `test.hits` and `test.visits` there). From f35edecebbd1427555dbb41e17606cab255eb0c2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 6 Jan 2021 23:27:00 +0300 Subject: [PATCH 210/291] Update ontime.md --- docs/en/getting-started/example-datasets/ontime.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index 5e499cafb2a..6e46cddba52 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -398,6 +398,8 @@ ORDER BY c DESC LIMIT 10; ``` +You can also play with the data in Playground, [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIERheU9mV2VlaywgY291bnQoKikgQVMgYwpGUk9NIG9udGltZQpXSEVSRSBZZWFyPj0yMDAwIEFORCBZZWFyPD0yMDA4CkdST1VQIEJZIERheU9mV2VlawpPUkRFUiBCWSBjIERFU0M7Cg==). + This performance test was created by Vadim Tkachenko. See: - https://www.percona.com/blog/2009/10/02/analyzing-air-traffic-performance-with-infobright-and-monetdb/ From 7e1d5b038723f54bb3d29ab5af5fac9da2e180a4 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 00:10:54 +0300 Subject: [PATCH 211/291] Update FunctionsStringHash.h --- src/Functions/FunctionsStringHash.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h index 907625ebaf5..72fb6e4c76d 100644 --- a/src/Functions/FunctionsStringHash.h +++ b/src/Functions/FunctionsStringHash.h @@ -31,8 +31,8 @@ public: static constexpr auto name = Name::name; static constexpr size_t default_shingle_size = 3; static constexpr size_t default_num_hashes = 6; - static constexpr size_t max_shingle_size = 1000; - static constexpr size_t max_num_hashes = 1000; + static constexpr size_t max_shingle_size = 25; + static constexpr size_t max_num_hashes = 25; static FunctionPtr create(const Context &) { return std::make_shared(); } From fdcfacda6033d8d1fe5236a7bf4218e4d2e7d3ee Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 01:55:01 +0300 Subject: [PATCH 212/291] Fix *If combinator with Nullable types sumIf(Nullable()) and similar unary functions (unary w/o If combinator) was working incorrectly, since it returns "sum" from the getName() helper, and so distributed query processing fails. The problem is in the optimization in AggregateFunctionIfNullUnary::add() for the unary functions. It pass only one column to write result to, instead of all passed arguments + result columns. While AggregateFunctionIf::add() assumes that it accepts arguments + result columns, and use last column as a result. Introduced-in: #16610 Fixes: #18210 --- .../AggregateFunctionIf.cpp | 30 +++++++++++++++---- .../01642_if_nullable_regression.reference | 5 ++++ .../01642_if_nullable_regression.sql | 7 +++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/01642_if_nullable_regression.reference create mode 100644 tests/queries/0_stateless/01642_if_nullable_regression.sql diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index 276abb90920..5e7e3844956 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -53,17 +53,35 @@ class AggregateFunctionIfNullUnary final private: size_t num_arguments; + /// The name of the nested function, including combinators (i.e. *If) + /// + /// getName() from the nested_function cannot be used because in case of *If combinator + /// with Nullable argument nested_function will point to the function w/o combinator. + /// (I.e. sumIf(Nullable, 1) -> sum()), and distributed query processing will fail. + /// + /// And nested_function cannot point to the function with *If since + /// due to optimization in the add() which pass only one column with the result, + /// and so AggregateFunctionIf::add() cannot be called this way + /// (it write to the last argument -- num_arguments-1). + /// + /// And to avoid extra level of indirection, the name of function is cached: + /// + /// AggregateFunctionIfNullUnary::add -> [ AggregateFunctionIf::add -> ] AggregateFunctionSum::add + String name; + using Base = AggregateFunctionNullBase>; public: String getName() const override { - return Base::getName(); + return name; } - AggregateFunctionIfNullUnary(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) - : Base(std::move(nested_function_), arguments, params), num_arguments(arguments.size()) + AggregateFunctionIfNullUnary(const String & name_, AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) + : Base(std::move(nested_function_), arguments, params) + , num_arguments(arguments.size()) + , name(name_) { if (num_arguments == 0) throw Exception("Aggregate function " + getName() + " require at least one argument", @@ -174,14 +192,14 @@ AggregateFunctionPtr AggregateFunctionIf::getOwnNullAdapter( { if (return_type_is_nullable) { - return std::make_shared>(nested_func, arguments, params); + return std::make_shared>(nested_function->getName(), nested_func, arguments, params); } else { if (serialize_flag) - return std::make_shared>(nested_func, arguments, params); + return std::make_shared>(nested_function->getName(), nested_func, arguments, params); else - return std::make_shared>(nested_func, arguments, params); + return std::make_shared>(nested_function->getName(), nested_func, arguments, params); } } else diff --git a/tests/queries/0_stateless/01642_if_nullable_regression.reference b/tests/queries/0_stateless/01642_if_nullable_regression.reference new file mode 100644 index 00000000000..66430601da2 --- /dev/null +++ b/tests/queries/0_stateless/01642_if_nullable_regression.reference @@ -0,0 +1,5 @@ +\N +\N +\N +0 +90 diff --git a/tests/queries/0_stateless/01642_if_nullable_regression.sql b/tests/queries/0_stateless/01642_if_nullable_regression.sql new file mode 100644 index 00000000000..9b307cf667a --- /dev/null +++ b/tests/queries/0_stateless/01642_if_nullable_regression.sql @@ -0,0 +1,7 @@ +SELECT sumIf(dummy, dummy) FROM remote('127.0.0.{1,2}', view(SELECT cast(Null AS Nullable(UInt8)) AS dummy FROM system.one)); +SELECT sumIf(dummy, 1) FROM remote('127.0.0.{1,2}', view(SELECT cast(Null AS Nullable(UInt8)) AS dummy FROM system.one)); +-- Before #16610 it returns 0 while with this patch it will return NULL +SELECT sumIf(dummy, dummy) FROM remote('127.0.0.{1,2}', view(SELECT cast(dummy AS Nullable(UInt8)) AS dummy FROM system.one)); +SELECT sumIf(dummy, 1) FROM remote('127.0.0.{1,2}', view(SELECT cast(dummy AS Nullable(UInt8)) AS dummy FROM system.one)); + +SELECT sumIf(n, 1) FROM remote('127.0.0.{1,2}', view(SELECT cast(* AS Nullable(UInt8)) AS n FROM system.numbers limit 10)) From d99133118dd5943f53824ab75d8d9c5f8cbef1ab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 04:03:53 +0300 Subject: [PATCH 213/291] Generate build id when ClickHouse is linked with lld --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7716fe82677..e77e3b3feb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -214,6 +214,12 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") endif () endif() +# Create BuildID when using lld. For other linkers it is created by default. +if (LINKER_NAME MATCHES "lld$") + # SHA1 is not cryptographically secure but it is the best what lld is offering. + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") +endif () + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd From 6f75901ffbde003b63051384c528f273b66f286a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 04:09:18 +0300 Subject: [PATCH 214/291] Update test --- tests/queries/0_stateless/01326_build_id.reference | 2 +- tests/queries/0_stateless/01326_build_id.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01326_build_id.reference b/tests/queries/0_stateless/01326_build_id.reference index b6a7d89c68e..d00491fd7e5 100644 --- a/tests/queries/0_stateless/01326_build_id.reference +++ b/tests/queries/0_stateless/01326_build_id.reference @@ -1 +1 @@ -16 +1 diff --git a/tests/queries/0_stateless/01326_build_id.sql b/tests/queries/0_stateless/01326_build_id.sql index f9bf2731374..4451ec360aa 100644 --- a/tests/queries/0_stateless/01326_build_id.sql +++ b/tests/queries/0_stateless/01326_build_id.sql @@ -1 +1 @@ -SELECT length(buildId()); +SELECT length(buildId()) >= 16; From 6f481d75129dfd2c573d865cf04cdad14a35b4dd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 05:56:57 +0300 Subject: [PATCH 215/291] Add integrity checks for ClickHouse binary --- CMakeLists.txt | 6 ++++++ base/daemon/BaseDaemon.cpp | 42 +++++++++++++++++++++++++++++++++++++ base/daemon/BaseDaemon.h | 4 ++++ programs/CMakeLists.txt | 4 ++++ programs/main.cpp | 10 +++++++++ programs/server/Server.cpp | 43 +++++++++++++++++++++++++++++++++++++- src/Common/Elf.cpp | 9 ++++++++ src/Common/Elf.h | 3 +++ 8 files changed, 120 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e77e3b3feb6..95f95464857 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,6 +220,12 @@ if (LINKER_NAME MATCHES "lld$") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1") endif () +# Add a section with the hash of the compiled machine code for integrity checks. +# Only for official builds, because adding a section can be time consuming (rewrite of several GB). +if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD) + set (USE_BINARY_HASH 1) +endif () + cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 1224d9171ea..2c20eccf914 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -56,6 +56,9 @@ #include #include #include +#include +#include +#include #if !defined(ARCADIA_BUILD) # include @@ -340,6 +343,32 @@ private: /// Write symbolized stack trace line by line for better grep-ability. stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); +#if defined(__linux__) + /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + if (daemon.stored_binary_hash.empty()) + { + LOG_FATAL(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == daemon.stored_binary_hash) + { + LOG_FATAL(log, "Checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + LOG_FATAL(log, "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file was changed just after startup;" + " - the file is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, daemon.stored_binary_hash); + } +#endif + /// Write crash to system.crash_log table if available. if (collectCrashLog) collectCrashLog(sig, thread_num, query_id, stack_trace); @@ -799,6 +828,13 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() #else build_id_info = "no build id"; #endif + +#if defined(__linux__) + std::string executable_path = getExecutablePath(); + + if (!executable_path.empty()) + stored_binary_hash = DB::Elf(executable_path).getBinaryHash(); +#endif } void BaseDaemon::logRevision() const @@ -1010,3 +1046,9 @@ void BaseDaemon::setupWatchdog() #endif } } + + +String BaseDaemon::getStoredBinaryHash() const +{ + return stored_binary_hash; +} diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 090d4997606..3c065970504 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -121,6 +121,9 @@ public: /// argv0 is needed to change process name (consequently, it is needed for scripts involving "pgrep", "pidof" to work correctly). void shouldSetupWatchdog(char * argv0_); + /// Hash of the binary for integrity checks. + String getStoredBinaryHash() const; + protected: virtual void logRevision() const; @@ -168,6 +171,7 @@ protected: Poco::Util::AbstractConfiguration * last_configuration = nullptr; String build_id_info; + String stored_binary_hash; std::vector handled_signals; diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a1b5467f234..9adca58b55a 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -318,6 +318,10 @@ else () if (USE_GDB_ADD_INDEX) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) endif() + + if (USE_BINARY_HASH) + add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .note.ClickHouse.hash=hash clickhouse COMMENT "Adding .note.ClickHouse.hash to clickhouse" VERBATIM) + endif() endif () if (ENABLE_TESTS AND USE_GTEST) diff --git a/programs/main.cpp b/programs/main.cpp index dee02c55832..cbb22b7a87b 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -18,6 +18,7 @@ #endif #include +#include #include #include @@ -62,6 +63,14 @@ int mainEntryClickHouseStatus(int argc, char ** argv); int mainEntryClickHouseRestart(int argc, char ** argv); #endif +int mainEntryClickHouseHashBinary(int, char **) +{ + /// Intentionally without newline. So you can run: + /// objcopy --add-section .note.ClickHouse.hash=<(./clickhouse hash-binary) clickhouse + std::cout << getHashOfLoadedBinaryHex(); + return 0; +} + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) namespace @@ -110,6 +119,7 @@ std::pair clickhouse_applications[] = {"status", mainEntryClickHouseStatus}, {"restart", mainEntryClickHouseRestart}, #endif + {"hash-binary", mainEntryClickHouseHashBinary}, }; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 76765c0374c..63caec25099 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -65,6 +65,8 @@ #include #include #include +#include +#include #include #include #include @@ -184,6 +186,7 @@ namespace ErrorCodes extern const int FAILED_TO_GETPWUID; extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; extern const int NETWORK_ERROR; + extern const int CORRUPTED_DATA; } @@ -436,7 +439,45 @@ int Server::main(const std::vector & /*args*/) #if defined(OS_LINUX) std::string executable_path = getExecutablePath(); - if (executable_path.empty()) + + if (!executable_path.empty()) + { + /// Integrity check based on checksum of the executable code. + /// Note: it is not intended to protect from malicious party, + /// because the reference checksum can be easily modified as well. + /// And we don't involve asymmetric encryption with PKI yet. + /// It's only intended to protect from faulty hardware. + /// Note: it is only based on machine code. + /// But there are other sections of the binary (e.g. exception handling tables) + /// that are interpreted (not executed) but can alter the behaviour of the program as well. + + String calculated_binary_hash = getHashOfLoadedBinaryHex(); + String stored_binary_hash = getStoredBinaryHash(); + + if (stored_binary_hash.empty()) + { + LOG_WARNING(log, "Calculated checksum of the binary: {}." + " There is no information about the reference checksum.", calculated_binary_hash); + } + else if (calculated_binary_hash == stored_binary_hash) + { + LOG_INFO(log, "Calculated checksum of the binary: {}, integrity check passed.", calculated_binary_hash); + } + else + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Calculated checksum of the ClickHouse binary ({0}) does not correspond" + " to the reference checksum stored in the binary ({1})." + " It may indicate one of the following:" + " - the file {2} was changed just after startup;" + " - the file {2} is damaged on disk due to faulty hardware;" + " - the loaded executable is damaged in memory due to faulty hardware;" + " - the file {2} was intentionally modified;" + " - logical error in code." + , calculated_binary_hash, stored_binary_hash, executable_path); + } + } + else executable_path = "/usr/bin/clickhouse"; /// It is used for information messages. /// After full config loaded diff --git a/src/Common/Elf.cpp b/src/Common/Elf.cpp index 0c2359b3418..ee78c988f69 100644 --- a/src/Common/Elf.cpp +++ b/src/Common/Elf.cpp @@ -151,6 +151,15 @@ String Elf::getBuildID(const char * nhdr_pos, size_t size) } +String Elf::getBinaryHash() const +{ + if (auto section = findSectionByName(".note.ClickHouse.hash")) + return {section->begin(), section->end()}; + else + return {}; +} + + const char * Elf::Section::name() const { if (!elf.section_names) diff --git a/src/Common/Elf.h b/src/Common/Elf.h index 632d7e6f0b1..90783ddc18d 100644 --- a/src/Common/Elf.h +++ b/src/Common/Elf.h @@ -59,6 +59,9 @@ public: String getBuildID() const; static String getBuildID(const char * nhdr_pos, size_t size); + /// Hash of the binary for integrity checks. + String getBinaryHash() const; + private: MMapReadBufferFromFile in; size_t elf_size; From 15113a5e5fe6deb8c84c25d2cc1d96f7f9772770 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 06:01:15 +0300 Subject: [PATCH 216/291] Add files --- src/Common/getHashOfLoadedBinary.cpp | 40 ++++++++++++++++++++++++++++ src/Common/getHashOfLoadedBinary.h | 15 +++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/Common/getHashOfLoadedBinary.cpp create mode 100644 src/Common/getHashOfLoadedBinary.h diff --git a/src/Common/getHashOfLoadedBinary.cpp b/src/Common/getHashOfLoadedBinary.cpp new file mode 100644 index 00000000000..b3f69a0a6bb --- /dev/null +++ b/src/Common/getHashOfLoadedBinary.cpp @@ -0,0 +1,40 @@ +#include +#include +#include +#include + + +static int callback(dl_phdr_info * info, size_t, void * data) +{ + SipHash & hash = *reinterpret_cast(data); + + for (size_t header_index = 0; header_index < info->dlpi_phnum; ++header_index) + { + const auto & phdr = info->dlpi_phdr[header_index]; + + if (phdr.p_type == PT_LOAD && (phdr.p_flags & PF_X)) + { + hash.update(phdr.p_filesz); + hash.update(reinterpret_cast(info->dlpi_addr + phdr.p_vaddr), phdr.p_filesz); + } + } + + return 1; /// Do not continue iterating. +} + + +SipHash getHashOfLoadedBinary() +{ + SipHash hash; + dl_iterate_phdr(callback, &hash); + return hash; +} + + +std::string getHashOfLoadedBinaryHex() +{ + SipHash hash = getHashOfLoadedBinary(); + std::array checksum; + hash.get128(checksum[0], checksum[1]); + return getHexUIntUppercase(checksum); +} diff --git a/src/Common/getHashOfLoadedBinary.h b/src/Common/getHashOfLoadedBinary.h new file mode 100644 index 00000000000..699a0da8b8f --- /dev/null +++ b/src/Common/getHashOfLoadedBinary.h @@ -0,0 +1,15 @@ +#include +#include + +/** Calculate hash of the executable loaded segments of the first loaded object. + * It can be used for integrity checks. + * Does not work when ClickHouse is build as multiple shared libraries. + * Note: we don't hash all loaded readonly segments, because some of them are modified by 'strip' + * and we want something that survives 'strip'. + * Note: program behaviour can be affected not only by machine code but also by the data in these segments, + * so the integrity check is going to be incomplete. + */ +SipHash getHashOfLoadedBinary(); +std::string getHashOfLoadedBinaryHex(); + + From 63f0348291eb899076bf22765fc6080a7840cd04 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 7 Jan 2021 11:31:23 +0800 Subject: [PATCH 217/291] Fix test --- tests/integration/test_alter_settings_on_cluster/test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_alter_settings_on_cluster/test.py b/tests/integration/test_alter_settings_on_cluster/test.py index d56ef4461aa..6ab3d446b59 100644 --- a/tests/integration/test_alter_settings_on_cluster/test.py +++ b/tests/integration/test_alter_settings_on_cluster/test.py @@ -48,6 +48,7 @@ def test_default_database_on_cluster(started_cluster): ) for node in [ch1, ch2]: - assert node.query("SHOW CREATE test_local_table FORMAT TSV").endswith( - "old_parts_lifetime = 100\n" - ) + assert node.query( + database="test_default_database", + sql="SHOW CREATE test_local_table FORMAT TSV", + ).endswith("old_parts_lifetime = 100\n") From 65b4008d1fa28094f67a0d1c70cb94331d29aa30 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 06:36:00 +0300 Subject: [PATCH 218/291] Update 01600_count_of_parts_metrics.sh --- tests/queries/0_stateless/01600_count_of_parts_metrics.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.sh b/tests/queries/0_stateless/01600_count_of_parts_metrics.sh index f55ee58ad15..703fe1703a6 100755 --- a/tests/queries/0_stateless/01600_count_of_parts_metrics.sh +++ b/tests/queries/0_stateless/01600_count_of_parts_metrics.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh verify_sql="SELECT COUNT(1) @@ -18,7 +19,7 @@ FROM system.parts ) as b USING (Parts,PartsActive,PartsInactive)" verify(){ -for i in $(seq 1 10) +for _ in $(seq 1 10) do result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" ) if [ "$result" = "1" ];then From 703e16db5afaf1f6220fd111d6b72749e2e08de3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 06:38:05 +0300 Subject: [PATCH 219/291] Update StorageMergeTree.cpp --- src/Storages/StorageMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 3310cfdf126..c959ee613a8 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -171,7 +171,7 @@ void StorageMergeTree::shutdown() { if (parts_info->state == DataPartState::Committed) { - committed_parts_count++; + ++committed_parts_count; } } From dd42f911291011ab64b62bf41194976540e557ec Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 06:40:24 +0300 Subject: [PATCH 220/291] Fix style --- src/Common/getHashOfLoadedBinary.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/getHashOfLoadedBinary.h b/src/Common/getHashOfLoadedBinary.h index 699a0da8b8f..3a4a22b1bf1 100644 --- a/src/Common/getHashOfLoadedBinary.h +++ b/src/Common/getHashOfLoadedBinary.h @@ -1,3 +1,5 @@ +#pragma once + #include #include @@ -11,5 +13,3 @@ */ SipHash getHashOfLoadedBinary(); std::string getHashOfLoadedBinaryHex(); - - From fe36a5582d53d4088e1bb0c58ca23b18f08bc41e Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:30:59 +0800 Subject: [PATCH 221/291] Update client-libraries.md --- docs/zh/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index e94eb8bcfc0..e2412f2b8de 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ Yandex**没有**维护下面列出的库，也没有做过任何广泛的测试 - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) From f96208ac7c3b461aa749666b47847502c8256805 Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:31:35 +0800 Subject: [PATCH 222/291] Update client-libraries.md --- docs/en/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index f3a6381aeca..c08eec61b1c 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) From c82602abffc0a4229cfcef7c9c4fd2175eaa64c8 Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:31:56 +0800 Subject: [PATCH 223/291] Update client-libraries.md --- docs/es/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/es/interfaces/third-party/client-libraries.md b/docs/es/interfaces/third-party/client-libraries.md index 818bdbbc6f0..b61ab1a5d9c 100644 --- a/docs/es/interfaces/third-party/client-libraries.md +++ b/docs/es/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: Client Libraries - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) From b5bc5d6798042dcb1be1fdfe83829b8d9550a85c Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:32:24 +0800 Subject: [PATCH 224/291] Update client-libraries.md --- docs/fr/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/fr/interfaces/third-party/client-libraries.md b/docs/fr/interfaces/third-party/client-libraries.md index 5a86d12a09c..7949aa1d7cf 100644 --- a/docs/fr/interfaces/third-party/client-libraries.md +++ b/docs/fr/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [clickhouse-chauffeur](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2 / phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov / clickhouse-PHP-client](https://packagist.org/packages/8bitov/clickhouse-php-client) From 7fce85f639fc46cc3b02cd3a2afa27d48e54b576 Mon Sep 17 00:00:00 2001 From: feng lv Date: Thu, 7 Jan 2021 06:32:33 +0000 Subject: [PATCH 225/291] replace PODArray with stack PODArray --- src/AggregateFunctions/AggregateFunctionWindowFunnel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index 96f0eb183be..77d99e55eb7 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -33,7 +33,7 @@ template struct AggregateFunctionWindowFunnelData { using TimestampEvent = std::pair; - using TimestampEvents = PODArray; + using TimestampEvents = PODArrayWithStackMemory; using Comparator = ComparePairFirst; bool sorted = true; From a347bb06006a15b54df984a5fdf890c0b90d339c Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:32:42 +0800 Subject: [PATCH 226/291] Update client-libraries.md --- docs/ja/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ja/interfaces/third-party/client-libraries.md b/docs/ja/interfaces/third-party/client-libraries.md index ffe7b641c38..c7bd368bc4c 100644 --- a/docs/ja/interfaces/third-party/client-libraries.md +++ b/docs/ja/interfaces/third-party/client-libraries.md @@ -15,6 +15,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" - [clickhouse-ドライバ](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-クライアント](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-クライアント](https://packagist.org/packages/8bitov/clickhouse-php-client) From 2de812af0f7e27afbc93c1bb54667808dbc15fde Mon Sep 17 00:00:00 2001 From: long2ice Date: Thu, 7 Jan 2021 14:32:59 +0800 Subject: [PATCH 227/291] Update client-libraries.md --- docs/ru/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index c07aab5826c..26e05b02509 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -13,6 +13,7 @@ toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u - [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) - [clickhouse-client](https://github.com/yurial/clickhouse-client) - [aiochclient](https://github.com/maximdanilchenko/aiochclient) + - [asynch](https://github.com/long2ice/asynch) - PHP - [smi2/phpclickhouse](https://packagist.org/packages/smi2/phpClickHouse) - [8bitov/clickhouse-php-client](https://packagist.org/packages/8bitov/clickhouse-php-client) From 36e1361cf8cb5195791414aa3831fae1c5c4875a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 15:29:34 +0300 Subject: [PATCH 228/291] Miscellaneous --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 104eedf060e..0b4297e37b3 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -840,7 +840,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ sync_guard.emplace(volume->getDisk(), to); if (!volume->getDisk()->exists(from)) - throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); + throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); if (volume->getDisk()->exists(to)) { From c920a57097a1117ff0d384752b93a13fc8375399 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 15:35:20 +0300 Subject: [PATCH 229/291] Update tests.md --- docs/en/development/tests.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 5b096bcc5fa..1cb8c0fdcdb 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -203,10 +203,10 @@ For production builds, gcc is used (it still generates slightly more efficient c We run functional and integration tests under ASan on per-commit basis. ### Valgrind (Memcheck) -We run functional tests under Valgrind overnight. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). +We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). ### Undefined behaviour sanitizer -We run functional and integration tests under ASan on per-commit basis. +We run functional and integration tests under USan on per-commit basis. ### Thread sanitizer We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis. @@ -233,6 +233,8 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. + ## Security Audit {#security-audit} People from Yandex Security Team do some basic overview of ClickHouse capabilities from the security standpoint. From c6c2536a2a274ee332c7300a53572b0345d4f7a9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 15:37:18 +0300 Subject: [PATCH 230/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 1cb8c0fdcdb..598ed5d1a42 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -269,7 +269,7 @@ These tests are automated by separate team. Due to high number of moving parts, ## Test Coverage {#test-coverage} -As of July 2018 we don’t track test coverage. +We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis. ## Test Automation {#test-automation} From fe96e81e7e258805ff6cd366564a591a517db8c2 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Thu, 7 Jan 2021 13:57:39 +0100 Subject: [PATCH 231/291] Fix errors add examples --- .../functions/array-functions.md | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index ee4449d46fa..dc7727bdfd8 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1290,22 +1290,65 @@ Note that the `arrayFirstIndex` is a [higher-order function](../../sql-reference ## arrayMin(\[func,\] arr1, …) {#array-min} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the min of the `func` values. If the function is omitted, it just returns the min of the array elements. Note that the `arrayMin` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMin([1, 2, 4]) AS res +┌─res─┐ +│ 1 │ +└─────┘ + + +SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -4 │ +└─────┘ +``` + ## arrayMax(\[func,\] arr1, …) {#array-max} -Returns the sum of the `func` values. If the function is omitted, it just returns the min of the array elements. +Returns the max of the `func` values. If the function is omitted, it just returns the max of the array elements. Note that the `arrayMax` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arrayMax([1, 2, 4]) AS res +┌─res─┐ +│ 4 │ +└─────┘ + + +SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res +┌─res─┐ +│ -1 │ +└─────┘ +``` + ## arraySum(\[func,\] arr1, …) {#array-sum} Returns the sum of the `func` values. If the function is omitted, it just returns the sum of the array elements. Note that the `arraySum` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. +Examples: +```sql +SELECT arraySum([2,3]) AS res +┌─res─┐ +│ 5 │ +└─────┘ + + +SELECT arraySum(x -> x*x, [2, 3]) AS res +┌─res─┐ +│ 13 │ +└─────┘ +``` + + ## arrayAvg(\[func,\] arr1, …) {#array-avg} Returns the average of the `func` values. If the function is omitted, it just returns the average of the array elements. From 10478811f972a8ffc3cca3418a8b573757377ca7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 15:59:05 +0300 Subject: [PATCH 232/291] Update tests.md --- docs/en/development/tests.md | 76 ++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 598ed5d1a42..8a416027818 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -88,7 +88,7 @@ You can also place pair of files `.sh` and `.reference` along with the tool to r ## Miscellaneous Tests {#miscellaneous-tests} -There are tests for external dictionaries located at `tests/external_dictionaries` and for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. +There are tests for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. There is separate test for quorum inserts. This test run ClickHouse cluster on separate servers and emulate various failure cases: network split, packet drop (between ClickHouse nodes, between ClickHouse and ZooKeeper, between ClickHouse server and client, etc.), `kill -9`, `kill -STOP` and `kill -CONT` , like [Jepsen](https://aphyr.com/tags/Jepsen). Then the test checks that all acknowledged inserts was written and all rejected inserts was not. @@ -185,38 +185,43 @@ For example, build with system packages is bad practice, because we cannot guara Though we cannot run all tests on all variant of builds, we want to check at least that various build variants are not broken. For this purpose we use build tests. +We also test that there are no translation units that are too long to compile or require too much RAM. + +We also test that there are no too large stack frames. + ## Testing for Protocol Compatibility {#testing-for-protocol-compatibility} When we extend ClickHouse network protocol, we test manually that old clickhouse-client works with new clickhouse-server and new clickhouse-client works with old clickhouse-server (simply by running binaries from corresponding packages). +We also test some cases automatically with integrational tests: +- if data written by old version of ClickHouse can be successfully read by the new version; +- do distributed queries work in a cluster with different ClickHouse versions. + ## Help from the Compiler {#help-from-the-compiler} Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall -Wextra -Werror` and with some additional enabled warnings. Although these options are not enabled for third-party libraries. Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build. -For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime. +For production builds, clang is used, but we also test make gcc builds. For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime. ## Sanitizers {#sanitizers} ### Address sanitizer -We run functional and integration tests under ASan on per-commit basis. +We run functional, integration, stress and unit tests under ASan on per-commit basis. + +### Thread sanitizer +We run functional, integration, stress and unit tests under TSan on per-commit basis. + +### Memory sanitizer +We run functional, integration, stress and unit tests under MSan on per-commit basis. + +### Undefined behaviour sanitizer +We run functional, integration, stress and unit tests under USan on per-commit basis. The code of some third-party libraries is not sanitized for UB. ### Valgrind (Memcheck) We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). -### Undefined behaviour sanitizer -We run functional and integration tests under USan on per-commit basis. - -### Thread sanitizer -We run functional tests under TSan on per-commit basis. We still don’t run integration tests under TSan on per-commit basis. - -### Memory sanitizer -Currently we still don’t use MSan. - -### Debug allocator -Debug version of `jemalloc` is used for debug build. - ## Fuzzing {#fuzzing} ClickHouse fuzzing is implemented both using [libFuzzer](https://llvm.org/docs/LibFuzzer.html) and random SQL queries. @@ -235,19 +240,50 @@ You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (o We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. +## Stress test + +Stress tests are another case of fuzzing. It runs all functional tests in parallel in random order with a single server. Results of the tests are not checked. + +It is checked that: +- server does not crash, no debug or sanitizer traps are triggered; +- there are no deadlocks; +- the database structure is consistent; +- server can successfully stop after the test and start again without exceptions. + +There are five variants (Debug, ASan, TSan, MSan, UBSan). + +## Thread Fuzzer + +Thread Fuzzer (please don't mix up with Thread Sanitizer) is another kind of fuzzing that allows to randomize thread order of execution. It helps to find even more special cases. + ## Security Audit {#security-audit} People from Yandex Security Team do some basic overview of ClickHouse capabilities from the security standpoint. ## Static Analyzers {#static-analyzers} -We run `PVS-Studio` on per-commit basis. We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). +We run `clang-tidy` and `PVS-Studio` on per-commit basis. `clang-static-analyzer` checks are also enabled. `clang-tidy` is also used for some style checks. + +We have evaluated `clang-tidy`, `Coverity`, `cppcheck`, `PVS-Studio`, `tscancode`, `CodeQL`. You will find instructions for usage in `tests/instructions/` directory. Also you can read [the article in russian](https://habr.com/company/yandex/blog/342018/). If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of the box. +We also use `shellcheck` for static analysis of shell scripts. + ## Hardening {#hardening} -`FORTIFY_SOURCE` is used by default. It is almost useless, but still makes sense in rare cases and we don’t disable it. +In debug build we are using custom allocator that does ASLR of user-level allocations. + +We also manually protect memory regions that are expected to be readonly after allocation. + +In debug build we also involve a customization of libc that ensures that no "harmful" (obsolete, insecure, not thread-safe) functions are called. + +Debug assertions are used extensively. + +In debug build, if exception with "logical error" code (implies a bug) is being thrown, the program is terminated prematurally. It allows to use exceptions in release build but make it an assertion in debug build. + +Debug version of jemalloc is used for debug builds. +Debug version of libc++ is used for debug builds. ## Code Style {#code-style} @@ -261,6 +297,8 @@ Alternatively you can try `uncrustify` tool to reformat your code. Configuration `CLion` has its own code formatter that has to be tuned for our code style. +We also use `codespell` to find typos in code. It is automated as well. + ## Metrica B2B Tests {#metrica-b2b-tests} Each ClickHouse release is tested with Yandex Metrica and AppMetrica engines. Testing and stable versions of ClickHouse are deployed on VMs and run with a small copy of Metrica engine that is processing fixed sample of input data. Then results of two instances of Metrica engine are compared together. @@ -271,6 +309,10 @@ These tests are automated by separate team. Due to high number of moving parts, We also track test coverage but only for functional tests and only for clickhouse-server. It is performed on daily basis. +## Tests for Tests + +There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky. + ## Test Automation {#test-automation} We run tests with Yandex internal CI and job automation system named “Sandbox”. From c4c10fddb22b832308d65523ac3d01900b159c0c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 15:59:59 +0300 Subject: [PATCH 233/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 8a416027818..ba591244472 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -317,7 +317,7 @@ There is automated check for flaky tests. It runs all new tests 100 times (for f We run tests with Yandex internal CI and job automation system named “Sandbox”. -Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored eternally. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. +Build jobs and tests are run in Sandbox on per commit basis. Resulting packages and test results are published in GitHub and can be downloaded by direct links. Artifacts are stored for several months. When you send a pull request on GitHub, we tag it as “can be tested” and our CI system will build ClickHouse packages (release, debug, with address sanitizer, etc) for you. We don’t use Travis CI due to the limit on time and computational power. We don’t use Jenkins. It was used before and now we are happy we are not using Jenkins. From 025ade6ef518eb3cb7c6c0e240efe88ab4630b5d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:00:54 +0300 Subject: [PATCH 234/291] Update tests.md --- docs/en/development/tests.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index ba591244472..cc7d266aaa3 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -84,8 +84,6 @@ If you want to improve performance of ClickHouse in some scenario, and if improv Some programs in `tests` directory are not prepared tests, but are test tools. For example, for `Lexer` there is a tool `src/Parsers/tests/lexer` that just do tokenization of stdin and writes colorized result to stdout. You can use these kind of tools as a code examples and for exploration and manual testing. -You can also place pair of files `.sh` and `.reference` along with the tool to run it on some predefined input - then script result can be compared to `.reference` file. These kind of tests are not automated. - ## Miscellaneous Tests {#miscellaneous-tests} There are tests for machine learned models in `tests/external_models`. These tests are not updated and must be transferred to integration tests. From 9de3a2f892900815e54fa378b335d69028a3e790 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:01:54 +0300 Subject: [PATCH 235/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index cc7d266aaa3..758e905a090 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -11,7 +11,7 @@ Functional tests are the most simple and convenient to use. Most of ClickHouse f Each functional test sends one or multiple queries to the running ClickHouse server and compares the result with reference. -Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and not available to general public. We tend to use only `stateless` tests and avoid adding new `stateful` tests. +Tests are located in `queries` directory. There are two subdirectories: `stateless` and `stateful`. Stateless tests run queries without any preloaded test data - they often create small synthetic datasets on the fly, within the test itself. Stateful tests require preloaded test data from Yandex.Metrica and it is available to general public. Each test can be one of two types: `.sql` and `.sh`. `.sql` test is the simple SQL script that is piped to `clickhouse-client --multiquery --testmode`. `.sh` test is a script that is run by itself. SQL tests are generally preferable to `.sh` tests. You should use `.sh` tests only when you have to test some feature that cannot be exercised from pure SQL, such as piping some input data into `clickhouse-client` or testing `clickhouse-local`. From d0c72f83cce76c32b9b88aef7df585813865250a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:04:41 +0300 Subject: [PATCH 236/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 758e905a090..46f9b0b10c7 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -236,7 +236,7 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). -We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. ## Stress test From 99fa3cbd3448de5b6df20635f6ae183448ca0f14 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:06:03 +0300 Subject: [PATCH 237/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 46f9b0b10c7..6c7620df1fb 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -215,7 +215,7 @@ We run functional, integration, stress and unit tests under TSan on per-commit b We run functional, integration, stress and unit tests under MSan on per-commit basis. ### Undefined behaviour sanitizer -We run functional, integration, stress and unit tests under USan on per-commit basis. The code of some third-party libraries is not sanitized for UB. +We run functional, integration, stress and unit tests under UBSan on per-commit basis. The code of some third-party libraries is not sanitized for UB. ### Valgrind (Memcheck) We used to run functional tests under Valgrind overnight, but don't do it anymore. It takes multiple hours. Currently there is one known false positive in `re2` library, see [this article](https://research.swtch.com/sparse). From 80bdc76a231a74c9f63fa09e87db64d3839ceeca Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:13:48 +0300 Subject: [PATCH 238/291] Update tests.md --- docs/en/development/tests.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 6c7620df1fb..21baab8312e 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -283,6 +283,14 @@ In debug build, if exception with "logical error" code (implies a bug) is being Debug version of jemalloc is used for debug builds. Debug version of libc++ is used for debug builds. +## Runtime Integrity Checks + +Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. + +It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). + +ClickHouse provides diagnostics that will help ops engineers to find faulty hardware. + ## Code Style {#code-style} Code style rules are described [here](style.md). From bb4adf0293171fe1a4889c9bc0eaaabbb447ff57 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:15:00 +0300 Subject: [PATCH 239/291] Update tests.md --- docs/en/development/tests.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 21baab8312e..d0eedcad9e2 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -285,12 +285,14 @@ Debug version of libc++ is used for debug builds. ## Runtime Integrity Checks -Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. +Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). ClickHouse provides diagnostics that will help ops engineers to find faulty hardware. +\* and it is not slow. + ## Code Style {#code-style} Code style rules are described [here](style.md). From 8d3e50ae1d1d1b99a33e5b68c65d2a22a2c23672 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:15:44 +0300 Subject: [PATCH 240/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index d0eedcad9e2..ee92a6d9467 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -287,7 +287,7 @@ Debug version of libc++ is used for debug builds. Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. -It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). +It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). ClickHouse provides diagnostics that will help ops engineers to find faulty hardware. From 263c0a940873a1bdeb8c65f10ccbd4306f9c7199 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:17:00 +0300 Subject: [PATCH 241/291] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index ee92a6d9467..0c7165916ed 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -287,7 +287,7 @@ Debug version of libc++ is used for debug builds. Data stored on disk is checksummed. Data in MergeTree tables is checksummed in three ways simultaneously* (compressed data blocks, uncompressed data blocks, the total checksum across blocks). Data transferred over network between client and server or between servers is also checksummed. Replication ensures bit-identical data on replicas. -It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). +It is required to protect from faulty hardware (bit rot on storage media, bit flips in RAM on server, bit flips in RAM of network controller, bit flips in RAM of network switch, bit flips in RAM of client, bit flips on the wire). Note that bit flips are common and likely to occur even for ECC RAM and in presense of TCP checksums (if you manage to run thousands of servers processing petabytes of data each day). [See the video (russian)](https://www.youtube.com/watch?v=ooBAQIe0KlQ). ClickHouse provides diagnostics that will help ops engineers to find faulty hardware. From c69931d352f68715b9bcb929534c785cb2c0523c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:22:28 +0300 Subject: [PATCH 242/291] Update tests.md --- docs/en/development/tests.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 0c7165916ed..47c2d7f7115 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -167,17 +167,14 @@ Precise query execution timings are not recorded and not compared due to high va ## Build Tests {#build-tests} -Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. Tests are located at `ci` directory. They run build from source inside Docker, Vagrant, and sometimes with `qemu-user-static` inside Docker. These tests are under development and test runs are not automated. +Build tests allow to check that build is not broken on various alternative configurations and on some foreign systems. These tests are automated as well. -Motivation: - -Normally we release and run all tests on a single variant of ClickHouse build. But there are alternative build variants that are not thoroughly tested. Examples: - -- build on FreeBSD -- build on Debian with libraries from system packages -- build with shared linking of libraries -- build on AArch64 platform -- build on PowerPc platform +Examples: +- cross-compile for Darwin x86_64 (Mac OS X) +- cross-compile for FreeBSD x86_64 +- cross-compile for Linux AArch64 +- build on Ubuntu with libraries from system packages (discouraged) +- build with shared linking of libraries (discouraged) For example, build with system packages is bad practice, because we cannot guarantee what exact version of packages a system will have. But this is really needed by Debian maintainers. For this reason we at least have to support this variant of build. Another example: shared linking is a common source of trouble, but it is needed for some enthusiasts. From 9b4160bbd0d66d9c06ae321c00e16438ef85ba4a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:31:50 +0300 Subject: [PATCH 243/291] Update tests.md --- docs/en/development/tests.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index 47c2d7f7115..fb453e55417 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -318,6 +318,14 @@ We also track test coverage but only for functional tests and only for clickhous There is automated check for flaky tests. It runs all new tests 100 times (for functional tests) or 10 times (for integration tests). If at least single time the test failed, it is considered flaky. +## Testflows + +[Testflows](https://testflows.com/) is an enterprise-grade testing framework. It is used by Altinity for some of the tests and we run these tests in our CI. + +## Yandex Checks (only for Yandex employees) + +These checks are importing ClickHouse code into Yandex internal monorepository, so ClickHouse codebase can be used as a library by other products at Yandex (YT and YDB). Note that clickhouse-server itself is not being build from internal repo and unmodified open-source build is used for Yandex applications. + ## Test Automation {#test-automation} We run tests with Yandex internal CI and job automation system named “Sandbox”. From cea1b7dba5ef20de832aba09afee8103fc08c163 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 16:37:06 +0300 Subject: [PATCH 244/291] Fix build --- programs/server/Server.cpp | 1 - src/Common/getHashOfLoadedBinary.cpp | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 63caec25099..2f8029fc39c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -452,7 +452,6 @@ int Server::main(const std::vector & /*args*/) /// that are interpreted (not executed) but can alter the behaviour of the program as well. String calculated_binary_hash = getHashOfLoadedBinaryHex(); - String stored_binary_hash = getStoredBinaryHash(); if (stored_binary_hash.empty()) { diff --git a/src/Common/getHashOfLoadedBinary.cpp b/src/Common/getHashOfLoadedBinary.cpp index b3f69a0a6bb..d401b0311be 100644 --- a/src/Common/getHashOfLoadedBinary.cpp +++ b/src/Common/getHashOfLoadedBinary.cpp @@ -1,6 +1,9 @@ +#include + +#if defined(__linux__) + #include #include -#include #include @@ -38,3 +41,18 @@ std::string getHashOfLoadedBinaryHex() hash.get128(checksum[0], checksum[1]); return getHexUIntUppercase(checksum); } + +#else + +SipHash getHashOfLoadedBinary() +{ + return {}; +} + + +std::string getHashOfLoadedBinaryHex() +{ + return {}; +} + +#endif From b85bc3f5eba36c40a35d311e18848693580216a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 16:37:55 +0300 Subject: [PATCH 245/291] Fix Arcadia --- src/Common/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/ya.make b/src/Common/ya.make index 5d2534c3547..6c0bdef6c9c 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -98,6 +98,7 @@ SRCS( formatIPv6.cpp formatReadable.cpp getExecutablePath.cpp + getHashOfLoadedBinary.cpp getMappedArea.cpp getMultipleKeysFromConfig.cpp getNumberOfPhysicalCPUCores.cpp From a08db94343232617ce5ab9e0e69b25415e0c01c0 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 7 Jan 2021 16:40:52 +0300 Subject: [PATCH 246/291] Revert "Add metrics for part number in MergeTree in ClickHouse" --- src/Common/CurrentMetrics.cpp | 3 -- src/Storages/MergeTree/MergeTreeData.cpp | 44 +----------------- src/Storages/StorageMergeTree.cpp | 19 -------- src/Storages/StorageReplicatedMergeTree.cpp | 19 -------- .../01600_count_of_parts_metrics.reference | 4 -- .../01600_count_of_parts_metrics.sh | 46 ------------------- 6 files changed, 2 insertions(+), 133 deletions(-) delete mode 100644 tests/queries/0_stateless/01600_count_of_parts_metrics.reference delete mode 100755 tests/queries/0_stateless/01600_count_of_parts_metrics.sh diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 009ad7e0521..d3a4a41046e 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -5,9 +5,6 @@ #define APPLY_FOR_METRICS(M) \ M(Query, "Number of executing queries") \ M(Merge, "Number of executing background merges") \ - M(Parts, "Total number of data parts") \ - M(PartsActive, "Number of active data parts") \ - M(PartsInactive, "Number of inactive data parts") \ M(PartMutation, "Number of mutations (ALTER DELETE/UPDATE)") \ M(ReplicatedFetch, "Number of data parts being fetched from replica") \ M(ReplicatedSend, "Number of data parts being sent to replicas") \ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 344d0e64cf9..897aea4363a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -74,9 +74,6 @@ namespace CurrentMetrics { extern const Metric DelayedInserts; extern const Metric BackgroundMovePoolTask; - extern const Metric Parts; - extern const Metric PartsActive; - extern const Metric PartsInactive; } @@ -884,8 +881,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); addPartContributionToDataVolume(part); - CurrentMetrics::add(CurrentMetrics::Parts); - CurrentMetrics::add(CurrentMetrics::PartsActive); }); } @@ -902,10 +897,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (!data_parts_indexes.insert(part).second) throw Exception("Part " + part->name + " already exists", ErrorCodes::DUPLICATE_DATA_PART); - addPartContributionToDataVolume(part); - CurrentMetrics::add(CurrentMetrics::Parts); - CurrentMetrics::add(CurrentMetrics::PartsActive); + addPartContributionToDataVolume(part); } if (has_non_adaptive_parts && has_adaptive_parts && !settings->enable_mixed_granularity_parts) @@ -938,9 +931,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) (*it)->remove_time.store((*it)->modification_time, std::memory_order_relaxed); modifyPartState(it, DataPartState::Outdated); removePartContributionToDataVolume(*it); - - CurrentMetrics::sub(CurrentMetrics::PartsActive); - CurrentMetrics::add(CurrentMetrics::PartsInactive); }; (*prev_jt)->assertState({DataPartState::Committed}); @@ -1108,9 +1098,6 @@ void MergeTreeData::removePartsFinally(const MergeTreeData::DataPartsVector & pa (*it)->assertState({DataPartState::Deleting}); data_parts_indexes.erase(it); - - CurrentMetrics::sub(CurrentMetrics::Parts); - CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } @@ -2011,9 +1998,6 @@ bool MergeTreeData::renameTempPartAndReplace( auto part_it = data_parts_indexes.insert(part).first; - CurrentMetrics::add(CurrentMetrics::Parts); - CurrentMetrics::add(CurrentMetrics::PartsInactive); - if (out_transaction) { out_transaction->precommitted_parts.insert(part); @@ -2029,13 +2013,9 @@ bool MergeTreeData::renameTempPartAndReplace( covered_part->remove_time.store(current_time, std::memory_order_relaxed); modifyPartState(covered_part, DataPartState::Outdated); removePartContributionToColumnSizes(covered_part); - reduce_bytes += covered_part->getBytesOnDisk(); reduce_rows += covered_part->rows_count; ++reduce_parts; - - CurrentMetrics::sub(CurrentMetrics::PartsActive); - CurrentMetrics::add(CurrentMetrics::PartsInactive); } decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); @@ -2043,9 +2023,6 @@ bool MergeTreeData::renameTempPartAndReplace( modifyPartState(part_it, DataPartState::Committed); addPartContributionToColumnSizes(part); addPartContributionToDataVolume(part); - - CurrentMetrics::add(CurrentMetrics::PartsActive); - CurrentMetrics::sub(CurrentMetrics::PartsInactive); } auto part_in_memory = asInMemoryPart(part); @@ -2089,9 +2066,6 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect { removePartContributionToColumnSizes(part); removePartContributionToDataVolume(part); - - CurrentMetrics::sub(CurrentMetrics::PartsActive); - CurrentMetrics::add(CurrentMetrics::PartsInactive); } if (part->state == IMergeTreeDataPart::State::Committed || clear_without_timeout) @@ -2118,9 +2092,6 @@ void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(con modifyPartState(part, IMergeTreeDataPart::State::Temporary); /// Erase immediately data_parts_indexes.erase(it_part); - - CurrentMetrics::sub(CurrentMetrics::Parts); - CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } @@ -2211,17 +2182,12 @@ restore_covered) { removePartContributionToDataVolume(part); removePartContributionToColumnSizes(part); - - CurrentMetrics::sub(CurrentMetrics::PartsActive); - CurrentMetrics::add(CurrentMetrics::PartsInactive); } modifyPartState(it_part, DataPartState::Deleting); part->renameToDetached(prefix); - data_parts_indexes.erase(it_part); - CurrentMetrics::sub(CurrentMetrics::Parts); - CurrentMetrics::sub(CurrentMetrics::PartsInactive); + data_parts_indexes.erase(it_part); if (restore_covered && part->info.level == 0) { @@ -3431,9 +3397,6 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: data.modifyPartState(covered_part, DataPartState::Outdated); data.removePartContributionToColumnSizes(covered_part); - - CurrentMetrics::sub(CurrentMetrics::PartsActive); - CurrentMetrics::add(CurrentMetrics::PartsInactive); } reduce_parts += covered_parts.size(); @@ -3443,9 +3406,6 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: data.modifyPartState(part, DataPartState::Committed); data.addPartContributionToColumnSizes(part); - - CurrentMetrics::add(CurrentMetrics::PartsActive); - CurrentMetrics::sub(CurrentMetrics::PartsInactive); } } data.decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c959ee613a8..8d1bee3e889 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -31,9 +31,6 @@ namespace CurrentMetrics { extern const Metric BackgroundPoolTask; - extern const Metric Parts; - extern const Metric PartsActive; - extern const Metric PartsInactive; } namespace DB @@ -162,22 +159,6 @@ void StorageMergeTree::shutdown() /// may have race condition between our remove call and background /// process. clearOldPartsFromFilesystem(true); - - auto lock = lockParts(); - DataPartsVector all_parts(data_parts_by_info.begin(), data_parts_by_info.end()); - - size_t committed_parts_count = 0; - for (const auto & parts_info : all_parts) - { - if (parts_info->state == DataPartState::Committed) - { - ++committed_parts_count; - } - } - - CurrentMetrics::sub(CurrentMetrics::Parts, all_parts.size()); - CurrentMetrics::sub(CurrentMetrics::PartsActive, committed_parts_count); - CurrentMetrics::sub(CurrentMetrics::PartsInactive, all_parts.size() - committed_parts_count); } catch (...) { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f1990dfd8cb..8a802037f61 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -81,9 +81,6 @@ namespace ProfileEvents namespace CurrentMetrics { extern const Metric BackgroundFetchesPoolTask; - extern const Metric Parts; - extern const Metric PartsActive; - extern const Metric PartsInactive; } namespace DB @@ -3686,22 +3683,6 @@ void StorageReplicatedMergeTree::shutdown() /// which will remove themselves in their descrutors. If so, we may have /// race condition between our remove call and background process. clearOldPartsFromFilesystem(true); - - auto lock = lockParts(); - DataPartsVector all_parts(data_parts_by_info.begin(), data_parts_by_info.end()); - - size_t committed_parts_count = 0; - for (const auto & parts_info : all_parts) - { - if (parts_info->state == DataPartState::Committed) - { - committed_parts_count++; - } - } - - CurrentMetrics::sub(CurrentMetrics::Parts, all_parts.size()); - CurrentMetrics::sub(CurrentMetrics::PartsActive, committed_parts_count); - CurrentMetrics::sub(CurrentMetrics::PartsInactive, all_parts.size() - committed_parts_count); } diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.reference b/tests/queries/0_stateless/01600_count_of_parts_metrics.reference deleted file mode 100644 index 98fb6a68656..00000000000 --- a/tests/queries/0_stateless/01600_count_of_parts_metrics.reference +++ /dev/null @@ -1,4 +0,0 @@ -1 -1 -1 -1 diff --git a/tests/queries/0_stateless/01600_count_of_parts_metrics.sh b/tests/queries/0_stateless/01600_count_of_parts_metrics.sh deleted file mode 100755 index 703fe1703a6..00000000000 --- a/tests/queries/0_stateless/01600_count_of_parts_metrics.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -verify_sql="SELECT COUNT(1) -FROM -(SELECT - SUM(IF(metric = 'Parts', value, 0)) AS Parts, - SUM(IF(metric = 'PartsActive', value, 0)) AS PartsActive, - SUM(IF(metric = 'PartsInactive', value, 0)) AS PartsInactive -FROM system.metrics) as a INNER JOIN -(SELECT - toInt64(SUM(1)) AS Parts, - toInt64(SUM(IF(active = 1, 1, 0))) AS PartsActive, - toInt64(SUM(IF(active = 0, 1, 0))) AS PartsInactive -FROM system.parts -) as b USING (Parts,PartsActive,PartsInactive)" - -verify(){ -for _ in $(seq 1 10) -do -result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" ) -if [ "$result" = "1" ];then - echo 1 - return -fi -done -echo 0 -} - -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test_table" 2>/dev/null -$CLICKHOUSE_CLIENT --query="CREATE TABLE test_table(data Date) ENGINE = MergeTree PARTITION BY toYear(data) ORDER BY data;" 2>/dev/null - -$CLICKHOUSE_CLIENT --query="INSERT INTO test_table VALUES ('1992-01-01')" 2>/dev/null -verify - -$CLICKHOUSE_CLIENT --query="INSERT INTO test_table VALUES ('1992-01-02')" 2>/dev/null -verify - -$CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE test_table FINAL" 2>/dev/null -verify - -$CLICKHOUSE_CLIENT --query="DROP TABLE test_table" 2>/dev/null -verify From ea047b951f272fb2ad5339403d5ee50dd04090d9 Mon Sep 17 00:00:00 2001 From: fastio Date: Thu, 7 Jan 2021 22:13:17 +0800 Subject: [PATCH 247/291] Expand macros for fetchPartition --- src/Storages/StorageReplicatedMergeTree.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8a802037f61..d683c61215a 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4958,8 +4958,13 @@ void StorageReplicatedMergeTree::fetchPartition( const String & from_, const Context & query_context) { - String auxiliary_zookeeper_name = extractZooKeeperName(from_); - String from = extractZooKeeperPath(from_); + Macros::MacrosExpansionInfo info; + info.expand_special_macros_only = false; + info.table_id = getStorageID(); + info.table_id.uuid = UUIDHelpers::Nil; + auto expand_from = query_context.getMacros()->expand(from_, info); + String auxiliary_zookeeper_name = extractZooKeeperName(expand_from); + String from = extractZooKeeperPath(expand_from); if (from.empty()) throw Exception("ZooKeeper path should not be empty", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From 46c3266c6a9f927ec5f914f42943cec15e65ea35 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 15:53:01 +0300 Subject: [PATCH 248/291] Respect MINSIGSTKSZ for alternative stack to fix under aarch64 And acording to sigaltstack(2) alignment is not required: When a signal handler is invoked on the alternate stack, the kernel automatically aligns the address given in ss.ss_sp to a suitable address boundary for the underlying hardware architecture. Fixes: #18785 --- src/Common/ThreadStatus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 3809f84711e..5105fff03b2 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -25,7 +25,7 @@ thread_local ThreadStatus * current_thread = nullptr; thread_local ThreadStatus * main_thread = nullptr; #if !defined(SANITIZER) && !defined(ARCADIA_BUILD) - alignas(4096) static thread_local char alt_stack[4096]; + alignas(4096) static thread_local char alt_stack[std::max(MINSIGSTKSZ, 4096)]; static thread_local bool has_alt_stack = false; #endif From 8d3ec1cce9b378a7d1153ac5c83c770116173cf4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 18:49:14 +0300 Subject: [PATCH 249/291] Fix fsync_part_directory (missing O_DIRECTORY) On linux you should pass O_DIRECTORY, otherwise open(2) will fail --- src/Common/FileSyncGuard.cpp | 32 ++++++++++++++++++++++++++++++++ src/Common/FileSyncGuard.h | 24 +++++++----------------- 2 files changed, 39 insertions(+), 17 deletions(-) create mode 100644 src/Common/FileSyncGuard.cpp diff --git a/src/Common/FileSyncGuard.cpp b/src/Common/FileSyncGuard.cpp new file mode 100644 index 00000000000..3c36b7c42e5 --- /dev/null +++ b/src/Common/FileSyncGuard.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include // O_RDWR + +/// OSX does not have O_DIRECTORY +#ifndef O_DIRECTORY +#define O_DIRECTORY O_RDWR +#endif + +namespace DB +{ + +FileSyncGuard::FileSyncGuard(const DiskPtr & disk_, const String & path) + : disk(disk_) + , fd(disk_->open(path, O_DIRECTORY)) +{} + +FileSyncGuard::~FileSyncGuard() +{ + try + { + disk->sync(fd); + disk->close(fd); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +} diff --git a/src/Common/FileSyncGuard.h b/src/Common/FileSyncGuard.h index 486b02d0f24..f7ea00e0d36 100644 --- a/src/Common/FileSyncGuard.h +++ b/src/Common/FileSyncGuard.h @@ -1,10 +1,14 @@ #pragma once -#include +#include +#include namespace DB { +class IDisk; +using DiskPtr = std::shared_ptr; + /// Helper class, that receives file descriptor and does fsync for it in destructor. /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end. /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version. @@ -15,22 +19,8 @@ public: /// NOTE: If you have already opened descriptor, it's preferred to use /// this constructor instead of constructor with path. FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} - - FileSyncGuard(const DiskPtr & disk_, const String & path) - : disk(disk_), fd(disk_->open(path, O_RDWR)) {} - - ~FileSyncGuard() - { - try - { - disk->sync(fd); - disk->close(fd); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } + FileSyncGuard(const DiskPtr & disk_, const std::string & path); + ~FileSyncGuard(); private: DiskPtr disk; From 52f56cd6ad22ca67fb8e97365961d3ba24abf9d5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 19:01:10 +0300 Subject: [PATCH 250/291] Add support of F_FULLSYNC on osx --- src/Common/FileSyncGuard.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/FileSyncGuard.cpp b/src/Common/FileSyncGuard.cpp index 3c36b7c42e5..53134bc3230 100644 --- a/src/Common/FileSyncGuard.cpp +++ b/src/Common/FileSyncGuard.cpp @@ -20,6 +20,10 @@ FileSyncGuard::~FileSyncGuard() { try { +#if defined(OS_DARWIN) + if (fcntl(fd, F_FULLFSYNC, 0)) + throwFromErrno("Cannot fcntl(F_FULLFSYNC)"); +#endif disk->sync(fd); disk->close(fd); } From 53268d9f0592c04bf0ca0feb1c80db903bcb34d1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 19:17:02 +0300 Subject: [PATCH 251/291] Fix fsync_after_insert by not closing fd from finalize() But this may cause troubles, another option will be to split finalize() into finalize(bool sync) and finalizeImpl() This was introduced in #8356 --- src/IO/WriteBufferFromFile.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/IO/WriteBufferFromFile.h b/src/IO/WriteBufferFromFile.h index 77530c323d2..b7d58638113 100644 --- a/src/IO/WriteBufferFromFile.h +++ b/src/IO/WriteBufferFromFile.h @@ -50,11 +50,6 @@ public: /// Close file before destruction of object. void close(); - void finalize() override - { - close(); - } - std::string getFileName() const override { return file_name; From 513a824f30df80ff24abfb5935b8d5b85207e3a3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 19:26:53 +0300 Subject: [PATCH 252/291] Fix fsync_part_directory for parts renames --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 0b4297e37b3..1fcdabd033a 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -835,10 +835,6 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ String from = getFullRelativePath(); String to = storage.relative_data_path + new_relative_path + "/"; - std::optional sync_guard; - if (storage.getSettings()->fsync_part_directory) - sync_guard.emplace(volume->getDisk(), to); - if (!volume->getDisk()->exists(from)) throw Exception("Part directory " + fullPath(volume->getDisk(), from) + " doesn't exist. Most likely it is a logical error.", ErrorCodes::FILE_DOESNT_EXIST); @@ -862,6 +858,10 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); volume->getDisk()->moveFile(from, to); relative_path = new_relative_path; + + std::optional sync_guard; + if (storage.getSettings()->fsync_part_directory) + sync_guard.emplace(volume->getDisk(), to); } From 798fb4e842ed88bfe788c5de9ccac9d9986586d3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 19:00:09 +0300 Subject: [PATCH 253/291] Smoke coverage of various fsync capabilities --- .../01643_merge_tree_fsync_smoke.reference | 12 ++++++ .../01643_merge_tree_fsync_smoke.sql | 37 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference create mode 100644 tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference new file mode 100644 index 00000000000..654db9dbc86 --- /dev/null +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference @@ -0,0 +1,12 @@ +default +1 +compact fsync_after_insert +1 +compact fsync_after_insert,fsync_part_directory +1 +wide fsync_after_insert +1 +wide fsync_after_insert,fsync_part_directory +1 +memory in_memory_parts_insert_sync +1 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql new file mode 100644 index 00000000000..21ebb607693 --- /dev/null +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -0,0 +1,37 @@ +drop table if exists data_01643; + +select 'default'; +create table data_01643 (key Int) engine=MergeTree() order by key; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'compact fsync_after_insert'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'compact fsync_after_insert,fsync_part_directory'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'wide fsync_after_insert'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'wide fsync_after_insert,fsync_part_directory'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; + +select 'memory in_memory_parts_insert_sync'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1; +insert into data_01643 values (1); +select * from data_01643; +drop table data_01643; From 0fc2020f2abe1d69e8c5dff4f38740e5ebc09ea7 Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 7 Jan 2021 19:38:58 +0300 Subject: [PATCH 254/291] fix gcc build --- contrib/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index be3d3f86348..12078cea263 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -18,7 +18,11 @@ if (WITH_COVERAGE) set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) separate_arguments(WITHOUT_COVERAGE_LIST) # disable coverage for contib files and build with optimisations - add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + if (COMPILER_CLANG) + add_compile_options(-O3 -DNDEBUG -finline-functions -finline-hint-functions ${WITHOUT_COVERAGE_LIST}) + else() + add_compile_options(-O3 -DNDEBUG -finline-functions ${WITHOUT_COVERAGE_LIST}) + endif() endif() if (SANITIZE STREQUAL "undefined") From b1f08f5c27435cf9dad36a771f6b3a2bf619c8b2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 20:26:18 +0300 Subject: [PATCH 255/291] Rename FileSyncGuard to DirectorySyncGuard --- src/Common/{FileSyncGuard.cpp => DirectorySyncGuard.cpp} | 6 +++--- src/Common/{FileSyncGuard.h => DirectorySyncGuard.h} | 8 ++++---- src/Storages/MergeTree/DataPartsExchange.cpp | 4 ++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 ++-- src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 6 +++--- src/Storages/MergeTree/MergeTreeDataWriter.cpp | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) rename src/Common/{FileSyncGuard.cpp => DirectorySyncGuard.cpp} (76%) rename src/Common/{FileSyncGuard.h => DirectorySyncGuard.h} (76%) diff --git a/src/Common/FileSyncGuard.cpp b/src/Common/DirectorySyncGuard.cpp similarity index 76% rename from src/Common/FileSyncGuard.cpp rename to src/Common/DirectorySyncGuard.cpp index 53134bc3230..f8f7c153f03 100644 --- a/src/Common/FileSyncGuard.cpp +++ b/src/Common/DirectorySyncGuard.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include // O_RDWR @@ -11,12 +11,12 @@ namespace DB { -FileSyncGuard::FileSyncGuard(const DiskPtr & disk_, const String & path) +DirectorySyncGuard::DirectorySyncGuard(const DiskPtr & disk_, const String & path) : disk(disk_) , fd(disk_->open(path, O_DIRECTORY)) {} -FileSyncGuard::~FileSyncGuard() +DirectorySyncGuard::~DirectorySyncGuard() { try { diff --git a/src/Common/FileSyncGuard.h b/src/Common/DirectorySyncGuard.h similarity index 76% rename from src/Common/FileSyncGuard.h rename to src/Common/DirectorySyncGuard.h index f7ea00e0d36..062d20324ed 100644 --- a/src/Common/FileSyncGuard.h +++ b/src/Common/DirectorySyncGuard.h @@ -13,14 +13,14 @@ using DiskPtr = std::shared_ptr; /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end. /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version. /// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496 -class FileSyncGuard +class DirectorySyncGuard { public: /// NOTE: If you have already opened descriptor, it's preferred to use /// this constructor instead of constructor with path. - FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} - FileSyncGuard(const DiskPtr & disk_, const std::string & path); - ~FileSyncGuard(); + DirectorySyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} + DirectorySyncGuard(const DiskPtr & disk_, const std::string & path); + ~DirectorySyncGuard(); private: DiskPtr disk; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f85f9501fcf..5d50f29756c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -398,7 +398,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( disk->createDirectories(part_download_path); - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, part_download_path); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 1fcdabd033a..5d0f79f4679 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -859,7 +859,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ volume->getDisk()->moveFile(from, to); relative_path = new_relative_path; - std::optional sync_guard; + std::optional sync_guard; if (storage.getSettings()->fsync_part_directory) sync_guard.emplace(volume->getDisk(), to); } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 64a0e7329ee..83ad57fa6ab 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include @@ -780,7 +780,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor gathering_column_names.clear(); } - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, new_part_tmp_path); @@ -1182,7 +1182,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor disk->createDirectories(new_part_tmp_path); - std::optional sync_guard; + std::optional sync_guard; if (data.getSettings()->fsync_part_directory) sync_guard.emplace(disk, new_part_tmp_path); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index c93d4bceba0..de4d70d5e3e 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include @@ -362,7 +362,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa new_data_part->minmax_idx = std::move(minmax_idx); new_data_part->is_temp = true; - std::optional sync_guard; + std::optional sync_guard; if (new_data_part->isStoredOnDisk()) { /// The name could be non-unique in case of stale files from previous runs. From 9a09044da6b582538336521a218b61c8307fae06 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 20:41:43 +0300 Subject: [PATCH 256/291] Add DirectorySyncGuard.cpp into ya.make --- src/Common/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/ya.make b/src/Common/ya.make index 5d2534c3547..d60c16f4be4 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -37,6 +37,7 @@ SRCS( CurrentMetrics.cpp CurrentThread.cpp DNSResolver.cpp + DirectorySyncGuard.cpp Dwarf.cpp Elf.cpp ErrorCodes.cpp From 32d8785a26a99ab84059b46725f4893b07e76994 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 21:36:38 +0300 Subject: [PATCH 257/291] Better kill #18847 --- base/daemon/BaseDaemon.cpp | 5 +++-- base/daemon/BaseDaemon.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 1224d9171ea..0ea5dea6406 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -493,8 +493,9 @@ void BaseDaemon::kill() { dumpCoverageReportIfPossible(); pid_file.reset(); - if (::raise(SIGKILL) != 0) - throw Poco::SystemException("cannot kill process"); + /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL. + /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker). + _exit(128 + SIGKILL); } std::string BaseDaemon::getDefaultCorePath() const diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 090d4997606..ed6b5d21434 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -60,7 +60,7 @@ public: static void terminate(); /// Forceful shutdown - void kill(); + [[noreturn]] void kill(); /// Cancellation request has been received. bool isCancelled() const diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index fd36f3a6fd6..e583c11c00d 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -227,9 +227,11 @@ BlockIO InterpreterSystemQuery::execute() break; case Type::KILL: context.checkAccess(AccessType::SYSTEM_SHUTDOWN); - if (kill(0, SIGKILL)) - throwFromErrno("System call kill(0, SIGKILL) failed", ErrorCodes::CANNOT_KILL); - break; + /// Exit with the same code as it is usually set by shell when process is terminated by SIGKILL. + /// It's better than doing 'raise' or 'kill', because they have no effect for 'init' process (with pid = 0, usually in Docker). + LOG_INFO(log, "Exit immediately as the SYSTEM KILL command has been issued."); + _exit(128 + SIGKILL); + // break; /// unreachable case Type::DROP_DNS_CACHE: context.checkAccess(AccessType::SYSTEM_DROP_DNS_CACHE); DNSResolver::instance().dropCache(); From ffa5bbd5fdf88a9dda6c715542173fdccb7c6e7d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 22:19:33 +0300 Subject: [PATCH 258/291] Add SYSTEM SUSPEND command #15979 --- src/Interpreters/InterpreterSystemQuery.cpp | 31 ++++++++++++++++++- src/Parsers/ASTSystemQuery.cpp | 15 ++++++++- src/Parsers/ASTSystemQuery.h | 4 ++- src/Parsers/ParserSystemQuery.cpp | 14 +++++++++ .../01643_system_suspend.reference | 1 + .../0_stateless/01643_system_suspend.sql | 5 +++ 6 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01643_system_suspend.reference create mode 100644 tests/queries/0_stateless/01643_system_suspend.sql diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index fd36f3a6fd6..2a9c4cdb259 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -221,21 +222,40 @@ BlockIO InterpreterSystemQuery::execute() switch (query.type) { case Type::SHUTDOWN: + { context.checkAccess(AccessType::SYSTEM_SHUTDOWN); if (kill(0, SIGTERM)) throwFromErrno("System call kill(0, SIGTERM) failed", ErrorCodes::CANNOT_KILL); break; + } case Type::KILL: + { context.checkAccess(AccessType::SYSTEM_SHUTDOWN); if (kill(0, SIGKILL)) throwFromErrno("System call kill(0, SIGKILL) failed", ErrorCodes::CANNOT_KILL); break; + } + case Type::SUSPEND: + { + auto command = fmt::format("kill -STOP {0} && sleep {1} && kill -CONT {0}", getpid(), query.seconds); + LOG_DEBUG(log, "Will run {}", command); + auto res = ShellCommand::execute(command); + res->in.close(); + WriteBufferFromOwnString out; + copyData(res->out, out); + copyData(res->err, out); + if (!out.str().empty()) + LOG_DEBUG(log, "The command returned output: {}", command, out.str()); + break; + } case Type::DROP_DNS_CACHE: + { context.checkAccess(AccessType::SYSTEM_DROP_DNS_CACHE); DNSResolver::instance().dropCache(); /// Reinitialize clusters to update their resolved_addresses system_context.reloadClusterConfig(); break; + } case Type::DROP_MARK_CACHE: context.checkAccess(AccessType::SYSTEM_DROP_MARK_CACHE); system_context.dropMarkCache(); @@ -251,12 +271,15 @@ BlockIO InterpreterSystemQuery::execute() break; #endif case Type::RELOAD_DICTIONARY: + { context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY); system_context.getExternalDictionariesLoader().loadOrReload( DatabaseCatalog::instance().resolveDictionaryName(query.target_dictionary)); ExternalDictionariesLoader::resetAll(); break; + } case Type::RELOAD_DICTIONARIES: + { context.checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY); executeCommandsAndThrowIfError( [&] () { system_context.getExternalDictionariesLoader().reloadAllTriedToLoad(); }, @@ -264,6 +287,7 @@ BlockIO InterpreterSystemQuery::execute() ); ExternalDictionariesLoader::resetAll(); break; + } case Type::RELOAD_EMBEDDED_DICTIONARIES: context.checkAccess(AccessType::SYSTEM_RELOAD_EMBEDDED_DICTIONARIES); system_context.getEmbeddedDictionaries().reload(); @@ -273,6 +297,7 @@ BlockIO InterpreterSystemQuery::execute() system_context.reloadConfig(); break; case Type::RELOAD_SYMBOLS: + { #if defined(__ELF__) && !defined(__FreeBSD__) context.checkAccess(AccessType::SYSTEM_RELOAD_SYMBOLS); (void)SymbolIndex::instance(true); @@ -280,6 +305,7 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception("SYSTEM RELOAD SYMBOLS is not supported on current platform", ErrorCodes::NOT_IMPLEMENTED); #endif + } case Type::STOP_MERGES: startStopAction(ActionLocks::PartsMerge, false); break; @@ -340,6 +366,7 @@ BlockIO InterpreterSystemQuery::execute() ErrorCodes::BAD_ARGUMENTS); break; case Type::FLUSH_LOGS: + { context.checkAccess(AccessType::SYSTEM_FLUSH_LOGS); executeCommandsAndThrowIfError( [&] () { if (auto query_log = context.getQueryLog()) query_log->flush(true); }, @@ -352,6 +379,7 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto opentelemetry_span_log = context.getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); } ); break; + } case Type::STOP_LISTEN_QUERIES: case Type::START_LISTEN_QUERIES: throw Exception(String(ASTSystemQuery::typeToString(query.type)) + " is not supported yet", ErrorCodes::NOT_IMPLEMENTED); @@ -586,7 +614,8 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() switch (query.type) { case Type::SHUTDOWN: [[fallthrough]]; - case Type::KILL: + case Type::KILL: [[fallthrough]]; + case Type::SUSPEND: { required_access.emplace_back(AccessType::SYSTEM_SHUTDOWN); break; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 0d6e15a3d8c..f3a43d7f3fd 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -22,6 +22,8 @@ const char * ASTSystemQuery::typeToString(Type type) return "SHUTDOWN"; case Type::KILL: return "KILL"; + case Type::SUSPEND: + return "SUSPEND"; case Type::DROP_DNS_CACHE: return "DROP DNS CACHE"; case Type::DROP_MARK_CACHE: @@ -146,7 +148,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, auto print_on_volume = [&] { - settings.ostr << " ON VOLUME " + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON VOLUME " << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(storage_policy) << (settings.hilite ? hilite_none : "") << "." @@ -182,9 +184,20 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &, print_database_table(); } else if (type == Type::RELOAD_DICTIONARY) + { print_database_dictionary(); + } else if (type == Type::DROP_REPLICA) + { print_drop_replica(); + } + else if (type == Type::SUSPEND) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FOR " + << (settings.hilite ? hilite_none : "") << seconds + << (settings.hilite ? hilite_keyword : "") << " SECOND" + << (settings.hilite ? hilite_none : ""); + } } diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 756b5b52600..ad7eb664659 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -20,6 +20,7 @@ public: UNKNOWN, SHUTDOWN, KILL, + SUSPEND, DROP_DNS_CACHE, DROP_MARK_CACHE, DROP_UNCOMPRESSED_CACHE, @@ -65,9 +66,10 @@ public: String table; String replica; String replica_zk_path; - bool is_drop_whole_replica; + bool is_drop_whole_replica{}; String storage_policy; String volume; + UInt64 seconds{}; String getID(char) const override { return "SYSTEM query"; } diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index b6a90b348a0..491037da9a9 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -169,6 +169,20 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & parseDatabaseAndTableName(pos, expected, res->database, res->table); break; + case Type::SUSPEND: + { + ASTPtr seconds; + if (!(ParserKeyword{"FOR"}.ignore(pos, expected) + && ParserUnsignedInteger().parse(pos, seconds, expected) + && ParserKeyword{"SECOND"}.ignore(pos, expected))) /// SECOND, not SECONDS to be consistent with INTERVAL parsing in SQL + { + return false; + } + + res->seconds = seconds->as()->value.get(); + break; + } + default: /// There are no [db.table] after COMMAND NAME break; diff --git a/tests/queries/0_stateless/01643_system_suspend.reference b/tests/queries/0_stateless/01643_system_suspend.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01643_system_suspend.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01643_system_suspend.sql b/tests/queries/0_stateless/01643_system_suspend.sql new file mode 100644 index 00000000000..c2cd37e6156 --- /dev/null +++ b/tests/queries/0_stateless/01643_system_suspend.sql @@ -0,0 +1,5 @@ +CREATE TEMPORARY TABLE t (x DateTime); +INSERT INTO t VALUES (now()); +SYSTEM SUSPEND FOR 1 SECOND; +INSERT INTO t VALUES (now()); +SELECT max(x) - min(x) >= 1 FROM t; From 9cbb9c6d7c7bde5709f84efae61d343db7154868 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 7 Jan 2021 22:22:41 +0300 Subject: [PATCH 259/291] Add SYSTEM SUSPEND command #15979, continued --- src/Interpreters/InterpreterSystemQuery.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 2a9c4cdb259..c0715409011 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -246,6 +246,7 @@ BlockIO InterpreterSystemQuery::execute() copyData(res->err, out); if (!out.str().empty()) LOG_DEBUG(log, "The command returned output: {}", command, out.str()); + res->wait(); break; } case Type::DROP_DNS_CACHE: From 7b9f670cc3daacabdb91ec1ba031261105d455cb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 7 Jan 2021 23:34:08 +0300 Subject: [PATCH 260/291] Fix DirectorySyncGuard under OSX --- src/Common/DirectorySyncGuard.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Common/DirectorySyncGuard.cpp b/src/Common/DirectorySyncGuard.cpp index f8f7c153f03..f279a0d25d5 100644 --- a/src/Common/DirectorySyncGuard.cpp +++ b/src/Common/DirectorySyncGuard.cpp @@ -11,6 +11,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_FSYNC; +} + DirectorySyncGuard::DirectorySyncGuard(const DiskPtr & disk_, const String & path) : disk(disk_) , fd(disk_->open(path, O_DIRECTORY)) @@ -22,7 +27,7 @@ DirectorySyncGuard::~DirectorySyncGuard() { #if defined(OS_DARWIN) if (fcntl(fd, F_FULLFSYNC, 0)) - throwFromErrno("Cannot fcntl(F_FULLFSYNC)"); + throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC); #endif disk->sync(fd); disk->close(fd); From f2ff785460ba5876fd608680a374861498c0bb91 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 02:18:57 +0300 Subject: [PATCH 261/291] Do not check bit flips for big buffers (since the size can be corrupted) --- src/Compression/CompressedReadBufferBase.cpp | 27 ++++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 8b16b68a999..74c902ce492 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -67,20 +67,25 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c buf[pos / 8] ^= 1 << pos % 8; }; - /// Check if the difference caused by single bit flip in data. - for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos) + /// If size is too huge, then this may be caused by corruption. + /// And anyway this is pretty heavy, so avoid burning too much CPU here. + if (size < 1<<20) { - flip_bit(data, bit_pos); - - auto checksum_of_data_with_flipped_bit = CityHash_v1_0_2::CityHash128(data, size); - if (expected_checksum == checksum_of_data_with_flipped_bit) + /// Check if the difference caused by single bit flip in data. + for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos) { - message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " - << message_hardware_failure; - throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); - } + flip_bit(data, bit_pos); - flip_bit(data, bit_pos); /// Restore + auto checksum_of_data_with_flipped_bit = CityHash_v1_0_2::CityHash128(data, size); + if (expected_checksum == checksum_of_data_with_flipped_bit) + { + message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " + << message_hardware_failure; + throw Exception(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + } + + flip_bit(data, bit_pos); /// Restore + } } /// Check if the difference caused by single bit flip in stored checksum. From 4f0068bdde847c6126ec92f64110fec2279d6149 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 7 Jan 2021 18:42:54 -0500 Subject: [PATCH 262/291] Fixing inproper usage of And step. --- .../ldap/external_user_directory/tests/authentications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index ced54e95115..b47a85e3e92 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -968,7 +968,7 @@ def verification_cooldown_performance(self, server, rbac=False, iterations=5000) no_vcd_time = repeat_requests(server=server, iterations=iterations, vcd_value="0", rbac=rbac) metric("login_with_vcd_value_0", units="seconds", value=no_vcd_time) - with And("Log the performance improvement as a percentage."): + with Then("Log the performance improvement as a percentage"): metric("percentage_improvement", units="%", value=100*(no_vcd_time - vcd_time)/vcd_time) @TestOutline From a3d19fa64df8fc70fc6b6e5a0bc434721ae571e7 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 8 Jan 2021 12:28:09 +0800 Subject: [PATCH 263/291] Correctly override default settings remotely --- src/Core/Settings.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +-- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- .../__init__.py | 0 .../test.py | 28 +++++++++++++++++++ 5 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_passing_max_partitions_to_read_remotely/__init__.py create mode 100644 tests/integration/test_passing_max_partitions_to_read_remotely/test.py diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e2aa4e2f2d8..d181820c911 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -353,7 +353,7 @@ class IColumn; M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ \ M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \ - M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited.", 0) \ + M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ \ diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 75760e145d3..4e1f307137a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -709,12 +709,12 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( auto max_partitions_to_read = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data.getSettings()->max_partitions_to_read; - if (max_partitions_to_read) + if (max_partitions_to_read > 0) { std::set partitions; for (auto & part_with_ranges : parts_with_ranges) partitions.insert(part_with_ranges.data_part->info.partition_id); - if (partitions.size() > max_partitions_to_read) + if (partitions.size() > size_t(max_partitions_to_read)) throw Exception( ErrorCodes::TOO_MANY_PARTITIONS, "Too many partitions to read. Current {}, max {}", diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 97c6114a0be..3e811336fc6 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -110,7 +110,7 @@ struct Settings; M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \ M(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \ - M(UInt64, max_partitions_to_read, 0, "Limit the max number of partitions that can be accessed in one query. 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ + M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/__init__.py b/tests/integration/test_passing_max_partitions_to_read_remotely/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_passing_max_partitions_to_read_remotely/test.py b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py new file mode 100644 index 00000000000..45b3dd00b2a --- /dev/null +++ b/tests/integration/test_passing_max_partitions_to_read_remotely/test.py @@ -0,0 +1,28 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance("ch1") +ch2 = cluster.add_instance("ch2") + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + ch1.query("CREATE DATABASE test_default_database;") + yield cluster + + finally: + cluster.shutdown() + + +def test_default_database_on_cluster(started_cluster): + ch1.query( + database="test_default_database", + sql="CREATE TABLE test_local_table ENGINE MergeTree PARTITION BY i ORDER BY tuple() SETTINGS max_partitions_to_read = 1 AS SELECT arrayJoin([1, 2]) i;", + ) + + assert ch2.query( + sql="SELECT * FROM remote('ch1:9000', test_default_database, test_local_table) ORDER BY i FORMAT TSV SETTINGS max_partitions_to_read = 0;", + ) == "1\n2\n" From 9e6cb0301532e691c81b5ae1ec2599b13157fedf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 07:31:29 +0300 Subject: [PATCH 264/291] Update index.md --- docs/zh/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index 2dc1e2712e5..6aaca0a9630 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -22,7 +22,7 @@ toc_title: "\u7CFB\u7EDF\u8868" 大多数系统表将数据存储在RAM中。 ClickHouse服务器在开始时创建此类系统表。 -与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), [crash_log](../../operations/system-tables/crash_log.md#system.crash_log) and [text_log](../../operations/system-tables/crash_log.md#system.text_log) 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 +与其他系统表不同，系统日志表 [metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log), [query_log](../../operations/system-tables/query_log.md#system_tables-query_log), [query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log), [trace_log](../../operations/system-tables/trace_log.md#system_tables-trace_log), [part_log](../../operations/system-tables/part_log.md#system.part_log), crash_log and text_log 默认采用[MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) 引擎并将其数据存储在存储文件系统中。 如果从文件系统中删除表，ClickHouse服务器会在下一次写入数据时再次创建空表。 如果系统表架构在新版本中发生更改，则ClickHouse会重命名当前表并创建一个新表。 用户可以通过在`/etc/clickhouse-server/config.d/`下创建与系统表同名的配置文件, 或者在`/etc/clickhouse-server/config.xml`中设置相应配置项，来自定义系统日志表的结构。可以自定义的配置项如下: From 723d52e08d8aa59c86572446f6bcd89ea35df6ae Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 07:32:19 +0300 Subject: [PATCH 265/291] Update index.md --- docs/en/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 350ec7204c0..5ad95b48ea3 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,7 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric-log.md), [query_log](../../operations/system-tables/query-log.md), [query_thread_log](../../operations/system-tables/query-thread-log.md), [trace_log](../../operations/system-tables/trace-log.md), [part_log](../../operations/system-tables/part-log.md), [crash_log](../../operations/system-tables/crash-log.md) and [text_log](../../operations/system-tables/text-log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), [crash_log](../../operations/system-tables/crash_log.md) and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: From c9d37b194e099878556b0caa9eae3b9af0020ef4 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 07:33:01 +0300 Subject: [PATCH 266/291] Update index.md --- docs/en/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 5ad95b48ea3..479bed38bdd 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -20,7 +20,7 @@ System tables: Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), [crash_log](../../operations/system-tables/crash_log.md) and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. +Unlike other system tables, the system log tables [metric_log](../../operations/system-tables/metric_log.md), [query_log](../../operations/system-tables/query_log.md), [query_thread_log](../../operations/system-tables/query_thread_log.md), [trace_log](../../operations/system-tables/trace_log.md), [part_log](../../operations/system-tables/part_log.md), crash_log and [text_log](../../operations/system-tables/text_log.md) are served by [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem by default. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. System log tables can be customized by creating a config file with the same name as the table under `/etc/clickhouse-server/config.d/`, or setting corresponding elements in `/etc/clickhouse-server/config.xml`. Elements can be customized are: From 10ea06edadf7bb0133075533f16c9c248a007ae3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 8 Jan 2021 08:09:30 +0300 Subject: [PATCH 267/291] Import strsignal from Musl --- base/daemon/BaseDaemon.cpp | 30 +----- base/glibc-compatibility/musl/strsignal.c | 125 ++++++++++++++++++++++ base/harmful/harmful.c | 2 +- 3 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 base/glibc-compatibility/musl/strsignal.c diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 2c20eccf914..b35959d6b35 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -83,28 +83,6 @@ static void call_default_signal_handler(int sig) raise(sig); } -// Apparently strsignal is not instrumented by MemorySanitizer, so we -// have to unpoison it to avoid msan reports inside fmt library when we -// print it. -const char * msan_strsignal(int sig) -{ - // no glibc in osx/freebsd -#if !defined(__GLIBC_PREREQ) -#define __GLIBC_PREREQ(x, y) 0 -#endif - - // glibc 2.32+ deprecates sys_siglist[] - // newer glibc is a problem only for unbundled build. -#if __GLIBC_PREREQ(2, 32) - const char * signal_name = sigdescr_np(sig); -#else - const char * signal_name = sys_siglist[sig]; -#endif - - __msan_unpoison_string(signal_name); - return signal_name; -} - static constexpr size_t max_query_id_size = 127; static const size_t signal_pipe_buf_size = @@ -309,13 +287,13 @@ private: { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, msan_strsignal(sig), sig); + thread_num, strsignal(sig), sig); } else { LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, query_id, msan_strsignal(sig), sig); + thread_num, query_id, strsignal(sig), sig); } String error_message; @@ -894,13 +872,13 @@ void BaseDaemon::handleSignal(int signal_id) onInterruptSignals(signal_id); } else - throw DB::Exception(std::string("Unsupported signal: ") + msan_strsignal(signal_id), 0); + throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); } void BaseDaemon::onInterruptSignals(int signal_id) { is_cancelled = true; - LOG_INFO(&logger(), "Received termination signal ({})", msan_strsignal(signal_id)); + LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); if (sigint_signals_counter >= 2) { diff --git a/base/glibc-compatibility/musl/strsignal.c b/base/glibc-compatibility/musl/strsignal.c new file mode 100644 index 00000000000..fee894e8550 --- /dev/null +++ b/base/glibc-compatibility/musl/strsignal.c @@ -0,0 +1,125 @@ +#include +#include + +#if (SIGHUP == 1) && (SIGINT == 2) && (SIGQUIT == 3) && (SIGILL == 4) \ + && (SIGTRAP == 5) && (SIGABRT == 6) && (SIGBUS == 7) && (SIGFPE == 8) \ + && (SIGKILL == 9) && (SIGUSR1 == 10) && (SIGSEGV == 11) && (SIGUSR2 == 12) \ + && (SIGPIPE == 13) && (SIGALRM == 14) && (SIGTERM == 15) && (SIGSTKFLT == 16) \ + && (SIGCHLD == 17) && (SIGCONT == 18) && (SIGSTOP == 19) && (SIGTSTP == 20) \ + && (SIGTTIN == 21) && (SIGTTOU == 22) && (SIGURG == 23) && (SIGXCPU == 24) \ + && (SIGXFSZ == 25) && (SIGVTALRM == 26) && (SIGPROF == 27) && (SIGWINCH == 28) \ + && (SIGPOLL == 29) && (SIGPWR == 30) && (SIGSYS == 31) + +#define sigmap(x) x + +#else + +static const char map[] = { + [SIGHUP] = 1, + [SIGINT] = 2, + [SIGQUIT] = 3, + [SIGILL] = 4, + [SIGTRAP] = 5, + [SIGABRT] = 6, + [SIGBUS] = 7, + [SIGFPE] = 8, + [SIGKILL] = 9, + [SIGUSR1] = 10, + [SIGSEGV] = 11, + [SIGUSR2] = 12, + [SIGPIPE] = 13, + [SIGALRM] = 14, + [SIGTERM] = 15, +#if defined(SIGSTKFLT) + [SIGSTKFLT] = 16, +#elif defined(SIGEMT) + [SIGEMT] = 16, +#endif + [SIGCHLD] = 17, + [SIGCONT] = 18, + [SIGSTOP] = 19, + [SIGTSTP] = 20, + [SIGTTIN] = 21, + [SIGTTOU] = 22, + [SIGURG] = 23, + [SIGXCPU] = 24, + [SIGXFSZ] = 25, + [SIGVTALRM] = 26, + [SIGPROF] = 27, + [SIGWINCH] = 28, + [SIGPOLL] = 29, + [SIGPWR] = 30, + [SIGSYS] = 31 +}; + +#define sigmap(x) ((x) >= sizeof map ? (x) : map[(x)]) + +#endif + +static const char strings[] = + "Unknown signal\0" + "Hangup\0" + "Interrupt\0" + "Quit\0" + "Illegal instruction\0" + "Trace/breakpoint trap\0" + "Aborted\0" + "Bus error\0" + "Arithmetic exception\0" + "Killed\0" + "User defined signal 1\0" + "Segmentation fault\0" + "User defined signal 2\0" + "Broken pipe\0" + "Alarm clock\0" + "Terminated\0" +#if defined(SIGSTKFLT) + "Stack fault\0" +#elif defined(SIGEMT) + "Emulator trap\0" +#else + "Unknown signal\0" +#endif + "Child process status\0" + "Continued\0" + "Stopped (signal)\0" + "Stopped\0" + "Stopped (tty input)\0" + "Stopped (tty output)\0" + "Urgent I/O condition\0" + "CPU time limit exceeded\0" + "File size limit exceeded\0" + "Virtual timer expired\0" + "Profiling timer expired\0" + "Window changed\0" + "I/O possible\0" + "Power failure\0" + "Bad system call\0" + "RT32" + "\0RT33\0RT34\0RT35\0RT36\0RT37\0RT38\0RT39\0RT40" + "\0RT41\0RT42\0RT43\0RT44\0RT45\0RT46\0RT47\0RT48" + "\0RT49\0RT50\0RT51\0RT52\0RT53\0RT54\0RT55\0RT56" + "\0RT57\0RT58\0RT59\0RT60\0RT61\0RT62\0RT63\0RT64" +#if _NSIG > 65 + "\0RT65\0RT66\0RT67\0RT68\0RT69\0RT70\0RT71\0RT72" + "\0RT73\0RT74\0RT75\0RT76\0RT77\0RT78\0RT79\0RT80" + "\0RT81\0RT82\0RT83\0RT84\0RT85\0RT86\0RT87\0RT88" + "\0RT89\0RT90\0RT91\0RT92\0RT93\0RT94\0RT95\0RT96" + "\0RT97\0RT98\0RT99\0RT100\0RT101\0RT102\0RT103\0RT104" + "\0RT105\0RT106\0RT107\0RT108\0RT109\0RT110\0RT111\0RT112" + "\0RT113\0RT114\0RT115\0RT116\0RT117\0RT118\0RT119\0RT120" + "\0RT121\0RT122\0RT123\0RT124\0RT125\0RT126\0RT127\0RT128" +#endif + ""; + +char *strsignal(int signum) +{ + const char *s = strings; + + signum = sigmap(signum); + if (signum - 1U >= _NSIG-1) signum = 0; + + for (; signum--; s++) for (; *s; s++); + + return (char *)s; +} diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 4d08174170f..a99db6186ac 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -164,7 +164,7 @@ TRAP(sigsuspend) TRAP(sleep) TRAP(srand48) //TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately. -TRAP(strsignal) +//TRAP(strsignal) // This function is imported from Musl and is thread safe. TRAP(strtok) TRAP(tcflow) TRAP(tcsendbreak) From a1d0c04e68b1bfed4b9e9219fbf61ae42b230489 Mon Sep 17 00:00:00 2001 From: fastio Date: Fri, 8 Jan 2021 13:10:00 +0800 Subject: [PATCH 268/291] fix build --- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d683c61215a..ebf1e43ca04 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4958,7 +4958,7 @@ void StorageReplicatedMergeTree::fetchPartition( const String & from_, const Context & query_context) { - Macros::MacrosExpansionInfo info; + Macros::MacroExpansionInfo info; info.expand_special_macros_only = false; info.table_id = getStorageID(); info.table_id.uuid = UUIDHelpers::Nil; From b4c6052dd98f81c49eb96fc2b4d9b0d15e72b861 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 08:16:06 +0300 Subject: [PATCH 269/291] Update index.md --- docs/zh/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index 6aaca0a9630..e1a56505569 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -50,7 +50,7 @@ toc_title: "\u7CFB\u7EDF\u8868" ``` -默认情况下，表增长是无限的。 要控制表的大小，可以使用 [TTL](../../sql-reference/statements/alter.md#manipulations-with-table-ttl) 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 +默认情况下，表增长是无限的。 要控制表的大小，可以使用 TTL 删除过期日志记录的设置。 你也可以使用分区功能 `MergeTree`-发动机表。 ## 系统指标的来源 {#system-tables-sources-of-system-metrics} From d0352d1cd6967634c800206fea0cf84a4e618672 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 08:17:02 +0300 Subject: [PATCH 270/291] Update index.md --- docs/zh/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/index.md b/docs/zh/operations/system-tables/index.md index e1a56505569..56067bc5057 100644 --- a/docs/zh/operations/system-tables/index.md +++ b/docs/zh/operations/system-tables/index.md @@ -29,7 +29,7 @@ toc_title: "\u7CFB\u7EDF\u8868" - `database`: 系统日志表所在的数据库。这个选项目前已经废弃。所有的系统日表都位于`system`库中。 - `table`: 系统日志表名。 - `partition_by`: 指定[PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md)表达式。 -- `ttl`: 指定系统日志表[TTL](../../sql-reference/statements/alter/ttl.md#manipulations-with-table-ttl)选项。 +- `ttl`: 指定系统日志表TTL选项。 - `flush_interval_milliseconds`: 指定系统日志表数据落盘时间。 - `engine`: 指定完整的表引擎定义。(以`ENGINE = `开始)。 这个选项与`partition_by`以及`ttl`冲突。如果两者一起设置，服务启动时会抛出异常并且退出。 From 4a2bf8b699d2c53ed9f054816b9e12dea5c5908e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 09:06:35 +0300 Subject: [PATCH 271/291] Update index.md --- docs/en/operations/system-tables/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index 479bed38bdd..5dc23aee686 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -27,7 +27,7 @@ System log tables can be customized by creating a config file with the same name - `database`: database the system log table belongs to. This option is deprecated now. All system log tables are under database `system`. - `table`: table to insert data. - `partition_by`: specify [PARTITION BY](../../engines/table-engines/mergetree-family/custom-partitioning-key.md) expression. -- `ttl`: specify table [TTL](../../sql-reference/statements/alter.md) expression. +- `ttl`: specify table [TTL](../../sql-reference/statements/alter/ttl.md) expression. - `flush_interval_milliseconds`: interval of flushing data to disk. - `engine`: provide full engine expression (starting with `ENGINE =` ) with parameters. This option is contradict with `partition_by` and `ttl`. If set together, the server would raise an exception and exit. From 0506d4bf47c79f50024e12967155878e2968de21 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Fri, 8 Jan 2021 15:41:15 +0800 Subject: [PATCH 272/291] Update contrib from ClickHouse-Extras/rocksdb --- contrib/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/rocksdb b/contrib/rocksdb index 8b966f0ca29..54a0decabbc 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit 8b966f0ca298fc1475bd09d9775f32dff0fdce0a +Subproject commit 54a0decabbcf4c0bb5cf7befa9c597f28289bff5 From 64cb6405ac89d25e0cab19023b61629bd69b0485 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 14:20:13 +0300 Subject: [PATCH 273/291] Drop IStorage::supportsSettings() (replaced with StorageFeatures::supports_settings) --- src/Storages/IStorage.h | 3 --- src/Storages/Kafka/StorageKafka.h | 1 - src/Storages/MergeTree/MergeTreeData.h | 1 - src/Storages/RabbitMQ/StorageRabbitMQ.h | 1 - src/Storages/StorageProxy.h | 1 - 5 files changed, 7 deletions(-) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 42ece547e1c..cfb4c4e9646 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -120,9 +120,6 @@ public: /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } - /// Returns true if the storage supports settings. - virtual bool supportsSettings() const { return false; } - /// Returns true if the blocks shouldn't be pushed to associated views on insert. virtual bool noPushingToViews() const { return false; } diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index 8ec8e718011..53871990810 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -36,7 +36,6 @@ class StorageKafka final : public ext::shared_ptr_helper, public I public: std::string getName() const override { return "Kafka"; } - bool supportsSettings() const override { return true; } bool noPushingToViews() const override { return true; } void startup() override; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index e5ffe8c025b..52c0b61b977 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -357,7 +357,6 @@ public: || merging_params.mode == MergingParams::VersionedCollapsing; } - bool supportsSettings() const override { return true; } NamesAndTypesList getVirtuals() const override; bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context &, const StorageMetadataPtr & metadata_snapshot) const override; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index a46da6072af..893c5167a97 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -29,7 +29,6 @@ class StorageRabbitMQ final: public ext::shared_ptr_helper, pub public: std::string getName() const override { return "RabbitMQ"; } - bool supportsSettings() const override { return true; } bool noPushingToViews() const override { return true; } void startup() override; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index b7b948af4ba..fed9dd04e76 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -25,7 +25,6 @@ public: bool supportsReplication() const override { return getNested()->supportsReplication(); } bool supportsParallelInsert() const override { return getNested()->supportsParallelInsert(); } bool supportsDeduplication() const override { return getNested()->supportsDeduplication(); } - bool supportsSettings() const override { return getNested()->supportsSettings(); } bool noPushingToViews() const override { return getNested()->noPushingToViews(); } bool hasEvenlyDistributedRead() const override { return getNested()->hasEvenlyDistributedRead(); } From c7d0c2d075d86f6a04a8c691af9c0613d670a4b5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 14:36:42 +0300 Subject: [PATCH 274/291] Add some comments for StorageFeatures --- src/Storages/StorageFactory.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index de9060769cb..b13d4d6f679 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -47,13 +47,17 @@ public: bool has_force_restore_data_flag; }; + /// Analog of the IStorage::supports*() helpers + /// (But the former cannot be replaced with StorageFeatures due to nesting) struct StorageFeatures { bool supports_settings = false; bool supports_skipping_indices = false; bool supports_sort_order = false; bool supports_ttl = false; + /// See also IStorage::supportsReplication() bool supports_replication = false; + /// See also IStorage::supportsDeduplication() bool supports_deduplication = false; AccessType source_access_type = AccessType::NONE; }; From 714d5a067a8bfb770925d05f7fa7f6577c5b5af6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 14:42:17 +0300 Subject: [PATCH 275/291] Expose supports_parallel_insert via system.table_engines --- .../MergeTree/registerStorageMergeTree.cpp | 1 + .../RocksDB/StorageEmbeddedRocksDB.cpp | 1 + src/Storages/StorageBuffer.cpp | 3 ++ src/Storages/StorageDistributed.cpp | 1 + src/Storages/StorageFactory.h | 3 ++ src/Storages/StorageMemory.cpp | 3 ++ src/Storages/StorageNull.cpp | 3 ++ .../System/StorageSystemTableEngines.cpp | 33 +++++++++++-------- 8 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index a2429cead3d..0854cc3653c 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -756,6 +756,7 @@ void registerStorageMergeTree(StorageFactory & factory) .supports_skipping_indices = true, .supports_sort_order = true, .supports_ttl = true, + .supports_parallel_insert = true, }; factory.registerStorage("MergeTree", create, features); diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 80b25793806..249026d1011 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -367,6 +367,7 @@ void registerStorageEmbeddedRocksDB(StorageFactory & factory) { StorageFactory::StorageFeatures features{ .supports_sort_order = true, + .supports_parallel_insert = true, }; factory.registerStorage("EmbeddedRocksDB", create, features); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index e24db51688e..34787556649 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -996,6 +996,9 @@ void registerStorageBuffer(StorageFactory & factory) StorageBuffer::Thresholds{max_time, max_rows, max_bytes}, destination_id, static_cast(args.local_context.getSettingsRef().insert_allow_materialized_columns)); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 1390455edb1..dd99d0f0f27 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1005,6 +1005,7 @@ void registerStorageDistributed(StorageFactory & factory) args.attach); }, { + .supports_parallel_insert = true, .source_access_type = AccessType::REMOTE, }); } diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index b13d4d6f679..18dd24e10db 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -59,6 +59,8 @@ public: bool supports_replication = false; /// See also IStorage::supportsDeduplication() bool supports_deduplication = false; + /// See also IStorage::supportsParallelInsert() + bool supports_parallel_insert = false; AccessType source_access_type = AccessType::NONE; }; @@ -89,6 +91,7 @@ public: .supports_ttl = false, .supports_replication = false, .supports_deduplication = false, + .supports_parallel_insert = false, .source_access_type = AccessType::NONE, }); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 93f00206e6b..8651caecdfa 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -303,6 +303,9 @@ void registerStorageMemory(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageMemory::create(args.table_id, args.columns, args.constraints); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 499f7329cd9..f324d502834 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -29,6 +29,9 @@ void registerStorageNull(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); return StorageNull::create(args.table_id, args.columns, args.constraints); + }, + { + .supports_parallel_insert = true, }); } diff --git a/src/Storages/System/StorageSystemTableEngines.cpp b/src/Storages/System/StorageSystemTableEngines.cpp index e63923f69b6..3f06faf6736 100644 --- a/src/Storages/System/StorageSystemTableEngines.cpp +++ b/src/Storages/System/StorageSystemTableEngines.cpp @@ -8,26 +8,31 @@ namespace DB NamesAndTypesList StorageSystemTableEngines::getNamesAndTypes() { - return {{"name", std::make_shared()}, - {"supports_settings", std::make_shared()}, - {"supports_skipping_indices", std::make_shared()}, - {"supports_sort_order", std::make_shared()}, - {"supports_ttl", std::make_shared()}, - {"supports_replication", std::make_shared()}, - {"supports_deduplication", std::make_shared()}}; + return { + {"name", std::make_shared()}, + {"supports_settings", std::make_shared()}, + {"supports_skipping_indices", std::make_shared()}, + {"supports_sort_order", std::make_shared()}, + {"supports_ttl", std::make_shared()}, + {"supports_replication", std::make_shared()}, + {"supports_deduplication", std::make_shared()}, + {"supports_parallel_insert", std::make_shared()}, + }; } void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const { for (const auto & pair : StorageFactory::instance().getAllStorages()) { - res_columns[0]->insert(pair.first); - res_columns[1]->insert(pair.second.features.supports_settings); - res_columns[2]->insert(pair.second.features.supports_skipping_indices); - res_columns[3]->insert(pair.second.features.supports_sort_order); - res_columns[4]->insert(pair.second.features.supports_ttl); - res_columns[5]->insert(pair.second.features.supports_replication); - res_columns[6]->insert(pair.second.features.supports_deduplication); + int i = 0; + res_columns[i++]->insert(pair.first); + res_columns[i++]->insert(pair.second.features.supports_settings); + res_columns[i++]->insert(pair.second.features.supports_skipping_indices); + res_columns[i++]->insert(pair.second.features.supports_sort_order); + res_columns[i++]->insert(pair.second.features.supports_ttl); + res_columns[i++]->insert(pair.second.features.supports_replication); + res_columns[i++]->insert(pair.second.features.supports_deduplication); + res_columns[i++]->insert(pair.second.features.supports_parallel_insert); } } From c312676368464c420fd58c298d3835d5e0d269af Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 14:48:34 +0300 Subject: [PATCH 276/291] Add a smoke test for system.table_engines --- .../queries/0_stateless/01645_system_table_engines.reference | 5 +++++ tests/queries/0_stateless/01645_system_table_engines.sql | 1 + 2 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/01645_system_table_engines.reference create mode 100644 tests/queries/0_stateless/01645_system_table_engines.sql diff --git a/tests/queries/0_stateless/01645_system_table_engines.reference b/tests/queries/0_stateless/01645_system_table_engines.reference new file mode 100644 index 00000000000..00dbd906ee7 --- /dev/null +++ b/tests/queries/0_stateless/01645_system_table_engines.reference @@ -0,0 +1,5 @@ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ diff --git a/tests/queries/0_stateless/01645_system_table_engines.sql b/tests/queries/0_stateless/01645_system_table_engines.sql new file mode 100644 index 00000000000..35bfb9d29c9 --- /dev/null +++ b/tests/queries/0_stateless/01645_system_table_engines.sql @@ -0,0 +1 @@ +SELECT * FROM system.table_engines WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') FORMAT PrettyCompactNoEscapes; From a8f13594edf7ca2072ef9ba5b908112320c53ef1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 14:51:07 +0300 Subject: [PATCH 277/291] Update documentation for system.table_engines --- docs/en/operations/system-tables/table_engines.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/en/operations/system-tables/table_engines.md b/docs/en/operations/system-tables/table_engines.md index 4ca1fc657ee..30122cb133e 100644 --- a/docs/en/operations/system-tables/table_engines.md +++ b/docs/en/operations/system-tables/table_engines.md @@ -11,6 +11,7 @@ This table contains the following columns (the column type is shown in brackets) - `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`. - `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](../../engines/table-engines/mergetree-family/replication.md). - `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. +- `supports_parallel_insert` (UInt8) — Flag that indicates if table engine supports parallel insert (see [`max_insert_threads`](../../operations/settings/settings.md#settings-max-insert-threads) setting). Example: @@ -21,11 +22,11 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') ``` ``` text -┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ -│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ -└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ ``` **See also** From 6dba945a2edfd8c220e53cde534d175f282c43fb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 15:18:53 +0300 Subject: [PATCH 278/291] untrap setenv() call (used for timezone) This will also fix integrity check (#18811), the reason it does not pops up on CI during PR testing because CI does not set YANDEX_OFFICIAL_BUILD, while it does set it for commits merged into upstream commits, and hence once it was merged the special build was failed. Follow-up for: #16366 --- base/harmful/harmful.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 4d08174170f..51112eb4cb3 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -142,7 +142,7 @@ TRAP(qecvt) TRAP(qfcvt) TRAP(register_printf_function) TRAP(seed48) -TRAP(setenv) +//TRAP(setenv) TRAP(setfsent) TRAP(setgrent) TRAP(sethostent) From ed26ccf2175695447506dec9d94da6dfe331db97 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 8 Jan 2021 15:22:34 +0300 Subject: [PATCH 279/291] Do not run integrity check for cross compiled binaries Follow-up for: #18811 Refs: #18785 --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95f95464857..853b2df7aca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -222,7 +222,8 @@ endif () # Add a section with the hash of the compiled machine code for integrity checks. # Only for official builds, because adding a section can be time consuming (rewrite of several GB). -if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD) +# And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary) +if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE)) set (USE_BINARY_HASH 1) endif () From 04c07d59bf3f9f78e2ec0ddf29542fc22d3d52d1 Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 8 Jan 2021 15:43:49 +0000 Subject: [PATCH 280/291] add performance test --- tests/performance/optimize_window_funnel.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/performance/optimize_window_funnel.xml diff --git a/tests/performance/optimize_window_funnel.xml b/tests/performance/optimize_window_funnel.xml new file mode 100644 index 00000000000..d28b21e2f5d --- /dev/null +++ b/tests/performance/optimize_window_funnel.xml @@ -0,0 +1,12 @@ + + CREATE TABLE action(uid UInt64, event String, time DateTime) ENGINE = MergeTree ORDER BY uid + + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'a', now() from numbers(10000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'b', now() + INTERVAL 6 hour from numbers(10000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'c', now() + INTERVAL 12 hour from numbers(10000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'd', now() + INTERVAL 18 hour from numbers(10000000) + + SELECT level, count() from (select windowFunnel(86400)(time, event='a', event='b', event='c', event='d') level from action group by uid) group by level FORMAT Null + + DROP TABLE IF EXISTS action + From 6be8338584dacbf6d0f834256ce06a246289cb8c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 8 Jan 2021 22:41:24 +0300 Subject: [PATCH 281/291] Update CompressedReadBufferBase.cpp --- src/Compression/CompressedReadBufferBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 74c902ce492..5be31913140 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -69,7 +69,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c /// If size is too huge, then this may be caused by corruption. /// And anyway this is pretty heavy, so avoid burning too much CPU here. - if (size < 1<<20) + if (size < (1ULL << 20)) { /// Check if the difference caused by single bit flip in data. for (size_t bit_pos = 0; bit_pos < size * 8; ++bit_pos) From 5426eae79a3b7daadf43dbc6778bd1d8467ddbf9 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Sat, 9 Jan 2021 00:26:33 +0300 Subject: [PATCH 282/291] Minor fix --- utils/github/backport.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/github/backport.py b/utils/github/backport.py index c51c84e6680..576e3b069c2 100644 --- a/utils/github/backport.py +++ b/utils/github/backport.py @@ -32,8 +32,8 @@ class Backport: branches.append(pull_request['headRefName']) return branches - def execute(self, repo, until_commit, number, run_cherrypick, find_lts=False): - repo = LocalRepo(repo, 'origin', self.default_branch_name) + def execute(self, repo, upstream, until_commit, number, run_cherrypick, find_lts=False): + repo = LocalRepo(repo, upstream, self.default_branch_name) all_branches = repo.get_release_branches() # [(branch_name, base_commit)] last_branches = set([branch[0] for branch in all_branches[-number:]]) @@ -42,7 +42,7 @@ class Backport: branches = [] # iterate over all branches to preserve their precedence. for branch in all_branches: - if branch in last_branches or branch in lts_branches: + if branch[0] in last_branches or branch[0] in lts_branches: branches.append(branch) if not branches: @@ -119,6 +119,7 @@ if __name__ == "__main__": parser.add_argument('--lts', action='store_true', help='consider branches with LTS') parser.add_argument('--dry-run', action='store_true', help='do not create or merge any PRs', default=False) parser.add_argument('--verbose', '-v', action='store_true', help='more verbose output', default=False) + parser.add_argument('--upstream', '-u', type=str, help='remote name of upstream in repository', default='origin') args = parser.parse_args() if args.verbose: @@ -128,4 +129,4 @@ if __name__ == "__main__": cherrypick_run = lambda token, pr, branch: CherryPick(token, 'ClickHouse', 'ClickHouse', 'core', pr, branch).execute(args.repo, args.dry_run) bp = Backport(args.token, 'ClickHouse', 'ClickHouse', 'core') - bp.execute(args.repo, args.til, args.number, cherrypick_run, args.lts) + bp.execute(args.repo, args.upstream, args.til, args.number, cherrypick_run, args.lts) From a24be01025ab2a3cfaf0b09f666e4f1833a7bcc7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 9 Jan 2021 00:59:03 +0300 Subject: [PATCH 283/291] Drop Kafka storage from 01645_system_table_engines for fasttest --- tests/queries/0_stateless/01645_system_table_engines.reference | 1 - tests/queries/0_stateless/01645_system_table_engines.sql | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01645_system_table_engines.reference b/tests/queries/0_stateless/01645_system_table_engines.reference index 00dbd906ee7..afe0584bea1 100644 --- a/tests/queries/0_stateless/01645_system_table_engines.reference +++ b/tests/queries/0_stateless/01645_system_table_engines.reference @@ -1,5 +1,4 @@ ┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┬─supports_parallel_insert─┐ │ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ 1 │ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ 0 │ │ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ └───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┴──────────────────────────┘ diff --git a/tests/queries/0_stateless/01645_system_table_engines.sql b/tests/queries/0_stateless/01645_system_table_engines.sql index 35bfb9d29c9..5e8eef5508b 100644 --- a/tests/queries/0_stateless/01645_system_table_engines.sql +++ b/tests/queries/0_stateless/01645_system_table_engines.sql @@ -1 +1 @@ -SELECT * FROM system.table_engines WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') FORMAT PrettyCompactNoEscapes; +SELECT * FROM system.table_engines WHERE name in ('MergeTree', 'ReplicatedCollapsingMergeTree') FORMAT PrettyCompactNoEscapes; From 9049599e367bcccd809f45af138452f965b3303a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 9 Jan 2021 05:15:40 +0300 Subject: [PATCH 284/291] Update optimize_window_funnel.xml --- tests/performance/optimize_window_funnel.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/performance/optimize_window_funnel.xml b/tests/performance/optimize_window_funnel.xml index d28b21e2f5d..0d928fd0f4e 100644 --- a/tests/performance/optimize_window_funnel.xml +++ b/tests/performance/optimize_window_funnel.xml @@ -1,10 +1,10 @@ CREATE TABLE action(uid UInt64, event String, time DateTime) ENGINE = MergeTree ORDER BY uid - INSERT INTO action SELECT arrayJoin(groupArray(number)), 'a', now() from numbers(10000000) - INSERT INTO action SELECT arrayJoin(groupArray(number)), 'b', now() + INTERVAL 6 hour from numbers(10000000) - INSERT INTO action SELECT arrayJoin(groupArray(number)), 'c', now() + INTERVAL 12 hour from numbers(10000000) - INSERT INTO action SELECT arrayJoin(groupArray(number)), 'd', now() + INTERVAL 18 hour from numbers(10000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'a', now() from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'b', now() + INTERVAL 6 hour from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'c', now() + INTERVAL 12 hour from numbers(1000000) + INSERT INTO action SELECT arrayJoin(groupArray(number)), 'd', now() + INTERVAL 18 hour from numbers(1000000) SELECT level, count() from (select windowFunnel(86400)(time, event='a', event='b', event='c', event='d') level from action group by uid) group by level FORMAT Null From 8fb64472f28ba9d4491399b60af34672de13a17d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 9 Jan 2021 06:28:54 +0300 Subject: [PATCH 285/291] External queries: fix the case of expr IN table #9756 --- ...t_transform_query_for_external_database.cpp | 18 ++++++++++++++++++ .../transformQueryForExternalDatabase.cpp | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 835aebab900..99dfc55ed69 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -80,6 +80,24 @@ TEST(TransformQueryForExternalDatabase, InWithSingleElement) state.context, state.columns); } +TEST(TransformQueryForExternalDatabase, InWithTable) +{ + const State & state = State::instance(); + + check("SELECT column FROM test.table WHERE 1 IN external_table", + R"(SELECT "column" FROM "test"."table")", + state.context, state.columns); + check("SELECT column FROM test.table WHERE 1 IN (x)", + R"(SELECT "column" FROM "test"."table")", + state.context, state.columns); + check("SELECT column, field, value FROM test.table WHERE column IN (field, value)", + R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN ("field", "value"))", + state.context, state.columns); + check("SELECT column FROM test.table WHERE column NOT IN hello AND column = 123", + R"(SELECT "column" FROM "test"."table" WHERE ("column" = 123))", + state.context, state.columns); +} + TEST(TransformQueryForExternalDatabase, Like) { const State & state = State::instance(); diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index f35fb1c8a34..42daf8cfc26 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -138,6 +138,12 @@ bool isCompatible(const IAST & node) if (name == "tuple" && function->arguments->children.size() <= 1) return false; + /// If the right hand side of IN is an identifier (example: x IN table), then it's not compatible. + if ((name == "in" || name == "notIn") + && (function->arguments->children.size() != 2 + || function->arguments->children[1]->as())) + return false; + for (const auto & expr : function->arguments->children) if (!isCompatible(*expr)) return false; From 32f10c38d963b4d12ada46bfdb96c99cd4e0a4e9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 9 Jan 2021 08:30:23 +0300 Subject: [PATCH 286/291] Query Fuzzer: return fail fast semantics --- programs/client/Client.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 80ad8da837c..ca2a3db193f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -801,7 +801,8 @@ private: connection->setDefaultDatabase(connection_parameters.default_database); ReadBufferFromFile in(queries_file); readStringUntilEOF(text, in); - processMultiQuery(text); + if (!processMultiQuery(text)) + break; } return; } @@ -984,7 +985,8 @@ private: if (query_fuzzer_runs) { - processWithFuzzing(full_query); + if (!processWithFuzzing(full_query)) + return false; } else { @@ -1034,7 +1036,8 @@ private: } - void processWithFuzzing(const String & text) + /// Returns false when server is not available. + bool processWithFuzzing(const String & text) { ASTPtr orig_ast; @@ -1052,7 +1055,7 @@ private: if (!orig_ast) { // Can't continue after a parsing error - return; + return true; } // Don't repeat inserts, the tables grow too big. Also don't repeat @@ -1147,7 +1150,7 @@ private: // Probably the server is dead because we found an assertion // failure. Fail fast. fmt::print(stderr, "Lost connection to the server\n"); - return; + return false; } // The server is still alive so we're going to continue fuzzing. @@ -1173,6 +1176,8 @@ private: fuzz_base = ast_to_process; } } + + return true; } void processTextAsSingleQuery(const String & text_) From 8a57f8f42a2d980ee7aba0855b841b17c582dcdf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 9 Jan 2021 08:32:10 +0300 Subject: [PATCH 287/291] Remove useless headers --- src/Core/BackgroundSchedulePool.cpp | 1 - src/Core/MySQL/Authentication.cpp | 1 - src/Storages/MergeTree/ReplicatedFetchList.h | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 6b568ea937f..b3aedd6c875 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -1,5 +1,4 @@ #include "BackgroundSchedulePool.h" -#include #include #include #include diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index b0f5f8ccae2..e685ad0394d 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include diff --git a/src/Storages/MergeTree/ReplicatedFetchList.h b/src/Storages/MergeTree/ReplicatedFetchList.h index 81d538abf9c..0ab631e53b4 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.h +++ b/src/Storages/MergeTree/ReplicatedFetchList.h @@ -3,9 +3,9 @@ #include #include #include -#include #include + namespace CurrentMetrics { extern const Metric ReplicatedFetch; From 3b11ad14da48a27c35fa2dac8176749c44bc771b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Jan 2021 03:33:45 +0300 Subject: [PATCH 288/291] Added issue template for fuzz reports --- .github/ISSUE_TEMPLATE/fuzzing-report.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/fuzzing-report.md diff --git a/.github/ISSUE_TEMPLATE/fuzzing-report.md b/.github/ISSUE_TEMPLATE/fuzzing-report.md new file mode 100644 index 00000000000..c27825ae006 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/fuzzing-report.md @@ -0,0 +1,16 @@ +--- +name: Assertion found via fuzzing +about: Create a report to help us improve ClickHouse +title: '' +labels: fuzz +assignees: '' + +--- + +(you don't have to strictly follow this form) + +**Describe the bug** +A link to the report + +**How to reproduce** +Try to reproduce the report and copy the tables and queries involved. From 7dd9155a625e6f6a6c8c9893dca07c0837ea239c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Jan 2021 03:36:47 +0300 Subject: [PATCH 289/291] Reorder issue templates --- .github/ISSUE_TEMPLATE/{question.md => 10_question.md} | 0 .../ISSUE_TEMPLATE/{feature-request.md => 20_feature-request.md} | 0 .../{unexpected-behaviour.md => 30_unexpected-behaviour.md} | 0 .github/ISSUE_TEMPLATE/{bug-report.md => 40_bug-report.md} | 0 .../ISSUE_TEMPLATE/{usability-issue.md => 45_usability-issue.md} | 0 .github/ISSUE_TEMPLATE/{build-issue.md => 50_build-issue.md} | 0 .../{documentation-issue.md => 60_documentation-issue.md} | 0 .../{performance-issue.md => 70_performance-issue.md} | 0 .../{backward-compatibility.md => 80_backward-compatibility.md} | 0 .../ISSUE_TEMPLATE/{fuzzing-report.md => 90_fuzzing-report.md} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename .github/ISSUE_TEMPLATE/{question.md => 10_question.md} (100%) rename .github/ISSUE_TEMPLATE/{feature-request.md => 20_feature-request.md} (100%) rename .github/ISSUE_TEMPLATE/{unexpected-behaviour.md => 30_unexpected-behaviour.md} (100%) rename .github/ISSUE_TEMPLATE/{bug-report.md => 40_bug-report.md} (100%) rename .github/ISSUE_TEMPLATE/{usability-issue.md => 45_usability-issue.md} (100%) rename .github/ISSUE_TEMPLATE/{build-issue.md => 50_build-issue.md} (100%) rename .github/ISSUE_TEMPLATE/{documentation-issue.md => 60_documentation-issue.md} (100%) rename .github/ISSUE_TEMPLATE/{performance-issue.md => 70_performance-issue.md} (100%) rename .github/ISSUE_TEMPLATE/{backward-compatibility.md => 80_backward-compatibility.md} (100%) rename .github/ISSUE_TEMPLATE/{fuzzing-report.md => 90_fuzzing-report.md} (100%) diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/10_question.md similarity index 100% rename from .github/ISSUE_TEMPLATE/question.md rename to .github/ISSUE_TEMPLATE/10_question.md diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/20_feature-request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature-request.md rename to .github/ISSUE_TEMPLATE/20_feature-request.md diff --git a/.github/ISSUE_TEMPLATE/unexpected-behaviour.md b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md similarity index 100% rename from .github/ISSUE_TEMPLATE/unexpected-behaviour.md rename to .github/ISSUE_TEMPLATE/30_unexpected-behaviour.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/40_bug-report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug-report.md rename to .github/ISSUE_TEMPLATE/40_bug-report.md diff --git a/.github/ISSUE_TEMPLATE/usability-issue.md b/.github/ISSUE_TEMPLATE/45_usability-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/usability-issue.md rename to .github/ISSUE_TEMPLATE/45_usability-issue.md diff --git a/.github/ISSUE_TEMPLATE/build-issue.md b/.github/ISSUE_TEMPLATE/50_build-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/build-issue.md rename to .github/ISSUE_TEMPLATE/50_build-issue.md diff --git a/.github/ISSUE_TEMPLATE/documentation-issue.md b/.github/ISSUE_TEMPLATE/60_documentation-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/documentation-issue.md rename to .github/ISSUE_TEMPLATE/60_documentation-issue.md diff --git a/.github/ISSUE_TEMPLATE/performance-issue.md b/.github/ISSUE_TEMPLATE/70_performance-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/performance-issue.md rename to .github/ISSUE_TEMPLATE/70_performance-issue.md diff --git a/.github/ISSUE_TEMPLATE/backward-compatibility.md b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md similarity index 100% rename from .github/ISSUE_TEMPLATE/backward-compatibility.md rename to .github/ISSUE_TEMPLATE/80_backward-compatibility.md diff --git a/.github/ISSUE_TEMPLATE/fuzzing-report.md b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/fuzzing-report.md rename to .github/ISSUE_TEMPLATE/90_fuzzing-report.md From 1fa331183774d32ca693963c5267d9917646eb92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Jan 2021 03:42:05 +0300 Subject: [PATCH 290/291] Reformulate some issue templates --- .github/ISSUE_TEMPLATE/10_question.md | 2 +- .github/ISSUE_TEMPLATE/30_unexpected-behaviour.md | 2 +- .github/ISSUE_TEMPLATE/45_usability-issue.md | 2 +- .github/ISSUE_TEMPLATE/80_backward-compatibility.md | 2 +- .github/ISSUE_TEMPLATE/90_fuzzing-report.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/10_question.md b/.github/ISSUE_TEMPLATE/10_question.md index a5015de8217..6e23fbdc605 100644 --- a/.github/ISSUE_TEMPLATE/10_question.md +++ b/.github/ISSUE_TEMPLATE/10_question.md @@ -1,6 +1,6 @@ --- name: Question -about: Ask question about ClickHouse +about: Ask a question about ClickHouse title: '' labels: question assignees: '' diff --git a/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md index 27ab217ca33..3630d95ba33 100644 --- a/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md +++ b/.github/ISSUE_TEMPLATE/30_unexpected-behaviour.md @@ -1,6 +1,6 @@ --- name: Unexpected behaviour -about: Create a report to help us improve ClickHouse +about: Some feature is working in non-obvious way title: '' labels: unexpected behaviour assignees: '' diff --git a/.github/ISSUE_TEMPLATE/45_usability-issue.md b/.github/ISSUE_TEMPLATE/45_usability-issue.md index 6a084a72619..b03b11606c1 100644 --- a/.github/ISSUE_TEMPLATE/45_usability-issue.md +++ b/.github/ISSUE_TEMPLATE/45_usability-issue.md @@ -1,6 +1,6 @@ --- name: Usability issue -about: Create a report to help us improve ClickHouse +about: Report something can be made more convenient to use title: '' labels: usability assignees: '' diff --git a/.github/ISSUE_TEMPLATE/80_backward-compatibility.md b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md index 8f87197e73d..a13e9508f70 100644 --- a/.github/ISSUE_TEMPLATE/80_backward-compatibility.md +++ b/.github/ISSUE_TEMPLATE/80_backward-compatibility.md @@ -1,6 +1,6 @@ --- name: Backward compatibility issue -about: Create a report to help us improve ClickHouse +about: Report the case when the behaviour of a new version can break existing use cases title: '' labels: backward compatibility assignees: '' diff --git a/.github/ISSUE_TEMPLATE/90_fuzzing-report.md b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md index c27825ae006..1d9a8a75d28 100644 --- a/.github/ISSUE_TEMPLATE/90_fuzzing-report.md +++ b/.github/ISSUE_TEMPLATE/90_fuzzing-report.md @@ -1,6 +1,6 @@ --- name: Assertion found via fuzzing -about: Create a report to help us improve ClickHouse +about: Potential issue has been found via Fuzzer or Stress tests title: '' labels: fuzz assignees: '' From 8387ac86ef64de634f172c5ae057e2d05e7d210c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 10 Jan 2021 03:44:10 +0300 Subject: [PATCH 291/291] Add another issue template --- .../35_incomplete_implementation.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/35_incomplete_implementation.md diff --git a/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md new file mode 100644 index 00000000000..6a014ce3c29 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/35_incomplete_implementation.md @@ -0,0 +1,30 @@ +--- +name: Incomplete implementation +about: Implementation of existing feature is not finished +title: '' +labels: unfinished code +assignees: '' + +--- + +(you don't have to strictly follow this form) + +**Describe the unexpected behaviour** +A clear and concise description of what works not as it is supposed to. + +**How to reproduce** +* Which ClickHouse server version to use +* Which interface to use, if matters +* Non-default settings, if any +* `CREATE TABLE` statements for all tables involved +* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary +* Queries to run that lead to unexpected result + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Error message and/or stacktrace** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here.