From cb7be14492ed358416a35172ecc43741af605642 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 May 2022 20:16:42 +0300 Subject: [PATCH 001/136] FR: Expose what triggered the merge in system.part_log #26255 --- src/Interpreters/PartLog.cpp | 31 ++++++++++++++++ src/Interpreters/PartLog.h | 15 ++++++++ src/Storages/MergeTree/MergeTreeData.cpp | 4 ++ .../02293_part_log_has_merge_reason.reference | 1 + .../02293_part_log_has_merge_reason.sql | 37 +++++++++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 tests/queries/0_stateless/02293_part_log_has_merge_reason.reference create mode 100644 tests/queries/0_stateless/02293_part_log_has_merge_reason.sql diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index ce9aa0c03d1..643fd192cad 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -16,6 +16,25 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) { + switch (merge_type) + { + case MergeType::REGULAR: + return REGULAR_MERGE; + case MergeType::TTL_DELETE: + return TTL_DELETE_MERGE; + case MergeType::TTL_RECOMPRESS: + return TTL_RECOMPRESS_MERGE; + } + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast(merge_type)); +} + NamesAndTypesList PartLogElement::getNamesAndTypes() { auto event_type_datatype = std::make_shared( @@ -30,11 +49,22 @@ NamesAndTypesList PartLogElement::getNamesAndTypes() } ); + auto merge_reason_datatype = std::make_shared( + DataTypeEnum8::Values + { + {"NotAMerge", static_cast(NOT_A_MERGE)}, + {"RegularMerge", static_cast(REGULAR_MERGE)}, + {"TTLDeleteMerge", static_cast(TTL_DELETE_MERGE)}, + {"TTLRecompressMerge", static_cast(TTL_RECOMPRESS_MERGE)}, + } + ); + ColumnsWithTypeAndName columns_with_type_and_name; return { {"query_id", std::make_shared()}, {"event_type", std::move(event_type_datatype)}, + {"merge_reason", std::move(merge_reason_datatype)}, {"event_date", std::make_shared()}, {"event_time", std::make_shared()}, @@ -71,6 +101,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); + columns[i++]->insert(merge_reason); columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 7582f6fe9e6..48a54c55b1c 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -20,9 +21,22 @@ struct PartLogElement MOVE_PART = 6, }; + enum MergeReasonType + { + /// merge_reason is relevant only for event_type = 'MERGE_PARTS', in other cases it is NOT_A_MERGE + NOT_A_MERGE = 1, + /// Just regular merge + REGULAR_MERGE = 2, + /// Merge assigned to delete some data from parts (with TTLMergeSelector) + TTL_DELETE_MERGE = 3, + /// Merge with recompression + TTL_RECOMPRESS_MERGE = 4, + }; + String query_id; Type event_type = NEW_PART; + MergeReasonType merge_reason = NOT_A_MERGE; time_t event_time = 0; Decimal64 event_time_microseconds = 0; @@ -54,6 +68,7 @@ struct PartLogElement static std::string name() { return "PartLog"; } + static MergeReasonType getMergeReasonType(MergeType merge_type); static NamesAndTypesList getNamesAndTypes(); static NamesAndAliases getNamesAndAliases() { return {}; } void appendToBlock(MutableColumns & columns) const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d84fb9d30d3..86787635b1b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6104,6 +6104,10 @@ try part_log_elem.event_type = type; + if (part_log_elem.event_type == PartLogElement::MERGE_PARTS) + if (merge_entry) + part_log_elem.merge_reason = PartLogElement::getMergeReasonType((*merge_entry)->merge_type); + part_log_elem.error = static_cast(execution_status.code); part_log_elem.exception = execution_status.message; diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql new file mode 100644 index 00000000000..db1f4c26af4 --- /dev/null +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS t_part_log_has_merge_type_table; + +CREATE TABLE t_part_log_has_merge_type_table +( + event_time DateTime, + UserID UInt64, + Comment String +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL event_time + toIntervalMonth(3) +SETTINGS min_bytes_for_wide_part = 0, merge_with_ttl_timeout = 1; + +INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1'); +INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); + +OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL; + +SYSTEM FLUSH LOGS; + +SELECT count(*) +FROM +( + SELECT + metadata_modification_time, + event_time + FROM system.tables AS l + INNER JOIN system.part_log AS r + ON l.name = r.table + WHERE (l.database = currentDatabase()) AND + (l.name = 't_part_log_has_merge_type_table') AND + (r.event_type = 'MergeParts') AND + (r.merge_reason = 'TTLDeleteMerge') +) +WHERE (metadata_modification_time <= event_time); + +DROP TABLE t_part_log_has_merge_type_table; From d6d249d964971bf17064a5fdee0d5b953e14a42a Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 May 2022 20:16:42 +0300 Subject: [PATCH 002/136] FR: Expose what triggered the merge in system.part_log #26255 --- docs/en/operations/system-tables/part_log.md | 6 +++++ src/Interpreters/PartLog.cpp | 9 ++++--- .../02293_part_log_has_merge_reason.reference | 2 +- .../02293_part_log_has_merge_reason.sql | 26 +++++++------------ 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md index 00eaca23862..1b567367c97 100644 --- a/docs/en/operations/system-tables/part_log.md +++ b/docs/en/operations/system-tables/part_log.md @@ -14,6 +14,11 @@ The `system.part_log` table contains the following columns: - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition). - `MUTATE_PART` — Mutating of a data part. - `MOVE_PART` — Moving the data part from the one disk to another one. +- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values: + - `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`. + - `REGULAR_MERGE` — Some regular merge. + - `TTL_DELETE_MERGE` — Cleaning up expired data. + - `TTL_RECOMPRESS_MERGE` — Recompressing data part with the. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision. @@ -46,6 +51,7 @@ Row 1: ────── query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31 event_type: NewPart +merge_reason: NotAMerge event_date: 2021-02-02 event_time: 2021-02-02 11:14:28 event_time_microseconds: 2021-02-02 11:14:28.861919 diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 643fd192cad..4474c22d464 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -21,14 +21,15 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } -PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) { +PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) +{ switch (merge_type) { - case MergeType::REGULAR: + case MergeType::Regular: return REGULAR_MERGE; - case MergeType::TTL_DELETE: + case MergeType::TTLDelete: return TTL_DELETE_MERGE; - case MergeType::TTL_RECOMPRESS: + case MergeType::TTLRecompress: return TTL_RECOMPRESS_MERGE; } diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference index d00491fd7e5..220107cf15b 100644 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference @@ -1 +1 @@ -1 +MergeParts TTLDeleteMerge diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql index db1f4c26af4..7ef86354e71 100644 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql @@ -8,30 +8,22 @@ CREATE TABLE t_part_log_has_merge_type_table ) ENGINE = MergeTree() ORDER BY tuple() -TTL event_time + toIntervalMonth(3) -SETTINGS min_bytes_for_wide_part = 0, merge_with_ttl_timeout = 1; +SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true; INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1'); INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +ALTER TABLE t_part_log_has_merge_type_table + MODIFY TTL event_time + INTERVAL 3 MONTH; + OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL; SYSTEM FLUSH LOGS; -SELECT count(*) -FROM -( - SELECT - metadata_modification_time, - event_time - FROM system.tables AS l - INNER JOIN system.part_log AS r - ON l.name = r.table - WHERE (l.database = currentDatabase()) AND - (l.name = 't_part_log_has_merge_type_table') AND - (r.event_type = 'MergeParts') AND - (r.merge_reason = 'TTLDeleteMerge') -) -WHERE (metadata_modification_time <= event_time); +SELECT + event_type, + merge_reason +FROM system.part_log +WHERE (table = 't_part_log_has_merge_type_table') AND (merge_reason = 'TTLDeleteMerge'); DROP TABLE t_part_log_has_merge_type_table; From c5b40a9c91fc40bf929e54ef23af2bb33ee8c928 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 12 May 2022 16:39:50 +0000 Subject: [PATCH 003/136] WIP on GROUPING function --- src/Core/NamesAndTypes.cpp | 14 ++++ src/Core/NamesAndTypes.h | 2 + .../registerFunctionsMiscellaneous.cpp | 2 + src/Interpreters/ActionsVisitor.cpp | 50 +++++++++++- src/Interpreters/ActionsVisitor.h | 3 + src/Interpreters/ExpressionAnalyzer.cpp | 7 ++ src/Interpreters/InterpreterSelectQuery.cpp | 4 + src/Parsers/ExpressionElementParsers.cpp | 16 ++++ src/Processors/QueryPlan/AggregatingStep.cpp | 28 ++++++- src/Storages/VirtualColumnUtils.cpp | 2 +- .../02293_grouping_function.reference | 81 +++++++++++++++++++ .../0_stateless/02293_grouping_function.sql | 79 ++++++++++++++++++ 12 files changed, 280 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02293_grouping_function.reference create mode 100644 tests/queries/0_stateless/02293_grouping_function.sql diff --git a/src/Core/NamesAndTypes.cpp b/src/Core/NamesAndTypes.cpp index bd24a9e82bd..72768ce23fb 100644 --- a/src/Core/NamesAndTypes.cpp +++ b/src/Core/NamesAndTypes.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -214,4 +215,17 @@ std::optional NamesAndTypesList::tryGetByName(const std::string } return {}; } + +size_t NamesAndTypesList::getPosByName(const std::string &name) const noexcept +{ + size_t pos = 0; + for (const NameAndTypePair & column : *this) + { + if (column.name == name) + break; + ++pos; + } + return pos; +} + } diff --git a/src/Core/NamesAndTypes.h b/src/Core/NamesAndTypes.h index 2719017a726..c7a51f51816 100644 --- a/src/Core/NamesAndTypes.h +++ b/src/Core/NamesAndTypes.h @@ -107,6 +107,8 @@ public: /// Try to get column by name, return empty optional if column not found std::optional tryGetByName(const std::string & name) const; + + size_t getPosByName(const std::string & name) const noexcept; }; using NamesAndTypesLists = std::vector; diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 9cd9c70da16..9fe1fa69b5e 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -83,6 +83,7 @@ void registerFunctionZooKeeperSessionUptime(FunctionFactory &); void registerFunctionGetOSKernelVersion(FunctionFactory &); void registerFunctionGetTypeSerializationStreams(FunctionFactory &); void registerFunctionFlattenTuple(FunctionFactory &); +void registerFunctionGrouping(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -172,6 +173,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionGetOSKernelVersion(factory); registerFunctionGetTypeSerializationStreams(factory); registerFunctionFlattenTuple(factory); + registerFunctionGrouping(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index c57b85951bc..b7efbc97cc9 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include #include #include @@ -12,6 +15,7 @@ #include #include #include +#include #include #include @@ -459,14 +463,23 @@ public: }; ActionsMatcher::Data::Data( - ContextPtr context_, SizeLimits set_size_limit_, size_t subquery_depth_, - const NamesAndTypesList & source_columns_, ActionsDAGPtr actions_dag, - PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, - bool no_subqueries_, bool no_makeset_, bool only_consts_, bool create_source_for_in_) + ContextPtr context_, + SizeLimits set_size_limit_, + size_t subquery_depth_, + const NamesAndTypesList & source_columns_, + const NamesAndTypesList & aggregation_keys_, + ActionsDAGPtr actions_dag, + PreparedSets & prepared_sets_, + SubqueriesForSets & subqueries_for_sets_, + bool no_subqueries_, + bool no_makeset_, + bool only_consts_, + bool create_source_for_in_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) , source_columns(source_columns_) + , aggregation_keys(aggregation_keys_) , prepared_sets(prepared_sets_) , subqueries_for_sets(subqueries_for_sets_) , no_subqueries(no_subqueries_) @@ -817,6 +830,35 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & return; } + if (node.name == "grouping") + { + auto arguments_column_name = data.getUniqueName("__grouping_args"); + { + ColumnWithTypeAndName column; + column.name = arguments_column_name; + column.type = std::make_shared(std::make_shared()); + Array arguments_to_keys_map; + for (auto const & arg : node.arguments->children) + { + size_t pos = data.aggregation_keys.getPosByName(arg->getColumnName()); + arguments_to_keys_map.push_back(pos); + } + auto arguments_column = ColumnArray::create(ColumnUInt64::create()); + arguments_column->insert(Field{arguments_to_keys_map}); + + column.column = ColumnConst::create(ColumnPtr(std::move(arguments_column)), 1); + + data.addColumn(column); + } + + data.addFunction( + FunctionFactory::instance().get("grouping", data.getContext()), + { "__grouping_set_map", arguments_column_name }, + column_name + ); + return; + } + SetPtr prepared_set; if (checkFunctionIsInOrGlobalInOperator(node)) { diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index d1558cb961c..313eae9fc8d 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -87,6 +88,7 @@ public: SizeLimits set_size_limit; size_t subquery_depth; const NamesAndTypesList & source_columns; + const NamesAndTypesList & aggregation_keys; PreparedSets & prepared_sets; SubqueriesForSets & subqueries_for_sets; bool no_subqueries; @@ -108,6 +110,7 @@ public: SizeLimits set_size_limit_, size_t subquery_depth_, const NamesAndTypesList & source_columns_, + const NamesAndTypesList & aggregation_keys_, ActionsDAGPtr actions_dag, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index e7325363c08..8c3ea878718 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -47,6 +47,7 @@ #include #include +#include #include #include @@ -442,6 +443,9 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) } } + if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) + aggregated_columns.emplace_back("__grouping_set_map", std::make_shared(aggregation_keys.size() + 1)); + if (group_asts.empty()) { select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, {}); @@ -577,6 +581,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ settings.size_limits_for_set, subquery_depth, sourceColumns(), + aggregation_keys, std::move(actions), prepared_sets, subqueries_for_sets, @@ -597,6 +602,7 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP settings.size_limits_for_set, subquery_depth, sourceColumns(), + aggregation_keys, std::move(actions), prepared_sets, subqueries_for_sets, @@ -618,6 +624,7 @@ void ExpressionAnalyzer::getRootActionsForHaving( settings.size_limits_for_set, subquery_depth, sourceColumns(), + aggregation_keys, std::move(actions), prepared_sets, subqueries_for_sets, diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6bfadc66352..c8e04777574 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -756,6 +757,9 @@ Block InterpreterSelectQuery::getSampleBlockImpl() res.insert({nullptr, type, aggregate.column_name}); } + if (analysis_result.use_grouping_set_key) + res.insert({ nullptr, std::make_shared(query_analyzer->aggregationKeys().size() + 1), "__grouping_set_map" }); + return res; } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 9150fee3bde..021d4356f41 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -801,6 +801,20 @@ namespace node = makeASTFunction("exists", subquery); return true; } + + bool parseGrouping(IParser::Pos & pos, ASTPtr & node, Expected & expected) + { + ASTPtr expr_list; + if (!ParserExpressionList(false, false).parse(pos, expr_list, expected)) + return false; + + auto res = std::make_shared(); + res->name = "grouping"; + res->arguments = expr_list; + res->children.push_back(res->arguments); + node = std::move(res); + return true; + } } @@ -886,6 +900,8 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else if (function_name_lowercase == "datediff" || function_name_lowercase == "date_diff" || function_name_lowercase == "timestampdiff" || function_name_lowercase == "timestamp_diff") parsed_special_function = parseDateDiff(pos, node, expected); + else if (function_name_lowercase == "grouping") + parsed_special_function = parseGrouping(pos, node, expected); if (parsed_special_function.has_value()) return parsed_special_function.value() && ParserToken(TokenType::ClosingRoundBracket).ignore(pos); diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index d7d62d07d92..87588facff2 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include namespace DB { @@ -33,7 +35,7 @@ static ITransformingStep::Traits getTraits() }; } -static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params) +static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params, size_t keys_size) { if (params.empty()) return block; @@ -48,6 +50,10 @@ static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & pa for (auto & col : block) res.insert(std::move(col)); + auto map_column = ColumnFixedString::create(keys_size + 1); + map_column->resize(rows); + res.insert({ColumnPtr(std::move(map_column)), std::make_shared(keys_size + 1), "__grouping_set_map"}); + return res; } @@ -63,7 +69,7 @@ AggregatingStep::AggregatingStep( bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_) - : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_), getTraits(), false) + : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_, params_.keys_size), getTraits(), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(std::move(final_)) @@ -210,7 +216,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B /// Here we create a DAG which fills missing keys and adds `__grouping_set` column auto dag = std::make_shared(header.getColumnsWithTypeAndName()); ActionsDAG::NodeRawConstPtrs index; - index.reserve(output_header.columns() + 1); + index.reserve(output_header.columns() + 2); auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0); const auto * grouping_node = &dag->addColumn( @@ -237,6 +243,22 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); } + { + std::string grouping_map; + grouping_map.reserve(params.keys_size + 1); + std::unordered_set key_set(grouping_sets_params[set_counter].used_keys.begin(), grouping_sets_params[set_counter].used_keys.end()); + for (auto key : params.keys) + grouping_map += key_set.contains(key) ? '1' : '0'; + grouping_map += '0'; + auto nested_column = ColumnFixedString::create(params.keys_size + 1); + nested_column->insertString(grouping_map); + auto grouping_map_col = ColumnConst::create(ColumnPtr(std::move(nested_column)), 0); + const auto * grouping_map_node = &dag->addColumn( + {ColumnPtr(std::move(grouping_map_col)), std::make_shared(grouping_map.length()), "__grouping_set_map"}); + grouping_map_node = &dag->materializeNode(*grouping_map_node); + index.push_back(grouping_map_node); + } + dag->getIndex().swap(index); auto expression = std::make_shared(dag, settings.getActionsSettings()); auto transform = std::make_shared(header, expression); diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index d0840778c0f..99f3b86ac26 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -157,7 +157,7 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; ActionsVisitor::Data visitor_data( - context, SizeLimits{}, 1, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); + context, SizeLimits{}, 1, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); ActionsVisitor(visitor_data).visit(node); actions = visitor_data.getActions(); auto expression_actions = std::make_shared(actions); diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference new file mode 100644 index 00000000000..5ea3ca4a15b --- /dev/null +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -0,0 +1,81 @@ +0 2 +0 2 +0 4 +1 4 +2 4 +3 4 +4 4 +5 4 +6 4 +7 4 +8 4 +9 4 +0 1 +0 1 +0 4 +1 4 +2 4 +3 4 +4 4 +5 4 +6 4 +7 4 +8 4 +9 4 +0 0 +0 1 +0 1 +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 +0 +0 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 10 0 +0 1 4 +1 1 4 +2 1 4 +3 1 4 +4 1 4 +5 1 4 +6 1 4 +7 1 4 +8 1 4 +9 1 4 +0 1 6 +1 1 6 +2 1 6 +3 1 6 +4 1 6 +5 1 6 +6 1 6 +7 1 6 +8 1 6 +9 1 6 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +0 +0 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql new file mode 100644 index 00000000000..65771fd479d --- /dev/null +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -0,0 +1,79 @@ +SELECT + number, + grouping(number, number % 2, number % 3) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; + +SELECT + number, + grouping(number, number % 3, number % 2) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; + +SELECT + number, + grouping(number, number % 2, number % 3) = 2 AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; + +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, grouping(number, number % 2, number % 3) = 2; + +SELECT + number, + count(), + grouping(number, number % 2, number % 3) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number, number % 2), + () + ) +ORDER BY (gr, number); + +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +HAVING grouping(number, number % 2, number % 3) = 4 +ORDER BY number +SETTINGS enable_optimize_predicate_expression = 0; + +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +HAVING grouping(number, number % 2, number % 3) = 2 +ORDER BY number +SETTINGS enable_optimize_predicate_expression = 0; From 92575fc3e544121feafdcb9a86319ee57d9d393c Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 12 May 2022 16:54:02 +0000 Subject: [PATCH 004/136] Add missing file --- src/Functions/grouping.cpp | 78 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/Functions/grouping.cpp diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp new file mode 100644 index 00000000000..19e810edbd2 --- /dev/null +++ b/src/Functions/grouping.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class FunctionGrouping : public IFunction +{ +public: + static constexpr auto name = "grouping"; + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + bool isVariadic() const override + { + return true; + } + + size_t getNumberOfArguments() const override + { + return 0; + } + + bool useDefaultImplementationForNulls() const override { return false; } + + bool isSuitableForConstantFolding() const override { return false; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + String getName() const override + { + return name; + } + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + //TODO: add assert for argument types + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * grouping_set_column = checkAndGetColumn(arguments[0].column.get()); + auto argument_keys_column = checkAndGetColumnConst(arguments[1].column.get()); + + LOG_DEBUG(&Poco::Logger::get("Grouping"), "Args: {}, rows: {}", arguments.size(), arguments[1].column->getFamilyName()); + auto result = std::make_shared()->createColumn(); + for (size_t i = 0; i < input_rows_count; ++i) + { + auto mask = grouping_set_column->getDataAt(i).toView(); + LOG_DEBUG(&Poco::Logger::get("Grouping"), "Mask: {}", mask); + auto indexes = (*argument_keys_column)[i].get(); + UInt64 value = 0; + for (auto index : indexes) + value = (value << 1) + (mask[index.get()] == '1' ? 1 : 0); + LOG_DEBUG(&Poco::Logger::get("Grouping"), "Mask: {}, Arg: {}, value: {}", mask, toString(indexes), value); + result->insert(Field(value)); + } + return result; + } + +}; + +void registerFunctionGrouping(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} From c330c7703b6f44a9e906a74f551d4d734d6c61cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Sat, 7 May 2022 23:49:36 +0200 Subject: [PATCH 005/136] WIP: Seconds as floating point --- src/Core/SettingsFields.cpp | 69 ++++++++++++++++++++++++++++++++----- src/Core/SettingsFields.h | 2 +- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index c545ae753de..261a2822da8 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { @@ -176,27 +178,75 @@ UInt64 SettingFieldMaxThreads::getAuto() return getNumberOfPhysicalCPUCores(); } +namespace +{ + Poco::Timespan::TimeDiff float64AsSecondsToTimespan(Float64 d) + { + if (!std::isnormal(d) || std::signbit(d)) + throw Exception( + ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a positive normal floating point number"); + return static_cast(d *= 1000000); + } -template -SettingFieldTimespan::SettingFieldTimespan(const Field & f) : SettingFieldTimespan(fieldToNumber(f)) +} + +template <> +SettingFieldSeconds::SettingFieldTimespan(const Field & f) : SettingFieldTimespan(float64AsSecondsToTimespan(fieldToNumber(f))) { } -template -SettingFieldTimespan & SettingFieldTimespan::operator=(const Field & f) +template <> +SettingFieldMilliseconds::SettingFieldTimespan(const Field & f) : SettingFieldTimespan(fieldToNumber(f)) +{ +} + +template <> +SettingFieldSeconds & SettingFieldSeconds::operator=(const Field & f) +{ + *this = Poco::Timespan{float64AsSecondsToTimespan(fieldToNumber(f))}; + return *this; +} + +template <> +SettingFieldMilliseconds & SettingFieldMilliseconds::operator=(const Field & f) { *this = fieldToNumber(f); return *this; } -template -String SettingFieldTimespan::toString() const +template <> +String SettingFieldSeconds::toString() const +{ + return ::DB::toString(static_cast(value.totalMicroseconds()) / microseconds_per_unit); +} + +template <> +String SettingFieldMilliseconds::toString() const { return ::DB::toString(operator UInt64()); } -template -void SettingFieldTimespan::parseFromString(const String & str) +template <> +SettingFieldSeconds::operator Field() const +{ + return static_cast(value.totalMicroseconds()) / microseconds_per_unit; +} + +template <> +SettingFieldMilliseconds::operator Field() const +{ + return operator UInt64(); +} + +template <> +void SettingFieldSeconds::parseFromString(const String & str) +{ + Float64 n = parse(str.data(), str.size()); + *this = Poco::Timespan{static_cast(n * microseconds_per_unit)}; +} + +template <> +void SettingFieldMilliseconds::parseFromString(const String & str) { *this = stringToNumber(str); } @@ -204,6 +254,9 @@ void SettingFieldTimespan::parseFromString(const String & str) template void SettingFieldTimespan::writeBinary(WriteBuffer & out) const { + /// Note that this is unchanged and returns UInt64 for both seconds and milliseconds for + /// compatibility reasons as it's only used the clients or servers older than + /// DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS auto num_units = operator UInt64(); writeVarUInt(num_units, out); } diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index 474786eb963..dcc99f4a2c0 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -124,7 +124,7 @@ struct SettingFieldTimespan operator std::chrono::duration() const { return std::chrono::duration_cast>(std::chrono::microseconds(value.totalMicroseconds())); } /// NOLINT explicit operator UInt64() const { return value.totalMicroseconds() / microseconds_per_unit; } - explicit operator Field() const { return operator UInt64(); } + explicit operator Field() const; Poco::Timespan::TimeDiff totalMicroseconds() const { return value.totalMicroseconds(); } Poco::Timespan::TimeDiff totalMilliseconds() const { return value.totalMilliseconds(); } From 002498bd2be9665fa090055d94141e534485e019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 13:52:41 +0200 Subject: [PATCH 006/136] Add tests to check decimal value in seconds --- src/Core/SettingsFields.cpp | 4 +- .../02294_decimal_second_errors.reference | 3 ++ .../02294_decimal_second_errors.sql | 9 ++++ ...loating_point_second_in_settings.reference | 8 ++++ ...02294_floating_point_second_in_settings.sh | 41 +++++++++++++++++++ 5 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02294_decimal_second_errors.reference create mode 100644 tests/queries/0_stateless/02294_decimal_second_errors.sql create mode 100644 tests/queries/0_stateless/02294_floating_point_second_in_settings.reference create mode 100755 tests/queries/0_stateless/02294_floating_point_second_in_settings.sh diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 261a2822da8..64971ccb685 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -182,9 +182,9 @@ namespace { Poco::Timespan::TimeDiff float64AsSecondsToTimespan(Float64 d) { - if (!std::isnormal(d) || std::signbit(d)) + if (std::signbit(d) || (d != 0.0 && !std::isnormal(d))) throw Exception( - ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a positive normal floating point number"); + ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a positive normal floating point number. Got {}", d); return static_cast(d *= 1000000); } diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.reference b/tests/queries/0_stateless/02294_decimal_second_errors.reference new file mode 100644 index 00000000000..e8183f05f5d --- /dev/null +++ b/tests/queries/0_stateless/02294_decimal_second_errors.reference @@ -0,0 +1,3 @@ +1 +1 +1 diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.sql b/tests/queries/0_stateless/02294_decimal_second_errors.sql new file mode 100644 index 00000000000..1beffc1e7e5 --- /dev/null +++ b/tests/queries/0_stateless/02294_decimal_second_errors.sql @@ -0,0 +1,9 @@ +SELECT 1 SETTINGS max_execution_time=NaN; -- { serverError 72 } +SELECT 1 SETTINGS max_execution_time=Infinity; -- { serverError 72 }; +SELECT 1 SETTINGS max_execution_time=-Infinity; -- { serverError 72 }; +SELECT 1 SETTINGS max_execution_time=-0.5; -- { serverError 72 }; +SELECT 1 SETTINGS max_execution_time=-0.000000000001; -- { serverError 72 }; +SELECT 1 SETTINGS max_execution_time=-0.0; -- { serverError 72 }; +SELECT 1 SETTINGS max_execution_time=0.0; +SELECT 1 SETTINGS max_execution_time=10.5; +SELECT 1 SETTINGS max_execution_time=10; diff --git a/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference b/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference new file mode 100644 index 00000000000..f6216e2486a --- /dev/null +++ b/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference @@ -0,0 +1,8 @@ +TCP CLIENT +maximum: 1.1 +TCP CLIENT WITH SETTINGS IN QUERY +maximum: 1.1 +HTTP CLIENT +maximum: 1.1 +TABLE: system.settings +max_execution_time 0.1 1 diff --git a/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh b/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh new file mode 100755 index 00000000000..78aece76e49 --- /dev/null +++ b/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e -o pipefail + + +MAX_TIMEOUT=1.1 # Use 1.1 because using 0.x truncates to 0 in older releases + +function check_output() { + MAXTIME_USED=$(echo "$1" | grep -Eo "maximum: [0-9]+\.[0-9]+" | head -n1 || true) + if [ "${MAXTIME_USED}" != "maximum: ${MAX_TIMEOUT}" ]; + then + echo "'$MAXTIME_USED' is not equal to 'maximum: ${MAX_TIMEOUT}'" + echo "OUTPUT: $1" + else + echo "$MAXTIME_USED" + fi +} + +# TCP CLIENT +echo "TCP CLIENT" +OUTPUT=$($CLICKHOUSE_CLIENT --max_execution_time $MAX_TIMEOUT -q "SELECT count() FROM system.numbers" 2>&1 || true) +check_output "${OUTPUT}" + +echo "TCP CLIENT WITH SETTINGS IN QUERY" +OUTPUT=$($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.numbers SETTINGS max_execution_time=$MAX_TIMEOUT" 2>&1 || true) +check_output "${OUTPUT}" + +# HTTP CLIENT +echo "HTTP CLIENT" +OUTPUT=$(${CLICKHOUSE_CURL_COMMAND} -q -sS "$CLICKHOUSE_URL&max_execution_time=$MAX_TIMEOUT" -d \ + "SELECT count() FROM system.numbers" || true) +check_output "${OUTPUT}" + +# CHECK system.settings +echo "TABLE: system.settings" +echo "SELECT name, value, changed from system.settings where name = 'max_execution_time'" | clickhouse-client --max_execution_time 0.1 From 6d4bac321edd55bbfebd739bfa68767c1258659f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 15:08:37 +0200 Subject: [PATCH 007/136] Style --- src/Core/SettingsFields.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 64971ccb685..f13fd85758f 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes { extern const int SIZE_OF_FIXED_STRING_DOESNT_MATCH; extern const int CANNOT_PARSE_BOOL; + extern const int CANNOT_PARSE_NUMBER; } From d492b1c44dae3e5efa1af9fcbf6020653d64f397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 16:11:45 +0200 Subject: [PATCH 008/136] Fix timeout and better comment --- src/Core/SettingsFields.cpp | 2 +- tests/queries/0_stateless/02127_connection_drain.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index f13fd85758f..14c831f08f1 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -256,7 +256,7 @@ template void SettingFieldTimespan::writeBinary(WriteBuffer & out) const { /// Note that this is unchanged and returns UInt64 for both seconds and milliseconds for - /// compatibility reasons as it's only used the clients or servers older than + /// compatibility reasons as it's only used by the clients or servers older than /// DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS auto num_units = operator UInt64(); writeVarUInt(num_units, out); diff --git a/tests/queries/0_stateless/02127_connection_drain.sh b/tests/queries/0_stateless/02127_connection_drain.sh index 523b02d9bd5..597497e9b78 100755 --- a/tests/queries/0_stateless/02127_connection_drain.sh +++ b/tests/queries/0_stateless/02127_connection_drain.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # sync drain for _ in {1..100}; do prev=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") - curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=-1 format Null" + curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=0 format Null" now=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") if [[ "$prev" != $(( now-2 )) ]]; then continue From ae81268d4d057da377d13eb04f16047d0ff2acf9 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 13 May 2022 14:55:50 +0000 Subject: [PATCH 009/136] Try to compute helper column lazy --- src/Core/Names.h | 1 + src/Functions/grouping.cpp | 9 +++-- src/Interpreters/ActionsVisitor.cpp | 28 +++++++++++++- src/Interpreters/ActionsVisitor.h | 3 ++ src/Interpreters/ExpressionAnalyzer.cpp | 22 ++++++++--- src/Interpreters/ExpressionAnalyzer.h | 2 + src/Interpreters/InterpreterSelectQuery.cpp | 4 +- src/Processors/QueryPlan/AggregatingStep.cpp | 40 ++++++++++---------- src/Storages/VirtualColumnUtils.cpp | 2 +- 9 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/Core/Names.h b/src/Core/Names.h index 3281daa560e..003168bde27 100644 --- a/src/Core/Names.h +++ b/src/Core/Names.h @@ -16,6 +16,7 @@ using NameOrderedSet = std::set; using NameToNameMap = std::unordered_map; using NameToNameSetMap = std::unordered_map; using NameToNameVector = std::vector>; +using NameToIndexMap = std::unordered_map; using NameWithAlias = std::pair; using NamesWithAliases = std::vector; diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp index 19e810edbd2..c1b349ce9da 100644 --- a/src/Functions/grouping.cpp +++ b/src/Functions/grouping.cpp @@ -1,5 +1,6 @@ #include #include +#include "Columns/ColumnsNumber.h" #include #include #include @@ -49,14 +50,16 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * grouping_set_column = checkAndGetColumn(arguments[0].column.get()); - auto argument_keys_column = checkAndGetColumnConst(arguments[1].column.get()); + auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); + auto grouping_set_map_column = checkAndGetColumnConst(arguments[1].column.get()); + auto argument_keys_column = checkAndGetColumnConst(arguments[2].column.get()); LOG_DEBUG(&Poco::Logger::get("Grouping"), "Args: {}, rows: {}", arguments.size(), arguments[1].column->getFamilyName()); auto result = std::make_shared()->createColumn(); for (size_t i = 0; i < input_rows_count; ++i) { - auto mask = grouping_set_column->getDataAt(i).toView(); + UInt64 set_index = grouping_set_column->get64(i); + auto mask = grouping_set_map_column->getDataAt(set_index).toView(); LOG_DEBUG(&Poco::Logger::get("Grouping"), "Mask: {}", mask); auto indexes = (*argument_keys_column)[i].get(); UInt64 value = 0; diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index b7efbc97cc9..2e2cc49af58 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -11,6 +12,7 @@ #include #include +#include #include #include #include @@ -468,6 +470,7 @@ ActionsMatcher::Data::Data( size_t subquery_depth_, const NamesAndTypesList & source_columns_, const NamesAndTypesList & aggregation_keys_, + const ColumnNumbersList & grouping_set_keys_, ActionsDAGPtr actions_dag, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, @@ -480,6 +483,7 @@ ActionsMatcher::Data::Data( , subquery_depth(subquery_depth_) , source_columns(source_columns_) , aggregation_keys(aggregation_keys_) + , grouping_set_keys(grouping_set_keys_) , prepared_sets(prepared_sets_) , subqueries_for_sets(subqueries_for_sets_) , no_subqueries(no_subqueries_) @@ -834,6 +838,28 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { auto arguments_column_name = data.getUniqueName("__grouping_args"); { + if (!data.hasColumn("__grouping_set_map")) + { + ColumnWithTypeAndName column; + column.name = "__grouping_set_map"; + size_t map_size = data.aggregation_keys.size() + 1; + column.type = std::make_shared(std::make_shared(map_size)); + Array maps_per_set; + for (auto & grouping_set : data.grouping_set_keys) + { + std::string key_map(map_size, '0'); + for (auto index : grouping_set) + key_map[index] = '1'; + auto map_column = ColumnFixedString::create(map_size); + map_column->insertString(key_map); + maps_per_set.push_back(key_map); + } + auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); + grouping_set_map_column->insert(maps_per_set); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + + data.addColumn(column); + } ColumnWithTypeAndName column; column.name = arguments_column_name; column.type = std::make_shared(std::make_shared()); @@ -853,7 +879,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data.addFunction( FunctionFactory::instance().get("grouping", data.getContext()), - { "__grouping_set_map", arguments_column_name }, + { "__grouping_set", "__grouping_set_map", arguments_column_name }, column_name ); return; diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 313eae9fc8d..b7d2905ac73 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -89,6 +90,7 @@ public: size_t subquery_depth; const NamesAndTypesList & source_columns; const NamesAndTypesList & aggregation_keys; + const ColumnNumbersList & grouping_set_keys; PreparedSets & prepared_sets; SubqueriesForSets & subqueries_for_sets; bool no_subqueries; @@ -111,6 +113,7 @@ public: size_t subquery_depth_, const NamesAndTypesList & source_columns_, const NamesAndTypesList & aggregation_keys_, + const ColumnNumbersList & grouping_set_keys_, ActionsDAGPtr actions_dag, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 8c3ea878718..1a2cb4ace1a 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -43,6 +43,8 @@ #include #include +#include +#include #include #include @@ -326,7 +328,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) { if (ASTPtr group_by_ast = select_query->groupBy()) { - NameSet unique_keys; + NameToIndexMap unique_keys; ASTs & group_asts = group_by_ast->children; /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column @@ -348,6 +350,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) group_elements_ast = group_ast_element->children; NamesAndTypesList grouping_set_list; + ColumnNumbers grouping_set_indexes_list; for (ssize_t j = 0; j < ssize_t(group_elements_ast.size()); ++j) { @@ -388,15 +391,21 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// Aggregation keys are unique. if (!unique_keys.contains(key.name)) { - unique_keys.insert(key.name); + unique_keys[key.name] = aggregation_keys.size(); + grouping_set_indexes_list.push_back(aggregation_keys.size()); aggregation_keys.push_back(key); /// Key is no longer needed, therefore we can save a little by moving it. aggregated_columns.push_back(std::move(key)); } + else + { + grouping_set_indexes_list.push_back(unique_keys[key.name]); + } } aggregation_keys_list.push_back(std::move(grouping_set_list)); + aggregation_keys_indexes_list.push_back(std::move(grouping_set_indexes_list)); } else { @@ -434,7 +443,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// Aggregation keys are uniqued. if (!unique_keys.contains(key.name)) { - unique_keys.insert(key.name); + unique_keys[key.name] = aggregation_keys.size(); aggregation_keys.push_back(key); /// Key is no longer needed, therefore we can save a little by moving it. @@ -443,8 +452,8 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) } } - if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) - aggregated_columns.emplace_back("__grouping_set_map", std::make_shared(aggregation_keys.size() + 1)); + // if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) + // aggregated_columns.emplace_back("__grouping_set_map", std::make_shared(aggregation_keys.size() + 1)); if (group_asts.empty()) { @@ -582,6 +591,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ subquery_depth, sourceColumns(), aggregation_keys, + aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, @@ -603,6 +613,7 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP subquery_depth, sourceColumns(), aggregation_keys, + aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, @@ -625,6 +636,7 @@ void ExpressionAnalyzer::getRootActionsForHaving( subquery_depth, sourceColumns(), aggregation_keys, + aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index b3704095c92..1200091efef 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -65,6 +66,7 @@ struct ExpressionAnalyzerData bool has_aggregation = false; NamesAndTypesList aggregation_keys; NamesAndTypesLists aggregation_keys_list; + ColumnNumbersList aggregation_keys_indexes_list; bool has_const_aggregation_keys = false; AggregateDescriptions aggregate_descriptions; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c8e04777574..5f165f9d535 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -757,8 +757,8 @@ Block InterpreterSelectQuery::getSampleBlockImpl() res.insert({nullptr, type, aggregate.column_name}); } - if (analysis_result.use_grouping_set_key) - res.insert({ nullptr, std::make_shared(query_analyzer->aggregationKeys().size() + 1), "__grouping_set_map" }); + // if (analysis_result.use_grouping_set_key) + // res.insert({ nullptr, std::make_shared(query_analyzer->aggregationKeys().size() + 1), "__grouping_set_map" }); return res; } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 87588facff2..9c2b5a44914 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -35,7 +35,7 @@ static ITransformingStep::Traits getTraits() }; } -static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params, size_t keys_size) +static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params) { if (params.empty()) return block; @@ -50,9 +50,9 @@ static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & pa for (auto & col : block) res.insert(std::move(col)); - auto map_column = ColumnFixedString::create(keys_size + 1); - map_column->resize(rows); - res.insert({ColumnPtr(std::move(map_column)), std::make_shared(keys_size + 1), "__grouping_set_map"}); + // auto map_column = ColumnFixedString::create(keys_size + 1); + // map_column->resize(rows); + // res.insert({ColumnPtr(std::move(map_column)), std::make_shared(keys_size + 1), "__grouping_set_map"}); return res; } @@ -69,7 +69,7 @@ AggregatingStep::AggregatingStep( bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_) - : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_, params_.keys_size), getTraits(), false) + : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_), getTraits(), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(std::move(final_)) @@ -243,21 +243,21 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); } - { - std::string grouping_map; - grouping_map.reserve(params.keys_size + 1); - std::unordered_set key_set(grouping_sets_params[set_counter].used_keys.begin(), grouping_sets_params[set_counter].used_keys.end()); - for (auto key : params.keys) - grouping_map += key_set.contains(key) ? '1' : '0'; - grouping_map += '0'; - auto nested_column = ColumnFixedString::create(params.keys_size + 1); - nested_column->insertString(grouping_map); - auto grouping_map_col = ColumnConst::create(ColumnPtr(std::move(nested_column)), 0); - const auto * grouping_map_node = &dag->addColumn( - {ColumnPtr(std::move(grouping_map_col)), std::make_shared(grouping_map.length()), "__grouping_set_map"}); - grouping_map_node = &dag->materializeNode(*grouping_map_node); - index.push_back(grouping_map_node); - } + // { + // std::string grouping_map; + // grouping_map.reserve(params.keys_size + 1); + // std::unordered_set key_set(grouping_sets_params[set_counter].used_keys.begin(), grouping_sets_params[set_counter].used_keys.end()); + // for (auto key : params.keys) + // grouping_map += key_set.contains(key) ? '1' : '0'; + // grouping_map += '0'; + // auto nested_column = ColumnFixedString::create(params.keys_size + 1); + // nested_column->insertString(grouping_map); + // auto grouping_map_col = ColumnConst::create(ColumnPtr(std::move(nested_column)), 0); + // const auto * grouping_map_node = &dag->addColumn( + // {ColumnPtr(std::move(grouping_map_col)), std::make_shared(grouping_map.length()), "__grouping_set_map"}); + // grouping_map_node = &dag->materializeNode(*grouping_map_node); + // index.push_back(grouping_map_node); + // } dag->getIndex().swap(index); auto expression = std::make_shared(dag, settings.getActionsSettings()); diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 99f3b86ac26..ef25612f63e 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -157,7 +157,7 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; ActionsVisitor::Data visitor_data( - context, SizeLimits{}, 1, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); + context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); ActionsVisitor(visitor_data).visit(node); actions = visitor_data.getActions(); auto expression_actions = std::make_shared(actions); From a2870ef65ec44ecf14ecb9ddc192d2705f7f01ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 17:28:05 +0200 Subject: [PATCH 010/136] Allow negative values for seconds --- src/Core/SettingsFields.cpp | 4 ++-- tests/queries/0_stateless/02127_connection_drain.sh | 2 +- .../0_stateless/02294_decimal_second_errors.reference | 3 +++ .../0_stateless/02294_decimal_second_errors.sql | 10 ++++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 14c831f08f1..a27013ed6cf 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -183,9 +183,9 @@ namespace { Poco::Timespan::TimeDiff float64AsSecondsToTimespan(Float64 d) { - if (std::signbit(d) || (d != 0.0 && !std::isnormal(d))) + if (d != 0.0 && !std::isnormal(d)) throw Exception( - ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a positive normal floating point number. Got {}", d); + ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a normal floating point number or zero. Got {}", d); return static_cast(d *= 1000000); } diff --git a/tests/queries/0_stateless/02127_connection_drain.sh b/tests/queries/0_stateless/02127_connection_drain.sh index 597497e9b78..523b02d9bd5 100755 --- a/tests/queries/0_stateless/02127_connection_drain.sh +++ b/tests/queries/0_stateless/02127_connection_drain.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # sync drain for _ in {1..100}; do prev=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") - curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=0 format Null" + curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select * from remote('127.{2,3}', view(select * from numbers(1e6))) limit 100 settings drain_timeout=-1 format Null" now=$(curl -d@- -sS "${CLICKHOUSE_URL}" <<<"select value from system.metrics where metric = 'SyncDrainedConnections'") if [[ "$prev" != $(( now-2 )) ]]; then continue diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.reference b/tests/queries/0_stateless/02294_decimal_second_errors.reference index e8183f05f5d..a9e2f17562a 100644 --- a/tests/queries/0_stateless/02294_decimal_second_errors.reference +++ b/tests/queries/0_stateless/02294_decimal_second_errors.reference @@ -1,3 +1,6 @@ 1 1 1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.sql b/tests/queries/0_stateless/02294_decimal_second_errors.sql index 1beffc1e7e5..7a073f64d9a 100644 --- a/tests/queries/0_stateless/02294_decimal_second_errors.sql +++ b/tests/queries/0_stateless/02294_decimal_second_errors.sql @@ -1,9 +1,11 @@ SELECT 1 SETTINGS max_execution_time=NaN; -- { serverError 72 } SELECT 1 SETTINGS max_execution_time=Infinity; -- { serverError 72 }; SELECT 1 SETTINGS max_execution_time=-Infinity; -- { serverError 72 }; -SELECT 1 SETTINGS max_execution_time=-0.5; -- { serverError 72 }; -SELECT 1 SETTINGS max_execution_time=-0.000000000001; -- { serverError 72 }; -SELECT 1 SETTINGS max_execution_time=-0.0; -- { serverError 72 }; + +-- Ok values +SELECT 1 SETTINGS max_execution_time=-0.5; +SELECT 1 SETTINGS max_execution_time=0.5; +SELECT 1 SETTINGS max_execution_time=-1; SELECT 1 SETTINGS max_execution_time=0.0; -SELECT 1 SETTINGS max_execution_time=10.5; +SELECT 1 SETTINGS max_execution_time=-0.0; SELECT 1 SETTINGS max_execution_time=10; From efb30bdf6446be2364aff687ed590cc8a02c8c46 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 13 May 2022 18:20:12 +0000 Subject: [PATCH 011/136] Correctly use __grouping_set_map column --- src/Functions/grouping.cpp | 12 +++++------- src/Interpreters/ActionsVisitor.cpp | 2 -- .../0_stateless/02293_grouping_function.reference | 12 ++++++++++++ .../queries/0_stateless/02293_grouping_function.sql | 10 ++++++++++ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp index c1b349ce9da..1849bd0e9a5 100644 --- a/src/Functions/grouping.cpp +++ b/src/Functions/grouping.cpp @@ -1,14 +1,11 @@ #include -#include #include "Columns/ColumnsNumber.h" #include #include -#include #include #include #include #include -#include namespace DB { @@ -54,18 +51,19 @@ public: auto grouping_set_map_column = checkAndGetColumnConst(arguments[1].column.get()); auto argument_keys_column = checkAndGetColumnConst(arguments[2].column.get()); - LOG_DEBUG(&Poco::Logger::get("Grouping"), "Args: {}, rows: {}", arguments.size(), arguments[1].column->getFamilyName()); + auto masks = (*grouping_set_map_column)[0].get(); + auto result = std::make_shared()->createColumn(); for (size_t i = 0; i < input_rows_count; ++i) { UInt64 set_index = grouping_set_column->get64(i); - auto mask = grouping_set_map_column->getDataAt(set_index).toView(); - LOG_DEBUG(&Poco::Logger::get("Grouping"), "Mask: {}", mask); + auto mask = masks[set_index].get(); + auto indexes = (*argument_keys_column)[i].get(); UInt64 value = 0; for (auto index : indexes) value = (value << 1) + (mask[index.get()] == '1' ? 1 : 0); - LOG_DEBUG(&Poco::Logger::get("Grouping"), "Mask: {}, Arg: {}, value: {}", mask, toString(indexes), value); + result->insert(Field(value)); } return result; diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 2e2cc49af58..40a5f055243 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -850,8 +850,6 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & std::string key_map(map_size, '0'); for (auto index : grouping_set) key_map[index] = '1'; - auto map_column = ColumnFixedString::create(map_size); - map_column->insertString(key_map); maps_per_set.push_back(key_map); } auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference index 5ea3ca4a15b..f08e6d0ea99 100644 --- a/tests/queries/0_stateless/02293_grouping_function.reference +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -79,3 +79,15 @@ 9 0 0 +0 0 +0 1 +0 1 +1 0 +2 0 +3 0 +4 0 +5 0 +6 0 +7 0 +8 0 +9 0 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index 65771fd479d..3555f9dabab 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -77,3 +77,13 @@ GROUP BY HAVING grouping(number, number % 2, number % 3) = 2 ORDER BY number SETTINGS enable_optimize_predicate_expression = 0; + +SELECT + number, + GROUPING(number, number % 2, number % 3) = 2 as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + GROUPING SETS ( + (number), + (number % 2)) +ORDER BY number, gr; From 23dfe4941b52c88759849bf4b147d8ce41b45a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 20:31:45 +0200 Subject: [PATCH 012/136] Fix clang tidy error --- src/Core/SettingsFields.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index a27013ed6cf..ccf6162ab8e 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -186,7 +186,7 @@ namespace if (d != 0.0 && !std::isnormal(d)) throw Exception( ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a normal floating point number or zero. Got {}", d); - return static_cast(d *= 1000000); + return static_cast(d * 1000000); } } From 369c18ad7f2b869550b9203a21b617feb66625ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 13 May 2022 23:09:17 +0200 Subject: [PATCH 013/136] Fix clang tidy not error --- src/Core/SettingsFields.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index ccf6162ab8e..7b820401468 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -202,14 +202,14 @@ SettingFieldMilliseconds::SettingFieldTimespan(const Field & f) : SettingFieldTi } template <> -SettingFieldSeconds & SettingFieldSeconds::operator=(const Field & f) +SettingFieldTimespan & SettingFieldSeconds::operator=(const Field & f) { *this = Poco::Timespan{float64AsSecondsToTimespan(fieldToNumber(f))}; return *this; } template <> -SettingFieldMilliseconds & SettingFieldMilliseconds::operator=(const Field & f) +SettingFieldTimespan & SettingFieldMilliseconds::operator=(const Field & f) { *this = fieldToNumber(f); return *this; From 6fc7dfea809eabb81111306d6dcb4f2ce53e53a5 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 13 May 2022 23:04:12 +0000 Subject: [PATCH 014/136] Support ordinary GROUP BY --- src/Functions/grouping.cpp | 23 ++++++++ src/Interpreters/ActionsVisitor.cpp | 56 +++++++++++++------ src/Interpreters/ActionsVisitor.h | 4 +- src/Interpreters/ExpressionAnalyzer.cpp | 21 +++++-- src/Interpreters/ExpressionAnalyzer.h | 2 + src/Interpreters/InterpreterSelectQuery.cpp | 4 -- src/Parsers/ASTSelectQuery.h | 2 + src/Processors/QueryPlan/AggregatingStep.cpp | 20 ------- src/Storages/VirtualColumnUtils.cpp | 2 +- ...02293_grouping_function_group_by.reference | 20 +++++++ .../02293_grouping_function_group_by.sql | 18 ++++++ 11 files changed, 123 insertions(+), 49 deletions(-) create mode 100644 tests/queries/0_stateless/02293_grouping_function_group_by.reference create mode 100644 tests/queries/0_stateless/02293_grouping_function_group_by.sql diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp index 1849bd0e9a5..eb63764947c 100644 --- a/src/Functions/grouping.cpp +++ b/src/Functions/grouping.cpp @@ -45,8 +45,31 @@ public: return std::make_shared(); } + ColumnPtr executeSingleGroupingSet(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + { + auto grouping_set_map_column = checkAndGetColumnConst(arguments[0].column.get()); + auto argument_keys_column = checkAndGetColumnConst(arguments[1].column.get()); + + auto aggregation_keys_number = (*grouping_set_map_column)[0].get(); + + auto result = std::make_shared()->createColumn(); + for (size_t i = 0; i < input_rows_count; ++i) + { + auto indexes = (*argument_keys_column)[i].get(); + UInt64 value = 0; + for (auto index : indexes) + value = (value << 1) + (index.get() < aggregation_keys_number ? 1 : 0); + + result->insert(Field(value)); + } + return result; + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { + if (arguments.size() == 2) + return executeSingleGroupingSet(arguments, input_rows_count); + auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); auto grouping_set_map_column = checkAndGetColumnConst(arguments[1].column.get()); auto argument_keys_column = checkAndGetColumnConst(arguments[2].column.get()); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 40a5f055243..5bececb70ae 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -477,7 +477,8 @@ ActionsMatcher::Data::Data( bool no_subqueries_, bool no_makeset_, bool only_consts_, - bool create_source_for_in_) + bool create_source_for_in_, + bool has_grouping_set_column_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) @@ -490,6 +491,7 @@ ActionsMatcher::Data::Data( , no_makeset(no_makeset_) , only_consts(only_consts_) , create_source_for_in(create_source_for_in_) + , has_grouping_set_column(has_grouping_set_column_) , visit_depth(0) , actions_stack(std::move(actions_dag), context_) , next_unique_suffix(actions_stack.getLastActions().getIndex().size() + 1) @@ -842,19 +844,28 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { ColumnWithTypeAndName column; column.name = "__grouping_set_map"; - size_t map_size = data.aggregation_keys.size() + 1; - column.type = std::make_shared(std::make_shared(map_size)); - Array maps_per_set; - for (auto & grouping_set : data.grouping_set_keys) + if (data.has_grouping_set_column) { - std::string key_map(map_size, '0'); - for (auto index : grouping_set) - key_map[index] = '1'; - maps_per_set.push_back(key_map); + size_t map_size = data.aggregation_keys.size() + 1; + column.type = std::make_shared(std::make_shared(map_size)); + Array maps_per_set; + for (auto & grouping_set : data.grouping_set_keys) + { + std::string key_map(map_size, '0'); + for (auto index : grouping_set) + key_map[index] = '1'; + maps_per_set.push_back(key_map); + } + auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); + grouping_set_map_column->insert(maps_per_set); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + } + else + { + column.type = std::make_shared(); + auto grouping_set_map_column = ColumnUInt64::create(1, data.aggregation_keys.size()); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); } - auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); - grouping_set_map_column->insert(maps_per_set); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); data.addColumn(column); } @@ -875,11 +886,22 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data.addColumn(column); } - data.addFunction( - FunctionFactory::instance().get("grouping", data.getContext()), - { "__grouping_set", "__grouping_set_map", arguments_column_name }, - column_name - ); + if (data.has_grouping_set_column) + { + data.addFunction( + FunctionFactory::instance().get("grouping", data.getContext()), + { "__grouping_set", "__grouping_set_map", arguments_column_name }, + column_name + ); + } + else + { + data.addFunction( + FunctionFactory::instance().get("grouping", data.getContext()), + { "__grouping_set_map", arguments_column_name }, + column_name + ); + } return; } diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index b7d2905ac73..3f7f6b5b127 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -97,6 +97,7 @@ public: bool no_makeset; bool only_consts; bool create_source_for_in; + bool has_grouping_set_column; size_t visit_depth; ScopeStack actions_stack; @@ -120,7 +121,8 @@ public: bool no_subqueries_, bool no_makeset_, bool only_consts_, - bool create_source_for_in_); + bool create_source_for_in_, + bool has_grouping_set_column_); /// Does result of the calculation already exists in the block. bool hasColumn(const String & column_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 1a2cb4ace1a..9c74693e6a2 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -333,8 +333,10 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column /// With set number, which is used as an additional key at the stage of merging aggregating data. - if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) + bool process_grouping_sets = select_query->group_by_with_grouping_sets && group_asts.size() > 1; + if (process_grouping_sets) aggregated_columns.emplace_back("__grouping_set", std::make_shared()); + need_grouping_set_column = select_query->group_by_with_rollup || select_query->group_by_with_cube || process_grouping_sets; for (ssize_t i = 0; i < static_cast(group_asts.size()); ++i) { @@ -452,8 +454,12 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) } } - // if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) - // aggregated_columns.emplace_back("__grouping_set_map", std::make_shared(aggregation_keys.size() + 1)); + if (!select_query->group_by_with_grouping_sets) + { + auto & list = aggregation_keys_indexes_list.emplace_back(); + for (size_t i = 0; i < aggregation_keys.size(); ++i) + list.push_back(i); + } if (group_asts.empty()) { @@ -598,7 +604,8 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ no_makeset_for_subqueries, false /* no_makeset */, only_consts, - !isRemoteStorage() /* create_source_for_in */); + !isRemoteStorage() /* create_source_for_in */, + need_grouping_set_column); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -620,7 +627,8 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP true /* no_makeset_for_subqueries, no_makeset implies no_makeset_for_subqueries */, true /* no_makeset */, only_consts, - !isRemoteStorage() /* create_source_for_in */); + !isRemoteStorage() /* create_source_for_in */, + need_grouping_set_column); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -643,7 +651,8 @@ void ExpressionAnalyzer::getRootActionsForHaving( no_makeset_for_subqueries, false /* no_makeset */, only_consts, - true /* create_source_for_in */); + true /* create_source_for_in */, + need_grouping_set_column); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 1200091efef..5db4fda0fcf 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -77,6 +77,8 @@ struct ExpressionAnalyzerData /// All new temporary tables obtained by performing the GLOBAL IN/JOIN subqueries. TemporaryTablesMapping external_tables; + + bool need_grouping_set_column = false; }; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 5f165f9d535..6bfadc66352 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -757,9 +756,6 @@ Block InterpreterSelectQuery::getSampleBlockImpl() res.insert({nullptr, type, aggregate.column_name}); } - // if (analysis_result.use_grouping_set_key) - // res.insert({ nullptr, std::make_shared(query_analyzer->aggregationKeys().size() + 1), "__grouping_set_map" }); - return res; } diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index 704aeeeea7c..b3f53de3c99 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -89,6 +89,8 @@ public: bool group_by_with_grouping_sets = false; bool limit_with_ties = false; + bool needGroupingSetColumn() const noexcept { return group_by_with_cube || group_by_with_rollup || group_by_with_grouping_sets; } + ASTPtr & refSelect() { return getExpression(Expression::SELECT); } ASTPtr & refTables() { return getExpression(Expression::TABLES); } ASTPtr & refPrewhere() { return getExpression(Expression::PREWHERE); } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 9c2b5a44914..b830c7899bb 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -50,10 +50,6 @@ static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & pa for (auto & col : block) res.insert(std::move(col)); - // auto map_column = ColumnFixedString::create(keys_size + 1); - // map_column->resize(rows); - // res.insert({ColumnPtr(std::move(map_column)), std::make_shared(keys_size + 1), "__grouping_set_map"}); - return res; } @@ -243,22 +239,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B index.push_back(dag->getIndex()[header.getPositionByName(col.name)]); } - // { - // std::string grouping_map; - // grouping_map.reserve(params.keys_size + 1); - // std::unordered_set key_set(grouping_sets_params[set_counter].used_keys.begin(), grouping_sets_params[set_counter].used_keys.end()); - // for (auto key : params.keys) - // grouping_map += key_set.contains(key) ? '1' : '0'; - // grouping_map += '0'; - // auto nested_column = ColumnFixedString::create(params.keys_size + 1); - // nested_column->insertString(grouping_map); - // auto grouping_map_col = ColumnConst::create(ColumnPtr(std::move(nested_column)), 0); - // const auto * grouping_map_node = &dag->addColumn( - // {ColumnPtr(std::move(grouping_map_col)), std::make_shared(grouping_map.length()), "__grouping_set_map"}); - // grouping_map_node = &dag->materializeNode(*grouping_map_node); - // index.push_back(grouping_map_node); - // } - dag->getIndex().swap(index); auto expression = std::make_shared(dag, settings.getActionsSettings()); auto transform = std::make_shared(header, expression); diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index ef25612f63e..c1824206b60 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -157,7 +157,7 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; ActionsVisitor::Data visitor_data( - context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false); + context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false, false); ActionsVisitor(visitor_data).visit(node); actions = visitor_data.getActions(); auto expression_actions = std::make_shared(actions); diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference new file mode 100644 index 00000000000..38578d6ad1d --- /dev/null +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -0,0 +1,20 @@ +0 1 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +8 1 +9 1 +0 1 1 +1 1 1 +2 1 1 +3 1 1 +4 1 1 +5 1 1 +6 1 1 +7 1 1 +8 1 1 +9 1 1 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql new file mode 100644 index 00000000000..5b12c34adac --- /dev/null +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -0,0 +1,18 @@ +SELECT + number, + grouping(number, number % 2, number % 3) = 6 +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 +ORDER BY number; + +SELECT + number, + grouping(number), + GROUPING(number % 2) +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 +ORDER BY number; From 644b7c01bd199def618a7259fa71d2dc505d40db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Sat, 14 May 2022 01:19:19 +0200 Subject: [PATCH 015/136] Lower the value of settings to match old behaviour Before the change to use floats, this settings were getting 0 as the parsed value, so keep the old value by setting to zero explicitly --- tests/queries/0_stateless/01287_max_execution_speed.sql | 4 ++-- .../0_stateless/01290_max_execution_speed_distributed.sql | 2 +- .../1_stateful/00156_max_execution_speed_sample_merge.sql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql index 7dbeab2d635..6e6c71e445c 100644 --- a/tests/queries/0_stateless/01287_max_execution_speed.sql +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -1,11 +1,11 @@ -- Tags: no-fasttest -SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0.1; +SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError 160 } SELECT 'Ok (1)'; SET min_execution_speed = 0; -SET min_execution_speed_bytes = 800000000000, timeout_before_checking_execution_speed = 0.1; +SET min_execution_speed_bytes = 800000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError 160 } SELECT 'Ok (2)'; SET min_execution_speed_bytes = 0; diff --git a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql index 8dcac23550d..d0dc554f425 100644 --- a/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql +++ b/tests/queries/0_stateless/01290_max_execution_speed_distributed.sql @@ -1,7 +1,7 @@ -- Tags: distributed SET max_execution_speed = 1000000; -SET timeout_before_checking_execution_speed = 0.001; +SET timeout_before_checking_execution_speed = 0; SET max_block_size = 100; SET log_queries=1; diff --git a/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql index 37e91296f14..e325c18200b 100644 --- a/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql +++ b/tests/queries/1_stateful/00156_max_execution_speed_sample_merge.sql @@ -1,4 +1,4 @@ -SET max_execution_speed = 4000000, timeout_before_checking_execution_speed = 0.001; +SET max_execution_speed = 4000000, timeout_before_checking_execution_speed = 0; CREATE TEMPORARY TABLE times (t DateTime); From e5b395e0542ad723a2ead3f0702f0f6f4203dbdc Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 16 May 2022 17:33:38 +0000 Subject: [PATCH 016/136] Support ROLLUP and CUBE in GROUPING function --- src/Functions/grouping.cpp | 65 ++++++++++- src/Interpreters/ActionsVisitor.cpp | 63 ++++++---- src/Interpreters/ActionsVisitor.h | 13 ++- src/Interpreters/ExpressionAnalyzer.cpp | 19 ++- src/Interpreters/ExpressionAnalyzer.h | 3 +- src/Processors/QueryPlan/AggregatingStep.cpp | 11 ++ src/Processors/QueryPlan/AggregatingStep.h | 2 + src/Processors/QueryPlan/CubeStep.cpp | 3 +- src/Processors/QueryPlan/RollupStep.cpp | 3 +- src/Processors/Transforms/CubeTransform.cpp | 5 +- src/Processors/Transforms/CubeTransform.h | 1 + src/Processors/Transforms/RollupTransform.cpp | 5 +- src/Processors/Transforms/RollupTransform.h | 1 + src/Storages/VirtualColumnUtils.cpp | 2 +- ...02293_grouping_function_group_by.reference | 108 ++++++++++++++++++ .../02293_grouping_function_group_by.sql | 50 ++++++++ 16 files changed, 315 insertions(+), 39 deletions(-) diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp index eb63764947c..c6c6061307d 100644 --- a/src/Functions/grouping.cpp +++ b/src/Functions/grouping.cpp @@ -1,8 +1,11 @@ #include -#include "Columns/ColumnsNumber.h" +#include #include #include +#include #include +#include +#include #include #include #include @@ -45,7 +48,7 @@ public: return std::make_shared(); } - ColumnPtr executeSingleGroupingSet(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const + ColumnPtr executeOrdinaryGroupBy(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { auto grouping_set_map_column = checkAndGetColumnConst(arguments[0].column.get()); auto argument_keys_column = checkAndGetColumnConst(arguments[1].column.get()); @@ -65,16 +68,70 @@ public: return result; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeRollup( + const ColumnUInt64 * grouping_set_column, + const ColumnConst & argument_keys_column, + UInt64 keys, + size_t input_rows_count) const + { + auto result = std::make_shared()->createColumn(); + for (size_t i = 0; i < input_rows_count; ++i) + { + UInt64 set_index = grouping_set_column->get64(i); + + auto indexes = argument_keys_column[i].get(); + UInt64 value = 0; + for (auto index : indexes) + value = (value << 1) + (index.get() < keys - set_index ? 1 : 0); + + result->insert(Field(value)); + } + return result; + } + + ColumnPtr executeCube( + const ColumnUInt64 * grouping_set_column, + const ColumnConst & argument_keys_column, + UInt64 keys, + size_t input_rows_count) const + { + static constexpr auto ONE = static_cast(1); + auto result = std::make_shared()->createColumn(); + auto mask_base = (ONE << keys) - 1; + for (size_t i = 0; i < input_rows_count; ++i) + { + UInt64 set_index = grouping_set_column->get64(i); + auto mask = mask_base - set_index; + auto indexes = argument_keys_column[i].get(); + UInt64 value = 0; + for (auto index : indexes) + value = (value << 1) + (mask & (ONE << (keys - index.get() - 1)) ? 1 : 0); + + result->insert(Field(value)); + } + return result; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & , size_t input_rows_count) const override { if (arguments.size() == 2) - return executeSingleGroupingSet(arguments, input_rows_count); + return executeOrdinaryGroupBy(arguments, input_rows_count); auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); auto grouping_set_map_column = checkAndGetColumnConst(arguments[1].column.get()); auto argument_keys_column = checkAndGetColumnConst(arguments[2].column.get()); auto masks = (*grouping_set_map_column)[0].get(); + auto grouping_set_map_elem_type = applyVisitor(FieldToDataType(), masks[0]); + if (!isString(grouping_set_map_elem_type)) + { + bool is_rollup = masks[0].get() == 0; + auto keys = masks[1].get(); + if (is_rollup) + return executeRollup(grouping_set_column, *argument_keys_column, keys, input_rows_count); + else + return executeCube(grouping_set_column, *argument_keys_column, keys, input_rows_count); + } auto result = std::make_shared()->createColumn(); for (size_t i = 0; i < input_rows_count; ++i) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 5bececb70ae..70493e5fefc 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -62,6 +62,7 @@ namespace ErrorCodes extern const int INCORRECT_ELEMENT_OF_SET; extern const int BAD_ARGUMENTS; extern const int DUPLICATE_COLUMN; + extern const int LOGICAL_ERROR; } static NamesAndTypesList::iterator findColumn(const String & name, NamesAndTypesList & cols) @@ -478,7 +479,7 @@ ActionsMatcher::Data::Data( bool no_makeset_, bool only_consts_, bool create_source_for_in_, - bool has_grouping_set_column_) + GroupByKind group_by_kind_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) @@ -491,7 +492,7 @@ ActionsMatcher::Data::Data( , no_makeset(no_makeset_) , only_consts(only_consts_) , create_source_for_in(create_source_for_in_) - , has_grouping_set_column(has_grouping_set_column_) + , group_by_kind(group_by_kind_) , visit_depth(0) , actions_stack(std::move(actions_dag), context_) , next_unique_suffix(actions_stack.getLastActions().getIndex().size() + 1) @@ -844,27 +845,47 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { ColumnWithTypeAndName column; column.name = "__grouping_set_map"; - if (data.has_grouping_set_column) + switch (data.group_by_kind) { - size_t map_size = data.aggregation_keys.size() + 1; - column.type = std::make_shared(std::make_shared(map_size)); - Array maps_per_set; - for (auto & grouping_set : data.grouping_set_keys) + case GroupByKind::GROUPING_SETS: { - std::string key_map(map_size, '0'); - for (auto index : grouping_set) - key_map[index] = '1'; - maps_per_set.push_back(key_map); + size_t map_size = data.aggregation_keys.size() + 1; + column.type = std::make_shared(std::make_shared(map_size)); + Array maps_per_set; + for (auto & grouping_set : data.grouping_set_keys) + { + std::string key_map(map_size, '0'); + for (auto index : grouping_set) + key_map[index] = '1'; + maps_per_set.push_back(key_map); + } + auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); + grouping_set_map_column->insert(maps_per_set); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + break; } - auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); - grouping_set_map_column->insert(maps_per_set); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); - } - else - { - column.type = std::make_shared(); - auto grouping_set_map_column = ColumnUInt64::create(1, data.aggregation_keys.size()); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + case GroupByKind::ROLLUP: + case GroupByKind::CUBE: + { + column.type = std::make_shared(std::make_shared()); + auto grouping_set_map_column = ColumnArray::create(ColumnUInt64::create()); + Array kind_and_keys_size; + kind_and_keys_size.push_back(data.group_by_kind == GroupByKind::ROLLUP ? 0 : 1); + kind_and_keys_size.push_back(data.aggregation_keys.size()); + grouping_set_map_column->insert(kind_and_keys_size); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + break; + } + case GroupByKind::ORDINARY: + { + column.type = std::make_shared(); + auto grouping_set_map_column = ColumnUInt64::create(1, data.aggregation_keys.size()); + column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); + break; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected kind of GROUP BY clause for GROUPING function: {}", data.group_by_kind); } data.addColumn(column); @@ -886,7 +907,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & data.addColumn(column); } - if (data.has_grouping_set_column) + if (data.group_by_kind != GroupByKind::ORDINARY) { data.addFunction( FunctionFactory::instance().get("grouping", data.getContext()), diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 3f7f6b5b127..5fd228ba836 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -78,6 +78,15 @@ class ASTIdentifier; class ASTFunction; class ASTLiteral; +enum class GroupByKind +{ + NONE, + ORDINARY, + ROLLUP, + CUBE, + GROUPING_SETS, +}; + /// Collect ExpressionAction from AST. Returns PreparedSets and SubqueriesForSets too. class ActionsMatcher { @@ -97,7 +106,7 @@ public: bool no_makeset; bool only_consts; bool create_source_for_in; - bool has_grouping_set_column; + GroupByKind group_by_kind; size_t visit_depth; ScopeStack actions_stack; @@ -122,7 +131,7 @@ public: bool no_makeset_, bool only_consts_, bool create_source_for_in_, - bool has_grouping_set_column_); + GroupByKind group_by_kind_); /// Does result of the calculation already exists in the block. bool hasColumn(const String & column_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 9c74693e6a2..f7f67c28f93 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -331,12 +331,19 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAGPtr & temp_actions) NameToIndexMap unique_keys; ASTs & group_asts = group_by_ast->children; + if (select_query->group_by_with_rollup) + group_by_kind = GroupByKind::ROLLUP; + else if (select_query->group_by_with_cube) + group_by_kind = GroupByKind::CUBE; + else if (select_query->group_by_with_grouping_sets && group_asts.size() > 1) + group_by_kind = GroupByKind::GROUPING_SETS; + else + group_by_kind = GroupByKind::ORDINARY; + /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column /// With set number, which is used as an additional key at the stage of merging aggregating data. - bool process_grouping_sets = select_query->group_by_with_grouping_sets && group_asts.size() > 1; - if (process_grouping_sets) + if (group_by_kind != GroupByKind::ORDINARY) aggregated_columns.emplace_back("__grouping_set", std::make_shared()); - need_grouping_set_column = select_query->group_by_with_rollup || select_query->group_by_with_cube || process_grouping_sets; for (ssize_t i = 0; i < static_cast(group_asts.size()); ++i) { @@ -605,7 +612,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ false /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - need_grouping_set_column); + group_by_kind); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -628,7 +635,7 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP true /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - need_grouping_set_column); + group_by_kind); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -652,7 +659,7 @@ void ExpressionAnalyzer::getRootActionsForHaving( false /* no_makeset */, only_consts, true /* create_source_for_in */, - need_grouping_set_column); + group_by_kind); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 5db4fda0fcf..fb28c08ad23 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -78,7 +79,7 @@ struct ExpressionAnalyzerData /// All new temporary tables obtained by performing the GLOBAL IN/JOIN subqueries. TemporaryTablesMapping external_tables; - bool need_grouping_set_column = false; + GroupByKind group_by_kind = GroupByKind::NONE; }; diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index b830c7899bb..0028088d03f 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -35,6 +35,17 @@ static ITransformingStep::Traits getTraits() }; } +Block appendGroupingSetColumn(Block header) +{ + Block res; + res.insert({std::make_shared(), "__grouping_set"}); + + for (auto & col : header) + res.insert(std::move(col)); + + return res; +} + static Block appendGroupingColumn(Block block, const GroupingSetsParamsList & params) { if (params.empty()) diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index b933daaa474..4dd3d956350 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -25,6 +25,8 @@ struct GroupingSetsParams using GroupingSetsParamsList = std::vector; +Block appendGroupingSetColumn(Block header); + /// Aggregation. See AggregatingTransform. class AggregatingStep : public ITransformingStep { diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 23c5115ec68..43a6491157a 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace DB @@ -22,7 +23,7 @@ static ITransformingStep::Traits getTraits() } CubeStep::CubeStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) , params(std::move(params_)) { /// Aggregation keys are distinct diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index acaeb2bc9a7..2961ef5ddbd 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { @@ -22,7 +23,7 @@ static ITransformingStep::Traits getTraits() } RollupStep::RollupStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) , params(std::move(params_)) { /// Aggregation keys are distinct diff --git a/src/Processors/Transforms/CubeTransform.cpp b/src/Processors/Transforms/CubeTransform.cpp index 456eccc732f..f185e7565ea 100644 --- a/src/Processors/Transforms/CubeTransform.cpp +++ b/src/Processors/Transforms/CubeTransform.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -9,7 +10,7 @@ namespace ErrorCodes } CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_) - : IAccumulatingTransform(std::move(header), params_->getHeader()) + : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) , keys(params->params.keys) { @@ -74,6 +75,8 @@ Chunk CubeTransform::generate() } finalizeChunk(gen_chunk); + if (!gen_chunk.empty()) + gen_chunk.addColumn(0, ColumnUInt64::create(gen_chunk.getNumRows(), grouping_set++)); return gen_chunk; } diff --git a/src/Processors/Transforms/CubeTransform.h b/src/Processors/Transforms/CubeTransform.h index 6d0e2338174..b6f60af6aca 100644 --- a/src/Processors/Transforms/CubeTransform.h +++ b/src/Processors/Transforms/CubeTransform.h @@ -28,6 +28,7 @@ private: Columns current_zero_columns; UInt64 mask = 0; + UInt64 grouping_set = 0; Chunk merge(Chunks && chunks, bool final); }; diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index fb51b5f6b45..2355e35f8fd 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -1,11 +1,12 @@ #include #include +#include namespace DB { RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr params_) - : IAccumulatingTransform(std::move(header), params_->getHeader()) + : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) , keys(params->params.keys) { @@ -57,6 +58,8 @@ Chunk RollupTransform::generate() } finalizeChunk(gen_chunk); + if (!gen_chunk.empty()) + gen_chunk.addColumn(0, ColumnUInt64::create(gen_chunk.getNumRows(), set_counter++)); return gen_chunk; } diff --git a/src/Processors/Transforms/RollupTransform.h b/src/Processors/Transforms/RollupTransform.h index fd435740a63..e60c7e12de1 100644 --- a/src/Processors/Transforms/RollupTransform.h +++ b/src/Processors/Transforms/RollupTransform.h @@ -23,6 +23,7 @@ private: Chunks consumed_chunks; Chunk rollup_chunk; size_t last_removed_key = 0; + size_t set_counter = 0; Chunk merge(Chunks && chunks, bool final); }; diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index c1824206b60..dd6c30e3c79 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -157,7 +157,7 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; ActionsVisitor::Data visitor_data( - context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false, false); + context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false, GroupByKind::NONE); ActionsVisitor(visitor_data).visit(node); actions = visitor_data.getActions(); auto expression_actions = std::make_shared(actions); diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 38578d6ad1d..0285611b9fa 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -18,3 +18,111 @@ 7 1 1 8 1 1 9 1 1 +0 0 +0 4 +0 6 +1 4 +1 6 +2 4 +2 6 +3 4 +3 6 +4 4 +4 6 +5 4 +5 6 +6 4 +6 6 +7 4 +7 6 +8 4 +8 6 +9 4 +9 6 +0 0 +0 4 +0 6 +1 4 +1 6 +2 4 +2 6 +3 4 +3 6 +4 4 +4 6 +5 4 +5 6 +6 4 +6 6 +7 4 +7 6 +8 4 +8 6 +9 4 +9 6 +0 0 +0 1 +0 1 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 +0 0 +0 1 +0 1 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 +0 5 +0 6 +1 5 +1 6 +2 5 +2 6 +3 5 +3 6 +4 5 +4 6 +5 5 +5 6 +6 5 +6 6 +7 5 +7 6 +8 5 +8 6 +9 5 +9 6 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index 5b12c34adac..1b0fcdb9289 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -16,3 +16,53 @@ GROUP BY number, number % 2 ORDER BY number; + +SELECT + number, + grouping(number, number % 2, number % 3) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 + WITH ROLLUP +ORDER BY + number, gr; + +SELECT + number, + grouping(number, number % 2, number % 3) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) +ORDER BY + number, gr; + +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 + WITH CUBE +ORDER BY + number, gr; + +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) +ORDER BY + number, gr; + +SELECT + number, + grouping(number, number % 2) + 3 as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) +HAVING grouping(number) != 0 +ORDER BY + number, gr; From 6356112a76c998e75ebfb6d1dbd9eeff593e200a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 18 May 2022 15:23:31 +0000 Subject: [PATCH 017/136] Refactor GROUPING function --- src/Core/ColumnNumbers.h | 3 + src/Functions/grouping.cpp | 159 ------------------ src/Functions/grouping.h | 151 +++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 - src/Interpreters/ActionsVisitor.cpp | 110 ++++-------- .../02293_grouping_function.reference | 84 ++++----- .../0_stateless/02293_grouping_function.sql | 27 ++- ...02293_grouping_function_group_by.reference | 80 ++++----- .../02293_grouping_function_group_by.sql | 13 +- 9 files changed, 297 insertions(+), 332 deletions(-) delete mode 100644 src/Functions/grouping.cpp create mode 100644 src/Functions/grouping.h diff --git a/src/Core/ColumnNumbers.h b/src/Core/ColumnNumbers.h index 29b4c49dc83..2c1f02f720d 100644 --- a/src/Core/ColumnNumbers.h +++ b/src/Core/ColumnNumbers.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -8,6 +9,8 @@ namespace DB { using ColumnNumbers = std::vector; +using ColumnNumbersSet = std::unordered_set; using ColumnNumbersList = std::vector; +using ColumnNumbersSetList = std::vector; } diff --git a/src/Functions/grouping.cpp b/src/Functions/grouping.cpp deleted file mode 100644 index c6c6061307d..00000000000 --- a/src/Functions/grouping.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -class FunctionGrouping : public IFunction -{ -public: - static constexpr auto name = "grouping"; - static FunctionPtr create(ContextPtr) - { - return std::make_shared(); - } - - bool isVariadic() const override - { - return true; - } - - size_t getNumberOfArguments() const override - { - return 0; - } - - bool useDefaultImplementationForNulls() const override { return false; } - - bool isSuitableForConstantFolding() const override { return false; } - - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - - String getName() const override - { - return name; - } - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - //TODO: add assert for argument types - return std::make_shared(); - } - - ColumnPtr executeOrdinaryGroupBy(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const - { - auto grouping_set_map_column = checkAndGetColumnConst(arguments[0].column.get()); - auto argument_keys_column = checkAndGetColumnConst(arguments[1].column.get()); - - auto aggregation_keys_number = (*grouping_set_map_column)[0].get(); - - auto result = std::make_shared()->createColumn(); - for (size_t i = 0; i < input_rows_count; ++i) - { - auto indexes = (*argument_keys_column)[i].get(); - UInt64 value = 0; - for (auto index : indexes) - value = (value << 1) + (index.get() < aggregation_keys_number ? 1 : 0); - - result->insert(Field(value)); - } - return result; - } - - ColumnPtr executeRollup( - const ColumnUInt64 * grouping_set_column, - const ColumnConst & argument_keys_column, - UInt64 keys, - size_t input_rows_count) const - { - auto result = std::make_shared()->createColumn(); - for (size_t i = 0; i < input_rows_count; ++i) - { - UInt64 set_index = grouping_set_column->get64(i); - - auto indexes = argument_keys_column[i].get(); - UInt64 value = 0; - for (auto index : indexes) - value = (value << 1) + (index.get() < keys - set_index ? 1 : 0); - - result->insert(Field(value)); - } - return result; - } - - ColumnPtr executeCube( - const ColumnUInt64 * grouping_set_column, - const ColumnConst & argument_keys_column, - UInt64 keys, - size_t input_rows_count) const - { - static constexpr auto ONE = static_cast(1); - auto result = std::make_shared()->createColumn(); - auto mask_base = (ONE << keys) - 1; - for (size_t i = 0; i < input_rows_count; ++i) - { - UInt64 set_index = grouping_set_column->get64(i); - auto mask = mask_base - set_index; - auto indexes = argument_keys_column[i].get(); - UInt64 value = 0; - for (auto index : indexes) - value = (value << 1) + (mask & (ONE << (keys - index.get() - 1)) ? 1 : 0); - - result->insert(Field(value)); - } - return result; - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & , size_t input_rows_count) const override - { - if (arguments.size() == 2) - return executeOrdinaryGroupBy(arguments, input_rows_count); - - auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); - auto grouping_set_map_column = checkAndGetColumnConst(arguments[1].column.get()); - auto argument_keys_column = checkAndGetColumnConst(arguments[2].column.get()); - - auto masks = (*grouping_set_map_column)[0].get(); - auto grouping_set_map_elem_type = applyVisitor(FieldToDataType(), masks[0]); - if (!isString(grouping_set_map_elem_type)) - { - bool is_rollup = masks[0].get() == 0; - auto keys = masks[1].get(); - if (is_rollup) - return executeRollup(grouping_set_column, *argument_keys_column, keys, input_rows_count); - else - return executeCube(grouping_set_column, *argument_keys_column, keys, input_rows_count); - } - - auto result = std::make_shared()->createColumn(); - for (size_t i = 0; i < input_rows_count; ++i) - { - UInt64 set_index = grouping_set_column->get64(i); - auto mask = masks[set_index].get(); - - auto indexes = (*argument_keys_column)[i].get(); - UInt64 value = 0; - for (auto index : indexes) - value = (value << 1) + (mask[index.get()] == '1' ? 1 : 0); - - result->insert(Field(value)); - } - return result; - } - -}; - -void registerFunctionGrouping(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h new file mode 100644 index 00000000000..a881616812b --- /dev/null +++ b/src/Functions/grouping.h @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "Core/ColumnNumbers.h" +#include "DataTypes/Serializations/ISerialization.h" +#include "base/types.h" + +namespace DB +{ + +class FunctionGroupingBase : public IFunction +{ +protected: + static constexpr UInt64 ONE = 1; + + const ColumnNumbers arguments_indexes; + +public: + FunctionGroupingBase(ColumnNumbers arguments_indexes_) + : arguments_indexes(std::move(arguments_indexes_)) + {} + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } + + bool useDefaultImplementationForNulls() const override { return false; } + + bool isSuitableForConstantFolding() const override { return false; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override + { + return std::make_shared(); + } + + template + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, size_t input_rows_count, AggregationKeyChecker checker) const + { + auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); + + auto result = std::make_shared()->createColumn(); + for (size_t i = 0; i < input_rows_count; ++i) + { + UInt64 set_index = grouping_set_column->get64(i); + + UInt64 value = 0; + for (auto index : arguments_indexes) + value = (value << 1) + (checker(set_index, index) ? 1 : 0); + + result->insert(Field(value)); + } + return result; + } +}; + +class FunctionGroupingOrdinary : public FunctionGroupingBase +{ +public: + explicit FunctionGroupingOrdinary(ColumnNumbers arguments_indexes_) + : FunctionGroupingBase(std::move(arguments_indexes_)) + {} + + String getName() const override { return "groupingOrdinary"; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + UInt64 value = (ONE << arguments_indexes.size()) - 1; + return ColumnUInt64::create(input_rows_count, value); + } +}; + +class FunctionGroupingForRollup : public FunctionGroupingBase +{ + const UInt64 aggregation_keys_number; + +public: + FunctionGroupingForRollup(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) + : FunctionGroupingBase(std::move(arguments_indexes_)) + , aggregation_keys_number(aggregation_keys_number_) + {} + + String getName() const override { return "groupingForRollup"; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + return FunctionGroupingBase::executeImpl(arguments, input_rows_count, + [this](UInt64 set_index, UInt64 arg_index) + { + return arg_index < aggregation_keys_number - set_index; + } + ); + } +}; + +class FunctionGroupingForCube : public FunctionGroupingBase +{ + const UInt64 aggregation_keys_number; + +public: + + FunctionGroupingForCube(ColumnNumbers arguments_indexes_, UInt64 aggregation_keys_number_) + : FunctionGroupingBase(arguments_indexes_) + , aggregation_keys_number(aggregation_keys_number_) + {} + + String getName() const override { return "groupingForCube"; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + return FunctionGroupingBase::executeImpl(arguments, input_rows_count, + [this](UInt64 set_index, UInt64 arg_index) + { + auto set_mask = (ONE << aggregation_keys_number) - 1 - set_index; + return set_mask & (ONE << (aggregation_keys_number - arg_index - 1)); + } + ); + } +}; + +class FunctionGroupingForGroupingSets : public FunctionGroupingBase +{ + ColumnNumbersSetList grouping_sets; +public: + FunctionGroupingForGroupingSets(ColumnNumbers arguments_indexes_, ColumnNumbersList const & grouping_sets_) + : FunctionGroupingBase(std::move(arguments_indexes_)) + { + for (auto const & set : grouping_sets_) + grouping_sets.emplace_back(set.begin(), set.end()); + } + + String getName() const override { return "groupingForGroupingSets"; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + return FunctionGroupingBase::executeImpl(arguments, input_rows_count, + [this](UInt64 set_index, UInt64 arg_index) + { + return grouping_sets[set_index].contains(arg_index); + } + ); + } +}; + +} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 9fe1fa69b5e..9cd9c70da16 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -83,7 +83,6 @@ void registerFunctionZooKeeperSessionUptime(FunctionFactory &); void registerFunctionGetOSKernelVersion(FunctionFactory &); void registerFunctionGetTypeSerializationStreams(FunctionFactory &); void registerFunctionFlattenTuple(FunctionFactory &); -void registerFunctionGrouping(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -173,7 +172,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionGetOSKernelVersion(factory); registerFunctionGetTypeSerializationStreams(factory); registerFunctionFlattenTuple(factory); - registerFunctionGrouping(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 70493e5fefc..4f44513a5ea 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,6 +6,7 @@ #include #include +#include #include #include @@ -839,89 +841,39 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (node.name == "grouping") { - auto arguments_column_name = data.getUniqueName("__grouping_args"); + ColumnNumbers arguments_indexes; + auto aggregation_keys_number = data.aggregation_keys.size(); + for (auto const & arg : node.arguments->children) { - if (!data.hasColumn("__grouping_set_map")) - { - ColumnWithTypeAndName column; - column.name = "__grouping_set_map"; - switch (data.group_by_kind) - { - case GroupByKind::GROUPING_SETS: - { - size_t map_size = data.aggregation_keys.size() + 1; - column.type = std::make_shared(std::make_shared(map_size)); - Array maps_per_set; - for (auto & grouping_set : data.grouping_set_keys) - { - std::string key_map(map_size, '0'); - for (auto index : grouping_set) - key_map[index] = '1'; - maps_per_set.push_back(key_map); - } - auto grouping_set_map_column = ColumnArray::create(ColumnFixedString::create(map_size)); - grouping_set_map_column->insert(maps_per_set); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); - break; - } - case GroupByKind::ROLLUP: - case GroupByKind::CUBE: - { - column.type = std::make_shared(std::make_shared()); - auto grouping_set_map_column = ColumnArray::create(ColumnUInt64::create()); - Array kind_and_keys_size; - kind_and_keys_size.push_back(data.group_by_kind == GroupByKind::ROLLUP ? 0 : 1); - kind_and_keys_size.push_back(data.aggregation_keys.size()); - grouping_set_map_column->insert(kind_and_keys_size); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); - break; - } - case GroupByKind::ORDINARY: - { - column.type = std::make_shared(); - auto grouping_set_map_column = ColumnUInt64::create(1, data.aggregation_keys.size()); - column.column = ColumnConst::create(std::move(grouping_set_map_column), 1); - break; - } - default: - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Unexpected kind of GROUP BY clause for GROUPING function: {}", data.group_by_kind); - } - - data.addColumn(column); - } - ColumnWithTypeAndName column; - column.name = arguments_column_name; - column.type = std::make_shared(std::make_shared()); - Array arguments_to_keys_map; - for (auto const & arg : node.arguments->children) - { - size_t pos = data.aggregation_keys.getPosByName(arg->getColumnName()); - arguments_to_keys_map.push_back(pos); - } - auto arguments_column = ColumnArray::create(ColumnUInt64::create()); - arguments_column->insert(Field{arguments_to_keys_map}); - - column.column = ColumnConst::create(ColumnPtr(std::move(arguments_column)), 1); - - data.addColumn(column); + size_t pos = data.aggregation_keys.getPosByName(arg->getColumnName()); + if (pos == aggregation_keys_number) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of GROUPING function {} is not a part of GROUP BY clause", arg->getColumnName()); + arguments_indexes.push_back(pos); } - if (data.group_by_kind != GroupByKind::ORDINARY) + switch (data.group_by_kind) { - data.addFunction( - FunctionFactory::instance().get("grouping", data.getContext()), - { "__grouping_set", "__grouping_set_map", arguments_column_name }, - column_name - ); - } - else - { - data.addFunction( - FunctionFactory::instance().get("grouping", data.getContext()), - { "__grouping_set_map", arguments_column_name }, - column_name - ); + case GroupByKind::GROUPING_SETS: + { + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.grouping_set_keys)), { "__grouping_set" }, column_name); + break; + } + case GroupByKind::ROLLUP: + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.aggregation_keys.size())), { "__grouping_set" }, column_name); + break; + case GroupByKind::CUBE: + { + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.aggregation_keys.size())), { "__grouping_set" }, column_name); + break; + } + case GroupByKind::ORDINARY: + { + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes))), {}, column_name); + break; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected kind of GROUP BY clause for GROUPING function: {}", data.group_by_kind); } return; } diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference index f08e6d0ea99..dbae7a11f2e 100644 --- a/tests/queries/0_stateless/02293_grouping_function.reference +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -1,27 +1,27 @@ -0 2 -0 2 -0 4 -1 4 -2 4 -3 4 -4 4 -5 4 -6 4 -7 4 -8 4 -9 4 0 1 0 1 -0 4 -1 4 -2 4 -3 4 -4 4 -5 4 -6 4 -7 4 -8 4 -9 4 +0 2 +1 2 +2 2 +3 2 +4 2 +5 2 +6 2 +7 2 +8 2 +9 2 +0 1 +0 2 +0 2 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +8 1 +9 1 0 0 0 1 0 1 @@ -47,26 +47,26 @@ 8 9 0 10 0 -0 1 4 -1 1 4 -2 1 4 -3 1 4 -4 1 4 -5 1 4 -6 1 4 -7 1 4 -8 1 4 -9 1 4 -0 1 6 -1 1 6 -2 1 6 -3 1 6 -4 1 6 -5 1 6 -6 1 6 -7 1 6 -8 1 6 -9 1 6 +0 1 2 +1 1 2 +2 1 2 +3 1 2 +4 1 2 +5 1 2 +6 1 2 +7 1 2 +8 1 2 +9 1 2 +0 1 3 +1 1 3 +2 1 3 +3 1 3 +4 1 3 +5 1 3 +6 1 3 +7 1 3 +8 1 3 +9 1 3 0 1 2 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index 3555f9dabab..4bbf620a619 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -7,11 +7,11 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, gr; +ORDER BY number, gr; -- { serverError BAD_ARGUMENTS } SELECT number, - grouping(number, number % 3, number % 2) AS gr + grouping(number, number % 2) AS gr FROM numbers(10) GROUP BY GROUPING SETS ( @@ -22,7 +22,18 @@ ORDER BY number, gr; SELECT number, - grouping(number, number % 2, number % 3) = 2 AS gr + grouping(number % 2, number) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; + +SELECT + number, + grouping(number, number % 2) = 1 AS gr FROM numbers(10) GROUP BY GROUPING SETS ( @@ -39,12 +50,12 @@ GROUP BY (number), (number % 2) ) -ORDER BY number, grouping(number, number % 2, number % 3) = 2; +ORDER BY number, grouping(number, number % 2) = 1; SELECT number, count(), - grouping(number, number % 2, number % 3) AS gr + grouping(number, number % 2) AS gr FROM numbers(10) GROUP BY GROUPING SETS ( @@ -62,7 +73,7 @@ GROUP BY (number), (number % 2) ) -HAVING grouping(number, number % 2, number % 3) = 4 +HAVING grouping(number, number % 2) = 2 ORDER BY number SETTINGS enable_optimize_predicate_expression = 0; @@ -74,13 +85,13 @@ GROUP BY (number), (number % 2) ) -HAVING grouping(number, number % 2, number % 3) = 2 +HAVING grouping(number, number % 2) = 1 ORDER BY number SETTINGS enable_optimize_predicate_expression = 0; SELECT number, - GROUPING(number, number % 2, number % 3) = 2 as gr + GROUPING(number, number % 2) = 1 as gr FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY GROUPING SETS ( diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 0285611b9fa..9f73523728d 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -19,47 +19,47 @@ 8 1 1 9 1 1 0 0 -0 4 -0 6 -1 4 -1 6 -2 4 -2 6 -3 4 -3 6 -4 4 -4 6 -5 4 -5 6 -6 4 -6 6 -7 4 -7 6 -8 4 -8 6 -9 4 -9 6 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 0 0 -0 4 -0 6 -1 4 -1 6 -2 4 -2 6 -3 4 -3 6 -4 4 -4 6 -5 4 -5 6 -6 4 -6 6 -7 4 -7 6 -8 4 -8 6 -9 4 -9 6 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 0 0 0 1 0 1 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index 1b0fcdb9289..e9a0338c35a 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -2,6 +2,15 @@ SELECT number, grouping(number, number % 2, number % 3) = 6 FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 +ORDER BY number; -- { serverError BAD_ARGUMENTS } + +SELECT + number, + grouping(number, number % 2) = 3 +FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, number % 2 @@ -19,7 +28,7 @@ ORDER BY number; SELECT number, - grouping(number, number % 2, number % 3) AS gr + grouping(number, number % 2) AS gr FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY number, @@ -30,7 +39,7 @@ ORDER BY SELECT number, - grouping(number, number % 2, number % 3) AS gr + grouping(number, number % 2) AS gr FROM remote('127.0.0.{2,3}', numbers(10)) GROUP BY ROLLUP(number, number % 2) From d4c66f4a4859fae200d740567572b11172b96a6d Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 19 May 2022 16:36:51 +0000 Subject: [PATCH 018/136] Code cleanup & fix GROUPING() with TOTALS --- src/Functions/grouping.h | 10 ++-- src/Interpreters/ActionsVisitor.cpp | 7 +++ src/Interpreters/InterpreterSelectQuery.cpp | 7 ++- src/Parsers/ASTSelectQuery.h | 34 +++++++------ src/Processors/QueryPlan/AggregatingStep.cpp | 2 +- src/Processors/QueryPlan/CubeStep.cpp | 23 ++++++++- src/Processors/QueryPlan/CubeStep.h | 1 + src/Processors/QueryPlan/RollupStep.cpp | 7 ++- src/Processors/QueryPlan/RollupStep.h | 1 + ...02293_grouping_function_group_by.reference | 48 +++++++++++++++++++ .../02293_grouping_function_group_by.sql | 38 +++++++++++++++ 11 files changed, 146 insertions(+), 32 deletions(-) diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h index a881616812b..934be18345d 100644 --- a/src/Functions/grouping.h +++ b/src/Functions/grouping.h @@ -1,14 +1,14 @@ -#include +#pragma once + #include #include #include #include +#include +#include #include #include -#include -#include "Core/ColumnNumbers.h" -#include "DataTypes/Serializations/ISerialization.h" -#include "base/types.h" +#include namespace DB { diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 4f44513a5ea..d22989219a4 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -65,6 +65,8 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int DUPLICATE_COLUMN; extern const int LOGICAL_ERROR; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; } static NamesAndTypesList::iterator findColumn(const String & name, NamesAndTypesList & cols) @@ -841,6 +843,11 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (node.name == "grouping") { + size_t arguments_size = node.arguments->children.size(); + if (arguments_size == 0) + throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function GROUPING expects at least one argument"); + if (arguments_size > 64) + throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Function GROUPING can have up to 64 arguments, but {} provided", arguments_size); ColumnNumbers arguments_indexes; auto aggregation_keys_number = data.aggregation_keys.size(); for (auto const & arg : node.arguments->children) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6bfadc66352..3b438ef9863 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1095,6 +1095,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

desc->type == ProjectionDescription::Type::Aggregate) { query_info.projection->aggregate_overflow_row = aggregate_overflow_row; @@ -1387,11 +1390,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

(header.getColumnsWithTypeAndName()); ActionsDAG::NodeRawConstPtrs index; - index.reserve(output_header.columns() + 2); + index.reserve(output_header.columns() + 1); auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0); const auto * grouping_node = &dag->addColumn( diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 43a6491157a..91c85a08412 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -1,7 +1,9 @@ #include #include +#include #include #include +#include namespace DB { @@ -24,6 +26,7 @@ static ITransformingStep::Traits getTraits() CubeStep::CubeStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) + , keys_size(params_->params.keys_size) , params(std::move(params_)) { /// Aggregation keys are distinct @@ -31,14 +34,30 @@ CubeStep::CubeStep(const DataStream & input_stream_, AggregatingTransformParamsP output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); } -void CubeStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) +ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number) +{ + auto dag = std::make_shared(header.getColumnsWithTypeAndName()); + + auto grouping_col = ColumnUInt64::create(1, grouping_set_number); + const auto * grouping_node = &dag->addColumn( + {ColumnPtr(std::move(grouping_col)), std::make_shared(), "__grouping_set"}); + + grouping_node = &dag->materializeNode(*grouping_node); + auto & index = dag->getIndex(); + index.insert(index.begin(), grouping_node); + + auto expression = std::make_shared(dag, settings.getActionsSettings()); + return std::make_shared(header, expression); +} + +void CubeStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { pipeline.resize(1); pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr { if (stream_type == QueryPipelineBuilder::StreamType::Totals) - return nullptr; + return addGroupingSetForTotals(header, settings, (UInt64(1) << keys_size) - 1); return std::make_shared(header, std::move(params)); }); diff --git a/src/Processors/QueryPlan/CubeStep.h b/src/Processors/QueryPlan/CubeStep.h index 1079bed5398..d3e26f9379f 100644 --- a/src/Processors/QueryPlan/CubeStep.h +++ b/src/Processors/QueryPlan/CubeStep.h @@ -21,6 +21,7 @@ public: const Aggregator::Params & getParams() const; private: + size_t keys_size; AggregatingTransformParamsPtr params; }; diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index 2961ef5ddbd..3b061f9c246 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -25,20 +25,23 @@ static ITransformingStep::Traits getTraits() RollupStep::RollupStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) , params(std::move(params_)) + , keys_size(params->params.keys_size) { /// Aggregation keys are distinct for (auto key : params->params.keys) output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); } -void RollupStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) +ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number); + +void RollupStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { pipeline.resize(1); pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr { if (stream_type == QueryPipelineBuilder::StreamType::Totals) - return nullptr; + return addGroupingSetForTotals(header, settings, keys_size); return std::make_shared(header, std::move(params)); }); diff --git a/src/Processors/QueryPlan/RollupStep.h b/src/Processors/QueryPlan/RollupStep.h index 7cd71fecdc1..3dce6f74d9f 100644 --- a/src/Processors/QueryPlan/RollupStep.h +++ b/src/Processors/QueryPlan/RollupStep.h @@ -20,6 +20,7 @@ public: private: AggregatingTransformParamsPtr params; + size_t keys_size; }; } diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 9f73523728d..021083db6eb 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -126,3 +126,51 @@ 8 6 9 5 9 6 +0 0 +0 1 +0 1 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 + +0 0 +0 0 +0 2 +0 3 +1 2 +1 3 +2 2 +2 3 +3 2 +3 3 +4 2 +4 3 +5 2 +5 3 +6 2 +6 3 +7 2 +7 3 +8 2 +8 3 +9 2 +9 3 + +0 0 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index e9a0338c35a..b30080b88af 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -75,3 +75,41 @@ GROUP BY HAVING grouping(number) != 0 ORDER BY number, gr; + +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) WITH TOTALS +HAVING grouping(number) != 0 +ORDER BY + number, gr; -- { serverError NOT_IMPLEMENTED } + +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) WITH TOTALS +ORDER BY + number, gr; + +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) WITH TOTALS +HAVING grouping(number) != 0 +ORDER BY + number, gr; -- { serverError NOT_IMPLEMENTED } + +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) WITH TOTALS +ORDER BY + number, gr; From 37b66c8a9e70cab6b04ab87d4366ee0d42858a7f Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 23 May 2022 12:48:48 +0000 Subject: [PATCH 019/136] Check format name on storage creation --- src/Formats/FormatFactory.cpp | 7 +++++++ src/Formats/FormatFactory.h | 3 +++ src/Storages/HDFS/StorageHDFS.cpp | 1 + src/Storages/StorageFile.cpp | 2 ++ src/Storages/StorageS3.cpp | 1 + src/Storages/StorageURL.cpp | 1 + .../02311_create_table_with_unknown_format.reference | 0 .../0_stateless/02311_create_table_with_unknown_format.sql | 4 ++++ 8 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/02311_create_table_with_unknown_format.reference create mode 100644 tests/queries/0_stateless/02311_create_table_with_unknown_format.sql diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4c1b23a75ab..33fd68e67f7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -585,6 +585,13 @@ bool FormatFactory::checkIfFormatHasAnySchemaReader(const String & name) return checkIfFormatHasSchemaReader(name) || checkIfFormatHasExternalSchemaReader(name); } +void FormatFactory::checkFormatName(const String & name) const +{ + auto it = dict.find(name); + if (it == dict.end()) + throw Exception("Unknown format " + name, ErrorCodes::UNKNOWN_FORMAT); +} + FormatFactory & FormatFactory::instance() { static FormatFactory ret; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index f7d3c23d3b4..0431d1ef8c9 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -210,6 +210,9 @@ public: bool isInputFormat(const String & name) const; bool isOutputFormat(const String & name) const; + /// Check that format with specified name exists and throw an exception otherwise. + void checkFormatName(const String & name) const; + private: FormatsDictionary dict; FileExtensionFormats file_extension_formats; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index d114bb67016..a90db60e341 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -146,6 +146,7 @@ StorageHDFS::StorageHDFS( , distributed_processing(distributed_processing_) , partition_by(partition_by_) { + FormatFactory::instance().checkFormatName(format_name); context_->getRemoteHostFilter().checkURL(Poco::URI(uri_)); checkHDFSURL(uri_); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 47e32337dfe..a3a1d061ecc 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -382,6 +382,8 @@ StorageFile::StorageFile(CommonArguments args) , compression_method(args.compression_method) , base_path(args.getContext()->getPath()) { + if (format_name != "Distributed") + FormatFactory::instance().checkFormatName(format_name); } void StorageFile::setStorageMetadata(CommonArguments args) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index d402dce5ede..d7e3d5e5374 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -609,6 +609,7 @@ StorageS3::StorageS3( , partition_by(partition_by_) , is_key_with_globs(uri_.key.find_first_of("*?{") != std::string::npos) { + FormatFactory::instance().checkFormatName(format_name); context_->getGlobalContext()->getRemoteHostFilter().checkURL(uri_.uri); StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 0db4fa75aba..1961711785d 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -74,6 +74,7 @@ IStorageURLBase::IStorageURLBase( , http_method(http_method_) , partition_by(partition_by_) { + FormatFactory::instance().checkFormatName(format_name); StorageInMemoryMetadata storage_metadata; if (columns_.empty()) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.reference b/tests/queries/0_stateless/02311_create_table_with_unknown_format.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql new file mode 100644 index 00000000000..5f43ecd1c65 --- /dev/null +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -0,0 +1,4 @@ +create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=S3('http://host:2020/test/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} +create table test_02311 (x UInt32) engine=HDFS('http://hdfs:9000/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} From e9187ec4b7938616957bbad06c4816fcd94d7777 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 23 May 2022 14:35:09 +0000 Subject: [PATCH 020/136] Overcommit: update defaults, exception message and add ProfileEvent --- programs/server/Server.cpp | 2 +- src/Common/MemoryTracker.cpp | 36 ++++++++++++++++--- src/Common/OvercommitTracker.cpp | 24 ++++++++++--- src/Common/OvercommitTracker.h | 12 ++++++- src/Common/ProfileEvents.cpp | 1 + src/Common/tests/gtest_overcommit_tracker.cpp | 20 +++++------ 6 files changed, 73 insertions(+), 22 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index defc66b0ed9..18ab96983eb 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1095,7 +1095,7 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); - UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 200); + UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 5'000'000); global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time); total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 0e7803aaa71..b5a27543b4e 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #ifdef MEMORY_TRACKER_DEBUG_CHECKS @@ -52,6 +53,30 @@ namespace DB } } +namespace +{ + +inline std::string_view toDescription(OvercommitResult result) +{ + switch (result) + { + case OvercommitResult::NONE: + return "Memory overcommit isn't used. OvercommitTracker isn't set."; + case OvercommitResult::DISABLED: + return "Memory overcommit isn't used. Waiting time or orvercommit denominator are set to zero."; + case OvercommitResult::MEMORY_FREED: + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "OvercommitResult::MEMORY_FREED shouldn't be asked for description"); + case OvercommitResult::SELECTED: + return "Query was selected to stop by OvercommitTracker."; + case OvercommitResult::TIMEOUTED: + return "Waiting timeout for memory to be freed is reached."; + case OvercommitResult::NOT_ENOUGH_FREED: + return "Memory overcommit has freed not enough memory."; + } +} + +} + namespace ProfileEvents { extern const Event QueryMemoryLimitExceeded; @@ -189,11 +214,11 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { - bool need_to_throw = true; + OvercommitResult overcommit_result = OvercommitResult::NONE; if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr != nullptr && query_tracker != nullptr) - need_to_throw = overcommit_tracker_ptr->needToStopQuery(query_tracker, size); + overcommit_result = overcommit_tracker_ptr->needToStopQuery(query_tracker, size); - if (need_to_throw) + if (overcommit_result != OvercommitResult::MEMORY_FREED) { /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); @@ -201,12 +226,13 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT const auto * description = description_ptr.load(std::memory_order_relaxed); throw DB::Exception( DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, - "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}. OvercommitTracker decision: {}.", description ? " " : "", description ? description : "", formatReadableSizeWithBinarySuffix(will_be), size, - formatReadableSizeWithBinarySuffix(current_hard_limit)); + formatReadableSizeWithBinarySuffix(current_hard_limit), + toDescription(overcommit_result)); } else { diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index dbacc0d81a4..0c03ba58e87 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -2,8 +2,14 @@ #include #include +#include #include +namespace ProfileEvents +{ + extern const Event MemoryOvercommitWaitTimeMicroseconds; +} + using namespace std::chrono_literals; constexpr std::chrono::microseconds ZERO_MICROSEC = 0us; @@ -24,7 +30,7 @@ void OvercommitTracker::setMaxWaitTime(UInt64 wait_time) max_wait_time = wait_time * 1us; } -bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) +OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { // NOTE: Do not change the order of locks // @@ -36,7 +42,7 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) std::unique_lock lk(overcommit_m); if (max_wait_time == ZERO_MICROSEC) - return true; + return OvercommitResult::DISABLED; pickQueryToExclude(); assert(cancellation_state != QueryCancellationState::NONE); @@ -50,7 +56,7 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) // picked_tracker to be not null pointer. assert(cancellation_state == QueryCancellationState::SELECTED); cancellation_state = QueryCancellationState::NONE; - return true; + return OvercommitResult::DISABLED; } if (picked_tracker == tracker) { @@ -58,17 +64,20 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) // It may happen even when current state is RUNNING, because // ThreadStatus::~ThreadStatus may call MemoryTracker::alloc. cancellation_state = QueryCancellationState::RUNNING; - return true; + return OvercommitResult::SELECTED; } allow_release = true; required_memory += amount; required_per_thread[tracker] = amount; + auto wait_start_time = std::chrono::system_clock::now(); bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]() { return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE; }); + auto wait_end_time = std::chrono::system_clock::now(); + ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; @@ -84,7 +93,12 @@ bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) // As we don't need to free memory, we can continue execution of the selected query. if (required_memory == 0 && cancellation_state == QueryCancellationState::SELECTED) reset(); - return timeout || still_need != 0; + if (timeout) + return OvercommitResult::TIMEOUTED; + if (still_need != 0) + return OvercommitResult::NOT_ENOUGH_FREED; + else + return OvercommitResult::MEMORY_FREED; } void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 37de75f4848..79ed36cd7fa 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -36,6 +36,16 @@ struct OvercommitRatio class MemoryTracker; +enum class OvercommitResult +{ + NONE, + DISABLED, + MEMORY_FREED, + SELECTED, + TIMEOUTED, + NOT_ENOUGH_FREED, +}; + enum class QueryCancellationState { NONE = 0, // Hard limit is not reached, there is no selected query to kill. @@ -54,7 +64,7 @@ struct OvercommitTracker : boost::noncopyable { void setMaxWaitTime(UInt64 wait_time); - bool needToStopQuery(MemoryTracker * tracker, Int64 amount); + OvercommitResult needToStopQuery(MemoryTracker * tracker, Int64 amount); void tryContinueQueryExecutionAfterFree(Int64 amount); diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7f3b9788c1f..72fefc3e31c 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -192,6 +192,7 @@ M(RealTimeMicroseconds, "Total (wall clock) time spent in processing (queries and other tasks) threads (not that this is a sum).") \ M(UserTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in user space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \ M(SystemTimeMicroseconds, "Total time spent in processing (queries and other tasks) threads executing CPU instructions in OS kernel space. This include time CPU pipeline was stalled due to cache misses, branch mispredictions, hyper-threading, etc.") \ + M(MemoryOvercommitWaitTimeMicroseconds, "Total time spent in waiting for memory to be freed in OvercommitTracker.") \ M(SoftPageFaults, "") \ M(HardPageFaults, "") \ \ diff --git a/src/Common/tests/gtest_overcommit_tracker.cpp b/src/Common/tests/gtest_overcommit_tracker.cpp index 542af815842..c56ecec669f 100644 --- a/src/Common/tests/gtest_overcommit_tracker.cpp +++ b/src/Common/tests/gtest_overcommit_tracker.cpp @@ -56,7 +56,7 @@ void free_not_continue_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -112,7 +112,7 @@ void free_continue_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -168,7 +168,7 @@ void free_continue_and_alloc_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -181,7 +181,7 @@ void free_continue_and_alloc_test(T & overcommit_tracker) MemoryTracker failed; std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); - stopped_next = overcommit_tracker.needToStopQuery(&failed, 100); + stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; } ).join(); @@ -228,7 +228,7 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -241,7 +241,7 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker) MemoryTracker failed; std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); - stopped_next = overcommit_tracker.needToStopQuery(&failed, 100); + stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; } )); @@ -296,7 +296,7 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -309,7 +309,7 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker) MemoryTracker failed; std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); - stopped_next = overcommit_tracker.needToStopQuery(&failed, 100); + stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; } )); @@ -364,7 +364,7 @@ void free_continue_2_test(T & overcommit_tracker) threads.push_back(std::thread( [&, i]() { - if (overcommit_tracker.needToStopQuery(&trackers[i], 100)) + if (overcommit_tracker.needToStopQuery(&trackers[i], 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } )); @@ -415,7 +415,7 @@ void query_stop_not_continue_test(T & overcommit_tracker) auto thread = std::thread( [&]() { - if (overcommit_tracker.needToStopQuery(&another, 100)) + if (overcommit_tracker.needToStopQuery(&another, 100) != OvercommitResult::MEMORY_FREED) ++need_to_stop; } ); From b73d49158d73245e8ee190dfefad196b017f9bcb Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 23 May 2022 17:01:45 +0200 Subject: [PATCH 021/136] Fix test --- .../0_stateless/02311_create_table_with_unknown_format.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql index 5f43ecd1c65..d046ffebca2 100644 --- a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=S3('http://host:2020/test/data', UnknownFormat); -- {serverError UNKNOWN_FORMAT} From 2f37ad7fb8502c2ad21cebff8a7ee22497528efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 24 May 2022 00:12:00 +0200 Subject: [PATCH 022/136] Improve comment --- src/Core/SettingsFields.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 7b820401468..d35865d35c6 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -255,9 +255,13 @@ void SettingFieldMilliseconds::parseFromString(const String & str) template void SettingFieldTimespan::writeBinary(WriteBuffer & out) const { - /// Note that this is unchanged and returns UInt64 for both seconds and milliseconds for - /// compatibility reasons as it's only used by the clients or servers older than - /// DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + /// Note that this returns an UInt64 (for both seconds and milliseconds units) for compatibility reasons as the value + /// for seconds used to be a integer (now a Float64) + /// This method is only used to communicate with clients or servers older than DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS + /// in which the value was passed as binary (as a UInt64) + /// Later versions pass the setting values as String (using toString() and parseFromString()) and there passing "1.2" will + /// lead to `1` on releases with integer seconds or `1.2` on more recent releases + /// See https://github.com/ClickHouse/ClickHouse/issues/36940 for more details auto num_units = operator UInt64(); writeVarUInt(num_units, out); } From cd4020fa2ff6ed0c283accbe9437be2a581d6b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Sat, 14 May 2022 12:31:52 +0200 Subject: [PATCH 023/136] Reduce flakiness --- .../02294_floating_point_second_in_settings.reference | 2 +- .../0_stateless/02294_floating_point_second_in_settings.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference b/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference index f6216e2486a..c12d45c746a 100644 --- a/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference +++ b/tests/queries/0_stateless/02294_floating_point_second_in_settings.reference @@ -5,4 +5,4 @@ maximum: 1.1 HTTP CLIENT maximum: 1.1 TABLE: system.settings -max_execution_time 0.1 1 +max_execution_time 30.5 1 diff --git a/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh b/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh index 78aece76e49..b5bf2deb974 100755 --- a/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh +++ b/tests/queries/0_stateless/02294_floating_point_second_in_settings.sh @@ -38,4 +38,4 @@ check_output "${OUTPUT}" # CHECK system.settings echo "TABLE: system.settings" -echo "SELECT name, value, changed from system.settings where name = 'max_execution_time'" | clickhouse-client --max_execution_time 0.1 +echo "SELECT name, value, changed from system.settings where name = 'max_execution_time'" | clickhouse-client --max_execution_time 30.5 From 9518c41dda147023d37c47b220c2650999411bea Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 25 May 2022 09:01:12 +0000 Subject: [PATCH 024/136] Try to fix tests --- .../test_allowed_url_from_config/test.py | 24 +++++++++---------- ...02311_create_table_with_unknown_format.sql | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 01a2a500ebf..da9d4404c82 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -33,7 +33,7 @@ def start_cluster(): def test_config_with_hosts(start_cluster): assert ( node1.query( - "CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', HDFS)" + "CREATE TABLE table_test_1_1 (word String) Engine=URL('http://host:80', CSV)" ) == "" ) @@ -44,7 +44,7 @@ def test_config_with_hosts(start_cluster): == "" ) assert "not allowed" in node1.query_and_get_error( - "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', S3)" + "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://host:123', CSV)" ) assert "not allowed" in node1.query_and_get_error( "CREATE TABLE table_test_1_4 (word String) Engine=URL('https://yandex2.ru', CSV)" @@ -60,7 +60,7 @@ def test_config_with_only_primary_hosts(start_cluster): ) assert ( node2.query( - "CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', S3)" + "CREATE TABLE table_test_2_2 (word String) Engine=URL('https://host:123', CSV)" ) == "" ) @@ -72,25 +72,25 @@ def test_config_with_only_primary_hosts(start_cluster): ) assert ( node2.query( - "CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', HDFS)" + "CREATE TABLE table_test_2_4 (word String) Engine=URL('https://yandex.ru:87', CSV)" ) == "" ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node2.query_and_get_error( "CREATE TABLE table_test_2_5 (word String) Engine=URL('https://host:234', CSV)" ) assert "not allowed" in node2.query_and_get_error( - "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', S3)" + "CREATE TABLE table_test_2_6 (word String) Engine=URL('https://yandex2.ru', CSV)" ) def test_config_with_only_regexp_hosts(start_cluster): assert ( node3.query( - "CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', HDFS)" + "CREATE TABLE table_test_3_1 (word String) Engine=URL('https://host:80', CSV)" ) == "" ) @@ -104,7 +104,7 @@ def test_config_with_only_regexp_hosts(start_cluster): "CREATE TABLE table_test_3_3 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node3.query_and_get_error( - "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', S3)" + "CREATE TABLE table_test_3_4 (word String) Engine=URL('https://yandex2.ru', CSV)" ) @@ -123,7 +123,7 @@ def test_config_without_allowed_hosts_section(start_cluster): ) assert ( node4.query( - "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://host', CSV)" ) == "" ) @@ -135,7 +135,7 @@ def test_config_without_allowed_hosts_section(start_cluster): ) assert ( node4.query( - "CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', S3)" + "CREATE TABLE table_test_4_5 (word String) Engine=URL('ftp://something.com', CSV)" ) == "" ) @@ -149,13 +149,13 @@ def test_config_without_allowed_hosts(start_cluster): "CREATE TABLE table_test_5_2 (word String) Engine=S3('https://host:80/bucket/key', CSV)" ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', HDFS)" + "CREATE TABLE table_test_5_3 (word String) Engine=URL('https://host', CSV)" ) assert "not allowed" in node5.query_and_get_error( "CREATE TABLE table_test_5_4 (word String) Engine=URL('https://yandex.ru', CSV)" ) assert "not allowed" in node5.query_and_get_error( - "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', S3)" + "CREATE TABLE table_test_5_5 (word String) Engine=URL('ftp://something.com', CSV)" ) diff --git a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql index d046ffebca2..54e388c3cf0 100644 --- a/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql +++ b/tests/queries/0_stateless/02311_create_table_with_unknown_format.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, use-hdfs, no-backward-compatibility-check:22.5 create table test_02311 (x UInt32) engine=File(UnknownFormat); -- {serverError UNKNOWN_FORMAT} create table test_02311 (x UInt32) engine=URL('http://some/url', UnknownFormat); -- {serverError UNKNOWN_FORMAT} From 0e5f13e53eee2a88ec66e81fea1f7c5519403395 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 13 May 2022 16:45:57 +0200 Subject: [PATCH 025/136] MergingSortedAlgorithm single column specialization --- src/Core/SortCursor.h | 2 +- .../Merges/Algorithms/MergingSortedAlgorithm.cpp | 9 +++++++++ .../Merges/Algorithms/MergingSortedAlgorithm.h | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 584556045e8..ba786467385 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -241,7 +241,7 @@ struct SimpleSortCursor : SortCursorHelper #endif { int non_jit_result = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); - result = (non_jit_result != 0 && ((non_jit_result > 0) == (direction > 0))); + result = non_jit_result * direction > 0; } return result; diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index c5a72bf66bc..bdf12680448 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -75,7 +75,10 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) if (has_collation) queue_with_collation = SortingHeap(cursors); else + if (description.size() > 1) queue_without_collation = SortingHeap(cursors); + else + queue_simple = SortingHeap(cursors); } void MergingSortedAlgorithm::consume(Input & input, size_t source_num) @@ -87,7 +90,10 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) if (has_collation) queue_with_collation.push(cursors[source_num]); else + if (description.size() > 1) queue_without_collation.push(cursors[source_num]); + else + queue_simple.push(cursors[source_num]); } IMergingAlgorithm::Status MergingSortedAlgorithm::merge() @@ -95,7 +101,10 @@ IMergingAlgorithm::Status MergingSortedAlgorithm::merge() if (has_collation) return mergeImpl(queue_with_collation); else + if (description.size() > 1) return mergeImpl(queue_without_collation); + else + return mergeImpl(queue_simple); } template diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h index cf3ec44f5fc..787bec7fca0 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.h @@ -49,6 +49,7 @@ private: SortCursorImpls cursors; + SortingHeap queue_simple; SortingHeap queue_without_collation; SortingHeap queue_with_collation; From 6c033f340bdab3dd5bae9466932fc0cbcef9766a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 16 May 2022 11:43:36 +0200 Subject: [PATCH 026/136] Fixed tests --- src/Core/SortCursor.h | 4 ++++ .../Merges/Algorithms/MergingSortedAlgorithm.cpp | 9 +++------ .../0_stateless/00147_alter_nested_default.reference | 2 +- tests/queries/0_stateless/00147_alter_nested_default.sql | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index ba786467385..5f3dbbfb688 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -222,6 +222,10 @@ struct SimpleSortCursor : SortCursorHelper bool ALWAYS_INLINE greaterAt(const SimpleSortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const { + if (unlikely(impl->sort_columns_size == 0)) { + return impl->order > rhs.impl->order; + } + const auto & desc = impl->desc[0]; int direction = desc.direction; int nulls_direction = desc.nulls_direction; diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index bdf12680448..1453105b068 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -74,8 +74,7 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) if (has_collation) queue_with_collation = SortingHeap(cursors); - else - if (description.size() > 1) + else if (description.size() > 1) queue_without_collation = SortingHeap(cursors); else queue_simple = SortingHeap(cursors); @@ -89,8 +88,7 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) if (has_collation) queue_with_collation.push(cursors[source_num]); - else - if (description.size() > 1) + else if (description.size() > 1) queue_without_collation.push(cursors[source_num]); else queue_simple.push(cursors[source_num]); @@ -100,8 +98,7 @@ IMergingAlgorithm::Status MergingSortedAlgorithm::merge() { if (has_collation) return mergeImpl(queue_with_collation); - else - if (description.size() > 1) + else if (description.size() > 1) return mergeImpl(queue_without_collation); else return mergeImpl(queue_simple); diff --git a/tests/queries/0_stateless/00147_alter_nested_default.reference b/tests/queries/0_stateless/00147_alter_nested_default.reference index db1e3c7677e..130fb4897ef 100644 --- a/tests/queries/0_stateless/00147_alter_nested_default.reference +++ b/tests/queries/0_stateless/00147_alter_nested_default.reference @@ -17,8 +17,8 @@ 2015-01-01 ['Hello','World'] [0,0] 2015-01-01 ['Hello2','World2'] [0,0] 2015-01-01 Hello 0 -2015-01-01 World 0 2015-01-01 Hello2 0 +2015-01-01 World 0 2015-01-01 World2 0 2015-01-01 Hello 0 2015-01-01 Hello2 0 diff --git a/tests/queries/0_stateless/00147_alter_nested_default.sql b/tests/queries/0_stateless/00147_alter_nested_default.sql index e0a180b43cf..54c99545364 100644 --- a/tests/queries/0_stateless/00147_alter_nested_default.sql +++ b/tests/queries/0_stateless/00147_alter_nested_default.sql @@ -22,8 +22,8 @@ SELECT * FROM alter_00147 ARRAY JOIN n WHERE n.x LIKE '%Hello%' ORDER BY n.x; OPTIMIZE TABLE alter_00147; -SELECT * FROM alter_00147; -SELECT * FROM alter_00147 ARRAY JOIN n; -SELECT * FROM alter_00147 ARRAY JOIN n WHERE n.x LIKE '%Hello%'; +SELECT * FROM alter_00147 ORDER BY n.x; +SELECT * FROM alter_00147 ARRAY JOIN n ORDER BY n.x; +SELECT * FROM alter_00147 ARRAY JOIN n WHERE n.x LIKE '%Hello%' ORDER BY n.x; DROP TABLE alter_00147; From 9a9df26eecbc535d9db91a341bccebec85297a3a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 17 May 2022 18:20:50 +0200 Subject: [PATCH 027/136] Fixed tests --- src/Core/SortCursor.h | 4 ---- .../Impl/PrometheusTextOutputFormat.cpp | 19 ++++++++++++++----- .../Algorithms/MergingSortedAlgorithm.cpp | 18 +++++++++--------- ...storage_merge_aliases_with_where.reference | 19 ++++++++++--------- ..._test_storage_merge_aliases_with_where.sql | 19 +++++++++---------- .../01410_nullable_key_and_index.reference | 6 +++--- .../01410_nullable_key_and_index.sql | 2 +- .../01521_alter_enum_and_reverse_read.sql | 2 +- ...t_merge_across_partitions_select_final.sql | 3 +-- .../0_stateless/01700_deltasum.reference | 6 +++--- tests/queries/0_stateless/01700_deltasum.sql | 16 ++++++++-------- .../01780_column_sparse_full.reference | 2 +- .../0_stateless/01780_column_sparse_full.sql | 2 +- .../01781_merge_tree_deduplication.reference | 2 +- .../01781_merge_tree_deduplication.sql | 2 +- .../02267_output_format_prometheus.reference | 2 +- 16 files changed, 64 insertions(+), 60 deletions(-) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index 5f3dbbfb688..ba786467385 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -222,10 +222,6 @@ struct SimpleSortCursor : SortCursorHelper bool ALWAYS_INLINE greaterAt(const SimpleSortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const { - if (unlikely(impl->sort_columns_size == 0)) { - return impl->order > rhs.impl->order; - } - const auto & desc = impl->desc[0]; int direction = desc.direction; int nulls_direction = desc.nulls_direction; diff --git a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp index 3d0a02c99e8..5b6a42d1a77 100644 --- a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -107,17 +108,25 @@ void PrometheusTextOutputFormat::fixupBucketLabels(CurrentMetric & metric) { String bucket_label = metric.type == "histogram" ? "le" : "quantile"; - std::sort(metric.values.begin(), metric.values.end(), + ::sort(metric.values.begin(), metric.values.end(), [&bucket_label](const auto & lhs, const auto & rhs) { + bool lhs_labels_contain_sum = lhs.labels.contains("sum"); + bool lhs_labels_contain_count = lhs.labels.contains("count"); + bool lhs_labels_contain_sum_and_count = lhs_labels_contain_sum || lhs_labels_contain_count; + + bool rhs_labels_contain_sum = rhs.labels.contains("sum"); + bool rhs_labels_contain_count = rhs.labels.contains("count"); + bool rhs_labels_contain_sum_and_count = rhs_labels_contain_sum || rhs_labels_contain_count; + /// rows with labels at the beginning and then `_sum` and `_count` - if (lhs.labels.contains("sum") && rhs.labels.contains("count")) + if (lhs_labels_contain_sum && rhs_labels_contain_count) return true; - if (lhs.labels.contains("count") && rhs.labels.contains("sum")) + else if (lhs_labels_contain_count && rhs_labels_contain_sum) return false; - if (rhs.labels.contains("sum") || rhs.labels.contains("count")) + else if (rhs_labels_contain_sum_and_count && !lhs_labels_contain_sum_and_count) return true; - if (lhs.labels.contains("sum") || lhs.labels.contains("count")) + else if (lhs_labels_contain_sum_and_count && !rhs_labels_contain_sum_and_count) return false; auto lit = lhs.labels.find(bucket_label); diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index 1453105b068..a744b2ae051 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -74,10 +74,10 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) if (has_collation) queue_with_collation = SortingHeap(cursors); - else if (description.size() > 1) - queue_without_collation = SortingHeap(cursors); - else + else if (description.size() == 1) queue_simple = SortingHeap(cursors); + else + queue_without_collation = SortingHeap(cursors); } void MergingSortedAlgorithm::consume(Input & input, size_t source_num) @@ -88,20 +88,20 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) if (has_collation) queue_with_collation.push(cursors[source_num]); - else if (description.size() > 1) - queue_without_collation.push(cursors[source_num]); - else + else if (description.size() == 1) queue_simple.push(cursors[source_num]); + else + queue_without_collation.push(cursors[source_num]); } IMergingAlgorithm::Status MergingSortedAlgorithm::merge() { if (has_collation) return mergeImpl(queue_with_collation); - else if (description.size() > 1) - return mergeImpl(queue_without_collation); - else + else if (description.size() == 1) return mergeImpl(queue_simple); + else + return mergeImpl(queue_without_collation); } template diff --git a/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.reference b/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.reference index 569b21af197..57eebd45287 100644 --- a/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.reference +++ b/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.reference @@ -4,40 +4,40 @@ SELECT * FROM tt_m order by a; 2 4 3 4 5 12 -SELECT * FROM tt_m WHERE b != 0 order by b; +SELECT * FROM tt_m WHERE b != 0 order by b, a; 1 1 2 4 3 4 5 12 -SELECT * FROM tt_m WHERE b != 1 order by b; +SELECT * FROM tt_m WHERE b != 1 order by b, a; 2 4 3 4 5 12 -SELECT * FROM tt_m WHERE b != a * 2 order by b; +SELECT * FROM tt_m WHERE b != a * 2 order by b, a; 1 1 3 4 5 12 -SELECT * FROM tt_m WHERE b / 2 != a order by b; +SELECT * FROM tt_m WHERE b / 2 != a order by b, a; 1 1 3 4 5 12 -SELECT b FROM tt_m WHERE b >= 0 order by b; +SELECT b FROM tt_m WHERE b >= 0 order by b, a; 1 4 4 12 SELECT b FROM tt_m WHERE b == 12; 12 -SELECT b FROM tt_m ORDER BY b; +SELECT b FROM tt_m ORDER BY b, a; 1 4 4 12 -SELECT b, count() FROM tt_m GROUP BY b order by b; +SELECT b, count() FROM tt_m GROUP BY b order by b; 1 1 4 2 12 1 -SELECT b FROM tt_m order by b LIMIT 1 BY b; +SELECT b FROM tt_m order by b LIMIT 1 BY b; 1 4 12 @@ -47,7 +47,8 @@ SELECT max(a) FROM tt_m group by b order by b; 1 3 5 -SELECT a FROM tt_m order by b LIMIT 1 BY b; +SELECT a FROM tt_m order by b, a; 1 2 +3 5 diff --git a/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.sql b/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.sql index 20a22eb48b1..61c7fe3681e 100644 --- a/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.sql +++ b/tests/queries/0_stateless/01214_test_storage_merge_aliases_with_where.sql @@ -17,18 +17,17 @@ INSERT INTO tt4 VALUES (5); -- { echo } SELECT * FROM tt_m order by a; -SELECT * FROM tt_m WHERE b != 0 order by b; -SELECT * FROM tt_m WHERE b != 1 order by b; -SELECT * FROM tt_m WHERE b != a * 2 order by b; -SELECT * FROM tt_m WHERE b / 2 != a order by b; +SELECT * FROM tt_m WHERE b != 0 order by b, a; +SELECT * FROM tt_m WHERE b != 1 order by b, a; +SELECT * FROM tt_m WHERE b != a * 2 order by b, a; +SELECT * FROM tt_m WHERE b / 2 != a order by b, a; -SELECT b FROM tt_m WHERE b >= 0 order by b; +SELECT b FROM tt_m WHERE b >= 0 order by b, a; SELECT b FROM tt_m WHERE b == 12; -SELECT b FROM tt_m ORDER BY b; -SELECT b, count() FROM tt_m GROUP BY b order by b; -SELECT b FROM tt_m order by b LIMIT 1 BY b; +SELECT b FROM tt_m ORDER BY b, a; +SELECT b, count() FROM tt_m GROUP BY b order by b; +SELECT b FROM tt_m order by b LIMIT 1 BY b; SELECT a FROM tt_m WHERE b = 12; SELECT max(a) FROM tt_m group by b order by b; -SELECT a FROM tt_m order by b LIMIT 1 BY b; - +SELECT a FROM tt_m order by b, a; diff --git a/tests/queries/0_stateless/01410_nullable_key_and_index.reference b/tests/queries/0_stateless/01410_nullable_key_and_index.reference index c5b2ef292ea..da88fbddd7a 100644 --- a/tests/queries/0_stateless/01410_nullable_key_and_index.reference +++ b/tests/queries/0_stateless/01410_nullable_key_and_index.reference @@ -54,14 +54,14 @@ 123 1 1 1 3 -2 \N -2 2 2 1 +2 2 2 7 2 \N -3 \N +2 \N 3 2 3 4 +3 \N 2 \N 2 \N 3 \N diff --git a/tests/queries/0_stateless/01410_nullable_key_and_index.sql b/tests/queries/0_stateless/01410_nullable_key_and_index.sql index 46a58152700..969432eba01 100644 --- a/tests/queries/0_stateless/01410_nullable_key_and_index.sql +++ b/tests/queries/0_stateless/01410_nullable_key_and_index.sql @@ -46,7 +46,7 @@ INSERT INTO nullable_minmax_index VALUES (1, 1), (2, 2), (3, 2), (2, 1); -- [1, INSERT INTO nullable_minmax_index VALUES (2, NULL), (3, NULL); -- [+Inf, +Inf] SET force_primary_key = 0; -SELECT * FROM nullable_minmax_index ORDER BY k; +SELECT * FROM nullable_minmax_index ORDER BY k, v; SET max_rows_to_read = 6; SELECT * FROM nullable_minmax_index WHERE v IS NULL; SET max_rows_to_read = 8; diff --git a/tests/queries/0_stateless/01521_alter_enum_and_reverse_read.sql b/tests/queries/0_stateless/01521_alter_enum_and_reverse_read.sql index 7caa4d34214..014790a61c1 100644 --- a/tests/queries/0_stateless/01521_alter_enum_and_reverse_read.sql +++ b/tests/queries/0_stateless/01521_alter_enum_and_reverse_read.sql @@ -8,6 +8,6 @@ ALTER TABLE enum_test MODIFY COLUMN e Enum8('IU' = 1, 'WS' = 2, 'PS' = 3); INSERT INTO enum_test SELECT '2020-10-09 00:00:00', 'h1', 'PS' from numbers(1); -SELECT * FROM enum_test ORDER BY timestamp DESC; +SELECT * FROM enum_test ORDER BY timestamp, e desc; DROP TABLE IF EXISTS enum_test; diff --git a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql index 90975b0d9c4..bffd12e5aca 100644 --- a/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql +++ b/tests/queries/0_stateless/01524_do_not_merge_across_partitions_select_final.sql @@ -11,8 +11,7 @@ INSERT INTO select_final SELECT toDate('2000-01-01'), number + 1, '' FROM number INSERT INTO select_final SELECT toDate('2020-01-01'), number, '' FROM numbers(2); INSERT INTO select_final SELECT toDate('2020-01-01'), number + 1, '' FROM numbers(2); - -SELECT * FROM select_final FINAL ORDER BY x; +SELECT * FROM select_final FINAL ORDER BY x, t; TRUNCATE TABLE select_final; diff --git a/tests/queries/0_stateless/01700_deltasum.reference b/tests/queries/0_stateless/01700_deltasum.reference index 6be953e2b2d..3a1c733a590 100644 --- a/tests/queries/0_stateless/01700_deltasum.reference +++ b/tests/queries/0_stateless/01700_deltasum.reference @@ -3,8 +3,8 @@ 7 7 7 -5 2 +5 2.25 -6.5 -7 +2.9 +6 diff --git a/tests/queries/0_stateless/01700_deltasum.sql b/tests/queries/0_stateless/01700_deltasum.sql index 9f1404c6845..7cd2b352e23 100644 --- a/tests/queries/0_stateless/01700_deltasum.sql +++ b/tests/queries/0_stateless/01700_deltasum.sql @@ -3,7 +3,7 @@ select deltaSum(arrayJoin([1, 2, 3, 0, 3, 4])); select deltaSum(arrayJoin([1, 2, 3, 0, 3, 4, 2, 3])); select deltaSum(arrayJoin([1, 2, 3, 0, 3, 3, 3, 3, 3, 4, 2, 3])); select deltaSum(arrayJoin([1, 2, 3, 0, 0, 0, 0, 3, 3, 3, 3, 3, 4, 2, 3])); -select deltaSumMerge(rows) from +select deltaSumMerge(rows) as delta_sum from ( select * from ( @@ -11,8 +11,8 @@ select deltaSumMerge(rows) from union all select deltaSumState(arrayJoin([4, 5])) as rows ) order by rows -); -select deltaSumMerge(rows) from +) order by delta_sum; +select deltaSumMerge(rows) as delta_sum from ( select * from ( @@ -20,9 +20,9 @@ select deltaSumMerge(rows) from union all select deltaSumState(arrayJoin([0, 1])) as rows ) order by rows -); +) order by delta_sum; select deltaSum(arrayJoin([2.25, 3, 4.5])); -select deltaSumMerge(rows) from +select deltaSumMerge(rows) as delta_sum from ( select * from ( @@ -30,8 +30,8 @@ select deltaSumMerge(rows) from union all select deltaSumState(arrayJoin([4.1, 5.1, 6.6])) as rows ) order by rows -); -select deltaSumMerge(rows) from +) order by delta_sum; +select deltaSumMerge(rows) as delta_sum from ( select * from ( @@ -41,4 +41,4 @@ select deltaSumMerge(rows) from union all select deltaSumState(arrayJoin([4, 6])) as rows ) order by rows -); +) order by delta_sum; diff --git a/tests/queries/0_stateless/01780_column_sparse_full.reference b/tests/queries/0_stateless/01780_column_sparse_full.reference index dbe815800a9..03160f24a41 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.reference +++ b/tests/queries/0_stateless/01780_column_sparse_full.reference @@ -39,7 +39,7 @@ all_2_2_0 u Default 0 0 0 0 0 0 0 0 0 -1 1 1 +1 0 ====== 58413 57920 diff --git a/tests/queries/0_stateless/01780_column_sparse_full.sql b/tests/queries/0_stateless/01780_column_sparse_full.sql index 59128223dba..c190e8f0df4 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.sql +++ b/tests/queries/0_stateless/01780_column_sparse_full.sql @@ -43,7 +43,7 @@ SELECT '======'; SELECT toUInt32(s) % 5 AS k, groupUniqArray(u % 4) FROM t_sparse_full WHERE s != '' GROUP BY k ORDER BY k; SELECT max(range(id % 10)[u]) FROM t_sparse_full; SELECT '======'; -SELECT id, u, s FROM remote('127.0.0.{1,2}', currentDatabase(), t_sparse_full) ORDER BY id LIMIT 5; +SELECT id, u, s FROM remote('127.0.0.{1,2}', currentDatabase(), t_sparse_full) ORDER BY id, u, s LIMIT 5; SELECT '======'; SELECT sum(u) FROM t_sparse_full GROUP BY id % 3 AS k WITH TOTALS ORDER BY k; SELECT '======'; diff --git a/tests/queries/0_stateless/01781_merge_tree_deduplication.reference b/tests/queries/0_stateless/01781_merge_tree_deduplication.reference index cb5a3f1ff52..c593d1764ea 100644 --- a/tests/queries/0_stateless/01781_merge_tree_deduplication.reference +++ b/tests/queries/0_stateless/01781_merge_tree_deduplication.reference @@ -45,8 +45,8 @@ 11 11 12 12 =============== -88 11 11 77 11 11 +88 11 11 77 12 12 =============== 1 1 33 diff --git a/tests/queries/0_stateless/01781_merge_tree_deduplication.sql b/tests/queries/0_stateless/01781_merge_tree_deduplication.sql index 1e3b464da30..4a148c53957 100644 --- a/tests/queries/0_stateless/01781_merge_tree_deduplication.sql +++ b/tests/queries/0_stateless/01781_merge_tree_deduplication.sql @@ -103,7 +103,7 @@ INSERT INTO merge_tree_deduplication (key, value, part) VALUES (11, '11', 88); - INSERT INTO merge_tree_deduplication (key, value) VALUES (11, '11'); -- not deduplicated INSERT INTO merge_tree_deduplication (key, value) VALUES (12, '12'); -- not deduplicated -SELECT part, key, value FROM merge_tree_deduplication ORDER BY key; +SELECT part, key, value FROM merge_tree_deduplication ORDER BY key, part; -- Alters.... diff --git a/tests/queries/0_stateless/02267_output_format_prometheus.reference b/tests/queries/0_stateless/02267_output_format_prometheus.reference index 3199a6ead89..e02c6278eb0 100644 --- a/tests/queries/0_stateless/02267_output_format_prometheus.reference +++ b/tests/queries/0_stateless/02267_output_format_prometheus.reference @@ -11,8 +11,8 @@ http_request_duration_seconds_count 144320 # HELP http_requests_total Total number of HTTP requests # TYPE http_requests_total counter -http_requests_total{code="200",method="post"} 1027 1395066363000 http_requests_total{code="400",method="post"} 3 1395066363000 +http_requests_total{code="200",method="post"} 1027 1395066363000 metric_without_timestamp_and_labels 12.47 From fbec38ddb92319d446db6e3fd625d2c194666df3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 May 2022 12:51:21 +0200 Subject: [PATCH 028/136] Fixed performance tests --- src/Core/SortCursor.h | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index ba786467385..ca6667e9898 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -222,11 +222,7 @@ struct SimpleSortCursor : SortCursorHelper bool ALWAYS_INLINE greaterAt(const SimpleSortCursor & rhs, size_t lhs_pos, size_t rhs_pos) const { - const auto & desc = impl->desc[0]; - int direction = desc.direction; - int nulls_direction = desc.nulls_direction; - - bool result = false; + int res = 0; #if USE_EMBEDDED_COMPILER if (impl->desc.compiled_sort_description && rhs.impl->desc.compiled_sort_description) @@ -234,17 +230,23 @@ struct SimpleSortCursor : SortCursorHelper assert(impl->raw_sort_columns_data.size() == rhs.impl->raw_sort_columns_data.size()); auto sort_description_func_typed = reinterpret_cast(impl->desc.compiled_sort_description); - int jit_result = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT - result = jit_result > 0; + res = sort_description_func_typed(lhs_pos, rhs_pos, impl->raw_sort_columns_data.data(), rhs.impl->raw_sort_columns_data.data()); /// NOLINT } else #endif { - int non_jit_result = impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); - result = non_jit_result * direction > 0; + const auto & desc = impl->desc[0]; + int direction = desc.direction; + int nulls_direction = desc.nulls_direction; + res = direction * impl->sort_columns[0]->compareAt(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[0]), nulls_direction); } - return result; + if (res > 0) + return true; + if (res < 0) + return false; + + return impl->order > rhs.impl->order; } }; From 83554d1f2d3b5f3cd80878b715a203f88c72140e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 25 May 2022 13:05:39 +0200 Subject: [PATCH 029/136] Fixed style --- .../Formats/Impl/PrometheusTextOutputFormat.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp index 5b6a42d1a77..a0af124edce 100644 --- a/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrometheusTextOutputFormat.cpp @@ -113,20 +113,20 @@ void PrometheusTextOutputFormat::fixupBucketLabels(CurrentMetric & metric) { bool lhs_labels_contain_sum = lhs.labels.contains("sum"); bool lhs_labels_contain_count = lhs.labels.contains("count"); - bool lhs_labels_contain_sum_and_count = lhs_labels_contain_sum || lhs_labels_contain_count; + bool lhs_labels_contain_sum_or_count = lhs_labels_contain_sum || lhs_labels_contain_count; bool rhs_labels_contain_sum = rhs.labels.contains("sum"); bool rhs_labels_contain_count = rhs.labels.contains("count"); - bool rhs_labels_contain_sum_and_count = rhs_labels_contain_sum || rhs_labels_contain_count; + bool rhs_labels_contain_sum_or_count = rhs_labels_contain_sum || rhs_labels_contain_count; /// rows with labels at the beginning and then `_sum` and `_count` if (lhs_labels_contain_sum && rhs_labels_contain_count) return true; else if (lhs_labels_contain_count && rhs_labels_contain_sum) return false; - else if (rhs_labels_contain_sum_and_count && !lhs_labels_contain_sum_and_count) + else if (rhs_labels_contain_sum_or_count && !lhs_labels_contain_sum_or_count) return true; - else if (lhs_labels_contain_sum_and_count && !rhs_labels_contain_sum_and_count) + else if (lhs_labels_contain_sum_or_count && !rhs_labels_contain_sum_or_count) return false; auto lit = lhs.labels.find(bucket_label); From 4c9812d4c1941ee48e1a2fcd260764160afa714e Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 25 May 2022 15:00:11 +0000 Subject: [PATCH 030/136] Allow to skip some of the first rows in CSV/TSV formats --- src/Core/Settings.h | 2 ++ src/Formats/FormatFactory.cpp | 2 ++ src/Formats/FormatSettings.h | 2 ++ .../Formats/Impl/CSVRowInputFormat.cpp | 6 ++++++ src/Processors/Formats/Impl/CSVRowInputFormat.h | 1 + .../Formats/Impl/TabSeparatedRowInputFormat.cpp | 6 ++++++ .../Formats/Impl/TabSeparatedRowInputFormat.h | 1 + .../02314_csv_tsv_skip_first_lines.reference | 16 ++++++++++++++++ .../02314_csv_tsv_skip_first_lines.sql | 12 ++++++++++++ 9 files changed, 48 insertions(+) create mode 100644 tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.reference create mode 100644 tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index bf9785fcc00..4d1952ec028 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -673,6 +673,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, input_format_json_read_bools_as_numbers, true, "Allow to parse bools as numbers in JSON input formats", 0) \ M(Bool, input_format_protobuf_flatten_google_wrappers, false, "Enable Google wrappers for regular non-nested columns, e.g. google.protobuf.StringValue 'str' for String column 'str'. For Nullable columns empty wrappers are recognized as defaults, and missing as nulls", 0) \ M(Bool, output_format_protobuf_nullables_with_google_wrappers, false, "When serializing Nullable columns with Google wrappers, serialize default values as empty wrappers. If turned off, default and null values are not serialized", 0) \ + M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified amount of lines in the beginning of data in CSV format", 0) \ + M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified amount of lines in the beginning of data in TSV format", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 644e4d3ecfd..b2bec88340c 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -66,6 +66,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.csv.null_representation = settings.format_csv_null_representation; format_settings.csv.input_format_arrays_as_nested_csv = settings.input_format_csv_arrays_as_nested_csv; format_settings.csv.input_format_use_best_effort_in_schema_inference = settings.input_format_csv_use_best_effort_in_schema_inference; + format_settings.csv.skip_first_lines = settings.input_format_csv_skip_first_lines; format_settings.hive_text.fields_delimiter = settings.input_format_hive_text_fields_delimiter; format_settings.hive_text.collection_items_delimiter = settings.input_format_hive_text_collection_items_delimiter; format_settings.hive_text.map_keys_delimiter = settings.input_format_hive_text_map_keys_delimiter; @@ -123,6 +124,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; format_settings.tsv.null_representation = settings.format_tsv_null_representation; format_settings.tsv.input_format_use_best_effort_in_schema_inference = settings.input_format_tsv_use_best_effort_in_schema_inference; + format_settings.tsv.skip_first_lines = settings.input_format_tsv_skip_first_lines; format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index e6f0a7d229e..eabfa2ad58b 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -109,6 +109,7 @@ struct FormatSettings String null_representation = "\\N"; char tuple_delimiter = ','; bool input_format_use_best_effort_in_schema_inference = true; + UInt64 skip_first_lines = 0; } csv; struct HiveText @@ -219,6 +220,7 @@ struct FormatSettings String null_representation = "\\N"; bool input_format_enum_as_number = false; bool input_format_use_best_effort_in_schema_inference = true; + UInt64 skip_first_lines = 0; } tsv; struct diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 0eaa02c97cb..bddd4203a5d 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -259,6 +259,12 @@ bool CSVFormatReader::readField( } } +void CSVFormatReader::skipPrefixBeforeHeader() +{ + for (size_t i = 0; i != format_settings.csv.skip_first_lines; ++i) + readRow(); +} + CSVSchemaReader::CSVSchemaReader(ReadBuffer & in_, bool with_names_, bool with_types_, const FormatSettings & format_setting_) : FormatWithNamesAndTypesSchemaReader( diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.h b/src/Processors/Formats/Impl/CSVRowInputFormat.h index 91a872378c8..20a92c07830 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -58,6 +58,7 @@ public: void skipTypes() override { skipHeaderRow(); } void skipFieldDelimiter() override; void skipRowEndDelimiter() override; + void skipPrefixBeforeHeader() override; std::vector readNames() override { return readHeaderRow(); } std::vector readTypes() override { return readHeaderRow(); } diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 0be8257f463..877ba224fd5 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -230,6 +230,12 @@ void TabSeparatedFormatReader::checkNullValueForNonNullable(DataTypePtr type) } } +void TabSeparatedFormatReader::skipPrefixBeforeHeader() +{ + for (size_t i = 0; i != format_settings.csv.skip_first_lines; ++i) + readRow(); +} + void TabSeparatedRowInputFormat::syncAfterError() { skipToUnescapedNextLineOrEOF(*in); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index abab5b02c96..3476b974c3b 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -43,6 +43,7 @@ public: void skipTypes() override { skipHeaderRow(); } void skipFieldDelimiter() override; void skipRowEndDelimiter() override; + void skipPrefixBeforeHeader() override; std::vector readRow(); std::vector readNames() override { return readRow(); } diff --git a/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.reference b/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.reference new file mode 100644 index 00000000000..7d8e0c662cd --- /dev/null +++ b/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.reference @@ -0,0 +1,16 @@ +c1 Nullable(Float64) +c2 Nullable(Float64) +c3 Nullable(Float64) +0 1 2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +c1 Nullable(Float64) +c2 Nullable(Float64) +c3 Nullable(Float64) +0 1 2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 diff --git a/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.sql b/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.sql new file mode 100644 index 00000000000..ff913a2a3ca --- /dev/null +++ b/tests/queries/0_stateless/02314_csv_tsv_skip_first_lines.sql @@ -0,0 +1,12 @@ +-- Tags: no-parallel + +insert into function file(data_02314.csv) select number, number + 1 from numbers(5) settings engine_file_truncate_on_insert=1; +insert into function file(data_02314.csv) select number, number + 1, number + 2 from numbers(5); +desc file(data_02314.csv) settings input_format_csv_skip_first_lines=5; +select * from file(data_02314.csv) settings input_format_csv_skip_first_lines=5; + +insert into function file(data_02314.tsv) select number, number + 1 from numbers(5) settings engine_file_truncate_on_insert=1; +insert into function file(data_02314.tsv) select number, number + 1, number + 2 from numbers(5); +desc file(data_02314.tsv) settings input_format_csv_skip_first_lines=5; +select * from file(data_02314.tsv) settings input_format_csv_skip_first_lines=5; + From ede6e2f43366bff404357d11ffb991b5d3e801e2 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 25 May 2022 15:10:20 +0000 Subject: [PATCH 031/136] Add docs for settings --- docs/en/interfaces/formats.md | 4 ++++ docs/en/operations/settings/settings.md | 12 ++++++++++++ src/Core/Settings.h | 4 ++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 31f948cbb00..36e2cf06927 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -149,6 +149,8 @@ Each element of [Nested](../sql-reference/data-types/nested-data-structures/nest In input data, ENUM values can be represented as names or as ids. First, we try to match the input value to the ENUM name. If we fail and the input value is a number, we try to match this number to ENUM id. If input data contains only ENUM ids, it's recommended to enable the setting [input_format_tsv_enum_as_number](../operations/settings/settings.md#settings-input_format_tsv_enum_as_number) to optimize ENUM parsing. +While data import, you can skip some first rows using setting [input_format_tsv_skip_first_lines](../operations/settings/settings.md#settings-input_format_tsv_skip_first_lines) + For example: ``` sql @@ -429,6 +431,8 @@ If input data contains only ENUM ids, it's recommended to enable the setting [in The CSV format supports the output of totals and extremes the same way as `TabSeparated`. +While data import, you can skip some first rows using setting [input_format_csv_skip_first_lines](../operations/settings/settings.md#settings-input_format_csv_skip_first_lines) + ## CSVWithNames {#csvwithnames} Also prints the header row with column names, similar to [TabSeparatedWithNames](#tabseparatedwithnames). diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 76fbc5f239d..77d01a735af 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -521,6 +521,18 @@ Result: └─────┴────────┘ ``` +## input_format_tsv_skip_first_lines {#settings-input_format_tsv_skip_first_lines} + +The number of lines to skip in the beginning of data in TSV input format. + +Default value: `0`. + +## input_format_csv_skip_first_lines {#settings-input_format_csv_skip_first_lines} + +The number of lines to skip in the beginning of data in CSV input format. + +Default value: `0`. + ## input_format_null_as_default {#settings-input-format-null-as-default} Enables or disables the initialization of [NULL](../../sql-reference/syntax.md#null-literal) fields with [default values](../../sql-reference/statements/create/table.md#create-default-values), if data type of these fields is not [nullable](../../sql-reference/data-types/nullable.md#data_type-nullable). diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4d1952ec028..f346a66f384 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -673,8 +673,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, input_format_json_read_bools_as_numbers, true, "Allow to parse bools as numbers in JSON input formats", 0) \ M(Bool, input_format_protobuf_flatten_google_wrappers, false, "Enable Google wrappers for regular non-nested columns, e.g. google.protobuf.StringValue 'str' for String column 'str'. For Nullable columns empty wrappers are recognized as defaults, and missing as nulls", 0) \ M(Bool, output_format_protobuf_nullables_with_google_wrappers, false, "When serializing Nullable columns with Google wrappers, serialize default values as empty wrappers. If turned off, default and null values are not serialized", 0) \ - M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified amount of lines in the beginning of data in CSV format", 0) \ - M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified amount of lines in the beginning of data in TSV format", 0) \ + M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified number of lines in the beginning of data in CSV format", 0) \ + M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified number of lines in the beginning of data in TSV format", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ From 6692b9c2ed4da7b126814d7911da885f595ff932 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 25 May 2022 12:11:44 -0400 Subject: [PATCH 032/136] showCertificate function implementation --- src/Functions/FunctionShowCertificate.cpp | 12 ++ src/Functions/FunctionShowCertificate.h | 189 ++++++++++++++++++++++ src/Functions/registerFunctions.cpp | 3 +- 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/Functions/FunctionShowCertificate.cpp create mode 100644 src/Functions/FunctionShowCertificate.h diff --git a/src/Functions/FunctionShowCertificate.cpp b/src/Functions/FunctionShowCertificate.cpp new file mode 100644 index 00000000000..e978f77244c --- /dev/null +++ b/src/Functions/FunctionShowCertificate.cpp @@ -0,0 +1,12 @@ +#include "FunctionShowCertificate.h" +#include + +namespace DB +{ + +void registerFunctionShowCertificate(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/FunctionShowCertificate.h b/src/Functions/FunctionShowCertificate.h new file mode 100644 index 00000000000..ce9b8f58103 --- /dev/null +++ b/src/Functions/FunctionShowCertificate.h @@ -0,0 +1,189 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if USE_SSL + #include + #include "Poco/Net/SSLManager.h" + #include "Poco/Crypto/X509Certificate.h" +#endif + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SUPPORT_IS_DISABLED; +} + +// showCertificate() +class FunctionShowCertificate : public IFunction +{ +public: + static constexpr auto name = "showCertificate"; + + static FunctionPtr create(ContextPtr) + { +#if !defined(USE_SSL) || USE_SSL == 0 + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support is disabled"); +#endif + return std::make_shared(); + } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName &) const override + { + return std::make_shared(std::make_shared(), std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + MutableColumnPtr keys = DataTypeString().createColumn(); + MutableColumnPtr values = DataTypeString().createColumn(); + MutableColumnPtr offsets = DataTypeNumber().createColumn(); + + if (input_rows_count) + { +#if USE_SSL + if (const X509 * cert = SSL_CTX_get0_certificate(Poco::Net::SSLManager::instance().defaultServerContext()->sslContext())) + { + BIO * b = BIO_new(BIO_s_mem()); + SCOPE_EXIT( + { + BIO_free(b); + }); + + keys->insert("version"); + values->insert(std::to_string(X509_get_version(cert) + 1)); + + { + char buf[1024] = {0}; + const ASN1_INTEGER * sn = cert->cert_info->serialNumber; + BIGNUM * bnsn = ASN1_INTEGER_to_BN(sn, nullptr); + SCOPE_EXIT( + { + BN_free(bnsn); + }); + if (BN_print(b, bnsn) > 0 && BIO_read(b, buf, sizeof(buf)) > 0) + { + keys->insert("serial_number"); + values->insert(buf); + } + + } + + { + const ASN1_BIT_STRING *sig = nullptr; + const X509_ALGOR *al = nullptr; + char buf[1024] = {0}; + X509_get0_signature(&sig, &al, cert); + if (al) + { + OBJ_obj2txt(buf, sizeof(buf), al->algorithm, 0); + keys->insert("signature_algo"); + values->insert(buf); + } + } + + char * issuer = X509_NAME_oneline(cert->cert_info->issuer, nullptr, 0); + if (issuer) + { + SCOPE_EXIT( + { + OPENSSL_free(issuer); + }); + keys->insert("issuer"); + values->insert(issuer); + } + + { + char buf[1024] = {0}; + if (ASN1_TIME_print(b, X509_get_notBefore(cert)) && BIO_read(b, buf, sizeof(buf)) > 0) + { + keys->insert("not_before"); + values->insert(buf); + } + } + + { + char buf[1024] = {0}; + if (ASN1_TIME_print(b, X509_get_notAfter(cert)) && BIO_read(b, buf, sizeof(buf)) > 0) + { + keys->insert("not_after"); + values->insert(buf); + } + } + + char * subject = X509_NAME_oneline(cert->cert_info->subject, nullptr, 0); + if (subject) + { + SCOPE_EXIT( + { + OPENSSL_free(subject); + }); + keys->insert("subject"); + values->insert(subject); + } + + if (X509_PUBKEY * pkey = X509_get_X509_PUBKEY(cert)) + { + char buf[1024] = {0}; + ASN1_OBJECT *ppkalg = nullptr; + const unsigned char *pk = nullptr; + int ppklen = 0; + X509_ALGOR *pa = nullptr; + if (X509_PUBKEY_get0_param(&ppkalg, &pk, &ppklen, &pa, pkey) && + i2a_ASN1_OBJECT(b, ppkalg) > 0 && BIO_read(b, buf, sizeof(buf)) > 0) + { + keys->insert("pkey_algo"); + values->insert(buf); + } + } + } + offsets->insert(keys->size()); +#endif + } + + size_t sz = keys->size(); + + if (sz && input_rows_count > 1) + { + keys->reserve(sz * input_rows_count); + values->reserve(sz * input_rows_count); + offsets->reserve(input_rows_count); + } + + for (size_t i = 1; i < input_rows_count; ++i) + { + for (size_t j = 0; j < sz; ++j) + { + keys->insertFrom(*keys, j); + values->insertFrom(*values, j); + } + offsets->insert(keys->size()); + } + + auto nested_column = ColumnArray::create( + ColumnTuple::create(Columns{std::move(keys), std::move(values)}), std::move(offsets)); + + return ColumnMap::create(nested_column); + } +}; + +} diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 0c67bf81d1e..bafaf61c2f5 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -68,7 +68,7 @@ void registerFunctionEncrypt(FunctionFactory & factory); void registerFunctionDecrypt(FunctionFactory & factory); void registerFunctionAESEncryptMysql(FunctionFactory & factory); void registerFunctionAESDecryptMysql(FunctionFactory & factory); - +void registerFunctionShowCertificate(FunctionFactory &); #endif void registerFunctions() @@ -135,6 +135,7 @@ void registerFunctions() registerFunctionDecrypt(factory); registerFunctionAESEncryptMysql(factory); registerFunctionAESDecryptMysql(factory); + registerFunctionShowCertificate(factory); #endif registerFunctionTid(factory); registerFunctionLogTrace(factory); From ea60a614d2dd166906efaff26f67e25c571b8a31 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 25 May 2022 20:33:13 +0200 Subject: [PATCH 033/136] Decrease namespace indent --- src/Functions/Regexps.h | 446 ++++++++++++++++++++-------------------- 1 file changed, 224 insertions(+), 222 deletions(-) diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index dc94b75211c..2611afedc14 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -38,254 +38,256 @@ namespace ErrorCodes namespace Regexps { - using Regexp = OptimizedRegularExpressionSingleThreaded; - using Pool = ObjectPoolMap; +using Regexp = OptimizedRegularExpressionSingleThreaded; +using Pool = ObjectPoolMap; - template - inline Regexp createRegexp(const std::string & pattern, int flags) +template +inline Regexp createRegexp(const std::string & pattern, int flags) +{ + if constexpr (like) + return {likePatternToRegexp(pattern), flags}; + else + return {pattern, flags}; +} + +template +inline int buildRe2Flags() +{ + int flags = OptimizedRegularExpression::RE_DOT_NL; + if constexpr (no_capture) + flags |= OptimizedRegularExpression::RE_NO_CAPTURE; + if constexpr (case_insensitive) + flags |= OptimizedRegularExpression::RE_CASELESS; + return flags; +} + +/** Returns holder of an object from Pool. + * You must hold the ownership while using the object. + * In destructor, it returns the object back to the Pool for further reuse. + */ +template +inline Pool::Pointer get(const std::string & pattern) +{ + /// the Singleton is thread-safe in C++11 + static Pool known_regexps; /// Different variables for different pattern parameters. + + return known_regexps.get(pattern, [&pattern] { - if constexpr (like) - return {likePatternToRegexp(pattern), flags}; - else - return {pattern, flags}; - } + const int flags = buildRe2Flags(); + ProfileEvents::increment(ProfileEvents::RegexpCreated); + return new Regexp{createRegexp(pattern, flags)}; + }); +} - template - inline int buildRe2Flags() - { - int flags = OptimizedRegularExpression::RE_DOT_NL; - if constexpr (no_capture) - flags |= OptimizedRegularExpression::RE_NO_CAPTURE; - if constexpr (case_insensitive) - flags |= OptimizedRegularExpression::RE_CASELESS; - return flags; - } - - /** Returns holder of an object from Pool. - * You must hold the ownership while using the object. - * In destructor, it returns the object back to the Pool for further reuse. - */ - template - inline Pool::Pointer get(const std::string & pattern) - { - /// the Singleton is thread-safe in C++11 - static Pool known_regexps; /// Different variables for different pattern parameters. - - return known_regexps.get(pattern, [&pattern] - { - const int flags = buildRe2Flags(); - ProfileEvents::increment(ProfileEvents::RegexpCreated); - return new Regexp{createRegexp(pattern, flags)}; - }); - } } #if USE_HYPERSCAN namespace MultiRegexps { - template - struct HyperscanDeleter +template +struct HyperscanDeleter +{ + template + void operator()(T * ptr) const { - template - void operator()(T * ptr) const - { - deleter(ptr); - } - }; + deleter(ptr); + } +}; - /// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception. - using CompilerError = std::unique_ptr>; - using ScratchPtr = std::unique_ptr>; - using DataBasePtr = std::unique_ptr>; +/// Helper unique pointers to correctly delete the allocated space when hyperscan cannot compile something and we throw an exception. +using CompilerError = std::unique_ptr>; +using ScratchPtr = std::unique_ptr>; +using DataBasePtr = std::unique_ptr>; - /// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher. - class Regexps +/// Database is thread safe across multiple threads and Scratch is not but we can copy it whenever we use it in the searcher. +class Regexps +{ +public: + Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { } + + hs_database_t * getDB() const { return db.get(); } + hs_scratch_t * getScratch() const { return scratch.get(); } + +private: + DataBasePtr db; + ScratchPtr scratch; +}; + +class RegexpsConstructor +{ +public: + RegexpsConstructor() = default; + + void setConstructor(std::function constructor_) { constructor = std::move(constructor_); } + + Regexps * operator()() { - public: - Regexps(hs_database_t * db_, hs_scratch_t * scratch_) : db{db_}, scratch{scratch_} { } - - hs_database_t * getDB() const { return db.get(); } - hs_scratch_t * getScratch() const { return scratch.get(); } - - private: - DataBasePtr db; - ScratchPtr scratch; - }; - - class RegexpsConstructor - { - public: - RegexpsConstructor() = default; - - void setConstructor(std::function constructor_) { constructor = std::move(constructor_); } - - Regexps * operator()() - { - std::unique_lock lock(mutex); - if (regexp) - return &*regexp; - regexp = constructor(); + std::unique_lock lock(mutex); + if (regexp) return &*regexp; - } + regexp = constructor(); + return &*regexp; + } - private: - std::function constructor; - std::optional regexp; - std::mutex mutex; - }; +private: + std::function constructor; + std::optional regexp; + std::mutex mutex; +}; - struct Pool +struct Pool +{ + /// Mutex for finding in map. + std::mutex mutex; + /// Patterns + possible edit_distance to database and scratch. + std::map, std::optional>, RegexpsConstructor> storage; +}; + +template +inline Regexps constructRegexps(const std::vector & str_patterns, std::optional edit_distance) +{ + (void)edit_distance; + /// Common pointers + std::vector patterns; + std::vector flags; + + /// Pointer for external edit distance compilation + std::vector ext_exprs; + std::vector ext_exprs_ptrs; + + patterns.reserve(str_patterns.size()); + flags.reserve(str_patterns.size()); + + if constexpr (CompileForEditDistance) { - /// Mutex for finding in map. - std::mutex mutex; - /// Patterns + possible edit_distance to database and scratch. - std::map, std::optional>, RegexpsConstructor> storage; - }; + ext_exprs.reserve(str_patterns.size()); + ext_exprs_ptrs.reserve(str_patterns.size()); + } - template - inline Regexps constructRegexps(const std::vector & str_patterns, std::optional edit_distance) + for (const StringRef ref : str_patterns) { - (void)edit_distance; - /// Common pointers - std::vector patterns; - std::vector flags; - - /// Pointer for external edit distance compilation - std::vector ext_exprs; - std::vector ext_exprs_ptrs; - - patterns.reserve(str_patterns.size()); - flags.reserve(str_patterns.size()); - + patterns.push_back(ref.data); + /* Flags below are the pattern matching flags. + * HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good + * performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode + * HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match. + * HS_FLAG_UTF8 is a flag where UTF8 literals are matched. + * HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice + * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag + */ + flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8); if constexpr (CompileForEditDistance) { - ext_exprs.reserve(str_patterns.size()); - ext_exprs_ptrs.reserve(str_patterns.size()); + /// Hyperscan currently does not support UTF8 matching with edit distance. + flags.back() &= ~HS_FLAG_UTF8; + ext_exprs.emplace_back(); + /// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance. + ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE; + ext_exprs.back().edit_distance = edit_distance.value(); + ext_exprs_ptrs.push_back(&ext_exprs.back()); } - - for (const StringRef ref : str_patterns) - { - patterns.push_back(ref.data); - /* Flags below are the pattern matching flags. - * HS_FLAG_DOTALL is a compile flag where matching a . will not exclude newlines. This is a good - * performance practice according to Hyperscan API. https://intel.github.io/hyperscan/dev-reference/performance.html#dot-all-mode - * HS_FLAG_ALLOWEMPTY is a compile flag where empty strings are allowed to match. - * HS_FLAG_UTF8 is a flag where UTF8 literals are matched. - * HS_FLAG_SINGLEMATCH is a compile flag where each pattern match will be returned only once. it is a good performance practice - * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag - */ - flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8); - if constexpr (CompileForEditDistance) - { - /// Hyperscan currently does not support UTF8 matching with edit distance. - flags.back() &= ~HS_FLAG_UTF8; - ext_exprs.emplace_back(); - /// HS_EXT_FLAG_EDIT_DISTANCE is a compile flag responsible for Levenstein distance. - ext_exprs.back().flags = HS_EXT_FLAG_EDIT_DISTANCE; - ext_exprs.back().edit_distance = edit_distance.value(); - ext_exprs_ptrs.push_back(&ext_exprs.back()); - } - } - hs_database_t * db = nullptr; - hs_compile_error_t * compile_error; - - std::unique_ptr ids; - - /// We mark the patterns to provide the callback results. - if constexpr (save_indices) - { - ids.reset(new unsigned int[patterns.size()]); - for (size_t i = 0; i < patterns.size(); ++i) - ids[i] = i + 1; - } - - hs_error_t err; - if constexpr (!CompileForEditDistance) - err = hs_compile_multi( - patterns.data(), - flags.data(), - ids.get(), - patterns.size(), - HS_MODE_BLOCK, - nullptr, - &db, - &compile_error); - else - err = hs_compile_ext_multi( - patterns.data(), - flags.data(), - ids.get(), - ext_exprs_ptrs.data(), - patterns.size(), - HS_MODE_BLOCK, - nullptr, - &db, - &compile_error); - - if (err != HS_SUCCESS) - { - /// CompilerError is a unique_ptr, so correct memory free after the exception is thrown. - CompilerError error(compile_error); - - if (error->expression < 0) - throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR); - else - throw Exception( - "Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message), - ErrorCodes::BAD_ARGUMENTS); - } - - ProfileEvents::increment(ProfileEvents::RegexpCreated); - - /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch - /// function which is faster than allocating scratch space each time in each thread. - hs_scratch_t * scratch = nullptr; - err = hs_alloc_scratch(db, &scratch); - - /// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch. - if (err != HS_SUCCESS) - throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - return Regexps{db, scratch}; } + hs_database_t * db = nullptr; + hs_compile_error_t * compile_error; - /// If CompileForEditDistance is False, edit_distance must be nullopt - /// Also, we use templates here because each instantiation of function - /// template has its own copy of local static variables which must not be the same - /// for different hyperscan compilations. - template - inline Regexps * get(const std::vector & patterns, std::optional edit_distance) + std::unique_ptr ids; + + /// We mark the patterns to provide the callback results. + if constexpr (save_indices) { - /// C++11 has thread-safe function-local static on most modern compilers. - static Pool known_regexps; /// Different variables for different pattern parameters. - - std::vector str_patterns; - str_patterns.reserve(patterns.size()); - for (const StringRef & ref : patterns) - str_patterns.push_back(ref.toString()); - - /// Get the lock for finding database. - std::unique_lock lock(known_regexps.mutex); - - auto it = known_regexps.storage.find({str_patterns, edit_distance}); - - /// If not found, compile and let other threads wait. - if (known_regexps.storage.end() == it) - { - it = known_regexps.storage - .emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple()) - .first; - it->second.setConstructor([&str_patterns = it->first.first, edit_distance]() - { - return constructRegexps(str_patterns, edit_distance); - }); - } - - /// Unlock before possible construction. - lock.unlock(); - return it->second(); + ids.reset(new unsigned int[patterns.size()]); + for (size_t i = 0; i < patterns.size(); ++i) + ids[i] = i + 1; } + + hs_error_t err; + if constexpr (!CompileForEditDistance) + err = hs_compile_multi( + patterns.data(), + flags.data(), + ids.get(), + patterns.size(), + HS_MODE_BLOCK, + nullptr, + &db, + &compile_error); + else + err = hs_compile_ext_multi( + patterns.data(), + flags.data(), + ids.get(), + ext_exprs_ptrs.data(), + patterns.size(), + HS_MODE_BLOCK, + nullptr, + &db, + &compile_error); + + if (err != HS_SUCCESS) + { + /// CompilerError is a unique_ptr, so correct memory free after the exception is thrown. + CompilerError error(compile_error); + + if (error->expression < 0) + throw Exception(String(error->message), ErrorCodes::LOGICAL_ERROR); + else + throw Exception( + "Pattern '" + str_patterns[error->expression] + "' failed with error '" + String(error->message), + ErrorCodes::BAD_ARGUMENTS); + } + + ProfileEvents::increment(ProfileEvents::RegexpCreated); + + /// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch + /// function which is faster than allocating scratch space each time in each thread. + hs_scratch_t * scratch = nullptr; + err = hs_alloc_scratch(db, &scratch); + + /// If not HS_SUCCESS, it is guaranteed that the memory would not be allocated for scratch. + if (err != HS_SUCCESS) + throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + return Regexps{db, scratch}; +} + +/// If CompileForEditDistance is False, edit_distance must be nullopt +/// Also, we use templates here because each instantiation of function +/// template has its own copy of local static variables which must not be the same +/// for different hyperscan compilations. +template +inline Regexps * get(const std::vector & patterns, std::optional edit_distance) +{ + /// C++11 has thread-safe function-local static on most modern compilers. + static Pool known_regexps; /// Different variables for different pattern parameters. + + std::vector str_patterns; + str_patterns.reserve(patterns.size()); + for (const StringRef & ref : patterns) + str_patterns.push_back(ref.toString()); + + /// Get the lock for finding database. + std::unique_lock lock(known_regexps.mutex); + + auto it = known_regexps.storage.find({str_patterns, edit_distance}); + + /// If not found, compile and let other threads wait. + if (known_regexps.storage.end() == it) + { + it = known_regexps.storage + .emplace(std::piecewise_construct, std::make_tuple(std::move(str_patterns), edit_distance), std::make_tuple()) + .first; + it->second.setConstructor([&str_patterns = it->first.first, edit_distance]() + { + return constructRegexps(str_patterns, edit_distance); + }); + } + + /// Unlock before possible construction. + lock.unlock(); + return it->second(); +} + } #endif // USE_HYPERSCAN From 49934a3dc865cc8131d94de4592d3bd4f21150c0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 25 May 2022 21:22:45 +0200 Subject: [PATCH 034/136] Cache compiled regexps when evaluating non-const needles Needles in a (non-const) needle column may repeat and this commit allows to skip compilation for known needles. Out of the different design alternatives (see below, if someone is interested), we now maintain - one global pattern cache, - with a fixed size of 42k elements currently, - and use LRU as eviction strategy. ------------------------------------------------------------------------ (sorry for the wall of text, dumping it here not for reading but just for reference) Write-up about considered design alternatives: 1. Keep the current global cache of const needles. For non-const needles, probe the cache but don't store values in it. Pros: need to maintain just a single cache, no problem with cache pollution assuming there are few distinct constant needles Cons: only useful if a non-const needle occurred as already as a const needle --> overall too simplistic 2. Keep the current global cache for const needles. For non-const needles, create a local (e.g. per-query) cache Pros: unlike (1.), non-const needles can be skipped even if they did not occur yet, no pollution of the const pattern cache when there are very many non-const needles (e.g. large / highly distinct needle columns). Cons: caches may explode "horizontally", i.e. we'll end up with the const cache + caches for Q1, Q2, ... QN, this makes it harder to control the overall space consumption, also patterns residing in different caches cannot be reused between queries, another difficulty is that the concept of "query" does not really exist at matching level - there are only column chunks and we'd potentially end up with 1 cache / chunk 3. Queries with const and non-const needles insert into the same global cache. Pros: the advantages of (2.) + allows to reuse compiled patterns accross parallel queries Cons: needs an eviction strategy to control cache size and pollution (and btw. (2.) also needs eviction strategies for the individual caches) 4. Queries with const needle use global cache, queries with non-const needle use a different global cache --> Overall similar to (3) but ignores the (likely) edge case that const and non-const needles overlap. In sum, (3.) seems the simplest and most beneficial approach. Eviction strategies: 0. Don't ever evict --> cache may grow infinitely and eventually make the system unusable (may even pose a DoS risk) 1. Flush the cache after a certain threshold is exceeded --> very simple but may lead to peridic performance drops 2. Use LRU --> more graceful performance degradation at threshold but comes with a (constant) performance overhead to maintain the LRU queue In sum, given that the pattern compilation in RE2 should be quite costly (pattern-to-DFA/NFA), LRU may be acceptable. --- src/Functions/FunctionsStringArray.h | 4 ++-- src/Functions/MatchImpl.h | 30 ++++++++++--------------- src/Functions/Regexps.h | 33 +++++++++++----------------- src/Functions/countMatches.h | 4 ++-- 4 files changed, 29 insertions(+), 42 deletions(-) diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index 2680816670f..6545c3e3549 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -448,7 +448,7 @@ public: class SplitByRegexpImpl { private: - Regexps::Pool::Pointer re; + Regexps::RegexpPtr re; OptimizedRegularExpression::MatchVec matches; Pos pos; @@ -532,7 +532,7 @@ public: class ExtractAllImpl { private: - Regexps::Pool::Pointer re; + Regexps::RegexpPtr re; OptimizedRegularExpression::MatchVec matches; size_t capture; diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 17bda74f8ab..9779eb8d608 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -166,7 +166,7 @@ struct MatchImpl } else { - auto regexp = Regexps::get(needle); + auto regexp = Regexps::get(needle); String required_substring; bool is_trivial; @@ -325,7 +325,7 @@ struct MatchImpl } else { - auto regexp = Regexps::get(needle); + auto regexp = Regexps::get(needle); String required_substring; bool is_trivial; @@ -479,22 +479,19 @@ struct MatchImpl } else { - // each row is expected to contain a different like/re2 pattern - // --> bypass the regexp cache, instead construct the pattern on-the-fly - const int flags = Regexps::buildRe2Flags(); - const auto & regexp = Regexps::Regexp(Regexps::createRegexp(needle, flags)); + auto regexp = Regexps::get(needle); - regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); + regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); if (required_substr.empty()) { - if (!regexp.getRE2()) /// An empty regexp. Always matches. + if (!regexp->getRE2()) /// An empty regexp. Always matches. { res[i] = !negate; } else { - const bool match = regexp.getRE2()->Match( + const bool match = regexp->getRE2()->Match( {reinterpret_cast(cur_haystack_data), cur_haystack_length}, 0, cur_haystack_length, @@ -524,7 +521,7 @@ struct MatchImpl const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0; const size_t end_pos = cur_haystack_length; - const bool match2 = regexp.getRE2()->Match( + const bool match2 = regexp->getRE2()->Match( {reinterpret_cast(cur_haystack_data), cur_haystack_length}, start_pos, end_pos, @@ -593,22 +590,19 @@ struct MatchImpl } else { - // each row is expected to contain a different like/re2 pattern - // --> bypass the regexp cache, instead construct the pattern on-the-fly - const int flags = Regexps::buildRe2Flags(); - const auto & regexp = Regexps::Regexp(Regexps::createRegexp(needle, flags)); + auto regexp = Regexps::get(needle); - regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); + regexp->getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix); if (required_substr.empty()) { - if (!regexp.getRE2()) /// An empty regexp. Always matches. + if (!regexp->getRE2()) /// An empty regexp. Always matches. { res[i] = !negate; } else { - const bool match = regexp.getRE2()->Match( + const bool match = regexp->getRE2()->Match( {reinterpret_cast(cur_haystack_data), cur_haystack_length}, 0, cur_haystack_length, @@ -638,7 +632,7 @@ struct MatchImpl const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0; const size_t end_pos = cur_haystack_length; - const bool match2 = regexp.getRE2()->Match( + const bool match2 = regexp->getRE2()->Match( {reinterpret_cast(cur_haystack_data), cur_haystack_length}, start_pos, end_pos, diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 2611afedc14..be3ce6cdeee 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -39,16 +39,8 @@ namespace ErrorCodes namespace Regexps { using Regexp = OptimizedRegularExpressionSingleThreaded; -using Pool = ObjectPoolMap; - -template -inline Regexp createRegexp(const std::string & pattern, int flags) -{ - if constexpr (like) - return {likePatternToRegexp(pattern), flags}; - else - return {pattern, flags}; -} +using Cache = LRUCache; +using RegexpPtr = Cache::MappedPtr; template inline int buildRe2Flags() @@ -61,22 +53,23 @@ inline int buildRe2Flags() return flags; } -/** Returns holder of an object from Pool. - * You must hold the ownership while using the object. - * In destructor, it returns the object back to the Pool for further reuse. - */ +/// Probes the cache of known compiled regexps for the given string pattern and returns a compiled regexp if +/// found. Otherwise, a new cache entry is created. template -inline Pool::Pointer get(const std::string & pattern) +inline RegexpPtr get(const String & pattern) { - /// the Singleton is thread-safe in C++11 - static Pool known_regexps; /// Different variables for different pattern parameters. + static Cache known_regexps(42'000); - return known_regexps.get(pattern, [&pattern] + auto [regexp_ptr, _] = known_regexps.getOrSet(pattern, [&pattern]() { const int flags = buildRe2Flags(); ProfileEvents::increment(ProfileEvents::RegexpCreated); - return new Regexp{createRegexp(pattern, flags)}; + if constexpr (like) + return std::make_shared(likePatternToRegexp(pattern), flags); + else + return std::make_shared(pattern, flags); }); + return regexp_ptr; } } diff --git a/src/Functions/countMatches.h b/src/Functions/countMatches.h index 6d60ca94c18..1d43b66d867 100644 --- a/src/Functions/countMatches.h +++ b/src/Functions/countMatches.h @@ -55,7 +55,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const ColumnConst * column_pattern = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get()); - Regexps::Pool::Pointer re = Regexps::get(column_pattern->getValue()); + Regexps::RegexpPtr re = Regexps::get(column_pattern->getValue()); OptimizedRegularExpression::MatchVec matches; const IColumn * column_haystack = arguments[0].column.get(); @@ -95,7 +95,7 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Error in FunctionCountMatches::getReturnTypeImpl()"); } - static uint64_t countMatches(StringRef src, Regexps::Pool::Pointer & re, OptimizedRegularExpression::MatchVec & matches) + static uint64_t countMatches(StringRef src, Regexps::RegexpPtr & re, OptimizedRegularExpression::MatchVec & matches) { /// Only one match is required, no need to copy more. static const unsigned matches_limit = 1; From 3c1b6609ae76aaba93fbf526b3892cf060698048 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 25 May 2022 21:23:35 +0000 Subject: [PATCH 035/136] Add comments and make tests more verbose --- src/Core/NamesAndTypes.h | 3 +- src/Functions/grouping.h | 12 +++ .../02293_grouping_function.reference | 83 ++++++++++++++++ .../0_stateless/02293_grouping_function.sql | 1 + ...02293_grouping_function_group_by.reference | 97 +++++++++++++++++++ .../02293_grouping_function_group_by.sql | 1 + 6 files changed, 196 insertions(+), 1 deletion(-) diff --git a/src/Core/NamesAndTypes.h b/src/Core/NamesAndTypes.h index c7a51f51816..b9c03aae0ca 100644 --- a/src/Core/NamesAndTypes.h +++ b/src/Core/NamesAndTypes.h @@ -105,9 +105,10 @@ public: /// Check that column contains in list bool contains(const String & name) const; - /// Try to get column by name, return empty optional if column not found + /// Try to get column by name, returns empty optional if column not found std::optional tryGetByName(const std::string & name) const; + /// Try to get column position by name, returns number of columns if column isn't found size_t getPosByName(const std::string & name) const noexcept; }; diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h index 934be18345d..7c8970667da 100644 --- a/src/Functions/grouping.h +++ b/src/Functions/grouping.h @@ -93,6 +93,12 @@ public: return FunctionGroupingBase::executeImpl(arguments, input_rows_count, [this](UInt64 set_index, UInt64 arg_index) { + // For ROLLUP(a, b, c) there will be following grouping set indexes: + // | GROUPING SET | INDEX | + // | (a, b, c) | 0 | + // | (a, b) | 1 | + // | (a) | 2 | + // | () | 3 | return arg_index < aggregation_keys_number - set_index; } ); @@ -117,6 +123,12 @@ public: return FunctionGroupingBase::executeImpl(arguments, input_rows_count, [this](UInt64 set_index, UInt64 arg_index) { + // For CUBE(a, b) there will be following grouping set indexes: + // | GROUPING SET | INDEX | + // | (a, b) | 0 | + // | (a) | 1 | + // | (b) | 2 | + // | () | 3 | auto set_mask = (ONE << aggregation_keys_number) - 1 - set_index; return set_mask & (ONE << (aggregation_keys_number - arg_index - 1)); } diff --git a/tests/queries/0_stateless/02293_grouping_function.reference b/tests/queries/0_stateless/02293_grouping_function.reference index dbae7a11f2e..e71d6812ab5 100644 --- a/tests/queries/0_stateless/02293_grouping_function.reference +++ b/tests/queries/0_stateless/02293_grouping_function.reference @@ -1,3 +1,14 @@ +-- { echoOn } +SELECT + number, + grouping(number, number % 2) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; 0 1 0 1 0 2 @@ -10,6 +21,16 @@ 7 2 8 2 9 2 +SELECT + number, + grouping(number % 2, number) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; 0 1 0 2 0 2 @@ -22,6 +43,16 @@ 7 1 8 1 9 1 +SELECT + number, + grouping(number, number % 2) = 1 AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, gr; 0 0 0 1 0 1 @@ -34,6 +65,15 @@ 7 0 8 0 9 0 +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY number, grouping(number, number % 2) = 1; 0 0 0 @@ -46,6 +86,18 @@ 7 8 9 +SELECT + number, + count(), + grouping(number, number % 2) AS gr +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number, number % 2), + () + ) +ORDER BY (gr, number); 0 10 0 0 1 2 1 1 2 @@ -67,6 +119,17 @@ 7 1 3 8 1 3 9 1 3 +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +HAVING grouping(number, number % 2) = 2 +ORDER BY number +SETTINGS enable_optimize_predicate_expression = 0; 0 1 2 @@ -77,8 +140,28 @@ 7 8 9 +SELECT + number +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +HAVING grouping(number, number % 2) = 1 +ORDER BY number +SETTINGS enable_optimize_predicate_expression = 0; 0 0 +SELECT + number, + GROUPING(number, number % 2) = 1 as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + GROUPING SETS ( + (number), + (number % 2)) +ORDER BY number, gr; 0 0 0 1 0 1 diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index 4bbf620a619..169fc09c324 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -9,6 +9,7 @@ GROUP BY ) ORDER BY number, gr; -- { serverError BAD_ARGUMENTS } +-- { echoOn } SELECT number, grouping(number, number % 2) AS gr diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.reference b/tests/queries/0_stateless/02293_grouping_function_group_by.reference index 021083db6eb..7f87aecd4bd 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.reference +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.reference @@ -1,3 +1,12 @@ +-- { echoOn } +SELECT + number, + grouping(number, number % 2) = 3 +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 +ORDER BY number; 0 1 1 1 2 1 @@ -8,6 +17,15 @@ 7 1 8 1 9 1 +SELECT + number, + grouping(number), + GROUPING(number % 2) +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 +ORDER BY number; 0 1 1 1 1 1 2 1 1 @@ -18,6 +36,16 @@ 7 1 1 8 1 1 9 1 1 +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 + WITH ROLLUP +ORDER BY + number, gr; 0 0 0 2 0 3 @@ -39,6 +67,14 @@ 8 3 9 2 9 3 +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) +ORDER BY + number, gr; 0 0 0 2 0 3 @@ -60,6 +96,16 @@ 8 3 9 2 9 3 +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + number, + number % 2 + WITH CUBE +ORDER BY + number, gr; 0 0 0 1 0 1 @@ -83,6 +129,14 @@ 8 3 9 2 9 3 +SELECT + number, + grouping(number, number % 2) AS gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) +ORDER BY + number, gr; 0 0 0 1 0 1 @@ -106,6 +160,15 @@ 8 3 9 2 9 3 +SELECT + number, + grouping(number, number % 2) + 3 as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) +HAVING grouping(number) != 0 +ORDER BY + number, gr; 0 5 0 6 1 5 @@ -126,6 +189,23 @@ 8 6 9 5 9 6 +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) WITH TOTALS +HAVING grouping(number) != 0 +ORDER BY + number, gr; -- { serverError NOT_IMPLEMENTED } +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + CUBE(number, number % 2) WITH TOTALS +ORDER BY + number, gr; 0 0 0 1 0 1 @@ -151,6 +231,23 @@ 9 3 0 0 +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) WITH TOTALS +HAVING grouping(number) != 0 +ORDER BY + number, gr; -- { serverError NOT_IMPLEMENTED } +SELECT + number, + grouping(number, number % 2) as gr +FROM remote('127.0.0.{2,3}', numbers(10)) +GROUP BY + ROLLUP(number, number % 2) WITH TOTALS +ORDER BY + number, gr; 0 0 0 2 0 3 diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index b30080b88af..9bf9d43478b 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -7,6 +7,7 @@ GROUP BY number % 2 ORDER BY number; -- { serverError BAD_ARGUMENTS } +-- { echoOn } SELECT number, grouping(number, number % 2) = 3 From 7cd7782e4f25b0a2aad9cfa69cd9ffcb4da1e6cc Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 25 May 2022 21:55:41 +0000 Subject: [PATCH 036/136] Process columns more efficiently in GROUPING() --- src/Functions/grouping.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Functions/grouping.h b/src/Functions/grouping.h index 7c8970667da..a49e946b2cb 100644 --- a/src/Functions/grouping.h +++ b/src/Functions/grouping.h @@ -43,18 +43,20 @@ public: template ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, size_t input_rows_count, AggregationKeyChecker checker) const { - auto grouping_set_column = checkAndGetColumn(arguments[0].column.get()); + const auto * grouping_set_column = checkAndGetColumn(arguments[0].column.get()); - auto result = std::make_shared()->createColumn(); + auto result = ColumnUInt64::create(); + auto & result_data = result->getData(); + result_data.reserve(input_rows_count); for (size_t i = 0; i < input_rows_count; ++i) { - UInt64 set_index = grouping_set_column->get64(i); + UInt64 set_index = grouping_set_column->getElement(i); UInt64 value = 0; for (auto index : arguments_indexes) value = (value << 1) + (checker(set_index, index) ? 1 : 0); - result->insert(Field(value)); + result_data.push_back(value); } return result; } From 16c6b6070344783c9fcbc935fc572da3207b07ec Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 25 May 2022 23:22:29 +0000 Subject: [PATCH 037/136] Introduce AggregationKeysInfo --- src/Interpreters/ActionsVisitor.cpp | 26 ++++++++--------- src/Interpreters/ActionsVisitor.h | 37 ++++++++++++++++++++----- src/Interpreters/ExpressionAnalyzer.cpp | 12 ++------ src/Interpreters/ExpressionAnalyzer.h | 5 ++++ src/Storages/VirtualColumnUtils.cpp | 7 ++++- 5 files changed, 56 insertions(+), 31 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index d22989219a4..99d2217cba3 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -473,9 +473,7 @@ ActionsMatcher::Data::Data( ContextPtr context_, SizeLimits set_size_limit_, size_t subquery_depth_, - const NamesAndTypesList & source_columns_, - const NamesAndTypesList & aggregation_keys_, - const ColumnNumbersList & grouping_set_keys_, + std::reference_wrapper source_columns_, ActionsDAGPtr actions_dag, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, @@ -483,22 +481,20 @@ ActionsMatcher::Data::Data( bool no_makeset_, bool only_consts_, bool create_source_for_in_, - GroupByKind group_by_kind_) + AggregationKeysInfo aggregation_keys_info_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) , source_columns(source_columns_) - , aggregation_keys(aggregation_keys_) - , grouping_set_keys(grouping_set_keys_) , prepared_sets(prepared_sets_) , subqueries_for_sets(subqueries_for_sets_) , no_subqueries(no_subqueries_) , no_makeset(no_makeset_) , only_consts(only_consts_) , create_source_for_in(create_source_for_in_) - , group_by_kind(group_by_kind_) , visit_depth(0) , actions_stack(std::move(actions_dag), context_) + , aggregation_keys_info(aggregation_keys_info_) , next_unique_suffix(actions_stack.getLastActions().getIndex().size() + 1) { } @@ -848,29 +844,31 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function GROUPING expects at least one argument"); if (arguments_size > 64) throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Function GROUPING can have up to 64 arguments, but {} provided", arguments_size); + auto keys_info = data.aggregation_keys_info; + auto aggregation_keys_number = keys_info.aggregation_keys.size(); + ColumnNumbers arguments_indexes; - auto aggregation_keys_number = data.aggregation_keys.size(); for (auto const & arg : node.arguments->children) { - size_t pos = data.aggregation_keys.getPosByName(arg->getColumnName()); + size_t pos = keys_info.aggregation_keys.getPosByName(arg->getColumnName()); if (pos == aggregation_keys_number) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of GROUPING function {} is not a part of GROUP BY clause", arg->getColumnName()); arguments_indexes.push_back(pos); } - switch (data.group_by_kind) + switch (keys_info.group_by_kind) { case GroupByKind::GROUPING_SETS: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.grouping_set_keys)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys)), { "__grouping_set" }, column_name); break; } case GroupByKind::ROLLUP: - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.aggregation_keys.size())), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); break; case GroupByKind::CUBE: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.aggregation_keys.size())), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number)), { "__grouping_set" }, column_name); break; } case GroupByKind::ORDINARY: @@ -880,7 +878,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & } default: throw Exception(ErrorCodes::LOGICAL_ERROR, - "Unexpected kind of GROUP BY clause for GROUPING function: {}", data.group_by_kind); + "Unexpected kind of GROUP BY clause for GROUPING function: {}", keys_info.group_by_kind); } return; } diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 5fd228ba836..5a74124192c 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -87,6 +87,33 @@ enum class GroupByKind GROUPING_SETS, }; +/* + * This class stores information about aggregation keys used in GROUP BY clause. + * It's used for providing information about aggregation to GROUPING function + * implementation. +*/ +struct AggregationKeysInfo +{ + AggregationKeysInfo( + std::reference_wrapper aggregation_keys_, + std::reference_wrapper grouping_set_keys_, + GroupByKind group_by_kind_) + : aggregation_keys(aggregation_keys_) + , grouping_set_keys(grouping_set_keys_) + , group_by_kind(group_by_kind_) + {} + + AggregationKeysInfo(const AggregationKeysInfo &) = default; + AggregationKeysInfo(AggregationKeysInfo &&) = default; + + // Names and types of all used keys + const NamesAndTypesList & aggregation_keys; + // Indexes of aggregation keys used in each grouping set (only for GROUP BY GROUPING SETS) + const ColumnNumbersList & grouping_set_keys; + + GroupByKind group_by_kind; +}; + /// Collect ExpressionAction from AST. Returns PreparedSets and SubqueriesForSets too. class ActionsMatcher { @@ -98,17 +125,15 @@ public: SizeLimits set_size_limit; size_t subquery_depth; const NamesAndTypesList & source_columns; - const NamesAndTypesList & aggregation_keys; - const ColumnNumbersList & grouping_set_keys; PreparedSets & prepared_sets; SubqueriesForSets & subqueries_for_sets; bool no_subqueries; bool no_makeset; bool only_consts; bool create_source_for_in; - GroupByKind group_by_kind; size_t visit_depth; ScopeStack actions_stack; + AggregationKeysInfo aggregation_keys_info; /* * Remember the last unique column suffix to avoid quadratic behavior @@ -121,9 +146,7 @@ public: ContextPtr context_, SizeLimits set_size_limit_, size_t subquery_depth_, - const NamesAndTypesList & source_columns_, - const NamesAndTypesList & aggregation_keys_, - const ColumnNumbersList & grouping_set_keys_, + std::reference_wrapper source_columns_, ActionsDAGPtr actions_dag, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, @@ -131,7 +154,7 @@ public: bool no_makeset_, bool only_consts_, bool create_source_for_in_, - GroupByKind group_by_kind_); + AggregationKeysInfo aggregation_keys_info_); /// Does result of the calculation already exists in the block. bool hasColumn(const String & column_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 3931180d941..699f73abd67 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -603,8 +603,6 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ settings.size_limits_for_set, subquery_depth, sourceColumns(), - aggregation_keys, - aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, @@ -612,7 +610,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ false /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - group_by_kind); + getAggregationKeysInfo()); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -626,8 +624,6 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP settings.size_limits_for_set, subquery_depth, sourceColumns(), - aggregation_keys, - aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, @@ -635,7 +631,7 @@ void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, ActionsDAGP true /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - group_by_kind); + getAggregationKeysInfo()); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -650,8 +646,6 @@ void ExpressionAnalyzer::getRootActionsForHaving( settings.size_limits_for_set, subquery_depth, sourceColumns(), - aggregation_keys, - aggregation_keys_indexes_list, std::move(actions), prepared_sets, subqueries_for_sets, @@ -659,7 +653,7 @@ void ExpressionAnalyzer::getRootActionsForHaving( false /* no_makeset */, only_consts, true /* create_source_for_in */, - group_by_kind); + getAggregationKeysInfo()); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index ecac8cba771..971fe753978 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -205,6 +205,11 @@ protected: NamesAndTypesList getColumnsAfterArrayJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns); NamesAndTypesList analyzeJoin(ActionsDAGPtr & actions, const NamesAndTypesList & src_columns); + + AggregationKeysInfo getAggregationKeysInfo() const noexcept + { + return { aggregation_keys, aggregation_keys_indexes_list, group_by_kind }; + } }; class SelectQueryExpressionAnalyzer; diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index dd6c30e3c79..40cf650f690 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -156,8 +156,13 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block auto actions = std::make_shared(block.getColumnsWithTypeAndName()); PreparedSets prepared_sets; SubqueriesForSets subqueries_for_sets; + const NamesAndTypesList source_columns; + const NamesAndTypesList aggregation_keys; + const ColumnNumbersList grouping_set_keys; + ActionsVisitor::Data visitor_data( - context, SizeLimits{}, 1, {}, {}, {}, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false, GroupByKind::NONE); + context, SizeLimits{}, 1, source_columns, std::move(actions), prepared_sets, subqueries_for_sets, true, true, true, false, + { aggregation_keys, grouping_set_keys, GroupByKind::NONE }); ActionsVisitor(visitor_data).visit(node); actions = visitor_data.getActions(); auto expression_actions = std::make_shared(actions); From 2dc160a4c334cd06aedbbad8ead5b4e7bf910f78 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Wed, 25 May 2022 20:56:36 -0400 Subject: [PATCH 038/136] style fix --- src/Functions/FunctionShowCertificate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionShowCertificate.h b/src/Functions/FunctionShowCertificate.h index ce9b8f58103..bb3c0fd8299 100644 --- a/src/Functions/FunctionShowCertificate.h +++ b/src/Functions/FunctionShowCertificate.h @@ -111,7 +111,7 @@ public: keys->insert("issuer"); values->insert(issuer); } - + { char buf[1024] = {0}; if (ASN1_TIME_print(b, X509_get_notBefore(cert)) && BIO_read(b, buf, sizeof(buf)) > 0) From 85c7b55b816674c9e653ba3c9c2ba4671e78effd Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 26 May 2022 16:54:46 +0200 Subject: [PATCH 039/136] Slightly better jepsen tests --- tests/jepsen.clickhouse-keeper/resources/keeper_config.xml | 1 + .../src/jepsen/clickhouse_keeper/db.clj | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml b/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml index 8c574718d88..2ab747fbd71 100644 --- a/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml +++ b/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml @@ -5,6 +5,7 @@ trace /var/log/clickhouse-keeper/clickhouse-keeper.log /var/log/clickhouse-keeper/clickhouse-keeper.err.log + never diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index 745d88e97f7..22c8bfd348d 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -10,11 +10,6 @@ [jepsen.control.util :as cu] [jepsen.os.ubuntu :as ubuntu])) -(defn get-clickhouse-sky - [version] - (c/exec :sky :get :-d common-prefix :-N :Backbone version) - (str common-prefix "/clickhouse")) - (defn get-clickhouse-url [url] (non-precise-cached-wget! url)) @@ -27,7 +22,6 @@ [source] (info "Downloading clickhouse from" source) (cond - (clojure.string/starts-with? source "rbtorrent:") (get-clickhouse-sky source) (clojure.string/starts-with? source "http") (get-clickhouse-url source) (.exists (io/file source)) (get-clickhouse-scp source) :else (throw (Exception. (str "Don't know how to download clickhouse from" source))))) From 25884c68f15967ec5dcd7021db5d795647e0e3a4 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Thu, 26 May 2022 20:46:26 -0400 Subject: [PATCH 040/136] http named collection source implemented for dictionary --- src/Dictionaries/HTTPDictionarySource.cpp | 71 +++++++++++++------ .../ExternalDataSourceConfiguration.cpp | 57 +++++++++++++++ .../ExternalDataSourceConfiguration.h | 6 ++ 3 files changed, 111 insertions(+), 23 deletions(-) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index cf8b60f3681..8f7ca5e7a51 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include "DictionarySourceFactory.h" @@ -228,45 +229,69 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) if (dict_struct.has_expressions) throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `http` does not support attribute expressions"); - auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); - - const auto & settings_config_prefix = config_prefix + ".http"; - const auto & credentials_prefix = settings_config_prefix + ".credentials"; - + auto settings_config_prefix = config_prefix + ".http"; Poco::Net::HTTPBasicCredentials credentials; - - if (config.has(credentials_prefix)) - { - credentials.setUsername(config.getString(credentials_prefix + ".user", "")); - credentials.setPassword(config.getString(credentials_prefix + ".password", "")); - } - - const auto & headers_prefix = settings_config_prefix + ".headers"; ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries; + String url; + String format; - if (config.has(headers_prefix)) + auto named_collection = created_from_ddl + ? getURLBasedDataSourceConfiguration(config, settings_config_prefix, global_context) + : std::nullopt; + if (named_collection) { - Poco::Util::AbstractConfiguration::Keys config_keys; - config.keys(headers_prefix, config_keys); + url = named_collection->configuration.url; + format = named_collection->configuration.format; - header_entries.reserve(config_keys.size()); - for (const auto & key : config_keys) + credentials.setUsername(named_collection->configuration.user); + credentials.setPassword(named_collection->configuration.password); + + header_entries.reserve(named_collection->configuration.headers.size()); + for (const auto & header : named_collection->configuration.headers) + header_entries.emplace_back(std::make_tuple(header.first, header.second.get())); + } + else + { + const auto & credentials_prefix = settings_config_prefix + ".credentials"; + + if (config.has(credentials_prefix)) { - const auto header_key = config.getString(headers_prefix + "." + key + ".name", ""); - const auto header_value = config.getString(headers_prefix + "." + key + ".value", ""); - header_entries.emplace_back(std::make_tuple(header_key, header_value)); + credentials.setUsername(config.getString(credentials_prefix + ".user", "")); + credentials.setPassword(config.getString(credentials_prefix + ".password", "")); } + + const auto & headers_prefix = settings_config_prefix + ".headers"; + + + if (config.has(headers_prefix)) + { + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(headers_prefix, config_keys); + + header_entries.reserve(config_keys.size()); + for (const auto & key : config_keys) + { + const auto header_key = config.getString(headers_prefix + "." + key + ".name", ""); + const auto header_value = config.getString(headers_prefix + "." + key + ".value", ""); + header_entries.emplace_back(std::make_tuple(header_key, header_value)); + } + } + + url = config.getString(settings_config_prefix + ".url", ""); + format =config.getString(settings_config_prefix + ".format", ""); } auto configuration = HTTPDictionarySource::Configuration { - .url = config.getString(settings_config_prefix + ".url", ""), - .format =config.getString(settings_config_prefix + ".format", ""), + .url = url, + .format = format, .update_field = config.getString(settings_config_prefix + ".update_field", ""), .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1), .header_entries = std::move(header_entries) //-V1030 }; + auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); + return std::make_unique(dict_struct, configuration, credentials, sample_block, context, created_from_ddl); }; factory.registerSource("http", create_table_source); diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index abd20e6e5fd..55eff117d5e 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -248,6 +248,63 @@ std::optional getExternalDataSourceConfiguration( return std::nullopt; } +std::optional getURLBasedDataSourceConfiguration( + const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context) +{ + URLBasedDataSourceConfiguration configuration; + auto collection_name = dict_config.getString(dict_config_prefix + ".name", ""); + if (!collection_name.empty()) + { + const auto & config = context->getConfigRef(); + const auto & collection_prefix = fmt::format("named_collections.{}", collection_name); + + if (!config.has(collection_prefix)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", collection_name); + + configuration.url = + dict_config.getString(dict_config_prefix + ".url", config.getString(collection_prefix + ".url", "")); + configuration.format = + dict_config.getString(dict_config_prefix + ".format", config.getString(collection_prefix + ".format", "")); + configuration.compression_method = + dict_config.getString(dict_config_prefix + ".compression", config.getString(collection_prefix + ".compression_method", "")); + configuration.structure = + dict_config.getString(dict_config_prefix + ".structure", config.getString(collection_prefix + ".structure", "")); + configuration.user = + dict_config.getString(dict_config_prefix + ".credentials.user", config.getString(collection_prefix + ".credentials.user", "")); + configuration.password = + dict_config.getString(dict_config_prefix + ".credentials.password", config.getString(collection_prefix + ".credentials.password", "")); + + String headers_prefix; + const Poco::Util::AbstractConfiguration *headers_config = nullptr; + if (dict_config.has(dict_config_prefix + ".headers")) + { + headers_prefix = dict_config_prefix + ".headers"; + headers_config = &dict_config; + } + else + { + headers_prefix = collection_prefix + ".headers"; + headers_config = &config; + } + + if (headers_config) + { + Poco::Util::AbstractConfiguration::Keys header_keys; + headers_config->keys(headers_prefix, header_keys); + headers_prefix += "."; + for (const auto & header : header_keys) + { + const auto header_prefix = headers_prefix + header; + configuration.headers.emplace_back( + std::make_pair(headers_config->getString(header_prefix + ".name"), headers_config->getString(header_prefix + ".value"))); + } + } + + return URLBasedDataSourceConfig{ .configuration = configuration }; + } + + return std::nullopt; +} ExternalDataSourcesByPriority getExternalDataSourceConfigurationByPriority( const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context, HasConfigKeyFunc has_config_key) diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index dfac101e22d..19301c360f0 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -103,6 +103,9 @@ struct URLBasedDataSourceConfiguration String compression_method = "auto"; String structure = "auto"; + String user; + String password; + std::vector> headers; String http_method; @@ -129,6 +132,9 @@ struct URLBasedDataSourceConfig std::optional getURLBasedDataSourceConfiguration(const ASTs & args, ContextPtr context); +std::optional getURLBasedDataSourceConfiguration( + const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context); + template bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context); From e1772c571c12574f97d9e5252a5b86bc934491f3 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 27 May 2022 07:10:20 +0000 Subject: [PATCH 041/136] Compress clickhouse logs --- .../src/jepsen/clickhouse_keeper/db.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index 22c8bfd348d..0df704f0d4a 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -135,8 +135,13 @@ (do (info node "Coordination files exists, going to compress") (c/cd data-dir - (c/exec :tar :czf "coordination.tar.gz" "coordination"))))) - (let [common-logs [stderr-file (str logs-dir "/clickhouse-keeper.log") (str data-dir "/coordination.tar.gz")] + (c/exec :tar :czf "coordination.tar.gz" "coordination")))) + (if (cu/exists? (str logs-dir "/clickhouse-keeper.log")) + (do + (info node "Clickhouse logs exist, going to compress") + (c/cd logs-dir + (c/exec :gzip "clickhouse-keeper.log"))) (info node "Logs are missing"))) + (let [common-logs [stderr-file (str logs-dir "/clickhouse-keeper.log.gz") (str data-dir "/coordination.tar.gz")] gdb-log (str logs-dir "/gdb.log")] (if (cu/exists? (str logs-dir "/gdb.log")) (conj common-logs gdb-log) From ca67e67a7432582ee5ff5837d31432ab0e585d56 Mon Sep 17 00:00:00 2001 From: zhanglistar Date: Fri, 27 May 2022 15:52:04 +0800 Subject: [PATCH 042/136] Fix a typo --- src/Interpreters/ActionsDAG.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 2fc9b51674f..eb073ee8752 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -39,7 +39,7 @@ void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const map.add("Result Type", result_type->getName()); if (!result_name.empty()) - map.add("Result Type", magic_enum::enum_name(type)); + map.add("Result Name", result_name); if (column) map.add("Column", column->getName()); From ff228d63e8eece4cdd504f000de1dd2260954941 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 27 May 2022 10:14:13 +0200 Subject: [PATCH 043/136] Fix typo --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index f9c7ae37157..be361bd1e3f 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -81,7 +81,7 @@ $ ./src/unit_tests_dbms --gtest_filter=LocalAddress* ## Performance Tests {#performance-tests} -Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/tests/performance-comparison` tool . See the readme file for invocation. +Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/test/performance-comparison` tool . See the readme file for invocation. Each test run one or multiple queries (possibly with combinations of parameters) in a loop. From 7ccf4f4db73fc5f28efeb36427a2eae0af05dcc3 Mon Sep 17 00:00:00 2001 From: PigInCloud <44889745+yjant@users.noreply.github.com> Date: Fri, 27 May 2022 18:30:20 +0800 Subject: [PATCH 044/136] Update insert-into.md I translated some untranslated text into Chinese --- .../sql-reference/statements/insert-into.md | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/zh/sql-reference/statements/insert-into.md b/docs/zh/sql-reference/statements/insert-into.md index 928107fa2b2..4f958e31b18 100644 --- a/docs/zh/sql-reference/statements/insert-into.md +++ b/docs/zh/sql-reference/statements/insert-into.md @@ -71,7 +71,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] FORMAT format_name data_set INSERT INTO [db.]table [(c1, c2, c3)] FORMAT Values (v11, v12, v13), (v21, v22, v23), ... ``` -ClickHouse会清除数据前所有的空白字符与一行摘要信息（如果需要的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。 +ClickHouse会清除数据前所有的空白字符与一个换行符（如果有换行符的话）。所以在进行查询时，我们建议您将数据放入到输入输出格式名称后的新的一行中去（如果数据是以空白字符开始的，这将非常重要）。 示例: @@ -83,6 +83,10 @@ INSERT INTO t FORMAT TabSeparated 在使用命令行客户端或HTTP客户端时，你可以将具体的查询语句与数据分开发送。更多具体信息，请参考«[客户端](../../interfaces/index.md#interfaces)»部分。 +### 限制 {#constraints} + +如果表中有一些[限制](../../sql-reference/statements/create/table.md#constraints),，数据插入时会逐行进行数据校验，如果这里面包含了不符合限制条件的数据，服务将会抛出包含限制信息的异常，这个语句也会被停止执行。 + ### 使用`SELECT`的结果写入 {#insert_query_insert-select} ``` sql @@ -96,6 +100,66 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... 系统不支持的其他用于修改数据的查询：`UPDATE`, `DELETE`, `REPLACE`, `MERGE`, `UPSERT`, `INSERT UPDATE`。 但是，您可以使用 `ALTER TABLE ... DROP PARTITION`查询来删除一些旧的数据。 +如果 `SELECT` 查询中包含了 [input()](../../sql-reference/table-functions/input.md) 函数，那么 `FORMAT` 必须出现在查询语句的最后。 + +如果某一列限制了值不能是NULL，那么插入NULL的时候就会插入这个列类型的默认数据，可以通过设置 [insert_null_as_default](../../operations/settings/settings.md#insert_null_as_default) 插入NULL。 + +### 从文件向表中插入数据 {#inserting-data-from-a-file} + +``` sql +INSERT INTO [db.]table [(c1, c2, c3)] FROM INFILE file_name [COMPRESSION type] FORMAT format_name +``` +使用上面的语句可以从客户端的文件上读取数据并插入表中，`file_name` 和 `type` 都是 `String` 类型，输入文件的[格式](../../interfaces/formats.md) 一定要在 `FORMAT` 语句中设置。 + +支持读取压缩文件。默认会去读文件的拓展名作为文件的压缩方式，或者也可以在 `COMPRESSION` 语句中指明，支持的文件压缩格式如下：`'none'`， `'gzip'`， `'deflate'`， `'br'`， `'xz'`， `'zstd'`， `'lz4'`， `'bz2'`。 + +这个功能在 [command-line client](../../interfaces/cli.md) 和 [clickhouse-local](../../operations/utilities/clickhouse-local.md) 是可用的。 + +**样例** + +```bash +echo 1,A > input.csv ; echo 2,B >> input.csv +clickhouse-client --query="CREATE TABLE table_from_file (id UInt32, text String) ENGINE=MergeTree() ORDER BY id;" +clickhouse-client --query="INSERT INTO table_from_file FROM INFILE 'input.csv' FORMAT CSV;" +clickhouse-client --query="SELECT * FROM table_from_file FORMAT PrettyCompact;" +``` + +结果: + +```text +┌─id─┬─text─┐ +│ 1 │ A │ +│ 2 │ B │ +└────┴──────┘ +``` + +### 插入表函数 {#inserting-into-table-function} + +数据可以通过 [table functions](../../sql-reference/table-functions/index.md) 方法插入。 +``` sql +INSERT INTO [TABLE] FUNCTION table_func ... +``` + +**例如** + +可以这样使用[remote](../../sql-reference/table-functions/index.md#remote) 表函数: + +``` sql +CREATE TABLE simple_table (id UInt32, text String) ENGINE=MergeTree() ORDER BY id; +INSERT INTO TABLE FUNCTION remote('localhost', default.simple_table) + VALUES (100, 'inserted via remote()'); +SELECT * FROM simple_table; +``` + +结果: + +``` text +┌──id─┬─text──────────────────┐ +│ 100 │ inserted via remote() │ +└─────┴───────────────────────┘ +``` + + ### 性能的注意事项 {#xing-neng-de-zhu-yi-shi-xiang} 在进行`INSERT`时将会对写入的数据进行一些处理，按照主键排序，按照月份对数据进行分区等。所以如果在您的写入数据中包含多个月份的混合数据时，将会显著的降低`INSERT`的性能。为了避免这种情况： @@ -108,4 +172,6 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - 数据总是被实时的写入。 - 写入的数据已经按照时间排序。 +也可以异步的、小规模的插入数据，这些数据会被合并成多个批次，然后安全地写入到表中，通过设置[async_insert](../../operations/settings/settings.md#async-insert)，可以使用异步插入的方式，请注意，异步插入的方式只支持HTTP协议，并且不支持数据去重。 + [来源文章](https://clickhouse.com/docs/en/query_language/insert_into/) From 2613149f6bf4f242bbbf2c3c8539b5176fd77286 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 20 Feb 2022 22:12:25 +0300 Subject: [PATCH 045/136] Fix converting types for UNION queries (may produce LOGICAL_ERROR) CI founds [1]: 2022.02.20 15:14:23.969247 [ 492 ] {} BaseDaemon: (version 22.3.1.1, build id: 6082C357CFA6FF99) (from thread 472) (query_id: a5187ff9-962a-4e7c-86f6-8d48850a47d6) (query: SELECT 0., round(avgWeighted(x, y)) FROM (SELECT toDate(toDate('214748364.8', '-922337203.6854775808', '-0.1', NULL) - NULL, 10.000100135803223, '-2147483647'), 255 AS x, -2147483647 AS y UNION ALL SELECT y, NULL AS x, 2147483646 AS y)) Received signal Aborted (6) [1]: https://s3.amazonaws.com/clickhouse-test-reports/0/26d0e5438c86e52a145aaaf4cb523c399989a878/fuzzer_astfuzzerdebug,actions//report.html The problem is that subqueries returns different headers: - first query -- x, y - second query -- y, x v2: Make order of columns strict only for UNION https://s3.amazonaws.com/clickhouse-test-reports/34775/9cc8c01a463d18c471853568b2f0af659a4e643f/stateless_tests__address__actions__[2/2].html Fixes: 00597_push_down_predicate_long v3: add no-backward-compatibility-check for the test Fixes: #37569 Resubmit: #34775 Signed-off-by: Azat Khuzhin (cherry picked from commit a813f5996e95e424193265bb090ef7a402497d6e) --- .../InterpreterSelectWithUnionQuery.cpp | 4 ++ src/Interpreters/SelectQueryOptions.h | 8 ++++ src/Interpreters/TreeRewriter.cpp | 30 ++++++++++++- .../02227_union_match_by_name.reference | 44 +++++++++++++++++++ .../0_stateless/02227_union_match_by_name.sql | 5 +++ 5 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02227_union_match_by_name.reference create mode 100644 tests/queries/0_stateless/02227_union_match_by_name.sql diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 7506c3013cb..94ebfd73513 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -46,6 +46,10 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_children) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + /// This is required for UNION to match headers correctly. + if (num_children > 1) + options.reorderColumns(); + /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, /// because names could be different. diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index 31ed9d8c686..b0183e2761b 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -31,6 +31,8 @@ struct SelectQueryOptions bool only_analyze = false; bool modify_inplace = false; bool remove_duplicates = false; + /// This is required for UNION to match headers correctly. + bool reorder_columns_as_required_header = false; bool ignore_quota = false; bool ignore_limits = false; /// This flag is needed to analyze query ignoring table projections. @@ -97,6 +99,12 @@ struct SelectQueryOptions return *this; } + SelectQueryOptions & reorderColumns(bool value = true) + { + reorder_columns_as_required_header = value; + return *this; + } + SelectQueryOptions & noSubquery() { subquery_depth = 0; diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index c90421d6f4f..11a392f3adf 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -422,7 +422,7 @@ void renameDuplicatedColumns(const ASTSelectQuery * select_query) /// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result. /// Also we have to remove duplicates in case of GLOBAL subqueries. Their results are placed into tables so duplicates are impossible. /// Also remove all INTERPOLATE columns which are not in SELECT anymore. -void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups) +void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups, bool reorder_columns_as_required_header) { ASTs & elements = select_query->select()->children; @@ -453,6 +453,29 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const NameSet remove_columns; + /// Resort columns according to required_result_columns. + if (reorder_columns_as_required_header && !required_result_columns.empty()) + { + std::unordered_map name_pos; + { + size_t pos = 0; + for (const auto & name : required_result_columns) + name_pos[name] = pos++; + } + std::sort(elements.begin(), elements.end(), [&](const auto & lhs, const auto & rhs) + { + String lhs_name = lhs->getAliasOrColumnName(); + String rhs_name = rhs->getAliasOrColumnName(); + size_t lhs_pos = name_pos.size(); + size_t rhs_pos = name_pos.size(); + if (auto it = name_pos.find(lhs_name); it != name_pos.end()) + lhs_pos = it->second; + if (auto it = name_pos.find(rhs_name); it != name_pos.end()) + rhs_pos = it->second; + return lhs_pos < rhs_pos; + }); + } + for (const auto & elem : elements) { String name = elem->getAliasOrColumnName(); @@ -465,6 +488,8 @@ void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const } else if (select_query->distinct || hasArrayJoin(elem)) { + /// ARRAY JOIN cannot be optimized out since it may change number of rows, + /// so as DISTINCT. new_elements.push_back(elem); } else @@ -1135,6 +1160,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( size_t subquery_depth = select_options.subquery_depth; bool remove_duplicates = select_options.remove_duplicates; + bool reorder_columns_as_required_header = select_options.reorder_columns_as_required_header; const auto & settings = getContext()->getSettingsRef(); @@ -1186,7 +1212,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) /// and before 'executeScalarSubqueries', 'analyzeAggregation', etc. to avoid excessive calculations. - removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates); + removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates, reorder_columns_as_required_header); /// Executing scalar subqueries - replacing them with constant values. executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze); diff --git a/tests/queries/0_stateless/02227_union_match_by_name.reference b/tests/queries/0_stateless/02227_union_match_by_name.reference new file mode 100644 index 00000000000..72c4987a3d2 --- /dev/null +++ b/tests/queries/0_stateless/02227_union_match_by_name.reference @@ -0,0 +1,44 @@ +-- { echo } +EXPLAIN header = 1, optimize = 0 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); +Expression (Projection) +Header: avgWeighted(x, y) Nullable(Float64) + Expression (Before ORDER BY) + Header: avgWeighted(x, y) Nullable(Float64) + Aggregating + Header: avgWeighted(x, y) Nullable(Float64) + Expression (Before GROUP BY) + Header: x Nullable(UInt8) + y UInt8 + Union + Header: x Nullable(UInt8) + y UInt8 + Expression (Conversion before UNION) + Header: x Nullable(UInt8) + y UInt8 + Expression (Projection) + Header: x UInt8 + y UInt8 + Expression (Before ORDER BY) + Header: 255 UInt8 + 1 UInt8 + dummy UInt8 + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Header: dummy UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 + Expression (Conversion before UNION) + Header: x Nullable(UInt8) + y UInt8 + Expression (Projection) + Header: x Nullable(Nothing) + y UInt8 + Expression (Before ORDER BY) + Header: NULL Nullable(Nothing) + 1 UInt8 + dummy UInt8 + SettingQuotaAndLimits (Set limits and quota after reading from storage) + Header: dummy UInt8 + ReadFromStorage (SystemOne) + Header: dummy UInt8 +SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); +255 diff --git a/tests/queries/0_stateless/02227_union_match_by_name.sql b/tests/queries/0_stateless/02227_union_match_by_name.sql new file mode 100644 index 00000000000..8ff33b6193d --- /dev/null +++ b/tests/queries/0_stateless/02227_union_match_by_name.sql @@ -0,0 +1,5 @@ +-- Tags: no-backward-compatibility-check:22.5.1.2079 + +-- { echo } +EXPLAIN header = 1, optimize = 0 SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); +SELECT avgWeighted(x, y) FROM (SELECT NULL, 255 AS x, 1 AS y UNION ALL SELECT y, NULL AS x, 1 AS y); From 32167cb6fbdcf66b3f9590c0d4b495b253d7b076 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 27 May 2022 14:36:12 +0200 Subject: [PATCH 046/136] Turn on s3 tests --- tests/ci/ci_config.py | 1 - tests/ci/functional_test_check.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 33430c11a53..df013af8141 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -274,7 +274,6 @@ CI_CONFIG = { }, "Stateless tests (release, s3 storage, actions)": { "required_build": "package_release", - "force_tests": True, }, "Stress test (address, actions)": { "required_build": "package_asan", diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index af16c1b68c3..6516a67d333 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -44,7 +44,6 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): if "wide parts enabled" in check_name: result.append("USE_POLYMORPHIC_PARTS=1") - # temporary if "s3 storage" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") From abc90fad8dba3521a29d044cadad599d9336dcec Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 27 May 2022 12:42:51 +0000 Subject: [PATCH 047/136] fix WITH FILL with negative itervals --- src/Processors/Transforms/FillingTransform.cpp | 4 ++-- tests/queries/0_stateless/02112_with_fill_interval.reference | 3 +++ tests/queries/0_stateless/02112_with_fill_interval.sql | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 9e5d57a2b43..a41b5660e0d 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -90,9 +90,9 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr & if (which.isDate() || which.isDate32()) { Int64 avg_seconds = get(descr.fill_step) * descr.step_kind->toAvgSeconds(); - if (avg_seconds < 86400) + if (std::abs(avg_seconds) < 86400) throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION, - "Value of step is to low ({} seconds). Must be >= 1 day", avg_seconds); + "Value of step is to low ({} seconds). Must be >= 1 day", std::abs(avg_seconds)); } if (which.isDate()) diff --git a/tests/queries/0_stateless/02112_with_fill_interval.reference b/tests/queries/0_stateless/02112_with_fill_interval.reference index fc6f9378bfa..4bb99803eb1 100644 --- a/tests/queries/0_stateless/02112_with_fill_interval.reference +++ b/tests/queries/0_stateless/02112_with_fill_interval.reference @@ -107,3 +107,6 @@ 2020-05-01 2 0 2020-05-01 3 0 2020-05-01 4 0 +1970-01-04 +1970-01-03 +1970-01-02 diff --git a/tests/queries/0_stateless/02112_with_fill_interval.sql b/tests/queries/0_stateless/02112_with_fill_interval.sql index f26ec7da8c9..d2416f9a84b 100644 --- a/tests/queries/0_stateless/02112_with_fill_interval.sql +++ b/tests/queries/0_stateless/02112_with_fill_interval.sql @@ -79,3 +79,6 @@ d WITH FILL id WITH FILL FROM 1 TO 5; DROP TABLE with_fill_date; + +SELECT d FROM (SELECT toDate(1) AS d) +ORDER BY d DESC WITH FILL FROM toDate(3) TO toDate(0) STEP INTERVAL -1 DAY; From be1c3c132b4ed2613835d86f14894110bf2e05a1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 27 May 2022 16:08:49 +0200 Subject: [PATCH 048/136] Fix some trash --- .../MergeTree/MergeTreeDeduplicationLog.cpp | 60 +++++++++++++------ .../MergeTree/MergeTreeDeduplicationLog.h | 16 ++++- src/Storages/StorageMergeTree.cpp | 8 ++- .../0_stateless/02263_lazy_mark_load.sh | 3 +- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp index 33960e2e1ff..320509f2e20 100644 --- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp +++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp @@ -3,9 +3,10 @@ #include #include #include -#include #include #include +#include +#include namespace DB { @@ -74,27 +75,27 @@ size_t getLogNumber(const std::string & path_str) MergeTreeDeduplicationLog::MergeTreeDeduplicationLog( const std::string & logs_dir_, size_t deduplication_window_, - const MergeTreeDataFormatVersion & format_version_) + const MergeTreeDataFormatVersion & format_version_, + DiskPtr disk_) : logs_dir(logs_dir_) , deduplication_window(deduplication_window_) , rotate_interval(deduplication_window_ * 2) /// actually it doesn't matter , format_version(format_version_) , deduplication_map(deduplication_window) + , disk(disk_) { - namespace fs = std::filesystem; - if (deduplication_window != 0 && !fs::exists(logs_dir)) - fs::create_directories(logs_dir); + if (deduplication_window != 0 && !disk->exists(logs_dir)) + disk->createDirectories(logs_dir); } void MergeTreeDeduplicationLog::load() { - namespace fs = std::filesystem; - if (!fs::exists(logs_dir)) + if (!disk->exists(logs_dir)) return; - for (const auto & p : fs::directory_iterator(logs_dir)) + for (auto it = disk->iterateDirectory(logs_dir); it->isValid(); it->next()) { - const auto & path = p.path(); + const auto & path = it->path(); auto log_number = getLogNumber(path); existing_logs[log_number] = {path, 0}; } @@ -124,19 +125,19 @@ void MergeTreeDeduplicationLog::load() /// Can happen in case we have unfinished log if (!current_writer) - current_writer = std::make_unique(existing_logs.rbegin()->second.path, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY); + current_writer = disk->writeFile(existing_logs.rbegin()->second.path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append); } } size_t MergeTreeDeduplicationLog::loadSingleLog(const std::string & path) { - ReadBufferFromFile read_buf(path); + auto read_buf = disk->readFile(path); size_t total_entries = 0; - while (!read_buf.eof()) + while (!read_buf->eof()) { MergeTreeDeduplicationLogRecord record; - readRecord(record, read_buf); + readRecord(record, *read_buf); if (record.operation == MergeTreeDeduplicationOp::DROP) deduplication_map.erase(record.block_id); else @@ -160,7 +161,7 @@ void MergeTreeDeduplicationLog::rotate() if (current_writer) current_writer->sync(); - current_writer = std::make_unique(log_description.path, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY); + current_writer = disk->writeFile(log_description.path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append); } void MergeTreeDeduplicationLog::dropOutdatedLogs() @@ -188,7 +189,7 @@ void MergeTreeDeduplicationLog::dropOutdatedLogs() for (auto itr = existing_logs.begin(); itr != existing_logs.end();) { size_t number = itr->first; - std::filesystem::remove(itr->second.path); + disk->removeFile(itr->second.path); itr = existing_logs.erase(itr); if (remove_from_value == number) break; @@ -297,15 +298,38 @@ void MergeTreeDeduplicationLog::setDeduplicationWindowSize(size_t deduplication_ rotate_interval = deduplication_window * 2; /// If settings was set for the first time with ALTER MODIFY SETTING query - if (deduplication_window != 0 && !std::filesystem::exists(logs_dir)) - std::filesystem::create_directories(logs_dir); + if (deduplication_window != 0 && !disk->exists(logs_dir)) + disk->createDirectories(logs_dir); deduplication_map.setMaxSize(deduplication_window); rotateAndDropIfNeeded(); /// Can happen in case we have unfinished log if (!current_writer) - current_writer = std::make_unique(existing_logs.rbegin()->second.path, DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY); + current_writer = disk->writeFile(existing_logs.rbegin()->second.path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Append); +} + + +void MergeTreeDeduplicationLog::shutdown() +{ + std::lock_guard lock(state_mutex); + if (stopped) + return; + + stopped = true; + current_writer->finalize(); +} + +MergeTreeDeduplicationLog::~MergeTreeDeduplicationLog() +{ + try + { + shutdown(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } } diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.h b/src/Storages/MergeTree/MergeTreeDeduplicationLog.h index eda217798b2..2c90c9f43b1 100644 --- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.h +++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.h @@ -1,8 +1,8 @@ #pragma once #include #include -#include #include +#include #include #include #include @@ -137,7 +137,8 @@ public: MergeTreeDeduplicationLog( const std::string & logs_dir_, size_t deduplication_window_, - const MergeTreeDataFormatVersion & format_version_); + const MergeTreeDataFormatVersion & format_version_, + DiskPtr disk_); /// Add part into in-memory hash table and to disk /// Return true and part info if insertion was successful. @@ -151,6 +152,10 @@ public: void load(); void setDeduplicationWindowSize(size_t deduplication_window_); + + void shutdown(); + + ~MergeTreeDeduplicationLog(); private: const std::string logs_dir; /// Size of deduplication window @@ -171,11 +176,16 @@ private: LimitedOrderedHashMap deduplication_map; /// Writer to the current log file - std::unique_ptr current_writer; + std::unique_ptr current_writer; /// Overall mutex because we can have a lot of cocurrent inserts std::mutex state_mutex; + /// Disk where log is stored + DiskPtr disk; + + bool stopped{false}; + /// Start new log void rotate(); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 9ee5213630e..2cb62801fd5 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -182,6 +182,9 @@ void StorageMergeTree::shutdown() background_operations_assignee.finish(); background_moves_assignee.finish(); + if (deduplication_log) + deduplication_log->shutdown(); + try { /// We clear all old parts after stopping all background operations. @@ -715,8 +718,9 @@ void StorageMergeTree::loadDeduplicationLog() if (settings->non_replicated_deduplication_window != 0 && format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) throw Exception("Deduplication for non-replicated MergeTree in old syntax is not supported", ErrorCodes::BAD_ARGUMENTS); - std::string path = getDataPaths()[0] + "/deduplication_logs"; - deduplication_log = std::make_unique(path, settings->non_replicated_deduplication_window, format_version); + auto disk = getDisks()[0]; + std::string path = fs::path(relative_data_path) / "deduplication_logs"; + deduplication_log = std::make_unique(path, settings->non_replicated_deduplication_window, format_version, disk); deduplication_log->load(); } diff --git a/tests/queries/0_stateless/02263_lazy_mark_load.sh b/tests/queries/0_stateless/02263_lazy_mark_load.sh index bdcada6dc34..27922afec24 100755 --- a/tests/queries/0_stateless/02263_lazy_mark_load.sh +++ b/tests/queries/0_stateless/02263_lazy_mark_load.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-s3-storage set -eo pipefail CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -31,4 +32,4 @@ ${CLICKHOUSE_CLIENT} -q "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --log_queries=1 --query_id "${QUERY_ID}" -q "SELECT * FROM lazy_mark_test WHERE n3==11" ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -q "select ProfileEvents['FileOpen'] from system.query_log where query_id = '${QUERY_ID}' and type = 'QueryFinish' and current_database = currentDatabase()" \ No newline at end of file +${CLICKHOUSE_CLIENT} -q "select ProfileEvents['FileOpen'] from system.query_log where query_id = '${QUERY_ID}' and type = 'QueryFinish' and current_database = currentDatabase()" From 5a296aec01795147138261af4f0ac005fa48a977 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 27 May 2022 16:34:16 +0200 Subject: [PATCH 049/136] Fix build --- src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp index 320509f2e20..07f6e3f3be7 100644 --- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp +++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp @@ -317,7 +317,11 @@ void MergeTreeDeduplicationLog::shutdown() return; stopped = true; - current_writer->finalize(); + if (current_writer) + { + current_writer->finalize(); + current_writer.reset(); + } } MergeTreeDeduplicationLog::~MergeTreeDeduplicationLog() From 2fb735ffc1afa09d55364a44c780c6047dfaac09 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 25 May 2022 22:32:34 +0300 Subject: [PATCH 050/136] tests: add echo for 01883_grouping_sets_crash Signed-off-by: Azat Khuzhin --- .../01883_grouping_sets_crash.reference | 78 +++++++++++++++++++ .../0_stateless/01883_grouping_sets_crash.sql | 4 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.reference b/tests/queries/0_stateless/01883_grouping_sets_crash.reference index 4d9e967b766..bdb7adec06c 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.reference +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.reference @@ -1,3 +1,14 @@ +-- { echoOn } +SELECT + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ('wo\0ldworldwo\0ldworld'), + (fact_3_id, fact_4_id)) +ORDER BY + fact_3_id, fact_4_id; 0 0 1 1 2 2 @@ -9,7 +20,23 @@ 8 8 9 9 10 10 +SELECT 'SECOND QUERY:'; SECOND QUERY: +SELECT + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + (fact_1_id, fact_2_id), + ((-9223372036854775808, NULL, (tuple(1.), (tuple(1.), 1048576), 65535))), + ((tuple(3.4028234663852886e38), (tuple(1024), -2147483647), NULL)), + (fact_3_id, fact_4_id)) +ORDER BY + (NULL, ('256', (tuple(NULL), NULL), NULL, NULL), NULL) ASC, + fact_1_id DESC NULLS FIRST, + fact_2_id DESC NULLS FIRST, + fact_4_id ASC; 0 0 0 0 0 0 @@ -32,7 +59,26 @@ SECOND QUERY: 8 8 9 9 10 10 +SELECT 'THIRD QUERY:'; THIRD QUERY: +SELECT + extractAllGroups(NULL, 'worldworldworldwo\0ldworldworldworldwo\0ld'), + fact_2_id, + fact_3_id, + fact_4_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + (sales_value), + (fact_1_id, fact_2_id), + ('wo\0ldworldwo\0ldworld'), + (fact_3_id, fact_4_id)) +ORDER BY + fact_1_id DESC NULLS LAST, + fact_1_id DESC NULLS FIRST, + fact_2_id ASC, + fact_3_id DESC NULLS FIRST, + fact_4_id ASC; \N 1 0 0 \N 2 0 0 \N 3 0 0 @@ -154,6 +200,11 @@ THIRD QUERY: \N 0 0 0 \N 0 0 0 \N 0 0 0 +SELECT fact_3_id +FROM grouping_sets +GROUP BY + GROUPING SETS ((fact_3_id, fact_4_id)) +ORDER BY fact_3_id ASC; 1 2 3 @@ -164,6 +215,24 @@ THIRD QUERY: 8 9 10 +-- Following two queries were fuzzed +SELECT 'w\0\0ldworldwo\0l\0world' +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_4_id), + ( NULL), + ( fact_3_id, fact_4_id)) +ORDER BY + NULL ASC, + NULL DESC NULLS FIRST, + fact_3_id ASC, + fact_3_id ASC NULLS LAST, + 'wo\0ldworldwo\0ldworld' ASC NULLS LAST, + 'w\0\0ldworldwo\0l\0world' DESC NULLS FIRST, + 'wo\0ldworldwo\0ldworld' ASC, + NULL ASC NULLS FIRST, + fact_4_id DESC NULLS LAST; w\0\0ldworldwo\0l\0world w\0\0ldworldwo\0l\0world w\0\0ldworldwo\0l\0world @@ -185,6 +254,15 @@ w\0\0ldworldwo\0l\0world w\0\0ldworldwo\0l\0world w\0\0ldworldwo\0l\0world w\0\0ldworldwo\0l\0world +SELECT fact_3_id +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( 'wo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworldwo\0ldworld'), + ( NULL), + ( fact_4_id), + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id ASC NULLS FIRST; 0 0 0 diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.sql b/tests/queries/0_stateless/01883_grouping_sets_crash.sql index cf56c8546ce..7810ad1d8da 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.sql +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.sql @@ -11,6 +11,7 @@ SELECT number % 100 AS sales_value FROM system.numbers limit 1000; +-- { echoOn } SELECT fact_3_id, fact_4_id @@ -96,4 +97,5 @@ GROUP BY ( fact_3_id, fact_4_id)) ORDER BY fact_3_id ASC NULLS FIRST; -DROP TABLE IF EXISTS grouping_sets; \ No newline at end of file +-- { echoOff } +DROP TABLE IF EXISTS grouping_sets; From 1f29b0a9019099a4ffdc06fe8d72b747c8ef5fb2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 26 May 2022 10:28:55 +0300 Subject: [PATCH 051/136] Rewrite queries GROUPING SETS (foo, bar) to GROUP BY foo, bar This is better then introducing separate SelectQueryExpressionAnalyzer::useGroupingSetKey(), since for optimize_aggregation_in_order that method will not be enough, because size of ManyExpressionActions will not match size of SortDescription, in ReadInOrderOptimizer::ReadInOrderOptimizer() And plus it is cleaner. v2: fix clang-tidy Signed-off-by: Azat Khuzhin --- src/Interpreters/ExpressionAnalyzer.h | 10 ++++++- .../GroupingSetsRewriterVisitor.cpp | 22 ++++++++++++++ .../GroupingSetsRewriterVisitor.h | 30 +++++++++++++++++++ src/Interpreters/TreeRewriter.cpp | 9 ++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/Interpreters/GroupingSetsRewriterVisitor.cpp create mode 100644 src/Interpreters/GroupingSetsRewriterVisitor.h diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 85efb3829d0..930891bffd6 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -326,7 +326,15 @@ public: bool hasGlobalSubqueries() { return has_global_subqueries; } bool hasTableJoin() const { return syntax->ast_join; } - bool useGroupingSetKey() const { return aggregation_keys_list.size() > 1; } + /// When there is only one group in GROUPING SETS + /// it is a special case that is equal to GROUP BY, i.e.: + /// + /// GROUPING SETS ((a, b)) -> GROUP BY a, b + /// + /// But it is rewritten by GroupingSetsRewriterVisitor to GROUP BY, + /// so instead of aggregation_keys_list.size() > 1, + /// !aggregation_keys_list.empty() can be used. + bool useGroupingSetKey() const { return !aggregation_keys_list.empty(); } const NamesAndTypesList & aggregationKeys() const { return aggregation_keys; } bool hasConstAggregationKeys() const { return has_const_aggregation_keys; } diff --git a/src/Interpreters/GroupingSetsRewriterVisitor.cpp b/src/Interpreters/GroupingSetsRewriterVisitor.cpp new file mode 100644 index 00000000000..e5b930376cd --- /dev/null +++ b/src/Interpreters/GroupingSetsRewriterVisitor.cpp @@ -0,0 +1,22 @@ +#include +#include +#include + + +namespace DB +{ + +void GroupingSetsRewriterData::visit(ASTSelectQuery & select_query, ASTPtr &) +{ + const ASTPtr group_by = select_query.groupBy(); + if (!group_by || !select_query.group_by_with_grouping_sets) + return; + + if (group_by->children.size() != 1) + return; + + select_query.setExpression(ASTSelectQuery::Expression::GROUP_BY, std::move(group_by->children.front())); + select_query.group_by_with_grouping_sets = false; +} + +} diff --git a/src/Interpreters/GroupingSetsRewriterVisitor.h b/src/Interpreters/GroupingSetsRewriterVisitor.h new file mode 100644 index 00000000000..3d56bf1917d --- /dev/null +++ b/src/Interpreters/GroupingSetsRewriterVisitor.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ASTSelectQuery; + +/// Rewrite GROUPING SETS with only one group, to GROUP BY. +/// +/// Examples: +/// - GROUPING SETS (foo) -> GROUP BY foo +/// - GROUPING SETS ((foo, bar)) -> GROUP BY foo, bar +/// +/// But not the following: +/// - GROUPING SETS (foo, bar) (since it has two groups (foo) and (bar)) +class GroupingSetsRewriterData +{ +public: + using TypeToVisit = ASTSelectQuery; + + static void visit(ASTSelectQuery & select_query, ASTPtr &); +}; + +using GroupingSetsRewriterMatcher = OneTypeMatcher; +using GroupingSetsRewriterVisitor = InDepthNodeVisitor; + +} diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index c90421d6f4f..a13a233324c 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,12 @@ namespace using LogAST = DebugASTLog; /// set to true to enable logs +void optimizeGroupingSets(ASTPtr & query) +{ + GroupingSetsRewriterVisitor::Data data; + GroupingSetsRewriterVisitor(data).visit(query); +} + /// Select implementation of a function based on settings. /// Important that it is done as query rewrite. It means rewritten query /// will be sent to remote servers during distributed query execution, @@ -1373,6 +1380,8 @@ void TreeRewriter::normalize( /// Common subexpression elimination. Rewrite rules. QueryNormalizer::Data normalizer_data(aliases, source_columns_set, ignore_alias, settings, allow_self_aliases); QueryNormalizer(normalizer_data).visit(query); + + optimizeGroupingSets(query); } } From 8a224239c1d922158b4dc9f5d6609dca836dfd06 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 25 May 2022 21:59:39 +0300 Subject: [PATCH 052/136] Prohibit optimize_aggregation_in_order with GROUPING SETS AggregatingStep ignores it anyway, and it leads to the following error in getSortDescriptionFromGroupBy(), like in [1]: 2022.05.24 04:29:29.279431 [ 3395 ] {26543564-8bc8-4a3a-b984-70a2adf0245d} : Logical error: 'Trying to get name of not a column: ExpressionList'. [1]: https://s3.amazonaws.com/clickhouse-test-reports/36914/67d3ac72d26ab74d69f03c03422349d4faae9e19/stateless_tests__ubsan__actions_.html v2: revert change to getSortDescriptionFromGroupBy() after GroupingSetsRewriterVisitor had been introduced Signed-off-by: Azat Khuzhin --- src/Interpreters/InterpreterSelectQuery.cpp | 9 ++- .../01883_grouping_sets_crash.reference | 73 +++++++++++++++++++ .../0_stateless/01883_grouping_sets_crash.sql | 26 +++++++ 3 files changed, 105 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d143295181e..75822043681 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2060,12 +2060,15 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (prewhere_info) query_info.prewhere_info = prewhere_info; + bool optimize_read_in_order = analysis_result.optimize_read_in_order; + bool optimize_aggregation_in_order = analysis_result.optimize_aggregation_in_order && !query_analyzer->useGroupingSetKey(); + /// Create optimizer with prepared actions. /// Maybe we will need to calc input_order_info later, e.g. while reading from StorageMerge. - if ((analysis_result.optimize_read_in_order || analysis_result.optimize_aggregation_in_order) + if ((optimize_read_in_order || optimize_aggregation_in_order) && (!query_info.projection || query_info.projection->complete)) { - if (analysis_result.optimize_read_in_order) + if (optimize_read_in_order) { if (query_info.projection) { @@ -2291,7 +2294,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac SortDescription group_by_sort_description; - if (group_by_info && settings.optimize_aggregation_in_order) + if (group_by_info && settings.optimize_aggregation_in_order && !query_analyzer->useGroupingSetKey()) group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); else group_by_info = nullptr; diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.reference b/tests/queries/0_stateless/01883_grouping_sets_crash.reference index bdb7adec06c..c7c3eb1c977 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.reference +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.reference @@ -285,3 +285,76 @@ ORDER BY fact_3_id ASC NULLS FIRST; 8 9 10 +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; +1 1 100 +2 2 100 +3 3 100 +4 4 100 +5 5 100 +6 6 100 +7 7 100 +8 8 100 +9 9 100 +10 10 100 +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + fact_3_id, + fact_4_id) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; +0 1 100 +0 2 100 +0 3 100 +0 4 100 +0 5 100 +0 6 100 +0 7 100 +0 8 100 +0 9 100 +0 10 100 +1 0 100 +2 0 100 +3 0 100 +4 0 100 +5 0 100 +6 0 100 +7 0 100 +8 0 100 +9 0 100 +10 0 100 +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_3_id ), + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; +1 0 100 +1 1 100 +2 0 100 +2 2 100 +3 0 100 +3 3 100 +4 0 100 +4 4 100 +5 0 100 +5 5 100 +6 0 100 +6 6 100 +7 0 100 +7 7 100 +8 0 100 +8 8 100 +9 0 100 +9 9 100 +10 0 100 +10 10 100 diff --git a/tests/queries/0_stateless/01883_grouping_sets_crash.sql b/tests/queries/0_stateless/01883_grouping_sets_crash.sql index 7810ad1d8da..4cbd7c8f225 100644 --- a/tests/queries/0_stateless/01883_grouping_sets_crash.sql +++ b/tests/queries/0_stateless/01883_grouping_sets_crash.sql @@ -97,5 +97,31 @@ GROUP BY ( fact_3_id, fact_4_id)) ORDER BY fact_3_id ASC NULLS FIRST; +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; + +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + fact_3_id, + fact_4_id) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; + +SELECT fact_3_id, fact_4_id, count() +FROM grouping_sets +GROUP BY + GROUPING SETS ( + ( fact_3_id ), + ( fact_3_id, fact_4_id)) +ORDER BY fact_3_id, fact_4_id +SETTINGS optimize_aggregation_in_order=1; + -- { echoOff } DROP TABLE IF EXISTS grouping_sets; From 60b9d81773d5a56486c5685ec3880d6caf207d1f Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 27 May 2022 16:30:29 +0000 Subject: [PATCH 053/136] Remove global_memory_usage_overcommit_max_wait_microseconds --- .../settings.md | 10 ----- docs/en/operations/settings/settings.md | 2 +- programs/server/Server.cpp | 2 - src/Common/MemoryTracker.cpp | 8 ++++ src/Common/MemoryTracker.h | 10 +++++ src/Common/OvercommitTracker.cpp | 11 ++---- src/Common/OvercommitTracker.h | 4 -- src/Common/tests/gtest_overcommit_tracker.cpp | 38 ++++++++++++------- src/Core/Settings.h | 2 +- src/Interpreters/ProcessList.cpp | 3 +- .../configs/global_overcommit_tracker.xml | 1 - .../test_global_overcommit_tracker/test.py | 12 ++++-- 12 files changed, 57 insertions(+), 46 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index fd5c2a187b5..f235fba84f7 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1745,13 +1745,3 @@ Possible values: - Positive integer. Default value: `10000`. - -## global_memory_usage_overcommit_max_wait_microseconds {#global_memory_usage_overcommit_max_wait_microseconds} - -Sets maximum waiting time for global overcommit tracker. - -Possible values: - -- Positive integer. - -Default value: `200`. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 76fbc5f239d..9367d70507f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4279,7 +4279,7 @@ Maximum time thread will wait for memory to be freed in the case of memory overc If the timeout is reached and memory is not freed, an exception is thrown. Read more about [memory overcommit](memory-overcommit.md). -Default value: `200`. +Default value: `5000000`. ## memory_overcommit_ratio_denominator_for_user diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 18ab96983eb..2c6ffccd39d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1095,8 +1095,6 @@ int Server::main(const std::vector & /*args*/) total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); - UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 5'000'000); - global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time); total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); // FIXME logging-related things need synchronization -- see the 'Logger * log' saved diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index b5a27543b4e..51f4c83dc23 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -82,6 +82,8 @@ namespace ProfileEvents extern const Event QueryMemoryLimitExceeded; } +using namespace std::chrono_literals; + static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); @@ -363,6 +365,12 @@ OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit) } +void MemoryTracker::setOvercommitWaitingTime(UInt64 wait_time) +{ + max_wait_time.store(wait_time * 1us, std::memory_order_relaxed); +} + + void MemoryTracker::resetCounters() { amount.store(0, std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 73af2ab8857..58bd3a460bd 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -73,6 +74,8 @@ private: /// This description will be used as prefix into log messages (if isn't nullptr) std::atomic description_ptr = nullptr; + std::atomic max_wait_time; + std::atomic overcommit_tracker = nullptr; bool updatePeak(Int64 will_be, bool log_memory_usage); @@ -186,6 +189,13 @@ public: OvercommitRatio getOvercommitRatio(); OvercommitRatio getOvercommitRatio(Int64 limit); + std::chrono::microseconds getOvercommitWaitingTime() + { + return max_wait_time.load(std::memory_order_relaxed); + } + + void setOvercommitWaitingTime(UInt64 wait_time); + void setOvercommitTracker(OvercommitTracker * tracker) noexcept { overcommit_tracker.store(tracker, std::memory_order_relaxed); diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 0c03ba58e87..3cef72eb8b4 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -15,8 +15,7 @@ using namespace std::chrono_literals; constexpr std::chrono::microseconds ZERO_MICROSEC = 0us; OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) - : max_wait_time(ZERO_MICROSEC) - , picked_tracker(nullptr) + : picked_tracker(nullptr) , cancellation_state(QueryCancellationState::NONE) , global_mutex(global_mutex_) , freed_memory(0) @@ -24,12 +23,6 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} -void OvercommitTracker::setMaxWaitTime(UInt64 wait_time) -{ - std::lock_guard guard(overcommit_m); - max_wait_time = wait_time * 1us; -} - OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { // NOTE: Do not change the order of locks @@ -41,6 +34,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int std::unique_lock global_lock(global_mutex); std::unique_lock lk(overcommit_m); + auto max_wait_time = tracker->getOvercommitWaitingTime(); + if (max_wait_time == ZERO_MICROSEC) return OvercommitResult::DISABLED; diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 79ed36cd7fa..80aaed68e37 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -62,8 +62,6 @@ enum class QueryCancellationState // is killed to free memory. struct OvercommitTracker : boost::noncopyable { - void setMaxWaitTime(UInt64 wait_time); - OvercommitResult needToStopQuery(MemoryTracker * tracker, Int64 amount); void tryContinueQueryExecutionAfterFree(Int64 amount); @@ -82,8 +80,6 @@ protected: std::mutex overcommit_m; std::condition_variable cv; - std::chrono::microseconds max_wait_time; - // Specifies memory tracker of the chosen to stop query. // If soft limit is not set, all the queries which reach hard limit must stop. // This case is represented as picked tracker pointer is set to nullptr and diff --git a/src/Common/tests/gtest_overcommit_tracker.cpp b/src/Common/tests/gtest_overcommit_tracker.cpp index c56ecec669f..d832a73ffd9 100644 --- a/src/Common/tests/gtest_overcommit_tracker.cpp +++ b/src/Common/tests/gtest_overcommit_tracker.cpp @@ -40,15 +40,17 @@ static constexpr UInt64 WAIT_TIME = 4'000'000; template void free_not_continue_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); + std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -96,15 +98,16 @@ TEST(OvercommitTracker, GlobalFreeNotContinue) template void free_continue_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -152,15 +155,16 @@ TEST(OvercommitTracker, GlobalFreeContinue) template void free_continue_and_alloc_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -179,6 +183,7 @@ void free_continue_and_alloc_test(T & overcommit_tracker) [&]() { MemoryTracker failed; + failed.setOvercommitWaitingTime(WAIT_TIME); std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; @@ -212,15 +217,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc) template void free_continue_and_alloc_2_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -239,6 +245,7 @@ void free_continue_and_alloc_2_test(T & overcommit_tracker) [&]() { MemoryTracker failed; + failed.setOvercommitWaitingTime(WAIT_TIME); std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; @@ -280,15 +287,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc2) template void free_continue_and_alloc_3_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -307,6 +315,7 @@ void free_continue_and_alloc_3_test(T & overcommit_tracker) [&]() { MemoryTracker failed; + failed.setOvercommitWaitingTime(WAIT_TIME); std::this_thread::sleep_for(1000ms); overcommit_tracker.tryContinueQueryExecutionAfterFree(5000); stopped_next = overcommit_tracker.needToStopQuery(&failed, 100) != OvercommitResult::MEMORY_FREED; @@ -348,15 +357,16 @@ TEST(OvercommitTracker, GlobalFreeContinueAndAlloc3) template void free_continue_2_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - static constexpr size_t THREADS = 5; std::vector trackers(THREADS); + for (auto & tracker : trackers) + tracker.setOvercommitWaitingTime(WAIT_TIME); std::atomic need_to_stop = 0; std::vector threads; threads.reserve(THREADS); MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); for (size_t i = 0; i < THREADS; ++i) @@ -404,14 +414,14 @@ TEST(OvercommitTracker, GlobalFreeContinue2) template void query_stop_not_continue_test(T & overcommit_tracker) { - overcommit_tracker.setMaxWaitTime(WAIT_TIME); - std::atomic need_to_stop = 0; MemoryTracker picked; + picked.setOvercommitWaitingTime(WAIT_TIME); overcommit_tracker.setCandidate(&picked); MemoryTracker another; + another.setOvercommitWaitingTime(WAIT_TIME); auto thread = std::thread( [&]() { diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 29427c673ac..9111e1d80da 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -371,7 +371,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ \ - M(UInt64, memory_usage_overcommit_max_wait_microseconds, 200, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \ \ M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 6c101143234..6b8894414ee 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -225,6 +225,8 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as if (settings.memory_tracker_fault_probability) thread_group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); + thread_group->memory_tracker.setOvercommitWaitingTime(settings.memory_usage_overcommit_max_wait_microseconds); + /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values /// since allocation and deallocation could happen in different threads } @@ -244,7 +246,6 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); user_process_list.user_memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator_for_user); user_process_list.user_memory_tracker.setDescription("(for user)"); - user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds); if (!user_process_list.user_throttler) { diff --git a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml index 590759bd15d..a05d8865a6b 100644 --- a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml +++ b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml @@ -1,4 +1,3 @@ 50000000 - 500 \ No newline at end of file diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py index d3d56e82f38..093549249ce 100644 --- a/tests/integration/test_global_overcommit_tracker/test.py +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -18,8 +18,8 @@ def start_cluster(): cluster.shutdown() -TEST_QUERY_A = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=1" -TEST_QUERY_B = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=2" +TEST_QUERY_A = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=1, memory_usage_overcommit_max_wait_microseconds=500" +TEST_QUERY_B = "SELECT number FROM numbers(1000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator_for_user=2, memory_usage_overcommit_max_wait_microseconds=500" def test_overcommited_is_killed(): @@ -46,8 +46,12 @@ def test_overcommited_is_killed(): finished = True assert ( - overcommited_killed and finished - ), "no overcommited task was killed or all tasks are killed" + overcommited_killed + ), "no overcommited task was killed" + + assert ( + finished + ), "all tasks are killed" node.query("DROP USER IF EXISTS A") node.query("DROP USER IF EXISTS B") From 41ef0044f068f02d9fac278ec07dad6826825687 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Fri, 27 May 2022 13:43:34 -0400 Subject: [PATCH 054/136] endpoint is added --- src/Dictionaries/HTTPDictionarySource.cpp | 13 ++++++++++++- src/Storages/ExternalDataSourceConfiguration.cpp | 2 ++ src/Storages/ExternalDataSourceConfiguration.h | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 8f7ca5e7a51..17592a8d9da 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -233,6 +233,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) Poco::Net::HTTPBasicCredentials credentials; ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries; String url; + String endpoint; String format; auto named_collection = created_from_ddl @@ -241,6 +242,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) if (named_collection) { url = named_collection->configuration.url; + endpoint = named_collection->configuration.endpoint; format = named_collection->configuration.format; credentials.setUsername(named_collection->configuration.user); @@ -278,12 +280,21 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) } url = config.getString(settings_config_prefix + ".url", ""); + endpoint = config.getString(settings_config_prefix + ".endpoint", ""); format =config.getString(settings_config_prefix + ".format", ""); } + if (url.ends_with('/')) + { + if (endpoint.starts_with('/')) + url.pop_back(); + } + else if (!endpoint.empty() && !endpoint.starts_with('/')) + url.push_back('/'); + auto configuration = HTTPDictionarySource::Configuration { - .url = url, + .url = url + endpoint, .format = format, .update_field = config.getString(settings_config_prefix + ".update_field", ""), .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1), diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index 55eff117d5e..f916ac8c2af 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -263,6 +263,8 @@ std::optional getURLBasedDataSourceConfiguration( configuration.url = dict_config.getString(dict_config_prefix + ".url", config.getString(collection_prefix + ".url", "")); + configuration.endpoint = + dict_config.getString(dict_config_prefix + ".endpoint", config.getString(collection_prefix + ".endpoint", "")); configuration.format = dict_config.getString(dict_config_prefix + ".format", config.getString(collection_prefix + ".format", "")); configuration.compression_method = diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 19301c360f0..4ed46e1b26c 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -99,6 +99,7 @@ getExternalDataSourceConfigurationByPriority(const Poco::Util::AbstractConfigura struct URLBasedDataSourceConfiguration { String url; + String endpoint; String format = "auto"; String compression_method = "auto"; String structure = "auto"; From db2fe33926af47aad3155407a5443768a307dd43 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 27 May 2022 20:43:59 +0200 Subject: [PATCH 055/136] Update test.py --- tests/integration/test_global_overcommit_tracker/test.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py index 093549249ce..871f9ca983e 100644 --- a/tests/integration/test_global_overcommit_tracker/test.py +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -45,13 +45,8 @@ def test_overcommited_is_killed(): if err == "": finished = True - assert ( - overcommited_killed - ), "no overcommited task was killed" - - assert ( - finished - ), "all tasks are killed" + assert overcommited_killed, "no overcommited task was killed" + assert finished, "all tasks are killed" node.query("DROP USER IF EXISTS A") node.query("DROP USER IF EXISTS B") From b8fa931630e2fc20ffdceef5ac1ce64b7c247dde Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 28 May 2022 03:34:00 +0200 Subject: [PATCH 056/136] Preparation: forked some submodules --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 55fd684fddb..aa68aa218b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -79,10 +79,10 @@ url = https://github.com/ClickHouse/snappy.git [submodule "contrib/cppkafka"] path = contrib/cppkafka - url = https://github.com/mfontanini/cppkafka.git + url = https://github.com/ClickHouse/cppkafka.git [submodule "contrib/brotli"] path = contrib/brotli - url = https://github.com/google/brotli.git + url = https://github.com/ClickHouse/brotli.git [submodule "contrib/h3"] path = contrib/h3 url = https://github.com/ClickHouse/h3 @@ -144,7 +144,7 @@ ignore = untracked [submodule "contrib/msgpack-c"] path = contrib/msgpack-c - url = https://github.com/msgpack/msgpack-c + url = https://github.com/ClickHouse/msgpack-c [submodule "contrib/libcpuid"] path = contrib/libcpuid url = https://github.com/ClickHouse/libcpuid.git From 7281d51e45599f52d609f71fb1effc7fa6127c14 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 28 May 2022 04:01:03 +0200 Subject: [PATCH 057/136] Remove recursive submodules --- contrib/arrow | 2 +- contrib/brotli | 2 +- contrib/cppkafka | 2 +- contrib/msgpack-c | 2 +- contrib/rapidjson | 2 +- contrib/snappy | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/arrow b/contrib/arrow index efdcd015cfd..6f274b737c6 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5 +Subproject commit 6f274b737c66a6c39bab0d3bdf6cf7d139ef06f5 diff --git a/contrib/brotli b/contrib/brotli index 63be8a99401..5bd78768449 160000 --- a/contrib/brotli +++ b/contrib/brotli @@ -1 +1 @@ -Subproject commit 63be8a99401992075c23e99f7c84de1c653e39e2 +Subproject commit 5bd78768449751a78d4b4c646b0612917986f5b1 diff --git a/contrib/cppkafka b/contrib/cppkafka index 5a119f689f8..64bd67db12b 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1 +Subproject commit 64bd67db12b9c705e9127439a5b05b351d9df7da diff --git a/contrib/msgpack-c b/contrib/msgpack-c index 46684265d50..790b3fe58eb 160000 --- a/contrib/msgpack-c +++ b/contrib/msgpack-c @@ -1 +1 @@ -Subproject commit 46684265d50b5d1b062d4c5c428ba08462844b1d +Subproject commit 790b3fe58ebded7a8bd130782ef28bec5784c248 diff --git a/contrib/rapidjson b/contrib/rapidjson index c4ef90ccdbc..b571bd5c1a3 160000 --- a/contrib/rapidjson +++ b/contrib/rapidjson @@ -1 +1 @@ -Subproject commit c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa +Subproject commit b571bd5c1a3b1fc931d77ae36932537a3c9018c3 diff --git a/contrib/snappy b/contrib/snappy index fb057edfed8..3786173af20 160000 --- a/contrib/snappy +++ b/contrib/snappy @@ -1 +1 @@ -Subproject commit fb057edfed820212076239fd32cb2ff23e9016bf +Subproject commit 3786173af204d21da97180977ad6ab4321138b3d From 19b8e1324a4b2e3ece26e1078a4531d179843fb0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 28 May 2022 04:05:10 +0200 Subject: [PATCH 058/136] Forbid recursive submodules --- utils/check-style/check-style | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 84ce7ae5742..406b36e9251 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -340,3 +340,6 @@ fi # Forbid files that differ only by character case find $ROOT_PATH | sort -f | uniq -i -c | awk '{ if ($1 > 1) print }' + +# Forbid recursive submodules +find $ROOT_PATH/contrib -name '.gitmodules' -size +0 | xargs cat | grep -P '.' && echo "Recursive submodules are forbidden." From b24346328d12ff6a9d989c843f92311fba3e0454 Mon Sep 17 00:00:00 2001 From: Vxider Date: Sat, 28 May 2022 08:22:34 +0000 Subject: [PATCH 059/136] fix parser when using table identifer --- src/Storages/WindowView/StorageWindowView.cpp | 133 +++++------------- src/Storages/WindowView/StorageWindowView.h | 3 +- ...7_window_view_parser_inner_table.reference | 4 +- ...indow_view_with_table_identifier.reference | 7 + ...01084_window_view_with_table_identifier.sh | 31 ++++ 5 files changed, 77 insertions(+), 101 deletions(-) create mode 100644 tests/queries/0_stateless/01084_window_view_with_table_identifier.reference create mode 100755 tests/queries/0_stateless/01084_window_view_with_table_identifier.sh diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index a44b8954e3c..0f4d5885d19 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -81,20 +81,18 @@ namespace ErrorCodes namespace { /// Fetch all window info and replace tumble or hop node names with windowID - struct FetchQueryInfoMatcher + struct WindowFunctionMatcher { - using Visitor = InDepthNodeVisitor; + using Visitor = InDepthNodeVisitor; using TypeToVisit = ASTFunction; struct Data { ASTPtr window_function; - String window_id_name; - String window_id_alias; String serialized_window_function; - String timestamp_column_name; bool is_tumble = false; bool is_hop = false; + bool check_duplicate_window = false; }; static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; } @@ -111,18 +109,17 @@ namespace temp_node->setAlias(""); if (!data.window_function) { - data.serialized_window_function = serializeAST(*temp_node); + if (data.check_duplicate_window) + data.serialized_window_function = serializeAST(*temp_node); t->name = "windowID"; - data.window_id_name = t->getColumnName(); - data.window_id_alias = t->alias; data.window_function = t->clone(); data.window_function->setAlias(""); - data.timestamp_column_name = t->arguments->children[0]->getColumnName(); } else { - if (serializeAST(*temp_node) != data.serialized_window_function) - throw Exception("WINDOW VIEW only support ONE TIME WINDOW FUNCTION", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW); + if (data.check_duplicate_window && serializeAST(*temp_node) != data.serialized_window_function) + throw Exception( + "WINDOW VIEW only support ONE TIME WINDOW FUNCTION", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW); t->name = "windowID"; } } @@ -190,24 +187,6 @@ namespace using ReplaceFunctionNowVisitor = InDepthNodeVisitor, true>; - struct ReplaceFunctionWindowMatcher - { - using Visitor = InDepthNodeVisitor; - - struct Data{}; - - static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; } - - static void visit(ASTPtr & ast, Data &) - { - if (auto * t = ast->as()) - { - if (t->name == "hop" || t->name == "tumble") - t->name = "windowID"; - } - } - }; - class ToIdentifierMatcher { public: @@ -267,7 +246,7 @@ namespace { if (auto * t = ast->as()) { - ast = std::make_shared(t->shortName()); + t->setShortName(t->shortName()); } } }; @@ -420,7 +399,7 @@ ASTPtr StorageWindowView::getCleanupQuery() ASTPtr function_less; function_less= makeASTFunction( "less", - std::make_shared(inner_window_id_column_name), + std::make_shared(window_id_name), std::make_shared(getCleanupBound())); auto alter_query = std::make_shared(); @@ -535,7 +514,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) { /// SELECT * FROM inner_table WHERE window_id_name == w_end /// (because we fire at the end of windows) - filter_function = makeASTFunction("equals", std::make_shared(inner_window_id_column_name), std::make_shared(watermark)); + filter_function = makeASTFunction("equals", std::make_shared(window_id_name), std::make_shared(watermark)); } else { @@ -554,7 +533,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) func_array ->arguments->children.push_back(std::make_shared(w_end)); w_end = addTime(w_end, window_kind, -slice_num_units, *time_zone); } - filter_function = makeASTFunction("has", func_array, std::make_shared(inner_window_id_column_name)); + filter_function = makeASTFunction("has", func_array, std::make_shared(window_id_name)); } auto syntax_result = TreeRewriter(getContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList()); @@ -569,7 +548,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) /// Adding window column DataTypes window_column_type{std::make_shared(), std::make_shared()}; ColumnWithTypeAndName column; - column.name = inner_window_column_name; + column.name = window_column_name; column.type = std::make_shared(std::move(window_column_type)); column.column = column.type->createColumnConst(0, Tuple{w_start, watermark}); auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column)); @@ -582,7 +561,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) /// Removing window id column auto new_header = builder.getHeader(); - new_header.erase(inner_window_id_column_name); + new_header.erase(window_id_name); auto convert_actions_dag = ActionsDAG::makeConvertingActions( builder.getHeader().getColumnsWithTypeAndName(), new_header.getColumnsWithTypeAndName(), @@ -736,15 +715,14 @@ ASTPtr StorageWindowView::getSourceTableSelectQuery() { auto query = select_query->clone(); DropTableIdentifierMatcher::Data drop_table_identifier_data; - DropTableIdentifierMatcher::Visitor drop_table_identifier_visitor(drop_table_identifier_data); - drop_table_identifier_visitor.visit(query); + DropTableIdentifierMatcher::Visitor(drop_table_identifier_data).visit(query); - FetchQueryInfoMatcher::Data query_info_data; - FetchQueryInfoMatcher::Visitor(query_info_data).visit(query); + WindowFunctionMatcher::Data query_info_data; + WindowFunctionMatcher::Visitor(query_info_data).visit(query); auto order_by = std::make_shared(); auto order_by_elem = std::make_shared(); - order_by_elem->children.push_back(std::make_shared(query_info_data.timestamp_column_name)); + order_by_elem->children.push_back(std::make_shared(timestamp_column_name)); order_by_elem->direction = 1; order_by->children.push_back(order_by_elem); modified_select.setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_by)); @@ -778,7 +756,7 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c = InterpreterSelectQuery(inner_select_query, getContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) .getSampleBlock(); - auto columns_list = std::make_shared(); + ASTPtr columns_list = InterpreterCreateQuery::formatColumns(t_sample_block.getNamesAndTypesList()); if (is_time_column_func_now) { @@ -786,31 +764,8 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c column_window->name = window_id_name; column_window->type = std::make_shared("UInt32"); columns_list->children.push_back(column_window); - inner_window_id_column_name = window_id_name; } - for (const auto & column : t_sample_block.getColumnsWithTypeAndName()) - { - ParserIdentifierWithOptionalParameters parser; - String sql = column.type->getName(); - ASTPtr ast = parseQuery(parser, sql.data(), sql.data() + sql.size(), "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - auto column_dec = std::make_shared(); - column_dec->name = column.name; - column_dec->type = ast; - columns_list->children.push_back(column_dec); - if (!is_time_column_func_now && inner_window_id_column_name.empty() && startsWith(column.name, "windowID")) - { - inner_window_id_column_name = column.name; - } - } - - if (inner_window_id_column_name.empty()) - throw Exception( - "The first argument of time window function should not be a constant value.", - ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW); - - inner_window_column_name = std::regex_replace(inner_window_id_column_name, std::regex("windowID"), is_tumble ? "tumble" : "hop"); - ToIdentifierMatcher::Data query_data; query_data.window_id_name = window_id_name; query_data.window_id_alias = window_id_alias; @@ -818,8 +773,8 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c ReplaceFunctionNowData time_now_data; ReplaceFunctionNowVisitor time_now_visitor(time_now_data); - ReplaceFunctionWindowMatcher::Data func_hop_data; - ReplaceFunctionWindowMatcher::Visitor func_window_visitor(func_hop_data); + WindowFunctionMatcher::Data window_data; + WindowFunctionMatcher::Visitor window_visitor(window_data); DropTableIdentifierMatcher::Data drop_table_identifier_data; DropTableIdentifierMatcher::Visitor drop_table_identifier_visitor(drop_table_identifier_data); @@ -836,7 +791,7 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c } drop_table_identifier_visitor.visit(node); /// tumble/hop -> windowID - func_window_visitor.visit(node); + window_visitor.visit(node); to_identifier_visitor.visit(node); node->setAlias(""); return node; @@ -1315,6 +1270,8 @@ ASTPtr StorageWindowView::initInnerQuery(ASTSelectQuery query, ContextPtr contex if (is_time_column_func_now) window_id_name = func_now_data.window_id_name; + window_column_name = std::regex_replace(window_id_name, std::regex("windowID"), is_tumble ? "tumble" : "hop"); + /// Parse final query (same as mergeable query but has tumble/hop instead of windowID) final_query = mergeable_query->clone(); ReplaceWindowIdMatcher::Data final_query_data; @@ -1331,16 +1288,15 @@ ASTPtr StorageWindowView::innerQueryParser(const ASTSelectQuery & query) // Parse stage mergeable ASTPtr result = query.clone(); - FetchQueryInfoMatcher::Data query_info_data; - FetchQueryInfoMatcher::Visitor(query_info_data).visit(result); + + WindowFunctionMatcher::Data query_info_data; + query_info_data.check_duplicate_window = true; + WindowFunctionMatcher::Visitor(query_info_data).visit(result); if (!query_info_data.is_tumble && !query_info_data.is_hop) throw Exception(ErrorCodes::INCORRECT_QUERY, "TIME WINDOW FUNCTION is not specified for {}", getName()); - window_id_name = query_info_data.window_id_name; - window_id_alias = query_info_data.window_id_alias; - timestamp_column_name = query_info_data.timestamp_column_name; is_tumble = query_info_data.is_tumble; // Parse time window function @@ -1350,6 +1306,14 @@ ASTPtr StorageWindowView::innerQueryParser(const ASTSelectQuery & query) arguments.at(1), window_kind, window_num_units, "Illegal type of second argument of function " + window_function.name + " should be Interval"); + window_id_alias = window_function.alias; + if (auto * node = arguments[0]->as()) + timestamp_column_name = node->shortName(); + + DropTableIdentifierMatcher::Data drop_identifier_data; + DropTableIdentifierMatcher::Visitor(drop_identifier_data).visit(query_info_data.window_function); + window_id_name = window_function.getColumnName(); + slide_kind = window_kind; slide_num_units = window_num_units; @@ -1614,31 +1578,6 @@ void StorageWindowView::writeIntoWindowView( void StorageWindowView::startup() { - if (is_time_column_func_now) - inner_window_id_column_name = window_id_name; - else - { - Aliases aliases; - QueryAliasesVisitor(aliases).visit(mergeable_query); - auto inner_query_normalized = mergeable_query->clone(); - QueryNormalizer::Data normalizer_data(aliases, {}, false, getContext()->getSettingsRef(), false); - QueryNormalizer(normalizer_data).visit(inner_query_normalized); - auto inner_select_query = std::static_pointer_cast(inner_query_normalized); - auto t_sample_block - = InterpreterSelectQuery(inner_select_query, getContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) - .getSampleBlock(); - for (const auto & column : t_sample_block.getColumnsWithTypeAndName()) - { - if (startsWith(column.name, "windowID")) - { - inner_window_id_column_name = column.name; - break; - } - } - } - - inner_window_column_name = std::regex_replace(inner_window_id_column_name, std::regex("windowID"), is_tumble ? "tumble" : "hop"); - DatabaseCatalog::instance().addDependency(select_table_id, getStorageID()); // Start the working thread diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index d9343aa03ac..b639a5924ad 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -238,8 +238,7 @@ private: Int64 slide_num_units; String window_id_name; String window_id_alias; - String inner_window_column_name; - String inner_window_id_column_name; + String window_column_name; String timestamp_column_name; StorageID select_table_id = StorageID::createEmpty(); diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference b/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference index 319cbef0914..53df91c5523 100644 --- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference +++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference @@ -10,7 +10,7 @@ CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, ||---FUNCTION--- CREATE TABLE test_01047.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---PARTITION--- -CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(____timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01047.`.inner.wv`\n(\n `count(a)` AggregateFunction(count, Int32),\n `windowID(____timestamp, toIntervalSecond(\'1\'))` UInt32\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---JOIN--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 @@ -26,7 +26,7 @@ CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, ||---FUNCTION--- CREATE TABLE test_01047.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---PARTITION--- -CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01047.`.inner.wv`\n(\n `count(a)` AggregateFunction(count, Int32),\n `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---JOIN--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01084_window_view_with_table_identifier.reference b/tests/queries/0_stateless/01084_window_view_with_table_identifier.reference new file mode 100644 index 00000000000..de722f47f08 --- /dev/null +++ b/tests/queries/0_stateless/01084_window_view_with_table_identifier.reference @@ -0,0 +1,7 @@ +1 1 1990-01-01 12:00:05 +1 2 1990-01-01 12:00:05 +1 3 1990-01-01 12:00:05 +1 4 1990-01-01 12:00:10 +1 5 1990-01-01 12:00:10 +1 6 1990-01-01 12:00:15 +1 7 1990-01-01 12:00:15 diff --git a/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh new file mode 100755 index 00000000000..76d19a4d488 --- /dev/null +++ b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery < Date: Sat, 28 May 2022 12:30:05 -0400 Subject: [PATCH 060/136] test is added --- .../integration/test_storage_dict/__init__.py | 0 .../test_storage_dict/configs/conf.xml | 16 ++++++++ tests/integration/test_storage_dict/test.py | 40 +++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 tests/integration/test_storage_dict/__init__.py create mode 100644 tests/integration/test_storage_dict/configs/conf.xml create mode 100644 tests/integration/test_storage_dict/test.py diff --git a/tests/integration/test_storage_dict/__init__.py b/tests/integration/test_storage_dict/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_storage_dict/configs/conf.xml b/tests/integration/test_storage_dict/configs/conf.xml new file mode 100644 index 00000000000..c2ecb518884 --- /dev/null +++ b/tests/integration/test_storage_dict/configs/conf.xml @@ -0,0 +1,16 @@ + + + + + http://nginx:80/test_dict + PUT + TSV + k String, v String + + + http://nginx:80/ + /test_dict + TabSeparated + + + diff --git a/tests/integration/test_storage_dict/test.py b/tests/integration/test_storage_dict/test.py new file mode 100644 index 00000000000..df224f08968 --- /dev/null +++ b/tests/integration/test_storage_dict/test.py @@ -0,0 +1,40 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +uuids = [] + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node1", main_configs=["configs/conf.xml"], with_nginx=True + ) + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def test_storage_dict(cluster): + node1 = cluster.instances["node1"] + + node1.query( + f"insert into table function url(urldb) values ('foo', 'bar')" + ) + result = node1.query( + f"select * from url(urldb)" + ) + assert result.strip() == "foo\tbar" + + node1.query( + f"create dictionary dict (k String, v String) primary key k source(http(name urldict)) layout(complex_key_hashed()) lifetime(min 0 max 100)" + ) + result = node1.query( + f"select * from dict" + ) + assert result.strip() == "foo\tbar" From 4f07c684da00ead69d24f0aa04b68f64b39b72db Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Sat, 28 May 2022 12:45:53 -0400 Subject: [PATCH 061/136] style fix --- tests/integration/test_storage_dict/test.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_storage_dict/test.py b/tests/integration/test_storage_dict/test.py index df224f08968..a5270a42114 100644 --- a/tests/integration/test_storage_dict/test.py +++ b/tests/integration/test_storage_dict/test.py @@ -23,18 +23,12 @@ def cluster(): def test_storage_dict(cluster): node1 = cluster.instances["node1"] - node1.query( - f"insert into table function url(urldb) values ('foo', 'bar')" - ) - result = node1.query( - f"select * from url(urldb)" - ) + node1.query(f"insert into table function url(urldb) values ('foo', 'bar')") + result = node1.query(f"select * from url(urldb)") assert result.strip() == "foo\tbar" node1.query( f"create dictionary dict (k String, v String) primary key k source(http(name urldict)) layout(complex_key_hashed()) lifetime(min 0 max 100)" ) - result = node1.query( - f"select * from dict" - ) + result = node1.query(f"select * from dict") assert result.strip() == "foo\tbar" From e33d986cf1a842e66d9376fb9bb8bfc6fa83934c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:20:32 +0200 Subject: [PATCH 062/136] Revert "Merge pull request #34924 from azat/tests-overlaps" This reverts commit f13e436d40db2a15892653cc8baff61d09b73c7a, reversing changes made to aa3c05e9d49bcc8f2bcc6d79242742de6b16e51a. --- tests/clickhouse-test | 31 ----- tests/queries/0_stateless/00746_sql_fuzzy.sh | 3 +- .../00816_long_concurrent_alter_column.sh | 63 ++++++---- tests/queries/0_stateless/00900_orc_load.sh | 4 +- .../00909_kill_not_initialized_query.sh | 2 +- .../00941_system_columns_race_condition.sh | 51 ++++---- .../00991_system_parts_race_condition_long.sh | 69 ++++++----- ...tem_parts_race_condition_zookeeper_long.sh | 87 ++++++-------- ...tem_parts_race_condition_drop_zookeeper.sh | 110 +++++++++-------- .../01001_rename_merge_race_condition.sh | 31 ++--- ...lter_nullable_adaptive_granularity_long.sh | 41 ++++--- .../01003_kill_query_race_condition.sh | 41 ++++--- .../0_stateless/01004_rename_deadlock.sh | 45 ++++--- .../0_stateless/01005_rwr_shard_deadlock.sh | 31 +++-- .../0_stateless/01007_r1r2_w_r2r1_deadlock.sh | 43 ++++--- .../01013_sync_replica_timeout_zookeeper.sh | 2 +- ...rrent_recreate_reattach_and_show_tables.sh | 89 +++++++++----- ...8_ddl_dictionaries_concurrent_requrests.sh | 102 ++++++++-------- .../01019_alter_materialized_view_atomic.sh | 13 +- ...ent_move_partition_from_table_zookeeper.sh | 50 +++++--- .../01054_cache_dictionary_bunch_update.sh | 28 +++-- ...cache_dictionary_datarace_exception_ptr.sh | 22 ++-- ...076_parallel_alter_replicated_zookeeper.sh | 64 +++++----- ...arallel_alter_add_drop_column_zookeeper.sh | 67 ++++++----- ...9_parallel_alter_detach_table_zookeeper.sh | 71 +++++------ ...79_parallel_alter_modify_zookeeper_long.sh | 73 ++++++------ .../01085_max_distributed_connections.sh | 2 +- .../01103_optimize_drop_race_zookeeper.sh | 57 ++++----- ...tart_replicas_rename_deadlock_zookeeper.sh | 45 +++---- .../0_stateless/01150_ddl_guard_rwr.sh | 35 +++--- .../0_stateless/01154_move_partition_long.sh | 112 ++++++++++-------- .../01164_detach_attach_partition_race.sh | 28 +++-- .../01175_distributed_ddl_output_mode_long.sh | 2 +- .../0_stateless/01249_flush_interactive.sh | 2 - ..._recreate_reattach_and_show_tables_long.sh | 87 +++++++++----- ...1_aggregate_state_exception_memory_leak.sh | 11 +- ...2_aggregate_state_exception_memory_leak.sh | 8 +- .../01305_replica_create_drop_zookeeper.sh | 15 ++- ...20_create_sync_race_condition_zookeeper.sh | 8 +- .../01412_cache_dictionary_race.sh | 46 +++---- .../01444_create_table_drop_database_race.sh | 18 +-- ...01454_storagememory_data_race_challenge.sh | 18 +-- .../01542_dictionary_load_exception_race.sh | 30 ++--- .../01593_concurrent_alter_mutations_kill.sh | 36 +++--- ...alter_mutations_kill_many_replicas_long.sh | 42 ++++--- .../01602_max_distributed_connections.sh | 11 +- .../0_stateless/01632_tinylog_read_write.sh | 35 +++--- .../01671_ddl_hang_timeout_long.sh | 26 ++-- .../01732_race_condition_storage_join_long.sh | 35 ++++-- ...nt_ttl_and_normal_merges_zookeeper_long.sh | 40 ++++--- .../02015_async_inserts_stress_long.sh | 20 ++-- .../02015_shard_crash_clang_12_build.sh | 28 +++-- .../0_stateless/02030_rocksdb_race_long.sh | 27 +++-- ...44_url_glob_parallel_connection_refused.sh | 9 +- .../02122_4letter_words_stress_zookeeper.sh | 30 ++--- .../02124_buffer_with_type_map_long.sh | 12 +- tests/queries/shell_config.sh | 38 +----- 57 files changed, 1148 insertions(+), 998 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c4ad314ff9e..a8cab282b0d 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -243,37 +243,6 @@ def get_transactions_list(args): return f"Cannot get list of transactions: {e}" -def get_processlist_after_test(args): - log_comment = args.testcase_basename - database = args.testcase_database - if args.replicated_database: - return clickhouse_execute_json( - args, - f""" - SELECT materialize((hostName(), tcpPort())) as host, * - FROM clusterAllReplicas('test_cluster_database_replicated', system.processes) - WHERE - query NOT LIKE '%system.processes%' AND - {'NOT is_all_data_sent AND' if args.suppport_system_processes_is_all_data_sent else ''} - Settings['log_comment'] = '{log_comment}' AND - current_database = '{database}' - """, - ) - else: - return clickhouse_execute_json( - args, - f""" - SELECT * - FROM system.processes - WHERE - query NOT LIKE '%system.processes%' AND - {'NOT is_all_data_sent AND' if args.suppport_system_processes_is_all_data_sent else ''} - Settings['log_comment'] = '{log_comment}' AND - current_database = '{database}' - """, - ) - - # collect server stacktraces using gdb def get_stacktraces_from_gdb(server_pid): try: diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index 72e80956f59..b534b1820ba 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -15,7 +14,7 @@ $CLICKHOUSE_CLIENT -q "select name from system.table_functions format TSV;" > "$ # if you want long run use: env SQL_FUZZY_RUNS=100000 clickhouse-test sql_fuzzy for SQL_FUZZY_RUN in $(seq "${SQL_FUZZY_RUNS:=5}"); do - env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" perl "$CURDIR"/00746_sql_fuzzy.pl | clickhouse_client_timeout 60 $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 + env SQL_FUZZY_RUN="$SQL_FUZZY_RUN" perl "$CURDIR"/00746_sql_fuzzy.pl | timeout 60 $CLICKHOUSE_CLIENT --format Null --max_execution_time 10 -n --ignore-error >/dev/null 2>&1 if [[ $($CLICKHOUSE_CLIENT -q "SELECT 'Still alive'") != 'Still alive' ]]; then break fi diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 8e04ce7ff1e..19d9b006cd7 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -7,51 +7,64 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " - DROP TABLE IF EXISTS concurrent_alter_column; - CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple(); -" +echo "DROP TABLE IF EXISTS concurrent_alter_column" | ${CLICKHOUSE_CLIENT} +echo "CREATE TABLE concurrent_alter_column (ts DATETIME) ENGINE = MergeTree PARTITION BY toStartOfDay(ts) ORDER BY tuple()" | ${CLICKHOUSE_CLIENT} + function thread1() { - for i in {1..500}; do - echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;" - done | ${CLICKHOUSE_CLIENT} -n + while true; do + for i in {1..500}; do echo "ALTER TABLE concurrent_alter_column ADD COLUMN c$i DOUBLE;"; done | ${CLICKHOUSE_CLIENT} -n --query_id=alter_00816_1 + done } function thread2() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN d" + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN d DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_2; + sleep "$(echo 0.0$RANDOM)"; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN d" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_2; + done } function thread3() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN e" + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN e DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_3; + sleep "$(echo 0.0$RANDOM)"; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN e" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_3; + done } function thread4() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_column DROP COLUMN f" + while true; do + echo "ALTER TABLE concurrent_alter_column ADD COLUMN f DOUBLE" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_4; + sleep "$(echo 0.0$RANDOM)"; + echo "ALTER TABLE concurrent_alter_column DROP COLUMN f" | ${CLICKHOUSE_CLIENT} --query_id=alter_00816_4; + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & wait -$CLICKHOUSE_CLIENT -q "DROP TABLE concurrent_alter_column NO DELAY" +echo "DROP TABLE concurrent_alter_column NO DELAY" | ${CLICKHOUSE_CLIENT} # NO DELAY has effect only for Atomic database + +# Wait for alters and check for deadlocks (in case of deadlock this loop will not finish) +while true; do + echo "SELECT * FROM system.processes WHERE query_id LIKE 'alter\\_00816\\_%'" | ${CLICKHOUSE_CLIENT} | grep -q -F 'alter' || break + sleep 1; +done + echo 'did not crash' diff --git a/tests/queries/0_stateless/00900_orc_load.sh b/tests/queries/0_stateless/00900_orc_load.sh index 3779092c511..b3f2c39e5d2 100755 --- a/tests/queries/0_stateless/00900_orc_load.sh +++ b/tests/queries/0_stateless/00900_orc_load.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -DATA_FILE=$CUR_DIR/data_orc/test_$CLICKHOUSE_TEST_UNIQUE_NAME.orc +DATA_FILE=$CUR_DIR/data_orc/test.orc ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_load (int Int32, smallint Int8, bigint Int64, float Float32, double Float64, date Date, y String, datetime64 DateTime64(3)) ENGINE = Memory" @@ -14,7 +14,7 @@ ${CLICKHOUSE_CLIENT} --query="select * from orc_load FORMAT ORC" > $DATA_FILE ${CLICKHOUSE_CLIENT} --query="truncate table orc_load" cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" -clickhouse_client_timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < $DATA_FILE +timeout 3 ${CLICKHOUSE_CLIENT} -q "insert into orc_load format ORC" < $DATA_FILE ${CLICKHOUSE_CLIENT} --query="select * from orc_load" ${CLICKHOUSE_CLIENT} --query="drop table orc_load" rm -rf "$DATA_FILE" diff --git a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh index 816bab4c491..531652a33e7 100755 --- a/tests/queries/0_stateless/00909_kill_not_initialized_query.sh +++ b/tests/queries/0_stateless/00909_kill_not_initialized_query.sh @@ -35,7 +35,7 @@ $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query='$query_to_kill' ASYNC" &>/dev/nul sleep 1 # Kill $query_for_pending SYNC. This query is not blocker, so it should be killed fast. -clickhouse_client_timeout 20 ${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null +timeout 20 ${CLICKHOUSE_CLIENT} -q "KILL QUERY WHERE query='$query_for_pending' SYNC" &>/dev/null # Both queries have to be killed, doesn't matter with SYNC or ASYNC kill for _ in {1..15} diff --git a/tests/queries/0_stateless/00941_system_columns_race_condition.sh b/tests/queries/0_stateless/00941_system_columns_race_condition.sh index 24f159c7d24..69dfb30cd2c 100755 --- a/tests/queries/0_stateless/00941_system_columns_race_condition.sh +++ b/tests/queries/0_stateless/00941_system_columns_race_condition.sh @@ -9,45 +9,40 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " - DROP TABLE IF EXISTS alter_table; - CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a; -" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table" +$CLICKHOUSE_CLIENT -q "CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a" function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns UNION ALL SELECT name FROM system.columns FORMAT Null" + while true; do $CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns UNION ALL SELECT name FROM system.columns FORMAT Null"; done } function thread2() { - $CLICKHOUSE_CLIENT -n --query " - ALTER TABLE alter_table ADD COLUMN h String; - ALTER TABLE alter_table MODIFY COLUMN h UInt64; - ALTER TABLE alter_table DROP COLUMN h; - " + while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done } -export -f thread1 -export -f thread2 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread1 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & -clickhouse_client_loop_timeout 15 thread2 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread1 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & +timeout 15 bash -c thread2 2> /dev/null & wait diff --git a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh index 731e96b055e..8243c6bde62 100755 --- a/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh +++ b/tests/queries/0_stateless/00991_system_parts_race_condition_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, long +# Tags: race # This test is disabled because it triggers internal assert in Thread Sanitizer. # Thread Sanitizer does not support for more than 64 mutexes to be locked in a single thread. @@ -11,68 +11,67 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " - DROP TABLE IF EXISTS alter_table; - CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1; -" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table" +$CLICKHOUSE_CLIENT -q "CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = MergeTree ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1" function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" + while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; done } function thread2() { - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;" + while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table ADD COLUMN h String '0'; ALTER TABLE alter_table MODIFY COLUMN h UInt64; ALTER TABLE alter_table DROP COLUMN h;"; done } function thread3() { - $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)" + while true; do $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; done } function thread4() { - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table FINAL" + while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table FINAL"; done } function thread5() { - $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table DELETE WHERE rand() % 2 = 1" + while true; do $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table DELETE WHERE rand() % 2 = 1"; done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 -export -f thread5 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; +export -f thread5; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & wait diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index 3cd61c9972a..8dbd10fc27b 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -9,95 +9,76 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; - CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') - ORDER BY a - PARTITION BY b % 10 - SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; - - CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') - ORDER BY a - PARTITION BY b % 10 - SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; + CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; + CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 " function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" + while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; done } function thread2() { - $CLICKHOUSE_CLIENT -nm --query " - ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; - ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; - ALTER TABLE alter_table0 DROP COLUMN h; - " + while true; do $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table0 ADD COLUMN h String DEFAULT '0'; ALTER TABLE alter_table0 MODIFY COLUMN h UInt64; ALTER TABLE alter_table0 DROP COLUMN h;"; done } function thread3() { - $CLICKHOUSE_CLIENT -q " - INSERT INTO alter_table0 - SELECT - rand(1), rand(2), 1 / rand(3), toString(rand(4)), - [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), - (rand(8), rand(9)) - FROM numbers(100000)" + while true; do $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table0 SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; done } function thread4() { - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL" + while true; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table0 FINAL"; done } function thread5() { - $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table0 DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288" + while true; do $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table0 DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288"; done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 -export -f thread5 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; +export -f thread5; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & wait check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum(c))" $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & - wait diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 3fe7b0443eb..d5d43d3c293 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -12,87 +12,93 @@ set -e function thread1() { # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null" + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.parts FORMAT Null"; + done } function thread2() { - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table_$REPLICA ADD COLUMN h String '0'; ALTER TABLE alter_table_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE alter_table_$REPLICA DROP COLUMN h;" + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE alter_table_$REPLICA ADD COLUMN h String '0'; ALTER TABLE alter_table_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE alter_table_$REPLICA DROP COLUMN h;"; + done } function thread3() { - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table_$REPLICA SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)" + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "INSERT INTO alter_table_$REPLICA SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(100000)"; + done } function thread4() { - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL"; + sleep 0.$RANDOM; + done } function thread5() { - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table_$REPLICA DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE alter_table_$REPLICA DELETE WHERE cityHash64(a,b,c,d,e,g) % 1048576 < 524288"; + sleep 0.$RANDOM; + done } function thread6() { - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -mn -q " - DROP TABLE IF EXISTS alter_table_$REPLICA; - - CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') - ORDER BY a - PARTITION BY b % 10 - SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0; - " - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS alter_table_$REPLICA; + CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + sleep 0.$RANDOM; + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 -export -f thread5 -export -f thread6 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; +export -f thread5; +export -f thread6; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2>&1 | grep "was not completely removed from ZooKeeper" & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from ZooKeeper" & wait diff --git a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh index 9b5f92628f7..253d06c038c 100755 --- a/tests/queries/0_stateless/01001_rename_merge_race_condition.sh +++ b/tests/queries/0_stateless/01001_rename_merge_race_condition.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, long +# Tags: race CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -7,32 +7,35 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1" -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2" -$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; +$CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt64) ENGINE = Memory"; function thread1() { - seq 1 1000 | { - sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' - } | $CLICKHOUSE_CLIENT -n + while true; do + seq 1 1000 | sed -r -e 's/.+/RENAME TABLE test1 TO test2; RENAME TABLE test2 TO test1;/' | $CLICKHOUSE_CLIENT -n + done } function thread2() { - $CLICKHOUSE_CLIENT --query "SELECT * FROM merge('$CLICKHOUSE_DATABASE', '^test[12]$')" + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM merge('$CLICKHOUSE_DATABASE', '^test[12]$')" + done } -export -f thread1 -export -f thread2 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & wait -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1" -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test1"; +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test2"; diff --git a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh index ee38cf4eb7e..b29a42a7356 100755 --- a/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh +++ b/tests/queries/0_stateless/01002_alter_nullable_adaptive_granularity_long.sh @@ -12,39 +12,48 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test (x UInt8, s String MATERIALIZED to function thread1() { - $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)" + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO test SELECT rand() FROM numbers(1000)"; + done } function thread2() { - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);" - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;" - sleep 0.0$RANDOM + while true; do + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x Nullable(UInt8);"; + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE test MODIFY COLUMN x UInt8;"; + sleep 0.0$RANDOM + done } function thread3() { - $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null" + while true; do + $CLICKHOUSE_CLIENT -n --query "SELECT count() FROM test FORMAT Null"; + done } function thread4() { - $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL" - sleep 0.1$RANDOM + while true; do + $CLICKHOUSE_CLIENT -n --query "OPTIMIZE TABLE test FINAL"; + sleep 0.1$RANDOM + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01003_kill_query_race_condition.sh b/tests/queries/0_stateless/01003_kill_query_race_condition.sh index f98897b1544..b71a5106383 100755 --- a/tests/queries/0_stateless/01003_kill_query_race_condition.sh +++ b/tests/queries/0_stateless/01003_kill_query_race_condition.sh @@ -9,37 +9,44 @@ set -e function thread1() { - $CLICKHOUSE_CLIENT --query_id=hello_01003 --query "SELECT count() FROM numbers(1000000000)" --format Null; + while true; do + $CLICKHOUSE_CLIENT --query_id=hello_01003 --query "SELECT count() FROM numbers(1000000000)" --format Null; + done } function thread2() { - $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello_01003'" --format Null - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id = 'hello_01003'" --format Null; + sleep 0.$RANDOM + done } function thread3() { - $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null + while true; do + $CLICKHOUSE_CLIENT --query "SHOW PROCESSLIST" --format Null; + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.processes" --format Null; + done } -export -f thread1 -export -f thread2 -export -f thread3 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01004_rename_deadlock.sh b/tests/queries/0_stateless/01004_rename_deadlock.sh index 4a3f23883c0..f0adf136e94 100755 --- a/tests/queries/0_stateless/01004_rename_deadlock.sh +++ b/tests/queries/0_stateless/01004_rename_deadlock.sh @@ -14,41 +14,48 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test2 (x UInt8) ENGINE = MergeTree ORDE function thread1() { - $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test1 TO test_tmp, test2 TO test1, test_tmp TO test2" + done } function thread2() { - $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM test1 UNION ALL SELECT * FROM test2" --format Null + done } function thread3() { - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null + while true; do + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables" --format Null + done } -export -f thread1 -export -f thread2 -export -f thread3 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & wait sleep 1 diff --git a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh index 948032f19e7..ef352606b69 100755 --- a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh +++ b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh @@ -12,31 +12,36 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE test1 (x UInt8) ENGINE = MergeTree ORDE function thread1() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE test1 MODIFY COLUMN x UInt8" + done } function thread2() { - $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', '$CLICKHOUSE_DATABASE', test1))" --format Null + while true; do + $CLICKHOUSE_CLIENT --query "SELECT x FROM test1 WHERE x IN (SELECT x FROM remote('127.0.0.2', '$CLICKHOUSE_DATABASE', test1))" --format Null + done } -export -f thread1 -export -f thread2 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh index 7fa5f5456a2..9f4b2241732 100755 --- a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh +++ b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: deadlock, long +# Tags: deadlock CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -16,39 +16,48 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE b (x UInt8) ENGINE = MergeTree ORDER BY function thread1() { - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - seq 1 100 | awk '{ print "SELECT x FROM a WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n + while true; do + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + seq 1 100 | awk '{ print "SELECT x FROM a WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n + done } function thread2() { - # NOTE: database = $CLICKHOUSE_DATABASE is unwanted - seq 1 100 | awk '{ print "SELECT x FROM b WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n + while true; do + # NOTE: database = $CLICKHOUSE_DATABASE is unwanted + seq 1 100 | awk '{ print "SELECT x FROM b WHERE x IN (SELECT toUInt8(count()) FROM system.tables);" }' | $CLICKHOUSE_CLIENT -n + done } function thread3() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x UInt8" + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE a MODIFY COLUMN x UInt8" + done } function thread4() { - $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x Nullable(UInt8)" - $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x UInt8" + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x Nullable(UInt8)" + $CLICKHOUSE_CLIENT --query "ALTER TABLE b MODIFY COLUMN x UInt8" + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh index 290708ccde4..55bbfb3ff11 100755 --- a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh +++ b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -19,7 +19,7 @@ ${CLICKHOUSE_CLIENT} -n -q " INSERT INTO $R1 VALUES (1) " -clickhouse_client_timeout 10s ${CLICKHOUSE_CLIENT} --receive_timeout 1 -n -q " +timeout 10s ${CLICKHOUSE_CLIENT} -n -q " SET receive_timeout=1; SYSTEM SYNC REPLICA $R2 " 2>&1 | grep -F -q "Code: 159. DB::Exception" && echo 'OK' || echo 'Failed!' diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index d1953dbfd0f..0a6888a5c69 100755 --- a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -7,67 +7,95 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) export CURR_DATABASE="test_lazy_01014_concurrent_${CLICKHOUSE_DATABASE}" + function recreate_lazy_func1() { - $CLICKHOUSE_CLIENT -nm -q " - DETACH TABLE $CURR_DATABASE.log; - ATTACH TABLE $CURR_DATABASE.log; - " + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; + "; + + while true; do + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.log; + "; + + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.log; + "; + done } function recreate_lazy_func2() { - $CLICKHOUSE_CLIENT -nm -q " - CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; - DROP TABLE $CURR_DATABASE.tlog; - " + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog; + "; + done } function recreate_lazy_func3() { - $CLICKHOUSE_CLIENT -nm -q " - ATTACH TABLE $CURR_DATABASE.slog; - DETACH TABLE $CURR_DATABASE.slog; - " + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; + "; + + while true; do + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.slog; + "; + + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.slog; + "; + done } function recreate_lazy_func4() { - $CLICKHOUSE_CLIENT -nm -q " - CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; - DROP TABLE $CURR_DATABASE.tlog2; - " + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog2; + "; + done } function show_tables_func() { - $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE database = '$CURR_DATABASE' FORMAT Null" + while true; do + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE database = '$CURR_DATABASE' FORMAT Null"; + done } -export -f recreate_lazy_func1 -export -f recreate_lazy_func2 -export -f recreate_lazy_func3 -export -f recreate_lazy_func4 -export -f show_tables_func +export -f recreate_lazy_func1; +export -f recreate_lazy_func2; +export -f recreate_lazy_func3; +export -f recreate_lazy_func4; +export -f show_tables_func; ${CLICKHOUSE_CLIENT} -n -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); - - CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; - CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; " TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT show_tables_func 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func1 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func2 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func3 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func4 2> /dev/null & +timeout $TIMEOUT bash -c show_tables_func 2> /dev/null & wait sleep 1 @@ -80,3 +108,4 @@ ${CLICKHOUSE_CLIENT} -q "ATTACH TABLE $CURR_DATABASE.tlog2;" 2>/dev/null ${CLICKHOUSE_CLIENT} -q "DROP DATABASE $CURR_DATABASE" echo "Test OK" + diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh index e98fb57c2a4..872b0a7c1a1 100755 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_concurrent_requrests.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -n -q " DROP DATABASE IF EXISTS database_for_dict; DROP TABLE IF EXISTS table_for_dict1; DROP TABLE IF EXISTS table_for_dict2; @@ -20,104 +20,96 @@ $CLICKHOUSE_CLIENT -mn -q " CREATE DATABASE database_for_dict; - CREATE DICTIONARY database_for_dict.dict1 (key_column UInt64, value_column String) - PRIMARY KEY key_column - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) - LIFETIME(MIN 1 MAX 5) - LAYOUT(FLAT()); + CREATE DICTIONARY database_for_dict.dict1 (key_column UInt64, value_column String) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict1' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) LIFETIME(MIN 1 MAX 5) LAYOUT(FLAT()); - CREATE DICTIONARY database_for_dict.dict2 (key_column UInt64, value_column String) - PRIMARY KEY key_column - SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) - LIFETIME(MIN 1 MAX 5) - LAYOUT(CACHE(SIZE_IN_CELLS 150)); + CREATE DICTIONARY database_for_dict.dict2 (key_column UInt64, value_column String) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict2' PASSWORD '' DB '$CLICKHOUSE_DATABASE')) LIFETIME(MIN 1 MAX 5) LAYOUT(CACHE(SIZE_IN_CELLS 150)); " function thread1() { - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.dictionaries FORMAT Null" + while true; do $CLICKHOUSE_CLIENT --query "SELECT * FROM system.dictionaries FORMAT Null"; done } function thread2() { - $CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict1" ||: + while true; do CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict1" ||: ; done } function thread3() { - $CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict2" ||: + while true; do CLICKHOUSE_CLIENT --query "ATTACH DICTIONARY database_for_dict.dict2" ||:; done } function thread4() { - $CLICKHOUSE_CLIENT -n -q " + while true; do $CLICKHOUSE_CLIENT -n -q " SELECT * FROM database_for_dict.dict1 FORMAT Null; SELECT * FROM database_for_dict.dict2 FORMAT Null; - " ||: + " ||: ; done } function thread5() { - $CLICKHOUSE_CLIENT -n -q " + while true; do $CLICKHOUSE_CLIENT -n -q " SELECT dictGetString('database_for_dict.dict1', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; SELECT dictGetString('database_for_dict.dict2', 'value_column', toUInt64(number)) from numbers(1000) FROM FORMAT Null; - " ||: + " ||: ; done } function thread6() { - $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict1" + while true; do $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict1"; done } function thread7() { - $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict2" + while true; do $CLICKHOUSE_CLIENT -q "DETACH DICTIONARY database_for_dict.dict2"; done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 -export -f thread5 -export -f thread6 -export -f thread7 +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; +export -f thread5; +export -f thread6; +export -f thread7; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread7 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread7 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread7 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread6 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread7 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread6 2> /dev/null & +timeout $TIMEOUT bash -c thread7 2> /dev/null & wait $CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh index aa3e1b612dc..54a7e940377 100755 --- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh +++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: long set -e @@ -30,15 +29,19 @@ EOF function alter_thread() { + trap 'exit' INT + ALTERS[0]="ALTER TABLE mv MODIFY QUERY SELECT v FROM src;" ALTERS[1]="ALTER TABLE mv MODIFY QUERY SELECT v * 2 as v FROM src;" - $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 -q "${ALTERS[$RANDOM % 2]}" - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT --allow_experimental_alter_materialized_view_structure=1 -q "${ALTERS[$RANDOM % 2]}" + sleep "$(echo 0.$RANDOM)"; + done } -export -f alter_thread -clickhouse_client_loop_timeout 10 alter_thread & +export -f alter_thread; +timeout 10 bash -c alter_thread & for _ in {1..100}; do # Retry (hopefully retriable (deadlock avoided)) errors. diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 19c97ae96ec..4c0afc4c439 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -15,46 +15,66 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = Repli function thread1() { - $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst" + while true; + do + $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" --query_id=query1 + done } function thread2() { - $CLICKHOUSE_CLIENT --query="INSERT INTO src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" + while true; + do + $CLICKHOUSE_CLIENT --query="INSERT INTO src SELECT number % 2, toString(number) FROM system.numbers LIMIT 100000" --query_id=query2 + done } function thread3() { - $CLICKHOUSE_CLIENT --query="SELECT * FROM src" > /dev/null + while true; + do + $CLICKHOUSE_CLIENT --query="SELECT * FROM src" --query_id=query3 1> /dev/null + done } function thread4() { - $CLICKHOUSE_CLIENT --query="SELECT * FROM dst" > /dev/null + while true; + do + $CLICKHOUSE_CLIENT --query="SELECT * FROM dst" --query_id=query4 1> /dev/null + done } function thread5() { - $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst" + while true; + do + $CLICKHOUSE_CLIENT --query="ALTER TABLE src MOVE PARTITION 1 TO TABLE dst;" --query_id=query5 + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 -export -f thread5 +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; +export -f thread5; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread5 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread4 2> /dev/null & +timeout $TIMEOUT bash -c thread5 2> /dev/null & wait echo "DROP TABLE src NO DELAY" | ${CLICKHOUSE_CLIENT} echo "DROP TABLE dst NO DELAY" | ${CLICKHOUSE_CLIENT} +sleep 5 + +# Check for deadlocks +echo "SELECT * FROM system.processes WHERE query_id LIKE 'query%'" | ${CLICKHOUSE_CLIENT} echo 'did not crash' diff --git a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh index 6df9cc3e258..04b1f8b65ce 100755 --- a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh +++ b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh @@ -18,44 +18,56 @@ $CLICKHOUSE_CLIENT --query="insert into test_01054.ints values (3, 3, 3, 3, 3, 3 function thread1() { + for _ in {1..100} + do RAND_NUMBER_THREAD1=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD1));" + done } function thread2() { + for _ in {1..100} + do RAND_NUMBER_THREAD2=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD2));" + done } function thread3() { + for _ in {1..100} + do RAND_NUMBER_THREAD3=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD3));" + done } function thread4() { + for _ in {1..100} + do RAND_NUMBER_THREAD4=$($CLICKHOUSE_CLIENT --query="SELECT rand() % 100;") $CLICKHOUSE_CLIENT --query="select dictGet('one_cell_cache_ints', 'i8', toUInt64($RAND_NUMBER_THREAD4));" + done } -export -f thread1 -export -f thread2 -export -f thread3 -export -f thread4 +export -f thread1; +export -f thread2; +export -f thread3; +export -f thread4; TIMEOUT=10 # shellcheck disable=SC2188 -clickhouse_client_loop_timeout $TIMEOUT thread1 > /dev/null 2>&1 & -clickhouse_client_loop_timeout $TIMEOUT thread2 > /dev/null 2>&1 & -clickhouse_client_loop_timeout $TIMEOUT thread3 > /dev/null 2>&1 & -clickhouse_client_loop_timeout $TIMEOUT thread4 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread1 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread2 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread3 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread4 > /dev/null 2>&1 & wait diff --git a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh index a1a38370554..17068dcbdf9 100755 --- a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh +++ b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh @@ -38,24 +38,30 @@ LAYOUT(CACHE(SIZE_IN_CELLS 10)); function thread1() { - # This query will be ended with exception, because source dictionary has UUID as a key type. - $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(1));" + for _ in {1..50} + do + # This query will be ended with exception, because source dictionary has UUID as a key type. + $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(1));" + done } function thread2() { - # This query will be ended with exception, because source dictionary has UUID as a key type. - $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(2));" + for _ in {1..50} + do + # This query will be ended with exception, because source dictionary has UUID as a key type. + $CLICKHOUSE_CLIENT --query="SELECT dictGetFloat64('dictdb_01076.dict_datarace', 'value', toUInt64(2));" + done } -export -f thread1 -export -f thread2 +export -f thread1; +export -f thread2; TIMEOUT=5 -clickhouse_client_loop_timeout $TIMEOUT thread1 > /dev/null 2>&1 & -clickhouse_client_loop_timeout $TIMEOUT thread2 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread1 > /dev/null 2>&1 & +timeout $TIMEOUT bash -c thread2 > /dev/null 2>&1 & wait diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index 6c9acd6d0cd..bbc16121cb6 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -21,12 +21,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT -nm --query " - CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') - ORDER BY key - SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0; - " + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" @@ -45,52 +40,59 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mut # Run mutation on random replica function correct_alter_thread() { - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1" - sleep 1 + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1"; + sleep 1 + done } # This thread add some data to table. function insert_thread() { + VALUES=(7 8 9) - REPLICA=$(($RANDOM % 5 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" + sleep 0.$RANDOM + done } function detach_attach_thread() { - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" - sleep 0.$RANDOM - sleep 0.$RANDOM - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$REPLICA" + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" + sleep 0.$RANDOM + sleep 0.$RANDOM + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_mutate_mt_$REPLICA" + done } echo "Starting alters" -export -f correct_alter_thread -export -f insert_thread -export -f detach_attach_thread +export -f correct_alter_thread; +export -f insert_thread; +export -f detach_attach_thread; # We assign a lot of mutations so timeout shouldn't be too big TIMEOUT=15 -clickhouse_client_loop_timeout $TIMEOUT detach_attach_thread 2> /dev/null & +timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index 77278ad0bf9..06d6ef6a94b 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -15,12 +15,7 @@ done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT -nm --query " - CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') - ORDER BY key - SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000; - " + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_1 SELECT number, number + 10 from numbers(100000)" @@ -32,54 +27,58 @@ done function alter_thread() { - REPLICA=$(($RANDOM % 3 + 1)) - ADD=$(($RANDOM % 5 + 1)) - # additionaly we don't wait anything for more heavy concurrency - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0" - DROP=$(($RANDOM % 5 + 1)) - # additionaly we don't wait anything for more heavy concurrency - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + ADD=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + DROP=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done } function optimize_thread() { - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_alter_add_drop_$REPLICA FINAL SETTINGS replication_alter_partitions_sync=0"; + sleep 0.$RANDOM + done } function insert_thread() { - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_$REPLICA VALUES($RANDOM, 7)" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_$REPLICA VALUES($RANDOM, 7)" + sleep 0.$RANDOM + done } echo "Starting alters" -export -f alter_thread -export -f optimize_thread -export -f insert_thread +export -f alter_thread; +export -f optimize_thread; +export -f insert_thread; TIMEOUT=30 # Sometimes we detach and attach tables -clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 20a5f9af108..1f316b4b389 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -12,24 +12,11 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT -nm --query " - CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') - ORDER BY key - SETTINGS - max_replicated_mutations_in_queue=1000, - number_of_free_entries_in_pool_to_execute_mutation=0, - max_replicated_merges_in_queue=1000, - temporary_directories_lifetime=10, - cleanup_delay_period=3, - cleanup_delay_period_random_add=0; - " + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done -$CLICKHOUSE_CLIENT -nm --query " - INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10); - INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40); -" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10)" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10, 40)" for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i" @@ -44,11 +31,12 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alt function correct_alter_thread() { TYPES=(Float64 String UInt8 UInt32) - REPLICA=$(($RANDOM % 3 + 1)) - TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} - # additionaly we don't wait anything for more heavy concurrency - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done } # This thread add some data to table. After we finish we can check, that @@ -56,38 +44,43 @@ function correct_alter_thread() # insert queries will fail sometime because of wrong types. function insert_thread() { + VALUES=(7.0 7 '7') - REPLICA=$(($RANDOM % 3 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM + done } function detach_attach_thread() { - REPLICA=$(($RANDOM % 3 + 1)) - $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_detach_$REPLICA" - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$REPLICA" + while true; do + REPLICA=$(($RANDOM % 3 + 1)) + $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_alter_detach_$REPLICA" + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$REPLICA" + done } echo "Starting alters" -export -f correct_alter_thread -export -f insert_thread -export -f detach_attach_thread +export -f correct_alter_thread; +export -f insert_thread; +export -f detach_attach_thread; TIMEOUT=15 # Sometimes we detach and attach tables -clickhouse_client_loop_timeout $TIMEOUT detach_attach_thread 2> /dev/null & +timeout $TIMEOUT bash -c detach_attach_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh index aef23c460d8..ba8d89aad3c 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh @@ -14,11 +14,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT -nm --query " - CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') - ORDER BY key - SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" @@ -40,10 +36,12 @@ INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alt function correct_alter_thread() { TYPES=(Float64 String UInt8 UInt32) - REPLICA=$(($RANDOM % 5 + 1)) - TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done } # This thread add some data to table. After we finish we can check, that @@ -51,49 +49,56 @@ function correct_alter_thread() # insert queries will fail sometime because of wrong types. function insert_thread() { + VALUES=(7.0 7 '7') - REPLICA=$(($RANDOM % 5 + 1)) - VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, $VALUE)" + sleep 0.$RANDOM + done } # Some select load, to be sure, that our selects work in concurrent execution with alters function select_thread() { - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) FROM concurrent_alter_mt_$REPLICA" 1>/dev/null - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) FROM concurrent_alter_mt_$REPLICA" 1>/dev/null + sleep 0.$RANDOM + done } echo "Starting alters" -export -f correct_alter_thread -export -f insert_thread -export -f select_thread +export -f correct_alter_thread; +export -f insert_thread; +export -f select_thread; TIMEOUT=30 # Selects should run successfully -clickhouse_client_loop_timeout $TIMEOUT select_thread & -clickhouse_client_loop_timeout $TIMEOUT select_thread & -clickhouse_client_loop_timeout $TIMEOUT select_thread & +timeout $TIMEOUT bash -c select_thread & +timeout $TIMEOUT bash -c select_thread & +timeout $TIMEOUT bash -c select_thread & -clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT correct_alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & + + +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01085_max_distributed_connections.sh b/tests/queries/0_stateless/01085_max_distributed_connections.sh index 8a74935c05d..34862289d1e 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections.sh @@ -18,6 +18,6 @@ while [[ $i -lt $retries ]]; do # 10 less then 20 seconds (20 streams), but long enough to cover possible load peaks # "$@" left to pass manual options (like --experimental_use_processors 0) during manual testing - clickhouse_client_timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break + timeout 10s ${CLICKHOUSE_CLIENT} "${opts[@]}" "$@" && break ((++i)) done diff --git a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh index d73ce1aeef4..95f8dfc0377 100755 --- a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh +++ b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh @@ -9,53 +9,54 @@ set -e function thread1() { - $CLICKHOUSE_CLIENT -q "INSERT INTO concurrent_optimize_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(10000)" + while true; do + $CLICKHOUSE_CLIENT -q "INSERT INTO concurrent_optimize_table SELECT rand(1), rand(2), 1 / rand(3), toString(rand(4)), [rand(5), rand(6)], rand(7) % 2 ? NULL : generateUUIDv4(), (rand(8), rand(9)) FROM numbers(10000)"; + done } function thread2() { - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE concurrent_optimize_table FINAL" - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE concurrent_optimize_table FINAL"; + sleep 0.$RANDOM; + done } function thread3() { - $CLICKHOUSE_CLIENT -mn -q " - DROP TABLE IF EXISTS concurrent_optimize_table; - CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') - ORDER BY a - PARTITION BY b % 10 - SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; - sleep 0.$RANDOM - sleep 0.$RANDOM - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS concurrent_optimize_table; + CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + sleep 0.$RANDOM; + sleep 0.$RANDOM; + sleep 0.$RANDOM; + done } -export -f thread1 -export -f thread2 -export -f thread3 +export -f thread1; +export -f thread2; +export -f thread3; TIMEOUT=15 -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread3 2> /dev/null & +timeout $TIMEOUT bash -c thread1 2> /dev/null & +timeout $TIMEOUT bash -c thread2 2> /dev/null & +timeout $TIMEOUT bash -c thread3 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index a51e786b058..e6b145aa57c 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -14,41 +14,46 @@ done function rename_thread_1() { - $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1 TO replica_01108_1_tmp, - replica_01108_2 TO replica_01108_2_tmp, - replica_01108_3 TO replica_01108_3_tmp, - replica_01108_4 TO replica_01108_4_tmp" - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1 TO replica_01108_1_tmp, + replica_01108_2 TO replica_01108_2_tmp, + replica_01108_3 TO replica_01108_3_tmp, + replica_01108_4 TO replica_01108_4_tmp"; + sleep 0.$RANDOM; + done } function rename_thread_2() { - $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1_tmp TO replica_01108_2, - replica_01108_2_tmp TO replica_01108_3, - replica_01108_3_tmp TO replica_01108_4, - replica_01108_4_tmp TO replica_01108_1" - sleep 0.$RANDOM + while true; do + $CLICKHOUSE_CLIENT -q "RENAME TABLE replica_01108_1_tmp TO replica_01108_2, + replica_01108_2_tmp TO replica_01108_3, + replica_01108_3_tmp TO replica_01108_4, + replica_01108_4_tmp TO replica_01108_1"; + sleep 0.$RANDOM; + done } function restart_replicas_loop() { - for i in $(seq 4); do - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}" - $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}_tmp" + while true; do + for i in $(seq 4); do + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}"; + $CLICKHOUSE_CLIENT -q "SYSTEM RESTART REPLICA replica_01108_${i}_tmp"; + done + sleep 0.$RANDOM; done - sleep 0.$RANDOM } -export -f rename_thread_1 -export -f rename_thread_2 +export -f rename_thread_1; +export -f rename_thread_2; export -f restart_replicas_loop TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT rename_thread_1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT rename_thread_2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT restart_replicas_loop 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT restart_replicas_loop 2> /dev/null & +timeout $TIMEOUT bash -c rename_thread_1 2> /dev/null & +timeout $TIMEOUT bash -c rename_thread_2 2> /dev/null & +timeout $TIMEOUT bash -c restart_replicas_loop 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh index 175df7fd9b9..50e6f91b49b 100755 --- a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -13,30 +13,31 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01150" $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t1 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" $CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t2 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" -function thread_detach_attach() -{ - $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 219' -e '(query: ' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 82' -e '(query: ' - sleep 0.0$RANDOM +function thread_detach_attach { + while true; do + $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 219' -e '(query: ' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F -e 'Received exception from server' -e 'Code: 82' -e '(query: ' + sleep 0.0$RANDOM + done } -function thread_rename() -{ - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' - sleep 0.0$RANDOM +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F -e 'Received exception from server' -e '(query: ' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + done } export -f thread_detach_attach export -f thread_rename -clickhouse_client_loop_timeout 20 thread_detach_attach & -clickhouse_client_loop_timeout 20 thread_rename & - +timeout 20 bash -c "thread_detach_attach" & +timeout 20 bash -c 'thread_rename' & wait sleep 1 diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index 4a0ed7808da..7cefac28e22 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -26,85 +26,99 @@ wait #function create_drop_thread() #{ -# REPLICA=$(($RANDOM % 16)) -# $CLICKHOUSE_CLIENT -q "DROP TABLE src_$REPLICA;" -# arr=("$@") -# engine=${arr[$RANDOM % ${#arr[@]}]} -# $CLICKHOUSE_CLIENT -q "CREATE TABLE src_$REPLICA (p UInt64, k UInt64, v UInt64) ENGINE=$engine PARTITION BY p % 10 ORDER BY k" -# sleep 0.$RANDOM +# while true; do +# REPLICA=$(($RANDOM % 16)) +# $CLICKHOUSE_CLIENT -q "DROP TABLE src_$REPLICA;" +# arr=("$@") +# engine=${arr[$RANDOM % ${#arr[@]}]} +# $CLICKHOUSE_CLIENT -q "CREATE TABLE src_$REPLICA (p UInt64, k UInt64, v UInt64) ENGINE=$engine PARTITION BY p % 10 ORDER BY k" +# sleep 0.$RANDOM; +# done #} function insert_thread() { - REPLICA=$(($RANDOM % 16)) - LIMIT=$(($RANDOM % 100)) - $CLICKHOUSE_CLIENT -q "INSERT INTO $1_$REPLICA SELECT * FROM generateRandom('p UInt64, k UInt64, v UInt64') LIMIT $LIMIT" 2>/dev/null + while true; do + REPLICA=$(($RANDOM % 16)) + LIMIT=$(($RANDOM % 100)) + $CLICKHOUSE_CLIENT -q "INSERT INTO $1_$REPLICA SELECT * FROM generateRandom('p UInt64, k UInt64, v UInt64') LIMIT $LIMIT" 2>/dev/null + done } function move_partition_src_dst_thread() { - FROM_REPLICA=$(($RANDOM % 16)) - TO_REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$FROM_REPLICA MOVE PARTITION $PARTITION TO TABLE dst_$TO_REPLICA" 2>/dev/null - sleep 0.$RANDOM + while true; do + FROM_REPLICA=$(($RANDOM % 16)) + TO_REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$FROM_REPLICA MOVE PARTITION $PARTITION TO TABLE dst_$TO_REPLICA" 2>/dev/null + sleep 0.$RANDOM; + done } function replace_partition_src_src_thread() { - FROM_REPLICA=$(($RANDOM % 16)) - TO_REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$TO_REPLICA REPLACE PARTITION $PARTITION FROM src_$FROM_REPLICA" 2>/dev/null - sleep 0.$RANDOM + while true; do + FROM_REPLICA=$(($RANDOM % 16)) + TO_REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE src_$TO_REPLICA REPLACE PARTITION $PARTITION FROM src_$FROM_REPLICA" 2>/dev/null + sleep 0.$RANDOM; + done } function drop_partition_thread() { - REPLICA=$(($RANDOM % 16)) - PARTITION=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PARTITION $PARTITION" 2>/dev/null - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 16)) + PARTITION=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PARTITION $PARTITION" 2>/dev/null + sleep 0.$RANDOM; + done } function optimize_thread() { - REPLICA=$(($RANDOM % 16)) - TABLE="src" - if (( RANDOM % 2 )); then - TABLE="dst" - fi - $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE ${TABLE}_$REPLICA" 2>/dev/null - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 16)) + TABLE="src" + if (( RANDOM % 2 )); then + TABLE="dst" + fi + $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE ${TABLE}_$REPLICA" 2>/dev/null + sleep 0.$RANDOM; + done } function drop_part_thread() { - REPLICA=$(($RANDOM % 16)) - part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1") - $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null - sleep 0.$RANDOM + while true; do + REPLICA=$(($RANDOM % 16)) + part=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.parts WHERE active AND database='$CLICKHOUSE_DATABASE' and table='dst_$REPLICA' ORDER BY rand() LIMIT 1") + $CLICKHOUSE_CLIENT -q "ALTER TABLE dst_$REPLICA DROP PART '$part'" 2>/dev/null + sleep 0.$RANDOM; + done } #export -f create_drop_thread; -export -f insert_thread -export -f move_partition_src_dst_thread -export -f replace_partition_src_src_thread -export -f drop_partition_thread -export -f optimize_thread -export -f drop_part_thread +export -f insert_thread; +export -f move_partition_src_dst_thread; +export -f replace_partition_src_src_thread; +export -f drop_partition_thread; +export -f optimize_thread; +export -f drop_part_thread; TIMEOUT=60 -#clickhouse_client_loop_timeout $TIMEOUT "create_drop_thread ${engines[@]}" & -clickhouse_client_loop_timeout $TIMEOUT insert_thread src & -clickhouse_client_loop_timeout $TIMEOUT insert_thread src & -clickhouse_client_loop_timeout $TIMEOUT insert_thread dst & -clickhouse_client_loop_timeout $TIMEOUT move_partition_src_dst_thread & -clickhouse_client_loop_timeout $TIMEOUT replace_partition_src_src_thread & -clickhouse_client_loop_timeout $TIMEOUT drop_partition_thread & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread & -clickhouse_client_loop_timeout $TIMEOUT drop_part_thread & +#timeout $TIMEOUT bash -c "create_drop_thread ${engines[@]}" & +timeout $TIMEOUT bash -c 'insert_thread src' & +timeout $TIMEOUT bash -c 'insert_thread src' & +timeout $TIMEOUT bash -c 'insert_thread dst' & +timeout $TIMEOUT bash -c move_partition_src_dst_thread & +timeout $TIMEOUT bash -c replace_partition_src_src_thread & +timeout $TIMEOUT bash -c drop_partition_thread & +timeout $TIMEOUT bash -c optimize_thread & +timeout $TIMEOUT bash -c drop_part_thread & wait check_replication_consistency "dst_" "count(), sum(p), sum(k), sum(v)" diff --git a/tests/queries/0_stateless/01164_detach_attach_partition_race.sh b/tests/queries/0_stateless/01164_detach_attach_partition_race.sh index 27658650b91..a7c87351ff4 100755 --- a/tests/queries/0_stateless/01164_detach_attach_partition_race.sh +++ b/tests/queries/0_stateless/01164_detach_attach_partition_race.sh @@ -12,30 +12,36 @@ $CLICKHOUSE_CLIENT -q "insert into mt values (3)" function thread_insert() { - $CLICKHOUSE_CLIENT -q "insert into mt values (rand())"; + while true; do + $CLICKHOUSE_CLIENT -q "insert into mt values (rand())"; + done } function thread_detach_attach() { - $CLICKHOUSE_CLIENT -q "alter table mt detach partition id 'all'"; - $CLICKHOUSE_CLIENT -q "alter table mt attach partition id 'all'"; + while true; do + $CLICKHOUSE_CLIENT -q "alter table mt detach partition id 'all'"; + $CLICKHOUSE_CLIENT -q "alter table mt attach partition id 'all'"; + done } function thread_drop_detached() { - $CLICKHOUSE_CLIENT --allow_drop_detached -q "alter table mt drop detached partition id 'all'"; + while true; do + $CLICKHOUSE_CLIENT --allow_drop_detached -q "alter table mt drop detached partition id 'all'"; + done } -export -f thread_insert -export -f thread_detach_attach -export -f thread_drop_detached +export -f thread_insert; +export -f thread_detach_attach; +export -f thread_drop_detached; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread_insert & -clickhouse_client_loop_timeout $TIMEOUT thread_detach_attach 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread_detach_attach 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT thread_drop_detached 2> /dev/null & +timeout $TIMEOUT bash -c thread_insert & +timeout $TIMEOUT bash -c thread_detach_attach 2> /dev/null & +timeout $TIMEOUT bash -c thread_detach_attach 2> /dev/null & +timeout $TIMEOUT bash -c thread_drop_detached 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh index e632841bd01..e4a23055ae6 100755 --- a/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh +++ b/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh @@ -36,7 +36,7 @@ function run_until_out_contains() RAND_COMMENT="01175_DDL_$RANDOM" LOG_COMMENT="${CLICKHOUSE_LOG_COMMENT}_$RAND_COMMENT" -CLICKHOUSE_CLIENT_WITH_SETTINGS=${CLICKHOUSE_CLIENT/--log_comment ${CLICKHOUSE_LOG_COMMENT}/--log_comment ${LOG_COMMENT}} +CLICKHOUSE_CLIENT_WITH_SETTINGS=${CLICKHOUSE_CLIENT/--log_comment=\'${CLICKHOUSE_LOG_COMMENT}\'/--log_comment=\'${LOG_COMMENT}\'} CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --output_format_parallel_formatting=0 " CLICKHOUSE_CLIENT_WITH_SETTINGS+=" --distributed_ddl_entry_format_version=2 " diff --git a/tests/queries/0_stateless/01249_flush_interactive.sh b/tests/queries/0_stateless/01249_flush_interactive.sh index 2ab85e2fb6c..89167002ed5 100755 --- a/tests/queries/0_stateless/01249_flush_interactive.sh +++ b/tests/queries/0_stateless/01249_flush_interactive.sh @@ -28,5 +28,3 @@ while true; do [[ $(test) == $(echo -ne "0\n1\n2\n3\n4\n---\n0\n1\n2\n3\n4\n---\n") ]] && break sleep 1 done - -clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh index 1235e013ff7..3c11dc5f772 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh @@ -10,67 +10,94 @@ export CURR_DATABASE="test_lazy_01294_concurrent_${CLICKHOUSE_DATABASE}" function recreate_lazy_func1() { - $CLICKHOUSE_CLIENT -nm -q " - DETACH TABLE $CURR_DATABASE.log; - ATTACH TABLE $CURR_DATABASE.log; + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; "; + + while true; do + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.log; + "; + + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.log; + "; + done } function recreate_lazy_func2() { - $CLICKHOUSE_CLIENT -nm -q " - CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; - DROP TABLE $CURR_DATABASE.tlog; - " + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog; + "; + done } function recreate_lazy_func3() { - $CLICKHOUSE_CLIENT -nm -q " - ATTACH TABLE $CURR_DATABASE.slog; - DETACH TABLE $CURR_DATABASE.slog; - " + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; + "; + + while true; do + $CLICKHOUSE_CLIENT -q " + ATTACH TABLE $CURR_DATABASE.slog; + "; + + $CLICKHOUSE_CLIENT -q " + DETACH TABLE $CURR_DATABASE.slog; + "; + done } function recreate_lazy_func4() { - $CLICKHOUSE_CLIENT -nm -q " - CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; - DROP TABLE $CURR_DATABASE.tlog2; - " + while true; do + $CLICKHOUSE_CLIENT -q " + CREATE TABLE $CURR_DATABASE.tlog2 (a UInt64, b UInt64) ENGINE = TinyLog; + "; + + $CLICKHOUSE_CLIENT -q " + DROP TABLE $CURR_DATABASE.tlog2; + "; + done } function test_func() { - for table in log tlog slog tlog2; do - $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null + while true; do + for table in log tlog slog tlog2; do + $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null + done done } -export -f recreate_lazy_func1 -export -f recreate_lazy_func2 -export -f recreate_lazy_func3 -export -f recreate_lazy_func4 -export -f test_func +export -f recreate_lazy_func1; +export -f recreate_lazy_func2; +export -f recreate_lazy_func3; +export -f recreate_lazy_func4; +export -f test_func; ${CLICKHOUSE_CLIENT} -n -q " DROP DATABASE IF EXISTS $CURR_DATABASE; CREATE DATABASE $CURR_DATABASE ENGINE = Lazy(1); - - CREATE TABLE $CURR_DATABASE.log (a UInt64, b UInt64) ENGINE = Log; - CREATE TABLE $CURR_DATABASE.slog (a UInt64, b UInt64) ENGINE = StripeLog; " TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func1 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func2 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func3 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT recreate_lazy_func4 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT test_func 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func1 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func2 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func3 2> /dev/null & +timeout $TIMEOUT bash -c recreate_lazy_func4 2> /dev/null & +timeout $TIMEOUT bash -c test_func 2> /dev/null & wait sleep 1 diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh index ed7e5937bd0..010c66b9bb1 100755 --- a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh @@ -5,6 +5,15 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +function test() +{ + for _ in {1..1000}; do + $CLICKHOUSE_CLIENT --max_memory_usage 1G <<< "SELECT uniqExactState(number) FROM system.numbers_mt GROUP BY number % 10"; + done +} + +export -f test; + # If the memory leak exists, it will lead to OOM fairly quickly. -clickhouse_client_loop_timeout 30 $CLICKHOUSE_CLIENT --max_memory_usage 1G -q 'SELECT uniqExactState(number) FROM system.numbers_mt GROUP BY number % 10' 2>&1 | grep -o -F 'Memory limit (for query) exceeded' | uniq +timeout 30 bash -c test 2>&1 | grep -o -F 'Memory limit (for query) exceeded' | uniq echo 'Ok' diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh index 52b635fe5df..7e10ab475d4 100755 --- a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh +++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh @@ -7,10 +7,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function test() { - $CLICKHOUSE_CLIENT --query "SELECT groupArrayIfState(('Hello, world' AS s) || s || s || s || s || s || s || s || s || s, NOT throwIf(number > 10000000, 'Ok')) FROM system.numbers_mt GROUP BY number % 10"; + for _ in {1..250}; do + $CLICKHOUSE_CLIENT --query "SELECT groupArrayIfState(('Hello, world' AS s) || s || s || s || s || s || s || s || s || s, NOT throwIf(number > 10000000, 'Ok')) FROM system.numbers_mt GROUP BY number % 10"; + done } -export -f test +export -f test; # If the memory leak exists, it will lead to OOM fairly quickly. -clickhouse_client_loop_timeout 30 test 2>&1 | grep -o -F 'Ok' | uniq +timeout 30 bash -c test 2>&1 | grep -o -F 'Ok' | uniq diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 9b4159fa002..1d2d4516b9c 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -9,17 +9,20 @@ set -e function thread() { - $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | - grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' + while true; do + $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 SYNC; + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now| were removed by another replica|Removing leftovers from table|Another replica was suddenly created|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|^\(query: ' + done } -export -f thread +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f thread; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread 1 & -clickhouse_client_loop_timeout $TIMEOUT thread 2 & +timeout $TIMEOUT bash -c 'thread 1' & +timeout $TIMEOUT bash -c 'thread 2' & wait diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index d538cafd79d..758ec4825e0 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -12,19 +12,19 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Diff function thread1() { - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;" + while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done } function thread2() { - $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA test_01320.r" 2>/dev/null + while true; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA test_01320.r" 2>/dev/null; done } export -f thread1 export -f thread2 -clickhouse_client_loop_timeout 10 thread1 & -clickhouse_client_loop_timeout 10 thread2 & +timeout 10 bash -c thread1 & +timeout 10 bash -c thread2 & wait diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh index 7d189518a58..165a461193d 100755 --- a/tests/queries/0_stateless/01412_cache_dictionary_race.sh +++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh @@ -26,41 +26,45 @@ LAYOUT(CACHE(SIZE_IN_CELLS 3)); function dict_get_thread() { - $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done } function drop_create_table_thread() { - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( - key_column UInt64, - second_column UInt8, - third_column String - ) - ENGINE MergeTree() ORDER BY tuple(); - INSERT INTO ordinary_db.table_for_dict_real SELECT number, number, toString(number) from numbers(2); - CREATE VIEW ordinary_db.view_for_dict AS SELECT key_column, second_column, third_column from ordinary_db.table_for_dict_real WHERE sleepEachRow(1) == 0; + while true; do + $CLICKHOUSE_CLIENT -n --query "CREATE TABLE ordinary_db.table_for_dict_real ( + key_column UInt64, + second_column UInt8, + third_column String + ) + ENGINE MergeTree() ORDER BY tuple(); + INSERT INTO ordinary_db.table_for_dict_real SELECT number, number, toString(number) from numbers(2); + CREATE VIEW ordinary_db.view_for_dict AS SELECT key_column, second_column, third_column from ordinary_db.table_for_dict_real WHERE sleepEachRow(1) == 0; " - sleep 10 + sleep 10 - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ordinary_db.table_for_dict_real" - sleep 10 + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ordinary_db.table_for_dict_real" + sleep 10 + done } -export -f dict_get_thread -export -f drop_create_table_thread +export -f dict_get_thread; +export -f drop_create_table_thread; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT bash -c drop_create_table_thread 2> /dev/null & +timeout $TIMEOUT bash -c drop_create_table_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh index 51989685e05..eb231e71525 100755 --- a/tests/queries/0_stateless/01444_create_table_drop_database_race.sh +++ b/tests/queries/0_stateless/01444_create_table_drop_database_race.sh @@ -11,14 +11,18 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function thread1() { - # ${CLICKHOUSE_CLIENT} --query="SHOW TABLES FROM test_01444" - ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS test_01444" 2>&1| grep -F "Code: " | grep -Fv "Code: 219" - ${CLICKHOUSE_CLIENT} --query="CREATE DATABASE IF NOT EXISTS test_01444" + while true; do +# ${CLICKHOUSE_CLIENT} --query="SHOW TABLES FROM test_01444" + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS test_01444" 2>&1| grep -F "Code: " | grep -Fv "Code: 219" + ${CLICKHOUSE_CLIENT} --query="CREATE DATABASE IF NOT EXISTS test_01444" + done } function thread2() { - ${CLICKHOUSE_CLIENT} --query="CREATE TABLE IF NOT EXISTS test_01444.t$RANDOM (x UInt8) ENGINE = MergeTree ORDER BY tuple()" 2>/dev/null + while true; do + ${CLICKHOUSE_CLIENT} --query="CREATE TABLE IF NOT EXISTS test_01444.t$RANDOM (x UInt8) ENGINE = MergeTree ORDER BY tuple()" 2>/dev/null + done } export -f thread1 @@ -26,9 +30,9 @@ export -f thread2 TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT thread1 & -clickhouse_client_loop_timeout $TIMEOUT thread2 & -clickhouse_client_loop_timeout $TIMEOUT thread2 & +timeout $TIMEOUT bash -c thread1 & +timeout $TIMEOUT bash -c thread2 & +timeout $TIMEOUT bash -c thread2 & wait diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh index 27fa74f332c..d83343b3cb3 100755 --- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh +++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh @@ -10,13 +10,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mem" $CLICKHOUSE_CLIENT -q "CREATE TABLE mem (x UInt64) engine = Memory" -function f() -{ +function f { + for _ in $(seq 1 300); do $CLICKHOUSE_CLIENT -q "SELECT count() FROM (SELECT * FROM mem SETTINGS max_threads=2) FORMAT Null;" + done } -function g() -{ +function g { + for _ in $(seq 1 100); do $CLICKHOUSE_CLIENT -n -q " INSERT INTO mem SELECT number FROM numbers(1000000); INSERT INTO mem SELECT number FROM numbers(1000000); @@ -29,13 +30,14 @@ function g() INSERT INTO mem VALUES (1); TRUNCATE TABLE mem; " + done } -export -f f -export -f g +export -f f; +export -f g; -clickhouse_client_loop_timeout 30 f > /dev/null & -clickhouse_client_loop_timeout 30 g > /dev/null & +timeout 30 bash -c f > /dev/null & +timeout 30 bash -c g > /dev/null & wait $CLICKHOUSE_CLIENT -q "DROP TABLE mem" diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh index 39ef530f6e9..981beb785a5 100755 --- a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -18,25 +18,27 @@ $CLICKHOUSE_CLIENT --query "CREATE DICTIONARY ordinary_db.dict1 ( key_column UIn function dict_get_thread() { - $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done } -export -f dict_get_thread +export -f dict_get_thread; TIMEOUT=10 -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index 30e1eff87da..acaa2cfcd25 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -13,32 +13,36 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_kill SELECT number, to function alter_thread { - TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}' and name='value'") - if [ "$TYPE" == "String" ]; then - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" - else - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" - fi + while true; do + TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}' and name='value'") + if [ "$TYPE" == "String" ]; then + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" + else + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_kill MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" + fi + done } function kill_mutation_thread { - # find any mutation and kill it - mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done=0 and database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_kill' LIMIT 1") - if [ ! -z "$mutation_id" ]; then - $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null - sleep 1 - fi + while true; do + # find any mutation and kill it + mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done=0 and database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_kill' LIMIT 1") + if [ ! -z "$mutation_id" ]; then + $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table='concurrent_mutate_kill' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null + sleep 1 + fi + done } -export -f alter_thread -export -f kill_mutation_thread +export -f alter_thread; +export -f kill_mutation_thread; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT kill_mutation_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c kill_mutation_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh index 3ee321371bf..f8f3ccd6dd6 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh @@ -27,34 +27,38 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SELECT sum(toUInt64(value)) FROM concurrent_kill_$i" done -function alter_thread() +function alter_thread { - REPLICA=$(($RANDOM % 5 + 1)) - TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_kill_$REPLICA' and database='${CLICKHOUSE_DATABASE}' and name='value'") - if [ "$TYPE" == "String" ]; then - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" - else - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" - fi + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=$($CLICKHOUSE_CLIENT --query "SELECT type FROM system.columns WHERE table='concurrent_kill_$REPLICA' and database='${CLICKHOUSE_DATABASE}' and name='value'") + if [ "$TYPE" == "String" ]; then + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2" + else + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_kill_$REPLICA MODIFY COLUMN value String SETTINGS replication_alter_partitions_sync=2" + fi + done } -function kill_mutation_thread() +function kill_mutation_thread { - # find any mutation and kill it - mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done = 0 and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}' LIMIT 1") - if [ ! -z "$mutation_id" ]; then - $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null - sleep 1 - fi + while true; do + # find any mutation and kill it + mutation_id=$($CLICKHOUSE_CLIENT --query "SELECT mutation_id FROM system.mutations WHERE is_done = 0 and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}' LIMIT 1") + if [ ! -z "$mutation_id" ]; then + $CLICKHOUSE_CLIENT --query "KILL MUTATION WHERE mutation_id='$mutation_id' and table like 'concurrent_kill_%' and database='${CLICKHOUSE_DATABASE}'" 1> /dev/null + sleep 1 + fi + done } -export -f alter_thread -export -f kill_mutation_thread +export -f alter_thread; +export -f kill_mutation_thread; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT alter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT kill_mutation_thread 2> /dev/null & +timeout $TIMEOUT bash -c alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c kill_mutation_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/01602_max_distributed_connections.sh b/tests/queries/0_stateless/01602_max_distributed_connections.sh index 0869d2d3a68..ed835a8768f 100755 --- a/tests/queries/0_stateless/01602_max_distributed_connections.sh +++ b/tests/queries/0_stateless/01602_max_distributed_connections.sh @@ -15,14 +15,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) i=0 retries=30 while [[ $i -lt $retries ]]; do - clickhouse_client_timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " + timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && break ((++i)) done i=0 retries=30 while [[ $i -lt $retries ]]; do - clickhouse_client_timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " + timeout 10 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 10 --query " SELECT sleep(1.5) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && break ((++i)) done @@ -30,13 +30,10 @@ done # If max_distributed_connections is low and async_socket_for_remote is disabled, # the concurrency of distributed queries will be also low. -clickhouse_client_timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " +timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " SELECT sleep(0.15) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=0 && echo 'Fail' -clickhouse_client_timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " +timeout 1 ${CLICKHOUSE_CLIENT} --max_threads 1 --max_distributed_connections 1 --async_socket_for_remote 0 --query " SELECT sleep(0.15) FROM remote('127.{1..10}', system.one) FORMAT Null" --prefer_localhost_replica=1 && echo 'Fail' -# FIXME -clickhouse_test_wait_queries 5 - echo 'Ok' diff --git a/tests/queries/0_stateless/01632_tinylog_read_write.sh b/tests/queries/0_stateless/01632_tinylog_read_write.sh index 42211168d45..3f41bcc5924 100755 --- a/tests/queries/0_stateless/01632_tinylog_read_write.sh +++ b/tests/queries/0_stateless/01632_tinylog_read_write.sh @@ -9,32 +9,35 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --multiquery --query "DROP TABLE IF EXISTS test; CREATE TABLE IF NOT EXISTS test (x UInt64, s Array(Nullable(String))) ENGINE = TinyLog;" -function thread_select() -{ - $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" - sleep 0.0$RANDOM +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM test FORMAT Null" + sleep 0.0$RANDOM + done } -function thread_insert() -{ - $CLICKHOUSE_CLIENT --query "INSERT INTO test VALUES (1, ['Hello'])" - sleep 0.0$RANDOM +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO test VALUES (1, ['Hello'])" + sleep 0.0$RANDOM + done } export -f thread_select export -f thread_insert + # Do randomized queries and expect nothing extraordinary happens. -clickhouse_client_loop_timeout 10 thread_select & -clickhouse_client_loop_timeout 10 thread_select & -clickhouse_client_loop_timeout 10 thread_select & -clickhouse_client_loop_timeout 10 thread_select & +timeout 10 bash -c 'thread_select' & +timeout 10 bash -c 'thread_select' & +timeout 10 bash -c 'thread_select' & +timeout 10 bash -c 'thread_select' & -clickhouse_client_loop_timeout 10 thread_insert & -clickhouse_client_loop_timeout 10 thread_insert & -clickhouse_client_loop_timeout 10 thread_insert & -clickhouse_client_loop_timeout 10 thread_insert & +timeout 10 bash -c 'thread_insert' & +timeout 10 bash -c 'thread_insert' & +timeout 10 bash -c 'thread_insert' & +timeout 10 bash -c 'thread_insert' & wait echo "Done" diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh index a06a9ec7057..93c4451524c 100755 --- a/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh +++ b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh @@ -5,24 +5,26 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -function thread_create_drop_table() -{ - REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null - sleep 0.0$RANDOM - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" +function thread_create_drop_table { + while true; do + REPLICA=$(($RANDOM % 10)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" + done } -function thread_alter_table() -{ - $CLICKHOUSE_CLIENT --query "ALTER TABLE $CLICKHOUSE_DATABASE.t1 on cluster test_shard_localhost ADD COLUMN newcol UInt32" >/dev/null 2>&1 - sleep 0.0$RANDOM +function thread_alter_table { + while true; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE $CLICKHOUSE_DATABASE.t1 on cluster test_shard_localhost ADD COLUMN newcol UInt32" >/dev/null 2>&1 + sleep 0.0$RANDOM + done } export -f thread_create_drop_table export -f thread_alter_table -clickhouse_client_loop_timeout 20 thread_create_drop_table & -clickhouse_client_loop_timeout 20 thread_alter_table & +timeout 20 bash -c "thread_create_drop_table" & +timeout 20 bash -c 'thread_alter_table' & wait sleep 1 diff --git a/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh b/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh index 1cd243675c6..5bb10220f7f 100755 --- a/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh +++ b/tests/queries/0_stateless/01732_race_condition_storage_join_long.sh @@ -11,34 +11,47 @@ set -o errexit set -o pipefail echo " - DROP TABLE IF EXISTS storage_join_race; - CREATE TABLE storage_join_race (x UInt64, y UInt64) Engine = Join(ALL, FULL, x); + DROP TABLE IF EXISTS storage_join_race; + CREATE TABLE storage_join_race (x UInt64, y UInt64) Engine = Join(ALL, FULL, x); " | $CLICKHOUSE_CLIENT -n function read_thread_big() { - $CLICKHOUSE_CLIENT -n -q "SELECT * FROM ( SELECT number AS x FROM numbers(100000) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null" + while true; do + echo " + SELECT * FROM ( SELECT number AS x FROM numbers(100000) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null; + " | $CLICKHOUSE_CLIENT -n + done } function read_thread_small() { - $CLICKHOUSE_CLIENT -n -q "SELECT * FROM ( SELECT number AS x FROM numbers(10) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null" + while true; do + echo " + SELECT * FROM ( SELECT number AS x FROM numbers(10) ) AS t1 ALL FULL JOIN storage_join_race USING (x) FORMAT Null; + " | $CLICKHOUSE_CLIENT -n + done } function read_thread_select() { - $CLICKHOUSE_CLIENT -n -q "SELECT * FROM storage_join_race FORMAT Null" + while true; do + echo " + SELECT * FROM storage_join_race FORMAT Null; + " | $CLICKHOUSE_CLIENT -n + done } -export -f read_thread_big -export -f read_thread_small -export -f read_thread_select +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f read_thread_big; +export -f read_thread_small; +export -f read_thread_select; TIMEOUT=20 -clickhouse_client_loop_timeout $TIMEOUT read_thread_big 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT read_thread_small 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT read_thread_select 2> /dev/null & +timeout $TIMEOUT bash -c read_thread_big 2> /dev/null & +timeout $TIMEOUT bash -c read_thread_small 2> /dev/null & +timeout $TIMEOUT bash -c read_thread_select 2> /dev/null & echo " INSERT INTO storage_join_race SELECT number AS x, number AS y FROM numbers (10000000); diff --git a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh index a29d0661621..a3682a3a74b 100755 --- a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh +++ b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh @@ -29,34 +29,38 @@ done function optimize_thread { - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE ttl_table$REPLICA FINAl" + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE ttl_table$REPLICA FINAl" + done } function insert_thread { - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" - $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + $CLICKHOUSE_CLIENT --optimize_on_insert=0 --query "INSERT INTO ttl_table$REPLICA SELECT now() + rand() % 5 - rand() % 3 FROM numbers(5)" + done } -export -f insert_thread -export -f optimize_thread +export -f insert_thread; +export -f optimize_thread; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT insert_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & +timeout $TIMEOUT bash -c optimize_thread 2> /dev/null & wait for i in $(seq 1 $NUM_REPLICAS); do diff --git a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh index c6f40d35855..81d70f79574 100755 --- a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh +++ b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh @@ -37,18 +37,24 @@ function insert3() function select1() { - ${CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts FORMAT Null" + while true; do + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts FORMAT Null" + done } function select2() { - ${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.asynchronous_inserts FORMAT Null" + while true; do + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM system.asynchronous_inserts FORMAT Null" + done } function truncate1() { - sleep 0.1 - ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE async_inserts" + while true; do + sleep 0.1 + ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE async_inserts" + done } ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts" @@ -69,9 +75,9 @@ for _ in {1..5}; do timeout $TIMEOUT bash -c insert3 & done -clickhouse_client_loop_timeout $TIMEOUT select1 & -clickhouse_client_loop_timeout $TIMEOUT select2 & -clickhouse_client_loop_timeout $TIMEOUT truncate1 & +timeout $TIMEOUT bash -c select1 & +timeout $TIMEOUT bash -c select2 & +timeout $TIMEOUT bash -c truncate1 & wait echo "OK" diff --git a/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh b/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh index 062b8512bff..57a263abac5 100755 --- a/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh +++ b/tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh @@ -19,24 +19,26 @@ $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zst function select_thread() { - $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM local" >/dev/null - $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM distributed" >/dev/null + while true; do + $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM local" >/dev/null + $CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM distributed" >/dev/null + done } -export -f select_thread +export -f select_thread; TIMEOUT=30 -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & +timeout $TIMEOUT bash -c select_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index ec20007892a..88c30852c86 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -11,29 +11,38 @@ set -o errexit set -o pipefail echo " - DROP TABLE IF EXISTS rocksdb_race; - CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); + DROP TABLE IF EXISTS rocksdb_race; + CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); " | $CLICKHOUSE_CLIENT -n function read_stat_thread() { - $CLICKHOUSE_CLIENT -n -q 'SELECT * FROM system.rocksdb FORMAT Null' + while true; do + echo " + SELECT * FROM system.rocksdb FORMAT Null; + " | $CLICKHOUSE_CLIENT -n + done } function truncate_thread() { - sleep 3s; - $CLICKHOUSE_CLIENT -n -q 'TRUNCATE TABLE rocksdb_race' + while true; do + sleep 3s; + echo " + TRUNCATE TABLE rocksdb_race; + " | $CLICKHOUSE_CLIENT -n + done } -export -f read_stat_thread -export -f truncate_thread +# https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout +export -f read_stat_thread; +export -f truncate_thread; TIMEOUT=20 -clickhouse_client_loop_timeout $TIMEOUT read_stat_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT truncate_thread 2> /dev/null & +timeout $TIMEOUT bash -c read_stat_thread 2> /dev/null & +timeout $TIMEOUT bash -c truncate_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh index 4b7d15c01c3..7e8579f7cbe 100755 --- a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh +++ b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh @@ -7,15 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) i=0 retries=5 -client_opts=( - --http_max_tries 1 - --max_execution_time 3 - --max_threads 10 - --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" - --format Null -) # Connecting to wrong address and checking for race condition while [[ $i -lt $retries ]]; do - clickhouse_client_timeout 4s ${CLICKHOUSE_CLIENT} "${client_opts[@]}" 2>/dev/null + timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null 2>/dev/null ((++i)) done diff --git a/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh b/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh index 84a1befe421..2deaf788ecf 100755 --- a/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh +++ b/tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh @@ -17,26 +17,28 @@ function four_letter_thread() function create_drop_thread() { - num=$(($RANDOM % 10 + 1)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table$num (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table$num', '0') ORDER BY key" - sleep 0.$RANDOM - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table$num" + while true; do + num=$(($RANDOM % 10 + 1)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table$num (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table$num', '0') ORDER BY key" + sleep 0.$RANDOM + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table$num" + done } -export -f four_letter_thread -export -f create_drop_thread +export -f four_letter_thread; +export -f create_drop_thread; TIMEOUT=15 -timeout $TIMEOUT four_letter_thread 2> /dev/null & -timeout $TIMEOUT four_letter_thread 2> /dev/null & -timeout $TIMEOUT four_letter_thread 2> /dev/null & -timeout $TIMEOUT four_letter_thread 2> /dev/null & +timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & +timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & +timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & +timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & -clickhouse_client_loop_timeout $TIMEOUT create_drop_thread 2> /dev/null & +timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & +timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & +timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & +timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null & wait diff --git a/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh b/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh index bc43f460289..1b2197ef943 100755 --- a/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh +++ b/tests/queries/0_stateless/02124_buffer_with_type_map_long.sh @@ -9,12 +9,16 @@ $CLICKHOUSE_CLIENT -q "CREATE TABLE t_buffer_map(m1 Map(String, UInt64), m2 Map( function insert1 { - $CLICKHOUSE_CLIENT -q "INSERT INTO t_buffer_map SELECT (range(10), range(10)), (range(10), range(10)) from numbers(100)" + while true; do + $CLICKHOUSE_CLIENT -q "INSERT INTO t_buffer_map SELECT (range(10), range(10)), (range(10), range(10)) from numbers(100)" + done } function select1 { - $CLICKHOUSE_CLIENT -q "SELECT * FROM t_buffer_map" 2> /dev/null > /dev/null + while true; do + $CLICKHOUSE_CLIENT -q "SELECT * FROM t_buffer_map" 2> /dev/null > /dev/null + done } TIMEOUT=10 @@ -22,8 +26,8 @@ TIMEOUT=10 export -f insert1 export -f select1 -clickhouse_client_loop_timeout $TIMEOUT insert1 & -clickhouse_client_loop_timeout $TIMEOUT select1 & +timeout $TIMEOUT bash -c insert1 & +timeout $TIMEOUT bash -c select1 & wait diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 7699e803f76..ce5947d95ed 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -19,9 +19,9 @@ export CLICKHOUSE_TEST_UNIQUE_NAME="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABAS [ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_BENCHMARK_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " [ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " -[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) " +[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' " [ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} " -[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) " +[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment='${CLICKHOUSE_LOG_COMMENT}' " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} # client @@ -129,37 +129,3 @@ function clickhouse_client_removed_host_parameter() # bash regex magic is arcane, but version dependant and weak; sed or awk are not really portable. $(echo "$CLICKHOUSE_CLIENT" | python3 -c "import sys, re; print(re.sub('--host(\s+|=)[^\s]+', '', sys.stdin.read()))") "$@" } - -function clickhouse_client_timeout() -{ - local timeout=$1 && shift - timeout -s INT "$timeout" "$@" -} -# Helper function to stop the clickhouse-client after SIGINT properly. -function clickhouse_client_loop_timeout() -{ - local timeout=$1 && shift - - local cmd - cmd="$(printf '%q ' "$@")" - - timeout -s INT "$timeout" bash -c "trap 'STOP_THE_LOOP=1' INT; while true; do [ ! -v STOP_THE_LOOP ] || break; $cmd; done" -} -# wait for queries to be finished -function clickhouse_test_wait_queries() -{ - local timeout=${1:-"600"} && shift - local query_id="wait-$CLICKHOUSE_TEST_UNIQUE_NAME" - local query="SELECT count() FROM system.processes WHERE current_database = '$CLICKHOUSE_DATABASE' AND query_id != '$query_id'" - local i=0 - (( timeout*=2 )) - while [[ "$(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query_id=$query_id" --data-binary "$query")" != "0" ]]; do - sleep 0.5 - - (( ++i )) - if [[ $i -gt $timeout ]]; then - echo "clickhouse_test_wait_queries: timeout exceeded" - exit 1 - fi - done -} From 1ae919a18aa37fdea552c3c589dfccbf1fd83263 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:23:58 +0200 Subject: [PATCH 063/136] Revert "Merge pull request #35865 from azat/clickhouse-test-left-queries" This reverts commit 18d094d79d239bbb8d14ea509b6dc7bf2a74e0fb, reversing changes made to 224f4dc620da606a7b5b52f855de607e43f0cc58. --- docker/test/stateless/run.sh | 2 - tests/clickhouse-test | 16 ---- .../01085_max_distributed_connections_http.sh | 2 - .../01502_log_tinylog_deadlock_race.sh | 78 ++++++++++--------- .../0_stateless/01675_data_type_coroutine.sh | 6 -- .../01675_data_type_coroutine_2.sh | 6 -- .../01731_async_task_queue_wait.sh | 2 - .../0_stateless/02104_overcommit_memory.sh | 23 +++--- .../02151_http_s_structure_set_eof.sh | 7 -- 9 files changed, 55 insertions(+), 87 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index e2ad91220d3..50632630f1f 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -92,8 +92,6 @@ function run_tests() if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') - # Cannot be used with replicated database, due to distributed_ddl_output_mode=none - ADDITIONAL_OPTIONS+=('--no-left-queries-check') ADDITIONAL_OPTIONS+=('--jobs') ADDITIONAL_OPTIONS+=('2') else diff --git a/tests/clickhouse-test b/tests/clickhouse-test index a8cab282b0d..eb97118aecc 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -375,7 +375,6 @@ class FailureReason(enum.Enum): RESULT_DIFF = "result differs with reference: " TOO_LONG = "Test runs too long (> 60s). Make it faster." INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" - LEFT_QUERIES = "Queries left in background after the test finished:" # SKIPPED reasons DISABLED = "disabled" @@ -828,21 +827,6 @@ class TestCase: description, ) - left_queries_check = args.no_left_queries_check is False - if self.tags and "no-left-queries-check" in self.tags: - left_queries_check = False - if left_queries_check: - processlist = get_processlist_after_test(self.testcase_args) - if processlist: - description += f"\n{json.dumps(processlist, indent=4)}\n" - return TestResult( - self.name, - TestStatus.FAIL, - FailureReason.LEFT_QUERIES, - total_time, - description, - ) - if os.path.exists(self.stdout_file): os.remove(self.stdout_file) if os.path.exists(self.stderr_file): diff --git a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh index 6e840f44930..0e40918257d 100755 --- a/tests/queries/0_stateless/01085_max_distributed_connections_http.sh +++ b/tests/queries/0_stateless/01085_max_distributed_connections_http.sh @@ -15,5 +15,3 @@ while [[ $i -lt $retries ]]; do timeout 1.8s ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_distributed_connections=2&max_threads=1" -d "$query" && break ((++i)) done - -clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh index 142b83f1e0b..b659d550fa4 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -10,40 +10,46 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -function thread_create() -{ - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" - sleep 0.0$RANDOM +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" + sleep 0.0$RANDOM + done } -function thread_drop() -{ - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" - sleep 0.0$RANDOM +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" + sleep 0.0$RANDOM + done } -function thread_rename() -{ - $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done } -function thread_select() -{ - $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } -function thread_insert() -{ - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: '| grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: '| grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } -function thread_insert_select() -{ - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } export -f thread_create @@ -59,18 +65,18 @@ export -f thread_insert_select function test_with_engine { echo "Testing $1" - clickhouse_client_loop_timeout 10 thread_create t1 $1 & - clickhouse_client_loop_timeout 10 thread_create t2 $1 & - clickhouse_client_loop_timeout 10 thread_drop t1 & - clickhouse_client_loop_timeout 10 thread_drop t2 & - clickhouse_client_loop_timeout 10 thread_rename t1 t2 & - clickhouse_client_loop_timeout 10 thread_rename t2 t1 & - clickhouse_client_loop_timeout 10 thread_select t1 & - clickhouse_client_loop_timeout 10 thread_select t2 & - clickhouse_client_loop_timeout 10 thread_insert t1 5 & - clickhouse_client_loop_timeout 10 thread_insert t2 10 & - clickhouse_client_loop_timeout 10 thread_insert_select t1 t2 & - clickhouse_client_loop_timeout 10 thread_insert_select t2 t1 & + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & wait echo "Done $1" diff --git a/tests/queries/0_stateless/01675_data_type_coroutine.sh b/tests/queries/0_stateless/01675_data_type_coroutine.sh index 687ff6ac473..4106d0d7f73 100755 --- a/tests/queries/0_stateless/01675_data_type_coroutine.sh +++ b/tests/queries/0_stateless/01675_data_type_coroutine.sh @@ -17,9 +17,3 @@ while [[ $counter -lt $retries ]]; do done echo 'Ok' - -# wait queries, since there is 'Maximum parse depth' error on the client -# and in this case it simply reset the connection and don't read everything -# from server, so there is no guarantee that the query is stopped when the -# client returns -clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/01675_data_type_coroutine_2.sh b/tests/queries/0_stateless/01675_data_type_coroutine_2.sh index 9c4ed81e345..501b9d4ab12 100755 --- a/tests/queries/0_stateless/01675_data_type_coroutine_2.sh +++ b/tests/queries/0_stateless/01675_data_type_coroutine_2.sh @@ -17,9 +17,3 @@ done #echo "I = ${I}" echo 'Ok' - -# wait queries, since there is 'Maximum parse depth' error on the client -# and in this case it simply reset the connection and don't read everything -# from server, so there is no guarantee that the query is stopped when the -# client returns -clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/01731_async_task_queue_wait.sh b/tests/queries/0_stateless/01731_async_task_queue_wait.sh index 6fdd676f336..e0babf3c6ff 100755 --- a/tests/queries/0_stateless/01731_async_task_queue_wait.sh +++ b/tests/queries/0_stateless/01731_async_task_queue_wait.sh @@ -8,5 +8,3 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # cancellation with async_socket_for_remote=1 (that ignores # max_distributed_connections) timeout --signal=SIGINT 1 ${CLICKHOUSE_CLIENT} --max_distributed_connections=1 --max_block_size=2 --interactive_delay=900000 -q "select number + sleep(0.3) as x from remote('127.{2,3}', system.numbers) settings max_block_size = 2" 2>&1 | grep "Empty task was returned from async task queue" || true - -clickhouse_test_wait_queries 60 diff --git a/tests/queries/0_stateless/02104_overcommit_memory.sh b/tests/queries/0_stateless/02104_overcommit_memory.sh index f2016dbc0c1..303c7cb44b6 100755 --- a/tests/queries/0_stateless/02104_overcommit_memory.sh +++ b/tests/queries/0_stateless/02104_overcommit_memory.sh @@ -5,19 +5,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u02104' -$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u02104 IDENTIFIED WITH no_password' -$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u02104' +$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password' +$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1' function overcommited() { - $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 \ - | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED" + while true; do + $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS memory_overcommit_ratio_denominator=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 \ + | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED" + done } function expect_execution() { - $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,memory_overcommit_ratio_denominator=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + while true; do + $CLICKHOUSE_CLIENT -u u02104 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,memory_overcommit_ratio_denominator=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + done } export -f overcommited @@ -27,9 +30,9 @@ function user_test() { for _ in {1..10}; do - clickhouse_client_loop_timeout 10 overcommited & - clickhouse_client_loop_timeout 10 expect_execution & - done + timeout 10 bash -c overcommited & + timeout 10 bash -c expect_execution & + done; wait } @@ -43,4 +46,4 @@ else echo "OVERCOMMITED WITH USER LIMIT WAS KILLED" fi -$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u02104' +$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1' diff --git a/tests/queries/0_stateless/02151_http_s_structure_set_eof.sh b/tests/queries/0_stateless/02151_http_s_structure_set_eof.sh index c3dfc4d03a8..448fa9bfede 100755 --- a/tests/queries/0_stateless/02151_http_s_structure_set_eof.sh +++ b/tests/queries/0_stateless/02151_http_s_structure_set_eof.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -26,9 +25,3 @@ timeout 0.15s ${CLICKHOUSE_CURL} -sS -F "s=@$tmp_file;" "${CLICKHOUSE_URL}&s_str echo $? timeout 0.15s ${CLICKHOUSE_CURL} -sS -F "s=@$tmp_file;" "${CLICKHOUSE_URL}&s_structure=key+Int&query=SELECT+dummy+IN+s&input_format_parallel_parsing=false" -o /dev/null echo $? - -# wait until the query above will start, -# so that clickhouse_test_wait_queries will see them. -sleep 5 - -clickhouse_test_wait_queries 60 From 9af6e237e135a7e35b0e3f1ee43d35659621b4c1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:25:56 +0200 Subject: [PATCH 064/136] remove retries --- tests/clickhouse-test | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index eb97118aecc..355f7b7a712 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -55,7 +55,6 @@ MESSAGES_TO_RETRY = [ "DB::Exception: New table appeared in database being dropped or detached. Try again", "is already started to be removing by another replica right now", "DB::Exception: Cannot enqueue query", - "line 1: wait_for: No record of process", # Something weird from bash internals, let's just retry "is executing longer than distributed_ddl_task_timeout", # FIXME ] From 70f39e08493532ea75c503a326c15f51ab9b2b59 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:27:40 +0200 Subject: [PATCH 065/136] Revert "Merge pull request #36731 from azat/fix-async_inserts_stress_long-test" This reverts commit d1d2f2c1a4979d17b7d58f591f56346bc79278f8, reversing changes made to 1ddb04b99294a050bd9c0e124c17617dd2f89011. --- tests/queries/0_stateless/02015_async_inserts_stress_long.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh index 81d70f79574..086419baa61 100755 --- a/tests/queries/0_stateless/02015_async_inserts_stress_long.sh +++ b/tests/queries/0_stateless/02015_async_inserts_stress_long.sh @@ -83,6 +83,3 @@ wait echo "OK" ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts"; - -# There is no other way to wait HTTP queries -clickhouse_test_wait_queries 30 From f3c2ca6a1378d09afd8e33696b623c125fd64266 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:28:38 +0200 Subject: [PATCH 066/136] Revert "Merge pull request #36634 from azat/01502_long_log_tinylog_deadlock_race-test" This reverts commit c04c62795e024dc115aa724d48f443550391bfae, reversing changes made to 88f05ac14ab60ac5353c06964c8470de21a5f627. --- .../01502_long_log_tinylog_deadlock_race.sh | 78 ++++++++++--------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh index a3dadf48c38..1087a7ed96b 100755 --- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh @@ -10,40 +10,46 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -function thread_create() -{ - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done } -function thread_drop() -{ - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done } -function thread_rename() -{ - $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' - sleep 0.0$RANDOM +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done } -function thread_select() -{ - $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } -function thread_insert() -{ - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } -function thread_insert_select() -{ - $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' - sleep 0.0$RANDOM +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -e 'Received exception from server' -e '^(query: ' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done } export -f thread_create @@ -59,18 +65,18 @@ export -f thread_insert_select function test_with_engine { echo "Testing $1" - clickhouse_client_loop_timeout 10 thread_create t1 $1 & - clickhouse_client_loop_timeout 10 thread_create t2 $1 & - clickhouse_client_loop_timeout 10 thread_drop t1 & - clickhouse_client_loop_timeout 10 thread_drop t2 & - clickhouse_client_loop_timeout 10 thread_rename t1 t2 & - clickhouse_client_loop_timeout 10 thread_rename t2 t1 & - clickhouse_client_loop_timeout 10 thread_select t1 & - clickhouse_client_loop_timeout 10 thread_select t2 & - clickhouse_client_loop_timeout 10 thread_insert t1 5 & - clickhouse_client_loop_timeout 10 thread_insert t2 10 & - clickhouse_client_loop_timeout 10 thread_insert_select t1 t2 & - clickhouse_client_loop_timeout 10 thread_insert_select t2 t1 & + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & wait echo "Done $1" From be498b0658a2905f4c3f7b7459c261ed19ac6b64 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 19:40:26 +0200 Subject: [PATCH 067/136] fix test --- .../0_stateless/01133_begin_commit_race.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01133_begin_commit_race.sh b/tests/queries/0_stateless/01133_begin_commit_race.sh index f64570950c7..7dadb35ccff 100755 --- a/tests/queries/0_stateless/01133_begin_commit_race.sh +++ b/tests/queries/0_stateless/01133_begin_commit_race.sh @@ -13,32 +13,40 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE mt (n Int64) ENGINE=MergeTree ORDER BY function begin_commit_readonly() { + while true; do $CLICKHOUSE_CLIENT --multiquery --query " SET wait_changes_become_visible_after_commit_mode='wait'; BEGIN TRANSACTION; COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION + done } function begin_rollback_readonly() { + while true; do $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=wait_unknown --multiquery --query " BEGIN TRANSACTION; SET TRANSACTION SNAPSHOT 42; ROLLBACK;" + done } function begin_insert_commit() { + while true; do $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=async --multiquery --query " BEGIN TRANSACTION; INSERT INTO mt VALUES ($RANDOM); COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION + done } function introspection() { + while true; do $CLICKHOUSE_CLIENT -q "SELECT * FROM system.transactions FORMAT Null" $CLICKHOUSE_CLIENT -q "SELECT transactionLatestSnapshot(), transactionOldestSnapshot() FORMAT Null" + done } export -f begin_commit_readonly @@ -48,10 +56,10 @@ export -f introspection TIMEOUT=20 -clickhouse_client_loop_timeout $TIMEOUT begin_commit_readonly & -clickhouse_client_loop_timeout $TIMEOUT begin_rollback_readonly & -clickhouse_client_loop_timeout $TIMEOUT begin_insert_commit & -clickhouse_client_loop_timeout $TIMEOUT introspection & +timeout $TIMEOUT bash -c begin_commit_readonly & +timeout $TIMEOUT bash -c begin_rollback_readonly & +timeout $TIMEOUT bash -c begin_insert_commit & +timeout $TIMEOUT bash -c introspection & wait From 9e3e7953535e35d9e3293925abf3eeae6d95422c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 20:44:58 +0200 Subject: [PATCH 068/136] fix --- tests/queries/0_stateless/02104_overcommit_memory.sh | 6 +++--- tests/queries/0_stateless/02235_remote_fs_cache_stress.sh | 2 +- tests/queries/0_stateless/02294_overcommit_overflow.sh | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02104_overcommit_memory.sh b/tests/queries/0_stateless/02104_overcommit_memory.sh index 303c7cb44b6..73d65eb40ed 100755 --- a/tests/queries/0_stateless/02104_overcommit_memory.sh +++ b/tests/queries/0_stateless/02104_overcommit_memory.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password' -$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1' +$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u02104 IDENTIFIED WITH no_password' +$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u02104' function overcommited() { @@ -46,4 +46,4 @@ else echo "OVERCOMMITED WITH USER LIMIT WAS KILLED" fi -$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1' +$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u02104' diff --git a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh index a5c0ee6ecff..bc1a4cbfdd1 100755 --- a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh +++ b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh @@ -63,7 +63,7 @@ SELECT indexOf(['a', 'b', NULL], toLowCardinality('a')); """ } -for i in `seq 1 32`; do go | grep -q "Exception" && echo 'FAIL' || echo 'OK' ||: & done +for _ in `seq 1 32`; do go | grep -q "Exception" && echo 'FAIL' || echo 'OK' ||: & done wait diff --git a/tests/queries/0_stateless/02294_overcommit_overflow.sh b/tests/queries/0_stateless/02294_overcommit_overflow.sh index 64a4dddc67f..0fe7882cb3d 100755 --- a/tests/queries/0_stateless/02294_overcommit_overflow.sh +++ b/tests/queries/0_stateless/02294_overcommit_overflow.sh @@ -11,14 +11,18 @@ $CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u02294' function query() { + while true; do $CLICKHOUSE_CLIENT -u u02294 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,memory_overcommit_ratio_denominator=2000000000000000000,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + done } export -f query +TIMEOUT=10 + for _ in {1..10}; do - clickhouse_client_loop_timeout 10 query & + timeout $TIMEOUT bash -c query & done wait From 313f4833d641993a4d090691c0f8d1921ded97a3 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 28 May 2022 23:21:55 +0200 Subject: [PATCH 069/136] fix --- tests/queries/0_stateless/00417_kill_query.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/00417_kill_query.sh b/tests/queries/0_stateless/00417_kill_query.sh index e40eb5c3f80..c3b57b8ef3f 100755 --- a/tests/queries/0_stateless/00417_kill_query.sh +++ b/tests/queries/0_stateless/00417_kill_query.sh @@ -22,4 +22,3 @@ $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 FORMAT TabSeparated" $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 0 SYNC FORMAT TabSeparated" $CLICKHOUSE_CLIENT -q "KILL QUERY WHERE 1 TEST" &>/dev/null -clickhouse_test_wait_queries 60 From 59a070778d56fbddf99e0c52c49bf9ad4051330a Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 29 May 2022 14:48:04 +0200 Subject: [PATCH 070/136] More quite logging for S3 tests --- tests/config/config.d/{logger.xml => logger_test.xml} | 0 tests/config/config.d/logger_trace.xml | 5 +++++ tests/config/install.sh | 5 ++++- 3 files changed, 9 insertions(+), 1 deletion(-) rename tests/config/config.d/{logger.xml => logger_test.xml} (100%) create mode 100644 tests/config/config.d/logger_trace.xml diff --git a/tests/config/config.d/logger.xml b/tests/config/config.d/logger_test.xml similarity index 100% rename from tests/config/config.d/logger.xml rename to tests/config/config.d/logger_test.xml diff --git a/tests/config/config.d/logger_trace.xml b/tests/config/config.d/logger_trace.xml new file mode 100644 index 00000000000..2c95dcf8fe5 --- /dev/null +++ b/tests/config/config.d/logger_trace.xml @@ -0,0 +1,5 @@ + + + trace + + diff --git a/tests/config/install.sh b/tests/config/install.sh index 3bc1967ab1f..fb015b4f931 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -40,7 +40,7 @@ ln -sf $SRC_PATH/config.d/transactions.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/encryption.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/CORS.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/zookeeper_log.xml $DEST_SERVER_PATH/config.d/ -ln -sf $SRC_PATH/config.d/logger.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/logger_test.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/named_collection.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/ssl_certs.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/filesystem_cache_log.xml $DEST_SERVER_PATH/config.d/ @@ -87,6 +87,9 @@ fi if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then ln -sf $SRC_PATH/config.d/s3_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ + # Too verbose logging in S3 tests + rm -f $DEST_SERVER_PATH/config.d/logger_test.xml + ln -sf $SRC_PATH/config.d/logger_trace.xml $DEST_SERVER_PATH/config.d/ fi if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then From 753bcee954a027c8f3ca93a9d02c1ab38feb9891 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 29 May 2022 18:38:09 +0200 Subject: [PATCH 071/136] Fix lazy mark load --- tests/queries/0_stateless/02263_lazy_mark_load.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02263_lazy_mark_load.sh b/tests/queries/0_stateless/02263_lazy_mark_load.sh index 27922afec24..b4716992a99 100755 --- a/tests/queries/0_stateless/02263_lazy_mark_load.sh +++ b/tests/queries/0_stateless/02263_lazy_mark_load.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-s3-storage +# Tags: no-s3-storage, no-random-settings set -eo pipefail CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -27,6 +27,7 @@ ENGINE = MergeTree ORDER BY n0 SETTINGS min_bytes_for_wide_part = 0; EOF +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES lazy_mark_test" ${CLICKHOUSE_CLIENT} -q "INSERT INTO lazy_mark_test select number, number % 3, number % 5, number % 10, number % 13, number % 15, number % 17, number % 18, number % 22, number % 25 from numbers(1000000)" ${CLICKHOUSE_CLIENT} -q "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --log_queries=1 --query_id "${QUERY_ID}" -q "SELECT * FROM lazy_mark_test WHERE n3==11" From c32b6076fba7ee91c613c8a36c5a2037a95ede94 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 30 May 2022 00:12:33 +0200 Subject: [PATCH 072/136] Remove stranges from code --- .../ObjectStorages/DiskObjectStorage.cpp | 7 +- .../DiskObjectStorageMetadata.cpp | 97 ++++++++----------- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 3 files changed, 48 insertions(+), 58 deletions(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 65b1d5a5bdf..8a5ebea29ec 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -237,7 +237,10 @@ void DiskObjectStorage::moveFile(const String & from_path, const String & to_pat metadata_helper->createFileOperationObject("rename", revision, object_metadata); } - metadata_disk->moveFile(from_path, to_path); + { + std::unique_lock lock(metadata_mutex); + metadata_disk->moveFile(from_path, to_path); + } } void DiskObjectStorage::moveFile(const String & from_path, const String & to_path) @@ -449,6 +452,8 @@ void DiskObjectStorage::removeMetadata(const String & path, std::vector LOG_WARNING(log, "Metadata file {} can't be read by reason: {}. Removing it forcibly.", backQuote(path), e.nested() ? e.nested()->message() : e.message()); + + std::unique_lock lock(metadata_mutex); metadata_disk->removeFile(path); } else diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 2e1ef31f8f0..39d3fa07134 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -74,70 +74,55 @@ DiskObjectStorageMetadata DiskObjectStorageMetadata::createAndStoreMetadataIfNot void DiskObjectStorageMetadata::load() { - try + const ReadSettings read_settings; + auto buf = metadata_disk->readFile(metadata_file_path, read_settings, 1024); /* reasonable buffer size for small file */ + + UInt32 version; + readIntText(version, *buf); + + if (version < VERSION_ABSOLUTE_PATHS || version > VERSION_READ_ONLY_FLAG) + throw Exception( + ErrorCodes::UNKNOWN_FORMAT, + "Unknown metadata file version. Path: {}. Version: {}. Maximum expected version: {}", + metadata_disk->getPath() + metadata_file_path, toString(version), toString(VERSION_READ_ONLY_FLAG)); + + assertChar('\n', *buf); + + UInt32 remote_fs_objects_count; + readIntText(remote_fs_objects_count, *buf); + assertChar('\t', *buf); + readIntText(total_size, *buf); + assertChar('\n', *buf); + remote_fs_objects.resize(remote_fs_objects_count); + + for (size_t i = 0; i < remote_fs_objects_count; ++i) { - const ReadSettings read_settings; - auto buf = metadata_disk->readFile(metadata_file_path, read_settings, 1024); /* reasonable buffer size for small file */ - - UInt32 version; - readIntText(version, *buf); - - if (version < VERSION_ABSOLUTE_PATHS || version > VERSION_READ_ONLY_FLAG) - throw Exception( - ErrorCodes::UNKNOWN_FORMAT, - "Unknown metadata file version. Path: {}. Version: {}. Maximum expected version: {}", - metadata_disk->getPath() + metadata_file_path, toString(version), toString(VERSION_READ_ONLY_FLAG)); - - assertChar('\n', *buf); - - UInt32 remote_fs_objects_count; - readIntText(remote_fs_objects_count, *buf); + String remote_fs_object_path; + size_t remote_fs_object_size; + readIntText(remote_fs_object_size, *buf); assertChar('\t', *buf); - readIntText(total_size, *buf); - assertChar('\n', *buf); - remote_fs_objects.resize(remote_fs_objects_count); - - for (size_t i = 0; i < remote_fs_objects_count; ++i) + readEscapedString(remote_fs_object_path, *buf); + if (version == VERSION_ABSOLUTE_PATHS) { - String remote_fs_object_path; - size_t remote_fs_object_size; - readIntText(remote_fs_object_size, *buf); - assertChar('\t', *buf); - readEscapedString(remote_fs_object_path, *buf); - if (version == VERSION_ABSOLUTE_PATHS) - { - if (!remote_fs_object_path.starts_with(remote_fs_root_path)) - throw Exception(ErrorCodes::UNKNOWN_FORMAT, - "Path in metadata does not correspond to root path. Path: {}, root path: {}, disk path: {}", - remote_fs_object_path, remote_fs_root_path, metadata_disk->getPath()); + if (!remote_fs_object_path.starts_with(remote_fs_root_path)) + throw Exception(ErrorCodes::UNKNOWN_FORMAT, + "Path in metadata does not correspond to root path. Path: {}, root path: {}, disk path: {}", + remote_fs_object_path, remote_fs_root_path, metadata_disk->getPath()); - remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); - } - assertChar('\n', *buf); - remote_fs_objects[i].relative_path = remote_fs_object_path; - remote_fs_objects[i].bytes_size = remote_fs_object_size; + remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); } - - readIntText(ref_count, *buf); assertChar('\n', *buf); - - if (version >= VERSION_READ_ONLY_FLAG) - { - readBoolText(read_only, *buf); - assertChar('\n', *buf); - } + remote_fs_objects[i].relative_path = remote_fs_object_path; + remote_fs_objects[i].bytes_size = remote_fs_object_size; } - catch (Exception & e) + + readIntText(ref_count, *buf); + assertChar('\n', *buf); + + if (version >= VERSION_READ_ONLY_FLAG) { - tryLogCurrentException(__PRETTY_FUNCTION__); - - if (e.code() == ErrorCodes::UNKNOWN_FORMAT) - throw; - - if (e.code() == ErrorCodes::MEMORY_LIMIT_EXCEEDED) - throw; - - throw Exception("Failed to read metadata file: " + metadata_file_path, ErrorCodes::UNKNOWN_FORMAT); + readBoolText(read_only, *buf); + assertChar('\n', *buf); } } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 16699260ea0..7623ff1976f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -400,7 +400,7 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam if (!allow_nullable_key && hasNullable(element.type)) throw Exception( - ErrorCodes::ILLEGAL_COLUMN, "{} key contains nullable columns, but `setting allow_nullable_key` is disabled", key_name); + ErrorCodes::ILLEGAL_COLUMN, "{} key contains nullable columns, but merge tree setting `allow_nullable_key` is disabled", key_name); } } From 6f35c2859261ed1a7e3fe6a1ce13954eee56bcff Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 30 May 2022 00:29:30 +0200 Subject: [PATCH 073/136] Fix style --- src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 39d3fa07134..deeb7f2b64a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -10,7 +10,6 @@ namespace ErrorCodes { extern const int UNKNOWN_FORMAT; extern const int PATH_ACCESS_DENIED; - extern const int MEMORY_LIMIT_EXCEEDED; } DiskObjectStorageMetadata DiskObjectStorageMetadata::readMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_) From 9ffa5f5e0d39ab7df981cf0fcc2952fedf53f60f Mon Sep 17 00:00:00 2001 From: flynn Date: Mon, 30 May 2022 08:10:16 +0000 Subject: [PATCH 074/136] fix typo --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 75a30a2d792..b79dba69e9c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -567,7 +567,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ - M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must me done via disk config), but allows to bypass cache for some queries if intended", 0) \ + M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must be done via disk config), but allows to bypass cache for some queries if intended", 0) \ M(UInt64, filesystem_cache_max_wait_sec, 5, "Allow to wait at most this number of seconds for download of current remote_fs_buffer_size bytes, and skip cache if exceeded", 0) \ M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ M(Bool, enable_filesystem_cache_log, false, "Allows to record the filesystem caching log for each query", 0) \ From 78bd47d8df8fcc2344a01c4cf1d59beef13ab2d3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 30 May 2022 11:32:14 +0300 Subject: [PATCH 075/136] Fix excessive LIST requests to coordinator for transactions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In [1] there was only few transactions, but lots of List for /test/clickhouse/txn/log: $ clickhouse-local --format TSVWithNamesAndTypes --file zookeeper_log.tsv.gz -q "select * except('path|session_id|event_time|thread_id|event_date|xid') apply(x->groupUniqArray(x)), path, session_id, min(event_time), max(event_time), count() from table where has_watch and type = 'Request' group by path, session_id order by count() desc limit 1 format Vertical" Row 1: ────── groupUniqArray(type): ['Request'] groupUniqArray(query_id): ['','62d75128-9031-48a5-87ba-aec3f0b591c6'] groupUniqArray(address): ['::1'] groupUniqArray(port): [9181] groupUniqArray(has_watch): [1] groupUniqArray(op_num): ['List'] groupUniqArray(data): [''] groupUniqArray(is_ephemeral): [0] groupUniqArray(is_sequential): [0] groupUniqArray(version): [] groupUniqArray(requests_size): [0] groupUniqArray(request_idx): [0] groupUniqArray(error): [] groupUniqArray(watch_type): [] groupUniqArray(watch_state): [] groupUniqArray(stat_version): [0] groupUniqArray(stat_cversion): [0] groupUniqArray(stat_dataLength): [0] groupUniqArray(stat_numChildren): [0] groupUniqArray(children): [[]] path: /test/clickhouse/txn/log session_id: 1 min(event_time): 2022-05-27 12:54:09.025897 max(event_time): 2022-05-27 13:37:12.846314 + + + + + numbers + + numbers_mt(2000000) + + + + needle_like + + simple patterns, all unique + '%' || toString(number) || '_' + simple patterns, low distinctness (10 patterns) + '%' || toString(number % 10) || '_' + + + + needle_match + + + '.*' || toString(number) || '.' + + '.*' || toString(number % 10) || '.' + + '([a-zA-Z][a-zA-Z0-9]*)://([^ /]+)(/[^ ]*)?([^ @]+)@([^ @]+)([0-9][0-9]?)/([0-9][0-9]?)/([0-9][0-9]([0-9][0-9])?)(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])' || toString(number) + + '([a-zA-Z][a-zA-Z0-9]*)://([^ /]+)(/[^ ]*)?([^ @]+)@([^ @]+)([0-9][0-9]?)/([0-9][0-9]?)/([0-9][0-9]([0-9][0-9])?)(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9])' || toString(number % 10) + + + + + + + + + select toString(number) as haystack, like(haystack, '%x_') + from(select * from {numbers}) + + + + select toString(number) as haystack, match(haystack, '.*x.') + from(select * from {numbers}) + + + + + + select toString(number) as haystack, {needle_like} as needle, like(haystack, needle) + from (select * from {numbers}); + + + + select toString(number) as haystack, {needle_match} as needle, match(haystack, needle) + from (select * from {numbers}); + + + From 9d04305a5a8614d61437c0ca1b598fde4014dca1 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 30 May 2022 23:00:28 +0200 Subject: [PATCH 091/136] Update Settings.h --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9111e1d80da..df8bbf0a307 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -371,7 +371,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ \ - M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit on user level. If timeout is reached and memory is not freed, exception is thrown.", 0) \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \ \ M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ From 30f8eb800a9b2249462f2a72e07ddbae60ec876d Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 30 May 2022 22:29:35 +0000 Subject: [PATCH 092/136] optimize function coalesce with two arguments --- src/Functions/coalesce.cpp | 12 ++++++++++-- tests/performance/conditional.xml | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index 96aa110a489..b2b234df515 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -112,7 +112,6 @@ public: auto is_not_null = FunctionFactory::instance().get("isNotNull", context); auto assume_not_null = FunctionFactory::instance().get("assumeNotNull", context); - auto multi_if = FunctionFactory::instance().get("multiIf", context); ColumnsWithTypeAndName multi_if_args; ColumnsWithTypeAndName tmp_args(1); @@ -144,7 +143,16 @@ public: if (multi_if_args.size() == 1) return multi_if_args.front().column; - ColumnPtr res = multi_if->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count); + /// If there was only two arguments (3 arguments passed to multiIf) + /// use function "if" instead, because it's implemented more efficient. + /// TODO: make "multiIf" the same efficient. + FunctionOverloadResolverPtr if_function; + if (multi_if_args.size() == 3) + if_function = FunctionFactory::instance().get("if", context); + else + if_function = FunctionFactory::instance().get("multiIf", context); + + ColumnPtr res = if_function->build(multi_if_args)->execute(multi_if_args, result_type, input_rows_count); /// if last argument is not nullable, result should be also not nullable if (!multi_if_args.back().column->isNullable() && res->isNullable()) diff --git a/tests/performance/conditional.xml b/tests/performance/conditional.xml index 91b6cb95ff2..e1d15852767 100644 --- a/tests/performance/conditional.xml +++ b/tests/performance/conditional.xml @@ -8,4 +8,7 @@ SELECT count() FROM zeros(10000000) WHERE NOT ignore(multiIf(rand() % 2, toDateTime(rand()), toDate(rand()))) SELECT count() FROM zeros(10000000) WHERE NOT ignore(if(rand() % 2, [toDateTime(rand())], [toDate(rand())])) SELECT count() FROM zeros(10000000) WHERE NOT ignore(multiIf(rand() % 2, [toDateTime(rand())], [toDate(rand())])) + + SELECT count() FROM numbers(50000000) WHERE NOT ignore(COALESCE(toNullable(number), 0)) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(multiIf(number % 7 = 0, 1, number % 3 = 0, 2, number % 5 = 0, 3, number % 11 = 0, 4, 5)) From a2857491c42d560e882debe6aa2f5563079a7841 Mon Sep 17 00:00:00 2001 From: yaqi-zhao Date: Fri, 27 May 2022 13:54:11 -0400 Subject: [PATCH 093/136] add avx512 support for mergetreereader --- src/Common/TargetSpecific.cpp | 3 ++ src/Common/TargetSpecific.h | 1 + .../MergeTree/MergeTreeRangeReader.cpp | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/Common/TargetSpecific.cpp b/src/Common/TargetSpecific.cpp index 43319eff44b..369c21490d4 100644 --- a/src/Common/TargetSpecific.cpp +++ b/src/Common/TargetSpecific.cpp @@ -16,6 +16,8 @@ UInt32 getSupportedArchs() result |= static_cast(TargetArch::AVX2); if (Cpu::CpuFlagsCache::have_AVX512F) result |= static_cast(TargetArch::AVX512F); + if (Cpu::CpuFlagsCache::have_AVX512BW) + result |= static_cast(TargetArch::AVX512BW); return result; } @@ -34,6 +36,7 @@ String toString(TargetArch arch) case TargetArch::AVX: return "avx"; case TargetArch::AVX2: return "avx2"; case TargetArch::AVX512F: return "avx512f"; + case TargetArch::AVX512BW: return "avx512bw"; } __builtin_unreachable(); diff --git a/src/Common/TargetSpecific.h b/src/Common/TargetSpecific.h index d7fa55fbb08..522dd6e43c3 100644 --- a/src/Common/TargetSpecific.h +++ b/src/Common/TargetSpecific.h @@ -80,6 +80,7 @@ enum class TargetArch : UInt32 AVX = (1 << 1), AVX2 = (1 << 2), AVX512F = (1 << 3), + AVX512BW = (1 << 4), }; /// Runtime detection. diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index d8dba458203..84a1ab91906 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -10,6 +11,7 @@ #include #endif + namespace DB { namespace ErrorCodes @@ -449,6 +451,33 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con { size_t count = 0; +#if defined(__AVX512F__) && defined(__AVX512BW__) /// check if avx512 instructions are compiled + if (isArchSupported(TargetArch::AVX512BW)) { + /// check if cpu support avx512 dynamically, haveAVX512BW contains check of haveAVX512F + const __m512i zero64 = _mm512_setzero_epi32(); + while (end - begin >= 64) + { + end -= 64; + const auto * pos = end; + UInt64 val = static_cast(_mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(pos)), zero64, _MM_CMPINT_EQ)); + val = ~val; + if (val == 0) + { + count += 64; + } else + { + count += __builtin_clzll(val); + return count; + } + } + while (end > begin && *(--end) == 0) + { + ++count; + } + return count; + } +#endif + #if defined(__SSE2__) && defined(__POPCNT__) const __m128i zero16 = _mm_setzero_si128(); while (end - begin >= 64) From 7e95bf31b24eafe4955dc66ef4b6a7e6c86f6fe1 Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Tue, 31 May 2022 09:26:26 +0200 Subject: [PATCH 094/136] more verbose sanity checks --- programs/server/Server.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 75deeeb2eb5..17a9e641a57 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -547,7 +547,7 @@ static void sanityChecks(Server & server) try { if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos) - server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded."); + server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check /sys/devices/system/clocksource/clocksource0/current_clocksource"); } catch (...) { @@ -556,7 +556,7 @@ static void sanityChecks(Server & server) try { if (readNumber("/proc/sys/vm/overcommit_memory") == 2) - server.context()->addWarningMessage("Linux memory overcommit is disabled."); + server.context()->addWarningMessage("Linux memory overcommit is disabled. Check /proc/sys/vm/overcommit_memory"); } catch (...) { @@ -565,7 +565,7 @@ static void sanityChecks(Server & server) try { if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos) - server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\"."); + server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\". Check /sys/kernel/mm/transparent_hugepage/enabled"); } catch (...) { @@ -574,7 +574,7 @@ static void sanityChecks(Server & server) try { if (readNumber("/proc/sys/kernel/pid_max") < 30000) - server.context()->addWarningMessage("Linux max PID is too low."); + server.context()->addWarningMessage("Linux max PID is too low. Check /proc/sys/kernel/pid_max"); } catch (...) { @@ -583,7 +583,7 @@ static void sanityChecks(Server & server) try { if (readNumber("/proc/sys/kernel/threads-max") < 30000) - server.context()->addWarningMessage("Linux threads max count is too low."); + server.context()->addWarningMessage("Linux threads max count is too low. Check /proc/sys/kernel/threads-max"); } catch (...) { @@ -591,7 +591,7 @@ static void sanityChecks(Server & server) std::string dev_id = getBlockDeviceId(data_path); if (getBlockDeviceType(dev_id) == BlockDeviceType::ROT && getBlockDeviceReadAheadBytes(dev_id) == 0) - server.context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded."); + server.context()->addWarningMessage("Rotational disk with disabled readahead is in use. Performance can be degraded. Used for data: " + String(data_path)); #endif try From 557bb2d23586a805bd2e5357f359b505bf565a83 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 30 May 2022 10:50:26 +0200 Subject: [PATCH 095/136] Disable amqp-cpp and cassandra build if libuv is disabled On MacOS/GCC, the libuv build is disabled due to a compiler bug. This is now propagated to dependent libraries amqp-cpp and cassandra. Oddly enough, the Mac/GCC build was broken since at least Jan 2022 without someone noticing. --- contrib/amqpcpp-cmake/CMakeLists.txt | 5 +++++ contrib/cassandra-cmake/CMakeLists.txt | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index 974d097e06f..6e655d3c255 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -5,6 +5,11 @@ if (NOT ENABLE_AMQPCPP) return() endif() +if (NOT TARGET ch_contrib::uv) + message(STATUS "Not using AMQP-CPP because libuv is disabled") + return() +endif() + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") set (SRCS diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 81c1fab3882..986ac438bb2 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -5,6 +5,11 @@ if (NOT ENABLE_CASSANDRA) return() endif() +if (NOT TARGET ch_contrib::uv) + message(STATUS "Not using cassandra because libuv is disabled") + return() +endif() + if (APPLE) set(CMAKE_MACOSX_RPATH ON) endif() From 026e073b0bb25dec684c292fa149a3ea7fc57ecd Mon Sep 17 00:00:00 2001 From: Sergei Trifonov Date: Tue, 31 May 2022 13:50:09 +0200 Subject: [PATCH 096/136] minor improvement --- programs/server/Server.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 17a9e641a57..311a9c284da 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -546,8 +546,9 @@ static void sanityChecks(Server & server) #if defined(OS_LINUX) try { - if (readString("/sys/devices/system/clocksource/clocksource0/current_clocksource").find("tsc") == std::string::npos) - server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check /sys/devices/system/clocksource/clocksource0/current_clocksource"); + const char * filename = "/sys/devices/system/clocksource/clocksource0/current_clocksource"; + if (readString(filename).find("tsc") == std::string::npos) + server.context()->addWarningMessage("Linux is not using a fast TSC clock source. Performance can be degraded. Check " + String(filename)); } catch (...) { @@ -555,8 +556,9 @@ static void sanityChecks(Server & server) try { - if (readNumber("/proc/sys/vm/overcommit_memory") == 2) - server.context()->addWarningMessage("Linux memory overcommit is disabled. Check /proc/sys/vm/overcommit_memory"); + const char * filename = "/proc/sys/vm/overcommit_memory"; + if (readNumber(filename) == 2) + server.context()->addWarningMessage("Linux memory overcommit is disabled. Check " + String(filename)); } catch (...) { @@ -564,8 +566,9 @@ static void sanityChecks(Server & server) try { - if (readString("/sys/kernel/mm/transparent_hugepage/enabled").find("[always]") != std::string::npos) - server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\". Check /sys/kernel/mm/transparent_hugepage/enabled"); + const char * filename = "/sys/kernel/mm/transparent_hugepage/enabled"; + if (readString(filename).find("[always]") != std::string::npos) + server.context()->addWarningMessage("Linux transparent hugepages are set to \"always\". Check " + String(filename)); } catch (...) { @@ -573,8 +576,9 @@ static void sanityChecks(Server & server) try { - if (readNumber("/proc/sys/kernel/pid_max") < 30000) - server.context()->addWarningMessage("Linux max PID is too low. Check /proc/sys/kernel/pid_max"); + const char * filename = "/proc/sys/kernel/pid_max"; + if (readNumber(filename) < 30000) + server.context()->addWarningMessage("Linux max PID is too low. Check " + String(filename)); } catch (...) { @@ -582,8 +586,9 @@ static void sanityChecks(Server & server) try { - if (readNumber("/proc/sys/kernel/threads-max") < 30000) - server.context()->addWarningMessage("Linux threads max count is too low. Check /proc/sys/kernel/threads-max"); + const char * filename = "/proc/sys/kernel/threads-max"; + if (readNumber(filename) < 30000) + server.context()->addWarningMessage("Linux threads max count is too low. Check " + String(filename)); } catch (...) { From 582be423298958b72bfc7c88da90bc6a58ce9dc0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 31 May 2022 08:31:00 +0000 Subject: [PATCH 097/136] Wait for leader election --- tests/integration/test_keeper_force_recovery/test.py | 12 +++++++++++- .../test_keeper_force_recovery_single_node/test.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_keeper_force_recovery/test.py b/tests/integration/test_keeper_force_recovery/test.py index e4f42ba21f6..5f1b7d1e4e4 100644 --- a/tests/integration/test_keeper_force_recovery/test.py +++ b/tests/integration/test_keeper_force_recovery/test.py @@ -132,7 +132,17 @@ def test_cluster_recovery(started_cluster): nodes[0].stop_clickhouse() - add_data(node_zks[1], "/test_force_recovery_extra", "somedataextra") + # we potentially killed the leader node so we give time for election + for _ in range(100): + try: + node_zks[1] = get_fake_zk(nodes[1].name, timeout=30.0) + add_data(node_zks[1], "/test_force_recovery_extra", "somedataextra") + break + except Exception as ex: + time.sleep(0.5) + print(f"Retrying create on {nodes[1].name}, exception {ex}") + else: + raise Exception(f"Failed creating a node on {nodes[1].name}") for node_zk in node_zks[2:CLUSTER_SIZE]: wait_and_assert_data(node_zk, "/test_force_recovery_extra", "somedataextra") diff --git a/tests/integration/test_keeper_force_recovery_single_node/test.py b/tests/integration/test_keeper_force_recovery_single_node/test.py index 1e58a25221e..0a554e33119 100644 --- a/tests/integration/test_keeper_force_recovery_single_node/test.py +++ b/tests/integration/test_keeper_force_recovery_single_node/test.py @@ -121,7 +121,17 @@ def test_cluster_recovery(started_cluster): nodes[0].stop_clickhouse() - add_data(node_zks[1], "/test_force_recovery_extra", "somedataextra") + # we potentially killed the leader node so we give time for election + for _ in range(100): + try: + node_zks[1] = get_fake_zk(nodes[1].name, timeout=30.0) + add_data(node_zks[1], "/test_force_recovery_extra", "somedataextra") + break + except Exception as ex: + time.sleep(0.5) + print(f"Retrying create on {nodes[1].name}, exception {ex}") + else: + raise Exception(f"Failed creating a node on {nodes[1].name}") for node_zk in node_zks[2:CLUSTER_SIZE]: wait_and_assert_data(node_zk, "/test_force_recovery_extra", "somedataextra") From c2087b3145d53d6569578c2dff35f90981f30571 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 31 May 2022 14:38:11 +0200 Subject: [PATCH 098/136] Fix --- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 3 ++- tests/integration/test_storage_rabbitmq/test.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 678416ddd42..6d0a3f4ab6c 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -1154,7 +1154,8 @@ void registerStorageRabbitMQ(StorageFactory & factory) if (!with_named_collection && !args.storage_def->settings) throw Exception(ErrorCodes::BAD_ARGUMENTS, "RabbitMQ engine must have settings"); - rabbitmq_settings->loadFromQuery(*args.storage_def); + if (args.storage_def->settings) + rabbitmq_settings->loadFromQuery(*args.storage_def); if (!rabbitmq_settings->rabbitmq_host_port.changed && !rabbitmq_settings->rabbitmq_address.changed) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index b18d9c26d88..c1bd136126f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -27,6 +27,7 @@ instance = cluster.add_instance( ], user_configs=["configs/users.xml"], with_rabbitmq=True, + stay_alive=True, ) @@ -2732,6 +2733,16 @@ def test_rabbitmq_predefined_configuration(rabbitmq_cluster): ) if result == "1\t2\n": break + instance.restart_clickhouse() + channel.basic_publish( + exchange="named", routing_key="", body=json.dumps({"key": 1, "value": 2}) + ) + while True: + result = instance.query( + "SELECT * FROM test.rabbitmq ORDER BY key", ignore_error=True + ) + if result == "1\t2\n": + break if __name__ == "__main__": From 4b427336e3654f0880cb510feb544f01c7b33c23 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 31 May 2022 09:37:34 -0400 Subject: [PATCH 099/136] tests with overridden and appended parameters --- tests/integration/test_storage_dict/configs/conf.xml | 9 +++++++++ tests/integration/test_storage_dict/test.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/integration/test_storage_dict/configs/conf.xml b/tests/integration/test_storage_dict/configs/conf.xml index c2ecb518884..e37ca358e63 100644 --- a/tests/integration/test_storage_dict/configs/conf.xml +++ b/tests/integration/test_storage_dict/configs/conf.xml @@ -12,5 +12,14 @@ /test_dict TabSeparated + + http://nginx:80/ + /test_dict + + + http://nginx:80/ + /test_dict + CSV + diff --git a/tests/integration/test_storage_dict/test.py b/tests/integration/test_storage_dict/test.py index a5270a42114..1ed974f267d 100644 --- a/tests/integration/test_storage_dict/test.py +++ b/tests/integration/test_storage_dict/test.py @@ -32,3 +32,15 @@ def test_storage_dict(cluster): ) result = node1.query(f"select * from dict") assert result.strip() == "foo\tbar" + + node1.query( + f"create dictionary dict1 (k String, v String) primary key k source(http(name urldict1 format TabSeparated)) layout(complex_key_hashed()) lifetime(min 0 max 100)" + ) + result = node1.query(f"select * from dict1") + assert result.strip() == "foo\tbar" + + node1.query( + f"create dictionary dict2 (k String, v String) primary key k source(http(name urldict2 format TabSeparated)) layout(complex_key_hashed()) lifetime(min 0 max 100)" + ) + result = node1.query(f"select * from dict2") + assert result.strip() == "foo\tbar" From 792adb05767a8010d40ae98991d1d25673f2be9c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 31 May 2022 13:53:45 +0000 Subject: [PATCH 100/136] Update jepsen and scp --- tests/jepsen.clickhouse-keeper/project.clj | 2 +- .../src/jepsen/clickhouse_keeper/db.clj | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/jepsen.clickhouse-keeper/project.clj b/tests/jepsen.clickhouse-keeper/project.clj index a168ef6920d..187e91cd44d 100644 --- a/tests/jepsen.clickhouse-keeper/project.clj +++ b/tests/jepsen.clickhouse-keeper/project.clj @@ -7,7 +7,7 @@ :main jepsen.clickhouse-keeper.main :plugins [[lein-cljfmt "0.7.0"]] :dependencies [[org.clojure/clojure "1.10.1"] - [jepsen "0.2.3"] + [jepsen "0.2.6"] [zookeeper-clj "0.9.4"] [org.apache.zookeeper/zookeeper "3.6.1" :exclusions [org.slf4j/slf4j-log4j12]]] :repl-options {:init-ns jepsen.clickhouse-keeper.main}) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index 0df704f0d4a..39d4cac8521 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -10,6 +10,25 @@ [jepsen.control.util :as cu] [jepsen.os.ubuntu :as ubuntu])) + +(ns jepsen.control.scp) + +(defn scp! + "Runs an SCP command by shelling out. Takes a conn-spec (used for port, key, + etc), a seq of sources, and a single destination, all as strings." + [conn-spec sources dest] + (apply util/sh "scp" "-rpC" + "-P" (str (:port conn-spec)) + (concat (when-let [k (:private-key-path conn-spec)] + ["-i" k]) + (if-not (:strict-host-key-checking conn-spec) + ["-o StrictHostKeyChecking=no"]) + sources + [dest])) + nil) + +(ns jepsen.clickhouse-keeper.db) + (defn get-clickhouse-url [url] (non-precise-cached-wget! url)) From 3e71a716f53862186a318c5357f01fb6e8eccb8b Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 31 May 2022 13:55:01 +0000 Subject: [PATCH 101/136] Enable only jepsen tests --- .github/workflows/pull_request.yml | 3195 +--------------------------- 1 file changed, 1 insertion(+), 3194 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f6e9880d088..375d3e1cd6e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -109,196 +109,8 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json - StyleCheck: - needs: DockerHubPush - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{ runner.temp }}/style_check - EOF - - name: Download changed images - # even if artifact does not exist, e.g. on `do not test` label or failed Docker job - continue-on-error: true - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Style Check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 style_check.py - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FastTest: - needs: DockerHubPush - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/fasttest - REPO_COPY=${{runner.temp}}/fasttest/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - EOF - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" - mkdir "$GITHUB_WORKSPACE" - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.TEMP_PATH }} - - name: Fast Test - run: | - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - CompatibilityCheck: - needs: [BuilderDebRelease] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/compatibility_check - REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: CompatibilityCheck - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - SplitBuildSmokeTest: - needs: [BuilderDebSplitted] - runs-on: [self-hosted, style-checker] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/split_build_check - REPO_COPY=${{runner.temp}}/split_build_check/ClickHouse - REPORTS_PATH=${{runner.temp}}/reports_dir - EOF - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Split build check - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 split_build_smoke_check.py - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -######################################################################################### -#################################### ORDINARY BUILDS #################################### -######################################################################################### - BuilderDebRelease: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_release - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - fetch-depth: 0 # for performance artifact - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" BuilderBinRelease: - needs: [DockerHubPush, FastTest] + needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: - name: Set envs @@ -342,3014 +154,9 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - # BuilderBinGCC: - # needs: [DockerHubPush, FastTest] - # runs-on: [self-hosted, builder] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/build_check - # IMAGES_PATH=${{runner.temp}}/images_path - # REPO_COPY=${{runner.temp}}/build_check/ClickHouse - # CACHES_PATH=${{runner.temp}}/../ccaches - # BUILD_NAME=binary_gcc - # EOF - # - name: Download changed images - # uses: actions/download-artifact@v2 - # with: - # name: changed_images - # path: ${{ runner.temp }}/images_path - # - name: Clear repository - # run: | - # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - # - name: Check out repository code - # uses: actions/checkout@v2 - # - name: Build - # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - # - name: Upload build URLs to artifacts - # if: ${{ success() || failure() }} - # uses: actions/upload-artifact@v2 - # with: - # name: ${{ env.BUILD_URLS }} - # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAarch64: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ runner.temp }}/images_path - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - fetch-depth: 0 # for performance artifact - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebAsan: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_asan - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebUBsan: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_ubsan - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebTsan: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_tsan - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebMsan: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_msan - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderDebDebug: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=package_debug - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -########################################################################################## -##################################### SPECIAL BUILDS ##################################### -########################################################################################## - BuilderDebSplitted: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_splitted - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinTidy: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_tidy - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwin: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinAarch64: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinFreeBSD: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_freebsd - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinDarwinAarch64: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_darwin_aarch64 - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" - BuilderBinPPC64: - needs: [DockerHubPush, FastTest] - runs-on: [self-hosted, builder] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse - CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_ppc64le - EOF - - name: Download changed images - uses: actions/download-artifact@v2 - with: - name: changed_images - path: ${{ env.IMAGES_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build - run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" - - name: Upload build URLs to artifacts - if: ${{ success() || failure() }} - uses: actions/upload-artifact@v2 - with: - name: ${{ env.BUILD_URLS }} - path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" -############################################################################################ -##################################### Docker images ####################################### -############################################################################################ - DockerServerImages: - needs: - - BuilderDebRelease - - BuilderDebAarch64 - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - with: - fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself - - name: Check docker clickhouse/clickhouse-server building - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push - python3 docker_server.py --release-type head --no-push --no-ubuntu \ - --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################ -##################################### BUILD REPORTER ####################################### -############################################################################################ - BuilderReport: - needs: - - BuilderBinRelease - - BuilderDebAarch64 - - BuilderDebAsan - - BuilderDebDebug - - BuilderDebMsan - - BuilderDebRelease - - BuilderDebTsan - - BuilderDebUBsan - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - CHECK_NAME=ClickHouse build check (actions) - REPORTS_PATH=${{runner.temp}}/reports_dir - TEMP_PATH=${{runner.temp}}/report_check - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - BuilderSpecialReport: - needs: - - BuilderBinAarch64 - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - - BuilderBinFreeBSD - # - BuilderBinGCC - - BuilderBinPPC64 - - BuilderBinTidy - - BuilderDebSplitted - runs-on: [self-hosted, style-checker] - if: ${{ success() || failure() }} - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/report_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=ClickHouse special build check (actions) - NEEDS_DATA_PATH=${{runner.temp}}/needs.json - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Report Builder - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cat > "$NEEDS_DATA_PATH" << 'EOF' - ${{ toJSON(needs) }} - EOF - cd "$GITHUB_WORKSPACE/tests/ci" - python3 build_report_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -########################### FUNCTIONAl STATELESS TESTS ####################################### -############################################################################################## - FunctionalStatelessTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, actions) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseDatabaseReplicated1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_database_replicated - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) - REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseWideParts: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_wide_parts - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, wide parts enabled, actions) - REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestReleaseS3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_s3_storage - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (release, s3 storage, actions) - REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (aarch64, actions) - REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (address, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (address, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (thread, actions) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (thread, actions) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (thread, actions) - REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (ubsan, actions) - REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse - KILL_TIMEOUT=10800 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan0: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (memory, actions) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan1: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (memory, actions) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestMsan2: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (memory, actions) - REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug0: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug1: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestDebug2: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests (debug, actions) - REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse - KILL_TIMEOUT=10800 - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_flaky_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests flaky check (address, actions) - REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - TestsBugfixCheck: - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/tests_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Tests bugfix validate check (actions) - KILL_TIMEOUT=3600 - REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Bugfix test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - - TEMP_PATH="${TEMP_PATH}/integration" \ - REPORTS_PATH="${REPORTS_PATH}/integration" \ - python3 integration_test_check.py "Integration tests bugfix validate check" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - TEMP_PATH="${TEMP_PATH}/stateless" \ - REPORTS_PATH="${REPORTS_PATH}/stateless" \ - python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ - --validate-bugfix --post-commit-status=file || echo 'ignore exit code' - - python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -############################ FUNCTIONAl STATEFUL TESTS ####################################### -############################################################################################## - FunctionalStatefulTestRelease: - needs: [BuilderDebRelease] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (release, actions) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAarch64: - needs: [BuilderDebAarch64] - runs-on: [self-hosted, func-tester-aarch64] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (aarch64, actions) - REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (address, actions) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (thread, actions) - REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (memory, actions) - REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (ubsan, actions) - REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - FunctionalStatefulTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateful_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateful tests (debug, actions) - REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse - KILL_TIMEOUT=3600 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Functional test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -######################################### STRESS TESTS ####################################### -############################################################################################## - StressTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (address, actions) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - StressTestTsan: - needs: [BuilderDebTsan] - # func testers have 16 cores + 128 GB memory - # while stress testers have 36 cores + 72 memory - # It would be better to have something like 32 + 128, - # but such servers almost unavailable as spot instances. - runs-on: [self-hosted, func-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_thread - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (thread, actions) - REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - StressTestMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_memory - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (memory, actions) - REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - StressTestUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_undefined - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (undefined, actions) - REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - StressTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stress_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stress test (debug, actions) - REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Stress test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 stress_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################## -##################################### AST FUZZERS ############################################ -############################################################################################## - ASTFuzzerTestAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (ASan, actions) - REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (TSan, actions) - REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestUBSan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (UBSan, actions) - REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestMSan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (MSan, actions) - REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - ASTFuzzerTestDebug: - needs: [BuilderDebDebug] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=AST fuzzer (debug, actions) - REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Fuzzer - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 ast_fuzzer_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -############################# INTEGRATION TESTS ############################################# -############################################################################################# - IntegrationTestsAsan0: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan1: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsAsan2: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=3 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan0: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (thread, actions) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan1: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (thread, actions) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan2: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (thread, actions) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsTsan3: - needs: [BuilderDebTsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (thread, actions) - REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release, actions) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsRelease1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_release - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests (release, actions) - REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=2 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - IntegrationTestsFlakyCheck: - needs: [BuilderDebAsan] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests flaky check (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### UNIT TESTS ############################################# -############################################################################################# - UnitTestsAsan: - needs: [BuilderDebAsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (asan, actions) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsReleaseClang: - needs: [BuilderBinRelease] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_asan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (release-clang, actions) - REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - # UnitTestsReleaseGCC: - # needs: [BuilderBinGCC] - # runs-on: [self-hosted, fuzzer-unit-tester] - # steps: - # - name: Set envs - # run: | - # cat >> "$GITHUB_ENV" << 'EOF' - # TEMP_PATH=${{runner.temp}}/unit_tests_asan - # REPORTS_PATH=${{runner.temp}}/reports_dir - # CHECK_NAME=Unit tests (release-gcc, actions) - # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse - # EOF - # - name: Download json reports - # uses: actions/download-artifact@v2 - # with: - # path: ${{ env.REPORTS_PATH }} - # - name: Clear repository - # run: | - # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - # - name: Check out repository code - # uses: actions/checkout@v2 - # - name: Unit test - # run: | - # sudo rm -fr "$TEMP_PATH" - # mkdir -p "$TEMP_PATH" - # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - # cd "$REPO_COPY/tests/ci" - # python3 unit_tests_check.py "$CHECK_NAME" - # - name: Cleanup - # if: always() - # run: | - # # shellcheck disable=SC2046 - # docker kill $(docker ps -q) ||: - # # shellcheck disable=SC2046 - # docker rm -f $(docker ps -a -q) ||: - # sudo rm -fr "$TEMP_PATH" - UnitTestsTsan: - needs: [BuilderDebTsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_tsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (tsan, actions) - REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsMsan: - needs: [BuilderDebMsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_msan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (msan, actions) - REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - UnitTestsUBsan: - needs: [BuilderDebUBsan] - runs-on: [self-hosted, fuzzer-unit-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/unit_tests_ubsan - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Unit tests (ubsan, actions) - REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Unit test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 unit_tests_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" -############################################################################################# -#################################### PERFORMANCE TESTS ###################################### -############################################################################################# - PerformanceComparison0: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison (actions) - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=0 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparison1: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison (actions) - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=1 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparison2: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison (actions) - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=2 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" - PerformanceComparison3: - needs: [BuilderDebRelease] - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/performance_comparison - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Performance Comparison (actions) - REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse - RUN_BY_HASH_NUM=3 - RUN_BY_HASH_TOTAL=4 - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Performance Comparison - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 performance_comparison_check.py "$CHECK_NAME" - - name: Cleanup - if: always() - run: | - # shellcheck disable=SC2046 - docker kill $(docker ps -q) ||: - # shellcheck disable=SC2046 - docker rm -f $(docker ps -a -q) ||: - sudo rm -fr "$TEMP_PATH" ############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# Jepsen: needs: [BuilderBinRelease] uses: ./.github/workflows/jepsen.yml - - FinishCheck: - needs: - - StyleCheck - - DockerHubPush - - DockerServerImages - - CheckLabels - - BuilderReport - - FastTest - - FunctionalStatelessTestDebug0 - - FunctionalStatelessTestDebug1 - - FunctionalStatelessTestDebug2 - - FunctionalStatelessTestRelease - - FunctionalStatelessTestReleaseDatabaseReplicated0 - - FunctionalStatelessTestReleaseDatabaseReplicated1 - - FunctionalStatelessTestReleaseWideParts - - FunctionalStatelessTestAarch64 - - FunctionalStatelessTestAsan0 - - FunctionalStatelessTestAsan1 - - FunctionalStatelessTestTsan0 - - FunctionalStatelessTestTsan1 - - FunctionalStatelessTestTsan2 - - FunctionalStatelessTestMsan0 - - FunctionalStatelessTestMsan1 - - FunctionalStatelessTestMsan2 - - FunctionalStatelessTestUBsan - - FunctionalStatefulTestDebug - - FunctionalStatefulTestRelease - - FunctionalStatefulTestAarch64 - - FunctionalStatefulTestAsan - - FunctionalStatefulTestTsan - - FunctionalStatefulTestMsan - - FunctionalStatefulTestUBsan - - FunctionalStatelessTestReleaseS3 - - StressTestDebug - - StressTestAsan - - StressTestTsan - - StressTestMsan - - StressTestUBsan - - ASTFuzzerTestDebug - - ASTFuzzerTestAsan - - ASTFuzzerTestTsan - - ASTFuzzerTestMSan - - ASTFuzzerTestUBSan - - IntegrationTestsAsan0 - - IntegrationTestsAsan1 - - IntegrationTestsAsan2 - - IntegrationTestsRelease0 - - IntegrationTestsRelease1 - - IntegrationTestsTsan0 - - IntegrationTestsTsan1 - - IntegrationTestsTsan2 - - IntegrationTestsTsan3 - - PerformanceComparison0 - - PerformanceComparison1 - - PerformanceComparison2 - - PerformanceComparison3 - - UnitTestsAsan - - UnitTestsTsan - - UnitTestsMsan - - UnitTestsUBsan - - UnitTestsReleaseClang - - SplitBuildSmokeTest - - CompatibilityCheck - - IntegrationTestsFlakyCheck - - Jepsen - runs-on: [self-hosted, style-checker] - steps: - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Finish label - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 finish_check.py From 49f815060aaa573d7b4534c957cbc8ce0daf7723 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 31 May 2022 15:18:44 +0000 Subject: [PATCH 102/136] Use tar for logs --- .../src/jepsen/clickhouse_keeper/db.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index 39d4cac8521..fcc28a5a272 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -155,12 +155,12 @@ (info node "Coordination files exists, going to compress") (c/cd data-dir (c/exec :tar :czf "coordination.tar.gz" "coordination")))) - (if (cu/exists? (str logs-dir "/clickhouse-keeper.log")) + (if (cu/exists? (str logs-dir)) (do - (info node "Clickhouse logs exist, going to compress") - (c/cd logs-dir - (c/exec :gzip "clickhouse-keeper.log"))) (info node "Logs are missing"))) - (let [common-logs [stderr-file (str logs-dir "/clickhouse-keeper.log.gz") (str data-dir "/coordination.tar.gz")] + (info node "Logs exist, going to compress") + (c/cd common-prefix + (c/exec :tar :czf "logs.tar.gz" "logs"))) (info node "Logs are missing"))) + (let [common-logs [(str common-prefix "/logs.tar.gz") (str data-dir "/coordination.tar.gz")] gdb-log (str logs-dir "/gdb.log")] (if (cu/exists? (str logs-dir "/gdb.log")) (conj common-logs gdb-log) From 544be03b41bb5368bdc65c9d9d3011033d1465e3 Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Tue, 31 May 2022 11:46:53 -0400 Subject: [PATCH 103/136] Update caches.md spelling --- docs/en/operations/caches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index f2427810184..27e81256315 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -5,7 +5,7 @@ sidebar_label: Caches # Cache Types {#cache-types} -When performing queries, ClichHouse uses different caches. +When performing queries, ClickHouse uses different caches. Main cache types: From 2476c6a988603ed5ba3a56692e640253455f8d70 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 25 May 2022 14:13:40 +0000 Subject: [PATCH 104/136] Fix error on joining with dictionary on some conditions --- src/Interpreters/HashJoin.cpp | 7 ++++++- src/Interpreters/TableJoin.cpp | 7 +++++++ .../queries/0_stateless/01391_join_on_dict_crash.reference | 1 + tests/queries/0_stateless/01391_join_on_dict_crash.sql | 4 ++-- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 521fbe139e7..1f3e7af0f45 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -538,6 +538,7 @@ void HashJoin::dataMapInit(MapsVariant & map) bool HashJoin::overDictionary() const { + assert(data->type != Type::DICT || table_join->getDictionaryReader()); return data->type == Type::DICT; } @@ -910,13 +911,14 @@ public: AddedColumns( const Block & block_with_columns_to_add, const Block & block, - const Block & saved_block_sample, + const Block & saved_block_sample_, const HashJoin & join, std::vector && join_on_keys_, bool is_asof_join, bool is_join_get_) : join_on_keys(join_on_keys_) , rows_to_add(block.rows()) + , saved_block_sample(saved_block_sample_) , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); @@ -959,6 +961,8 @@ public: template void appendFromBlock(const Block & block, size_t row_num) { + assertBlocksHaveEqualStructure(saved_block_sample, block, "appendFromBlock"); + if constexpr (has_defaults) applyLazyDefaults(); @@ -1024,6 +1028,7 @@ private: size_t lazy_defaults_count = 0; /// for ASOF const IColumn * left_asof_key = nullptr; + Block saved_block_sample; bool is_join_get; diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 10a27b9efc5..87502e5965e 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -479,6 +479,13 @@ bool TableJoin::tryInitDictJoin(const Block & sample_block, ContextPtr context) src_names.push_back(original); dst_columns.push_back({col.name, col.type}); } + else + { + /// Can't extract column from dictionary table + /// TODO: Sometimes it should be possible to recunstruct required column, + /// e.g. if it's an expression depending on dictionary attributes + return false; + } } dictionary_reader = std::make_shared(dict_name, src_names, dst_columns, context); diff --git a/tests/queries/0_stateless/01391_join_on_dict_crash.reference b/tests/queries/0_stateless/01391_join_on_dict_crash.reference index 573541ac970..aa47d0d46d4 100644 --- a/tests/queries/0_stateless/01391_join_on_dict_crash.reference +++ b/tests/queries/0_stateless/01391_join_on_dict_crash.reference @@ -1 +1,2 @@ 0 +0 diff --git a/tests/queries/0_stateless/01391_join_on_dict_crash.sql b/tests/queries/0_stateless/01391_join_on_dict_crash.sql index ac3ca94ddf6..854da04b334 100644 --- a/tests/queries/0_stateless/01391_join_on_dict_crash.sql +++ b/tests/queries/0_stateless/01391_join_on_dict_crash.sql @@ -20,8 +20,8 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' DB 'db_01391' t LIFETIME(MIN 1 MAX 1) LAYOUT(HASHED()); -select click_country_id from t cc -left join d on toUInt32(d.id) = cc.click_city_id; +SELECT click_country_id FROM t AS cc LEFT JOIN d ON toUInt32(d.id) = cc.click_city_id; +SELECT click_country_id FROM t AS cc LEFT JOIN d ON d.country_id < 99 AND d.id = cc.click_city_id; DROP DICTIONARY d; DROP TABLE t; From 7f4ddb16670ec6c0943e700071ab9ed0e6377abe Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 30 May 2022 12:59:47 +0000 Subject: [PATCH 105/136] Fix assert for 02244_lowcardinality_hash_join --- src/Interpreters/HashJoin.cpp | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 1f3e7af0f45..c52c01930a4 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -911,14 +911,14 @@ public: AddedColumns( const Block & block_with_columns_to_add, const Block & block, - const Block & saved_block_sample_, + const Block & saved_block_sample, const HashJoin & join, std::vector && join_on_keys_, bool is_asof_join, bool is_join_get_) : join_on_keys(join_on_keys_) , rows_to_add(block.rows()) - , saved_block_sample(saved_block_sample_) + , sample_block(block_with_columns_to_add) , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); @@ -959,24 +959,27 @@ public: } template - void appendFromBlock(const Block & block, size_t row_num) + void appendFromBlock(Block block, size_t row_num) { - assertBlocksHaveEqualStructure(saved_block_sample, block, "appendFromBlock"); - if constexpr (has_defaults) applyLazyDefaults(); + for (size_t j = 0, size = right_indexes.size(); j < size; ++j) + { + ColumnWithTypeAndName & column_from_block = block.getByPosition(right_indexes[j]); + if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) + { + JoinCommon::changeLowCardinalityInplace(column_from_block); + } + } + assertBlocksHaveEqualStructure(sample_block, block, "appendFromBlock"); + if (is_join_get) { /// If it's joinGetOrNull, we need to wrap not-nullable columns in StorageJoin. for (size_t j = 0, size = right_indexes.size(); j < size; ++j) { - auto column_from_block = block.getByPosition(right_indexes[j]); - if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) - { - JoinCommon::changeLowCardinalityInplace(column_from_block); - } - + const auto & column_from_block = block.getByPosition(right_indexes[j]); if (auto * nullable_col = typeid_cast(columns[j].get()); nullable_col && !column_from_block.column->isNullable()) nullable_col->insertFromNotNullable(*column_from_block.column, row_num); @@ -988,11 +991,7 @@ public: { for (size_t j = 0, size = right_indexes.size(); j < size; ++j) { - auto column_from_block = block.getByPosition(right_indexes[j]); - if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) - { - JoinCommon::changeLowCardinalityInplace(column_from_block); - } + const auto & column_from_block = block.getByPosition(right_indexes[j]); columns[j]->insertFrom(*column_from_block.column, row_num); } } @@ -1028,7 +1027,7 @@ private: size_t lazy_defaults_count = 0; /// for ASOF const IColumn * left_asof_key = nullptr; - Block saved_block_sample; + Block sample_block; bool is_join_get; From 673bc84bfc4b70a3163c74d45cd63746cb92d1f4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 30 May 2022 12:59:58 +0000 Subject: [PATCH 106/136] Reformat 02244_lowcardinality_hash_join --- .../02244_lowcardinality_hash_join.sql | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/02244_lowcardinality_hash_join.sql b/tests/queries/0_stateless/02244_lowcardinality_hash_join.sql index f2a601adf06..4ea949e4255 100644 --- a/tests/queries/0_stateless/02244_lowcardinality_hash_join.sql +++ b/tests/queries/0_stateless/02244_lowcardinality_hash_join.sql @@ -8,20 +8,16 @@ CREATE TABLE lc_table INSERT INTO lc_table VALUES('x'); -SELECT * -FROM lc_table -INNER JOIN lc_table AS lc_table2 ON lc_table.col = lc_table2.col; +SELECT * FROM lc_table INNER JOIN lc_table AS lc_table2 +ON lc_table.col = lc_table2.col; -SELECT * -FROM lc_table -INNER JOIN lc_table AS lc_table2 ON CAST(lc_table.col AS String) = CAST(lc_table2.col AS String); +SELECT * FROM lc_table INNER JOIN lc_table AS lc_table2 +ON CAST(lc_table.col AS String) = CAST(lc_table2.col AS String); -SELECT * -FROM lc_table -INNER JOIN lc_table AS lc_table2 ON (lc_table.col = lc_table2.col) OR (lc_table.col = lc_table2.col); +SELECT * FROM lc_table INNER JOIN lc_table AS lc_table2 +ON (lc_table.col = lc_table2.col) OR (lc_table.col = lc_table2.col); -SELECT * -FROM lc_table -INNER JOIN lc_table AS lc_table2 ON (CAST(lc_table.col AS String) = CAST(lc_table2.col AS String)) OR (CAST(lc_table.col AS String) = CAST(lc_table2.col AS String)); +SELECT * FROM lc_table INNER JOIN lc_table AS lc_table2 +ON (CAST(lc_table.col AS String) = CAST(lc_table2.col AS String)) OR (CAST(lc_table.col AS String) = CAST(lc_table2.col AS String)); DROP TABLE IF EXISTS lc_table; From e7be677fca158279578f6712dc1f407d21c3879e Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 30 May 2022 15:57:10 +0000 Subject: [PATCH 107/136] Assert structure match up to locard in appendFromBlock --- src/Interpreters/HashJoin.cpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index c52c01930a4..9f770b5cd46 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -918,7 +918,7 @@ public: bool is_join_get_) : join_on_keys(join_on_keys_) , rows_to_add(block.rows()) - , sample_block(block_with_columns_to_add) + , sample_block(saved_block_sample) , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); @@ -958,6 +958,33 @@ public: return ColumnWithTypeAndName(std::move(columns[i]), type_name[i].type, type_name[i].qualified_name); } + static void assertBlockEqualsStructureUpToLowCard(const Block & lhs_block, const Block & rhs_block) + { + if (lhs_block.columns() != rhs_block.columns()) + throw Exception("Different number of columns in blocks", ErrorCodes::LOGICAL_ERROR); + + for (size_t i = 0; i < lhs_block.columns(); ++i) + { + const auto & lhs = lhs_block.getByPosition(i); + const auto & rhs = rhs_block.getByPosition(i); + if (lhs.name != rhs.name) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]", + lhs_block.dumpStructure(), rhs_block.dumpStructure()); + + const auto & ltype = recursiveRemoveLowCardinality(lhs.type); + const auto & rtype = recursiveRemoveLowCardinality(rhs.type); + if (!ltype->equals(*rtype)) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]", + lhs_block.dumpStructure(), rhs_block.dumpStructure()); + + const auto & lcol = recursiveRemoveLowCardinality(lhs.column); + const auto & rcol = recursiveRemoveLowCardinality(rhs.column); + if (lcol->getDataType() != rcol->getDataType()) + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Block structure mismatch: [{}] != [{}]", + lhs_block.dumpStructure(), rhs_block.dumpStructure()); + } + } + template void appendFromBlock(Block block, size_t row_num) { @@ -972,7 +999,9 @@ public: JoinCommon::changeLowCardinalityInplace(column_from_block); } } - assertBlocksHaveEqualStructure(sample_block, block, "appendFromBlock"); + + /// Like assertBlocksHaveEqualStructure but doesn't check low cardinality + assertBlockEqualsStructureUpToLowCard(sample_block, block); if (is_join_get) { From ca0bd754b59897878924dcdc11ee0f5449fc5449 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 30 May 2022 15:58:57 +0000 Subject: [PATCH 108/136] Add no-backward-compatibility-check to 01391_join_on_dict_crash.sql --- tests/queries/0_stateless/01391_join_on_dict_crash.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01391_join_on_dict_crash.sql b/tests/queries/0_stateless/01391_join_on_dict_crash.sql index 854da04b334..13ebd080621 100644 --- a/tests/queries/0_stateless/01391_join_on_dict_crash.sql +++ b/tests/queries/0_stateless/01391_join_on_dict_crash.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-backward-compatibility-check DROP DATABASE IF EXISTS db_01391; CREATE DATABASE db_01391; From c5ac6294ae7d75c70f6691746e9c78fdcb2afc69 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 May 2022 16:16:13 +0000 Subject: [PATCH 109/136] Display entires for failed tests at the top of report --- tests/ci/report.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ci/report.py b/tests/ci/report.py index c8bd32bf642..83d89e628a2 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -165,6 +165,11 @@ def create_test_html_report( num_fails = 0 has_test_time = False has_test_logs = False + + if with_raw_logs: + # Display entires with logs at the top (they correspond to failed tests) + test_result.sort(key=lambda result: len(result) <= 3) + for result in test_result: test_name = result[0] test_status = result[1] From b11749ca2c97700c2c6d1c96f1e1b6ad8c190065 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 31 May 2022 16:45:29 +0000 Subject: [PATCH 110/136] Make GROUPING function skip constant folding --- src/Interpreters/ActionsVisitor.cpp | 3 ++ .../02315_grouping_constant_folding.reference | 29 +++++++++++++++++++ .../02315_grouping_constant_folding.sql | 13 +++++++++ 3 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/02315_grouping_constant_folding.reference create mode 100644 tests/queries/0_stateless/02315_grouping_constant_folding.sql diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 99d2217cba3..eacdd221d6e 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -839,6 +839,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (node.name == "grouping") { + if (data.only_consts) + return; // Can not perform constant folding, because this function can be executed only after GROUP BY + size_t arguments_size = node.arguments->children.size(); if (arguments_size == 0) throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, "Function GROUPING expects at least one argument"); diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference new file mode 100644 index 00000000000..5aa979b1453 --- /dev/null +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference @@ -0,0 +1,29 @@ +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); +1 0 0 3 +1 0 2 3 +1 0 4 3 +1 0 6 3 +1 0 8 3 +1 1 1 3 +1 1 3 3 +1 1 5 3 +1 1 7 3 +1 1 9 3 +5 0 0 2 +5 1 0 2 +10 0 0 0 +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); +1 0 0 3 +1 0 2 3 +1 0 4 3 +1 0 6 3 +1 0 8 3 +1 1 1 3 +1 1 3 3 +1 1 5 3 +1 1 7 3 +1 1 9 3 +5 0 0 2 +5 1 0 2 +10 0 0 0 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql new file mode 100644 index 00000000000..c4ef087a308 --- /dev/null +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS test02315; + +CREATE TABLE test02315(a UInt64, b UInt64) ENGINE=MergeTree() ORDER BY (a, b); + +INSERT INTO test02315 SELECT number % 2 as a, number as b FROM numbers(10); + +-- { echoOn } +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a), ()) ORDER BY (amount, a, b); + +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b); + +-- { echoOff } +DROP TABLE test02315; From 284c9bc68bddab2badad7a699bb552bb28808663 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 31 May 2022 16:02:06 +0000 Subject: [PATCH 111/136] Rollback some changes from appendFromBlock --- src/Interpreters/HashJoin.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 9f770b5cd46..07151b2bb1b 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -986,29 +986,29 @@ public: } template - void appendFromBlock(Block block, size_t row_num) + void appendFromBlock(const Block & block, size_t row_num) { if constexpr (has_defaults) applyLazyDefaults(); - for (size_t j = 0, size = right_indexes.size(); j < size; ++j) - { - ColumnWithTypeAndName & column_from_block = block.getByPosition(right_indexes[j]); - if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) - { - JoinCommon::changeLowCardinalityInplace(column_from_block); - } - } - +#ifndef NDEBUG /// Like assertBlocksHaveEqualStructure but doesn't check low cardinality assertBlockEqualsStructureUpToLowCard(sample_block, block); +#else + UNUSED(assertBlockEqualsStructureUpToLowCard); +#endif if (is_join_get) { /// If it's joinGetOrNull, we need to wrap not-nullable columns in StorageJoin. for (size_t j = 0, size = right_indexes.size(); j < size; ++j) { - const auto & column_from_block = block.getByPosition(right_indexes[j]); + auto column_from_block = block.getByPosition(right_indexes[j]); + if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) + { + JoinCommon::changeLowCardinalityInplace(column_from_block); + } + if (auto * nullable_col = typeid_cast(columns[j].get()); nullable_col && !column_from_block.column->isNullable()) nullable_col->insertFromNotNullable(*column_from_block.column, row_num); @@ -1020,7 +1020,11 @@ public: { for (size_t j = 0, size = right_indexes.size(); j < size; ++j) { - const auto & column_from_block = block.getByPosition(right_indexes[j]); + auto column_from_block = block.getByPosition(right_indexes[j]); + if (type_name[j].type->lowCardinality() != column_from_block.type->lowCardinality()) + { + JoinCommon::changeLowCardinalityInplace(column_from_block); + } columns[j]->insertFrom(*column_from_block.column, row_num); } } From 26609a18751b38f0f866e3a3ad3f5309cc951136 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Tue, 31 May 2022 21:41:10 +0200 Subject: [PATCH 112/136] Style fixes --- src/Common/TargetSpecific.cpp | 2 +- src/Storages/MergeTree/MergeTreeRangeReader.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Common/TargetSpecific.cpp b/src/Common/TargetSpecific.cpp index 369c21490d4..c52c8c2bcf0 100644 --- a/src/Common/TargetSpecific.cpp +++ b/src/Common/TargetSpecific.cpp @@ -17,7 +17,7 @@ UInt32 getSupportedArchs() if (Cpu::CpuFlagsCache::have_AVX512F) result |= static_cast(TargetArch::AVX512F); if (Cpu::CpuFlagsCache::have_AVX512BW) - result |= static_cast(TargetArch::AVX512BW); + result |= static_cast(TargetArch::AVX512BW); return result; } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 84a1ab91906..56bd62ee271 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -452,7 +452,8 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con size_t count = 0; #if defined(__AVX512F__) && defined(__AVX512BW__) /// check if avx512 instructions are compiled - if (isArchSupported(TargetArch::AVX512BW)) { + if (isArchSupported(TargetArch::AVX512BW)) + { /// check if cpu support avx512 dynamically, haveAVX512BW contains check of haveAVX512F const __m512i zero64 = _mm512_setzero_epi32(); while (end - begin >= 64) @@ -461,10 +462,9 @@ size_t MergeTreeRangeReader::ReadResult::numZerosInTail(const UInt8 * begin, con const auto * pos = end; UInt64 val = static_cast(_mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(pos)), zero64, _MM_CMPINT_EQ)); val = ~val; - if (val == 0) - { + if (val == 0) count += 64; - } else + else { count += __builtin_clzll(val); return count; From 6cf9405f09a5f50ef4f959d23b96b060c8d1fb51 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 1 Jun 2022 00:50:28 +0000 Subject: [PATCH 113/136] fix optimize_monotonous_functions_in_order_by in distributed queries --- src/Interpreters/SelectQueryOptions.h | 2 +- src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp | 9 ++++++++- src/Storages/ProjectionsDescription.cpp | 4 ++-- ...ize_monotonous_functions_in_order_by_remote.reference | 2 ++ ..._optimize_monotonous_functions_in_order_by_remote.sql | 6 ++++++ 5 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.reference create mode 100644 tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.sql diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index b0183e2761b..032befe1993 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -135,7 +135,7 @@ struct SelectQueryOptions return *this; } - SelectQueryOptions & ignoreASTOptimizationsAlias(bool value = true) + SelectQueryOptions & ignoreASTOptimizations(bool value = true) { ignore_ast_optimizations = value; return *this; diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index ed9cdf547c2..854a677afb9 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -46,8 +46,15 @@ std::unique_ptr createLocalPlan( checkStackSize(); auto query_plan = std::make_unique(); + /// Do not apply AST optimizations, because query + /// is already optimized and some optimizations + /// can be applied only for non-distributed tables + /// and we can produce query, inconsistent with remote plans. auto interpreter = InterpreterSelectQuery( - query_ast, context, SelectQueryOptions(processed_stage).setShardInfo(shard_num, shard_count)); + query_ast, context, + SelectQueryOptions(processed_stage) + .setShardInfo(shard_num, shard_count) + .ignoreASTOptimizations()); interpreter.setProperClientInfo(); if (coordinator) diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index 5e9966a2794..ad35c5d420b 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -109,7 +109,7 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const InterpreterSelectQuery select( result.query_ast, query_context, storage, {}, /// Here we ignore ast optimizations because otherwise aggregation keys may be removed from result header as constants. - SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias().ignoreASTOptimizationsAlias()); + SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias().ignoreASTOptimizations()); result.required_columns = select.getRequiredColumns(); result.sample_block = select.getSampleBlock(); @@ -221,7 +221,7 @@ ProjectionDescription ProjectionDescription::getMinMaxCountProjection( InterpreterSelectQuery select( result.query_ast, query_context, storage, {}, /// Here we ignore ast optimizations because otherwise aggregation keys may be removed from result header as constants. - SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias().ignoreASTOptimizationsAlias()); + SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias().ignoreASTOptimizations()); result.required_columns = select.getRequiredColumns(); result.sample_block = select.getSampleBlock(); diff --git a/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.reference b/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.reference new file mode 100644 index 00000000000..aa47d0d46d4 --- /dev/null +++ b/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.sql b/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.sql new file mode 100644 index 00000000000..6a5e4a0ae65 --- /dev/null +++ b/tests/queries/0_stateless/02315_optimize_monotonous_functions_in_order_by_remote.sql @@ -0,0 +1,6 @@ +SET prefer_localhost_replica = 1; +SET optimize_monotonous_functions_in_order_by = 1; + +SELECT * +FROM cluster(test_cluster_two_shards_localhost, system, one) +ORDER BY toDateTime(dummy); From 32d267ec6c1999cf89e0f1714c3fae1c0b177a0f Mon Sep 17 00:00:00 2001 From: Paul Loyd Date: Sun, 22 May 2022 20:17:16 +0800 Subject: [PATCH 114/136] Stop removing UTF-8 BOM in RowBinary* formats Fixes #37420 --- src/Processors/Formats/Impl/BinaryRowInputFormat.cpp | 1 + src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 10 +++++++++- .../Formats/Impl/CustomSeparatedRowInputFormat.cpp | 1 + .../Formats/Impl/JSONCompactEachRowRowInputFormat.cpp | 1 + .../Formats/Impl/TabSeparatedRowInputFormat.cpp | 10 +++++++++- .../Formats/RowInputFormatWithNamesAndTypes.cpp | 9 ++++++--- .../Formats/RowInputFormatWithNamesAndTypes.h | 5 ++++- .../0_stateless/02306_rowbinary_has_no_bom.reference | 1 + .../queries/0_stateless/02306_rowbinary_has_no_bom.sh | 10 ++++++++++ 9 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02306_rowbinary_has_no_bom.reference create mode 100755 tests/queries/0_stateless/02306_rowbinary_has_no_bom.sh diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index d3de2fbf494..91bebd0daa4 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -18,6 +18,7 @@ BinaryRowInputFormat::BinaryRowInputFormat(ReadBuffer & in_, Block header, Param header, in_, params_, + true, with_names_, with_types_, format_settings_, diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index bddd4203a5d..8f5591e6aa8 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -41,7 +41,15 @@ CSVRowInputFormat::CSVRowInputFormat( bool with_types_, const FormatSettings & format_settings_, std::unique_ptr format_reader_) - : RowInputFormatWithNamesAndTypes(header_, in_, params_, with_names_, with_types_, format_settings_, std::move(format_reader_)) + : RowInputFormatWithNamesAndTypes( + header_, + in_, + params_, + false, + with_names_, + with_types_, + format_settings_, + std::move(format_reader_)) { const String bad_delimiters = " \t\"'.UL"; if (bad_delimiters.find(format_settings.csv.delimiter) != String::npos) diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp index 56a639a0e30..61488a94ccd 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp @@ -47,6 +47,7 @@ CustomSeparatedRowInputFormat::CustomSeparatedRowInputFormat( header_, *buf_, params_, + false, with_names_, with_types_, format_settings_, diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index 0b7cc6669be..1bc5223a712 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -28,6 +28,7 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat( header_, in_, params_, + false, with_names_, with_types_, format_settings_, diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 877ba224fd5..a50302697e6 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -40,7 +40,15 @@ TabSeparatedRowInputFormat::TabSeparatedRowInputFormat( bool with_types_, bool is_raw_, const FormatSettings & format_settings_) - : RowInputFormatWithNamesAndTypes(header_, in_, params_, with_names_, with_types_, format_settings_, std::make_unique(in_, format_settings_, is_raw_)) + : RowInputFormatWithNamesAndTypes( + header_, + in_, + params_, + false, + with_names_, + with_types_, + format_settings_, + std::make_unique(in_, format_settings_, is_raw_)) { } diff --git a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.cpp b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.cpp index 0a20ac9eefd..a3dcbe914bb 100644 --- a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.cpp +++ b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.cpp @@ -17,6 +17,7 @@ RowInputFormatWithNamesAndTypes::RowInputFormatWithNamesAndTypes( const Block & header_, ReadBuffer & in_, const Params & params_, + bool is_binary_, bool with_names_, bool with_types_, const FormatSettings & format_settings_, @@ -24,6 +25,7 @@ RowInputFormatWithNamesAndTypes::RowInputFormatWithNamesAndTypes( : RowInputFormatWithDiagnosticInfo(header_, in_, params_) , format_settings(format_settings_) , data_types(header_.getDataTypes()) + , is_binary(is_binary_) , with_names(with_names_) , with_types(with_types_) , format_reader(std::move(format_reader_)) @@ -38,10 +40,11 @@ void RowInputFormatWithNamesAndTypes::readPrefix() if (getCurrentUnitNumber() != 0) return; - if (with_names || with_types || data_types.at(0)->textCanContainOnlyValidUTF8()) + /// Search and remove BOM only in textual formats (CSV, TSV etc), not in binary ones (RowBinary*). + /// Also, we assume that column name or type cannot contain BOM, so, if format has header, + /// then BOM at beginning of stream cannot be confused with name or type of field, and it is safe to skip it. + if (!is_binary && (with_names || with_types || data_types.at(0)->textCanContainOnlyValidUTF8())) { - /// We assume that column name or type cannot contain BOM, so, if format has header, - /// then BOM at beginning of stream cannot be confused with name or type of field, and it is safe to skip it. skipBOMIfExists(*in); } diff --git a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h index 72fbc444dfc..9fc8b2083df 100644 --- a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h +++ b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h @@ -24,13 +24,15 @@ class FormatWithNamesAndTypesReader; class RowInputFormatWithNamesAndTypes : public RowInputFormatWithDiagnosticInfo { protected: - /** with_names - in the first line the header with column names + /** is_binary - it is a binary format (e.g. don't search for BOM) + * with_names - in the first line the header with column names * with_types - in the second line the header with column names */ RowInputFormatWithNamesAndTypes( const Block & header_, ReadBuffer & in_, const Params & params_, + bool is_binary_, bool with_names_, bool with_types_, const FormatSettings & format_settings_, @@ -51,6 +53,7 @@ private: bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) override; void tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) override; + bool is_binary; bool with_names; bool with_types; std::unique_ptr format_reader; diff --git a/tests/queries/0_stateless/02306_rowbinary_has_no_bom.reference b/tests/queries/0_stateless/02306_rowbinary_has_no_bom.reference new file mode 100644 index 00000000000..2e006a3fdd7 --- /dev/null +++ b/tests/queries/0_stateless/02306_rowbinary_has_no_bom.reference @@ -0,0 +1 @@ +1651760768976141295 diff --git a/tests/queries/0_stateless/02306_rowbinary_has_no_bom.sh b/tests/queries/0_stateless/02306_rowbinary_has_no_bom.sh new file mode 100755 index 00000000000..b784d7c3984 --- /dev/null +++ b/tests/queries/0_stateless/02306_rowbinary_has_no_bom.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +echo "DROP TABLE IF EXISTS table_with_uint64" | ${CLICKHOUSE_CURL} -d@- -sS "${CLICKHOUSE_URL}" +echo "CREATE TABLE table_with_uint64(no UInt64) ENGINE = MergeTree ORDER BY no" | ${CLICKHOUSE_CURL} -d@- -sS "${CLICKHOUSE_URL}" +echo -en '\xef\xbb\xbf\x00\xab\x3b\xec\x16' | ${CLICKHOUSE_CURL} --data-binary @- "${CLICKHOUSE_URL}&query=INSERT+INTO+table_with_uint64(no)+FORMAT+RowBinary" +echo "SELECT * FROM table_with_uint64" | ${CLICKHOUSE_CURL} -d@- -sS "${CLICKHOUSE_URL}" From e55c2d07a3dd5d46793e9fdb1430ab29ded05efd Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 1 Jun 2022 06:27:24 +0000 Subject: [PATCH 115/136] Revert "Enable only jepsen tests" This reverts commit 3e71a716f53862186a318c5357f01fb6e8eccb8b. --- .github/workflows/pull_request.yml | 3195 +++++++++++++++++++++++++++- 1 file changed, 3194 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 375d3e1cd6e..f6e9880d088 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -109,8 +109,196 @@ jobs: with: name: changed_images path: ${{ runner.temp }}/changed_images.json + StyleCheck: + needs: DockerHubPush + runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{ runner.temp }}/style_check + EOF + - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Style Check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 style_check.py + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FastTest: + needs: DockerHubPush + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/fasttest + REPO_COPY=${{runner.temp}}/fasttest/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + EOF + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" + mkdir "$GITHUB_WORKSPACE" + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.TEMP_PATH }} + - name: Fast Test + run: | + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + CompatibilityCheck: + needs: [BuilderDebRelease] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/compatibility_check + REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: CompatibilityCheck + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + SplitBuildSmokeTest: + needs: [BuilderDebSplitted] + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/split_build_check + REPO_COPY=${{runner.temp}}/split_build_check/ClickHouse + REPORTS_PATH=${{runner.temp}}/reports_dir + EOF + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Split build check + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 split_build_smoke_check.py + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +######################################################################################### +#################################### ORDINARY BUILDS #################################### +######################################################################################### + BuilderDebRelease: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_release + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # for performance artifact + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" BuilderBinRelease: - needs: [DockerHubPush] + needs: [DockerHubPush, FastTest] runs-on: [self-hosted, builder] steps: - name: Set envs @@ -154,9 +342,3014 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + # BuilderBinGCC: + # needs: [DockerHubPush, FastTest] + # runs-on: [self-hosted, builder] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/build_check + # IMAGES_PATH=${{runner.temp}}/images_path + # REPO_COPY=${{runner.temp}}/build_check/ClickHouse + # CACHES_PATH=${{runner.temp}}/../ccaches + # BUILD_NAME=binary_gcc + # EOF + # - name: Download changed images + # uses: actions/download-artifact@v2 + # with: + # name: changed_images + # path: ${{ runner.temp }}/images_path + # - name: Clear repository + # run: | + # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + # - name: Check out repository code + # uses: actions/checkout@v2 + # - name: Build + # run: | + # git -C "$GITHUB_WORKSPACE" submodule sync + # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + # - name: Upload build URLs to artifacts + # if: ${{ success() || failure() }} + # uses: actions/upload-artifact@v2 + # with: + # name: ${{ env.BUILD_URLS }} + # path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + # - name: Cleanup + # if: always() + # run: | + # # shellcheck disable=SC2046 + # docker kill $(docker ps -q) ||: + # # shellcheck disable=SC2046 + # docker rm -f $(docker ps -a -q) ||: + # sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAarch64: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/images_path + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # for performance artifact + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebAsan: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_asan + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebUBsan: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_ubsan + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebTsan: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_tsan + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebMsan: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_msan + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderDebDebug: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=package_debug + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +########################################################################################## +##################################### SPECIAL BUILDS ##################################### +########################################################################################## + BuilderDebSplitted: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_splitted + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinTidy: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_tidy + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwin: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinAarch64: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinFreeBSD: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_freebsd + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinDarwinAarch64: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_darwin_aarch64 + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" + BuilderBinPPC64: + needs: [DockerHubPush, FastTest] + runs-on: [self-hosted, builder] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/build_check + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + CACHES_PATH=${{runner.temp}}/../ccaches + BUILD_NAME=binary_ppc64le + EOF + - name: Download changed images + uses: actions/download-artifact@v2 + with: + name: changed_images + path: ${{ env.IMAGES_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build + run: | + git -C "$GITHUB_WORKSPACE" submodule sync + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME" + - name: Upload build URLs to artifacts + if: ${{ success() || failure() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ env.BUILD_URLS }} + path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" "$CACHES_PATH" +############################################################################################ +##################################### Docker images ####################################### +############################################################################################ + DockerServerImages: + needs: + - BuilderDebRelease + - BuilderDebAarch64 + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + with: + fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself + - name: Check docker clickhouse/clickhouse-server building + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push --no-ubuntu \ + --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################ +##################################### BUILD REPORTER ####################################### +############################################################################################ + BuilderReport: + needs: + - BuilderBinRelease + - BuilderDebAarch64 + - BuilderDebAsan + - BuilderDebDebug + - BuilderDebMsan + - BuilderDebRelease + - BuilderDebTsan + - BuilderDebUBsan + runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=ClickHouse build check (actions) + REPORTS_PATH=${{runner.temp}}/reports_dir + TEMP_PATH=${{runner.temp}}/report_check + NEEDS_DATA_PATH=${{runner.temp}}/needs.json + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Report Builder + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cat > "$NEEDS_DATA_PATH" << 'EOF' + ${{ toJSON(needs) }} + EOF + cd "$GITHUB_WORKSPACE/tests/ci" + python3 build_report_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + BuilderSpecialReport: + needs: + - BuilderBinAarch64 + - BuilderBinDarwin + - BuilderBinDarwinAarch64 + - BuilderBinFreeBSD + # - BuilderBinGCC + - BuilderBinPPC64 + - BuilderBinTidy + - BuilderDebSplitted + runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/report_check + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=ClickHouse special build check (actions) + NEEDS_DATA_PATH=${{runner.temp}}/needs.json + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Report Builder + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cat > "$NEEDS_DATA_PATH" << 'EOF' + ${{ toJSON(needs) }} + EOF + cd "$GITHUB_WORKSPACE/tests/ci" + python3 build_report_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################## +########################### FUNCTIONAl STATELESS TESTS ####################################### +############################################################################################## + FunctionalStatelessTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, actions) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseDatabaseReplicated0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_database_replicated + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) + REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseDatabaseReplicated1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_database_replicated + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, DatabaseReplicated, actions) + REPO_COPY=${{runner.temp}}/stateless_database_replicated/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseWideParts: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_wide_parts + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, wide parts enabled, actions) + REPO_COPY=${{runner.temp}}/stateless_wide_parts/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (aarch64, actions) + REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (address, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (address, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (thread, actions) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (thread, actions) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (thread, actions) + REPO_COPY=${{runner.temp}}/stateless_tsan/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (ubsan, actions) + REPO_COPY=${{runner.temp}}/stateless_ubsan/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan0: + needs: [BuilderDebMsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (memory, actions) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan1: + needs: [BuilderDebMsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (memory, actions) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestMsan2: + needs: [BuilderDebMsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (memory, actions) + REPO_COPY=${{runner.temp}}/stateless_memory/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug0: + needs: [BuilderDebDebug] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug1: + needs: [BuilderDebDebug] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestDebug2: + needs: [BuilderDebDebug] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (debug, actions) + REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse + KILL_TIMEOUT=10800 + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestFlakyCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_flaky_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests flaky check (address, actions) + REPO_COPY=${{runner.temp}}/stateless_flaky_asan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + TestsBugfixCheck: + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/tests_bugfix_check + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Tests bugfix validate check (actions) + KILL_TIMEOUT=3600 + REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Bugfix test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + + TEMP_PATH="${TEMP_PATH}/integration" \ + REPORTS_PATH="${REPORTS_PATH}/integration" \ + python3 integration_test_check.py "Integration tests bugfix validate check" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + TEMP_PATH="${TEMP_PATH}/stateless" \ + REPORTS_PATH="${REPORTS_PATH}/stateless" \ + python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################## +############################ FUNCTIONAl STATEFUL TESTS ####################################### +############################################################################################## + FunctionalStatefulTestRelease: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (release, actions) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAarch64: + needs: [BuilderDebAarch64] + runs-on: [self-hosted, func-tester-aarch64] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (aarch64, actions) + REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (address, actions) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestTsan: + needs: [BuilderDebTsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (thread, actions) + REPO_COPY=${{runner.temp}}/stateful_tsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_msan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (memory, actions) + REPO_COPY=${{runner.temp}}/stateful_msan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (ubsan, actions) + REPO_COPY=${{runner.temp}}/stateful_ubsan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + FunctionalStatefulTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateful_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateful tests (debug, actions) + REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################## +######################################### STRESS TESTS ####################################### +############################################################################################## + StressTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (address, actions) + REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + StressTestTsan: + needs: [BuilderDebTsan] + # func testers have 16 cores + 128 GB memory + # while stress testers have 36 cores + 72 memory + # It would be better to have something like 32 + 128, + # but such servers almost unavailable as spot instances. + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_thread + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (thread, actions) + REPO_COPY=${{runner.temp}}/stress_thread/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + StressTestMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_memory + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (memory, actions) + REPO_COPY=${{runner.temp}}/stress_memory/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + StressTestUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_undefined + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (undefined, actions) + REPO_COPY=${{runner.temp}}/stress_undefined/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + StressTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stress_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stress test (debug, actions) + REPO_COPY=${{runner.temp}}/stress_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Stress test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 stress_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################## +##################################### AST FUZZERS ############################################ +############################################################################################## + ASTFuzzerTestAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/ast_fuzzer_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=AST fuzzer (ASan, actions) + REPO_COPY=${{runner.temp}}/ast_fuzzer_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Fuzzer + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 ast_fuzzer_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ASTFuzzerTestTsan: + needs: [BuilderDebTsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/ast_fuzzer_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=AST fuzzer (TSan, actions) + REPO_COPY=${{runner.temp}}/ast_fuzzer_tsan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Fuzzer + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 ast_fuzzer_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ASTFuzzerTestUBSan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/ast_fuzzer_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=AST fuzzer (UBSan, actions) + REPO_COPY=${{runner.temp}}/ast_fuzzer_ubsan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Fuzzer + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 ast_fuzzer_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ASTFuzzerTestMSan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/ast_fuzzer_msan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=AST fuzzer (MSan, actions) + REPO_COPY=${{runner.temp}}/ast_fuzzer_msan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Fuzzer + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 ast_fuzzer_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + ASTFuzzerTestDebug: + needs: [BuilderDebDebug] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/ast_fuzzer_debug + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=AST fuzzer (debug, actions) + REPO_COPY=${{runner.temp}}/ast_fuzzer_debug/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Fuzzer + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 ast_fuzzer_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################# +############################# INTEGRATION TESTS ############################################# +############################################################################################# + IntegrationTestsAsan0: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan, actions) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan1: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan, actions) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsAsan2: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (asan, actions) + REPO_COPY=${{runner.temp}}/integration_tests_asan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=3 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan0: + needs: [BuilderDebTsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (thread, actions) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan1: + needs: [BuilderDebTsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (thread, actions) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan2: + needs: [BuilderDebTsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (thread, actions) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsTsan3: + needs: [BuilderDebTsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (thread, actions) + REPO_COPY=${{runner.temp}}/integration_tests_tsan/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release, actions) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsRelease1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_release + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests (release, actions) + REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=2 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + IntegrationTestsFlakyCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan_flaky_check + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests flaky check (asan, actions) + REPO_COPY=${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################# +#################################### UNIT TESTS ############################################# +############################################################################################# + UnitTestsAsan: + needs: [BuilderDebAsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (asan, actions) + REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + UnitTestsReleaseClang: + needs: [BuilderBinRelease] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (release-clang, actions) + REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + # UnitTestsReleaseGCC: + # needs: [BuilderBinGCC] + # runs-on: [self-hosted, fuzzer-unit-tester] + # steps: + # - name: Set envs + # run: | + # cat >> "$GITHUB_ENV" << 'EOF' + # TEMP_PATH=${{runner.temp}}/unit_tests_asan + # REPORTS_PATH=${{runner.temp}}/reports_dir + # CHECK_NAME=Unit tests (release-gcc, actions) + # REPO_COPY=${{runner.temp}}/unit_tests_asan/ClickHouse + # EOF + # - name: Download json reports + # uses: actions/download-artifact@v2 + # with: + # path: ${{ env.REPORTS_PATH }} + # - name: Clear repository + # run: | + # sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + # - name: Check out repository code + # uses: actions/checkout@v2 + # - name: Unit test + # run: | + # sudo rm -fr "$TEMP_PATH" + # mkdir -p "$TEMP_PATH" + # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + # cd "$REPO_COPY/tests/ci" + # python3 unit_tests_check.py "$CHECK_NAME" + # - name: Cleanup + # if: always() + # run: | + # # shellcheck disable=SC2046 + # docker kill $(docker ps -q) ||: + # # shellcheck disable=SC2046 + # docker rm -f $(docker ps -a -q) ||: + # sudo rm -fr "$TEMP_PATH" + UnitTestsTsan: + needs: [BuilderDebTsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_tsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (tsan, actions) + REPO_COPY=${{runner.temp}}/unit_tests_tsan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + UnitTestsMsan: + needs: [BuilderDebMsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_msan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (msan, actions) + REPO_COPY=${{runner.temp}}/unit_tests_msan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + UnitTestsUBsan: + needs: [BuilderDebUBsan] + runs-on: [self-hosted, fuzzer-unit-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/unit_tests_ubsan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Unit tests (ubsan, actions) + REPO_COPY=${{runner.temp}}/unit_tests_ubsan/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Unit test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 unit_tests_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" +############################################################################################# +#################################### PERFORMANCE TESTS ###################################### +############################################################################################# + PerformanceComparison0: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison (actions) + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=0 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparison1: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison (actions) + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=1 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparison2: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison (actions) + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=2 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" + PerformanceComparison3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/performance_comparison + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Performance Comparison (actions) + REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse + RUN_BY_HASH_NUM=3 + RUN_BY_HASH_TOTAL=4 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Performance Comparison + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 performance_comparison_check.py "$CHECK_NAME" + - name: Cleanup + if: always() + run: | + # shellcheck disable=SC2046 + docker kill $(docker ps -q) ||: + # shellcheck disable=SC2046 + docker rm -f $(docker ps -a -q) ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################# ###################################### JEPSEN TESTS ######################################### ############################################################################################# Jepsen: needs: [BuilderBinRelease] uses: ./.github/workflows/jepsen.yml + + FinishCheck: + needs: + - StyleCheck + - DockerHubPush + - DockerServerImages + - CheckLabels + - BuilderReport + - FastTest + - FunctionalStatelessTestDebug0 + - FunctionalStatelessTestDebug1 + - FunctionalStatelessTestDebug2 + - FunctionalStatelessTestRelease + - FunctionalStatelessTestReleaseDatabaseReplicated0 + - FunctionalStatelessTestReleaseDatabaseReplicated1 + - FunctionalStatelessTestReleaseWideParts + - FunctionalStatelessTestAarch64 + - FunctionalStatelessTestAsan0 + - FunctionalStatelessTestAsan1 + - FunctionalStatelessTestTsan0 + - FunctionalStatelessTestTsan1 + - FunctionalStatelessTestTsan2 + - FunctionalStatelessTestMsan0 + - FunctionalStatelessTestMsan1 + - FunctionalStatelessTestMsan2 + - FunctionalStatelessTestUBsan + - FunctionalStatefulTestDebug + - FunctionalStatefulTestRelease + - FunctionalStatefulTestAarch64 + - FunctionalStatefulTestAsan + - FunctionalStatefulTestTsan + - FunctionalStatefulTestMsan + - FunctionalStatefulTestUBsan + - FunctionalStatelessTestReleaseS3 + - StressTestDebug + - StressTestAsan + - StressTestTsan + - StressTestMsan + - StressTestUBsan + - ASTFuzzerTestDebug + - ASTFuzzerTestAsan + - ASTFuzzerTestTsan + - ASTFuzzerTestMSan + - ASTFuzzerTestUBSan + - IntegrationTestsAsan0 + - IntegrationTestsAsan1 + - IntegrationTestsAsan2 + - IntegrationTestsRelease0 + - IntegrationTestsRelease1 + - IntegrationTestsTsan0 + - IntegrationTestsTsan1 + - IntegrationTestsTsan2 + - IntegrationTestsTsan3 + - PerformanceComparison0 + - PerformanceComparison1 + - PerformanceComparison2 + - PerformanceComparison3 + - UnitTestsAsan + - UnitTestsTsan + - UnitTestsMsan + - UnitTestsUBsan + - UnitTestsReleaseClang + - SplitBuildSmokeTest + - CompatibilityCheck + - IntegrationTestsFlakyCheck + - Jepsen + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Finish label + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 finish_check.py From df0a1d523ee82e25f4eee92ad1e9fc013b9c42ce Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 1 Jun 2022 06:32:52 +0000 Subject: [PATCH 116/136] add comment for overwrite --- .../src/jepsen/clickhouse_keeper/db.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index fcc28a5a272..c354e36e430 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -13,6 +13,9 @@ (ns jepsen.control.scp) +;; We need to overwrite Jepsen's implementation of scp! because it +;; doesn't use strict-host-key-checking + (defn scp! "Runs an SCP command by shelling out. Takes a conn-spec (used for port, key, etc), a seq of sources, and a single destination, all as strings." From 249fe561f4373a3c381a8c8ffc21e66ab476119a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 09:42:57 +0200 Subject: [PATCH 117/136] Fix build with -DENABLE_LIBRARIES=0 / -DENABLE_REPLXX=0 Replxx: When disabled via -DENABLE_LIBRARIES=0 or -DENABLE_REPLXX (the latter was undocumented) the build broke because replxx symbols were used since [0] in header LineReader.h. This header should in theory stay clean of replxx but doesn't for efficiency reasons. This change makes compilation of replxx mandatory. As replxx is quite small, I guess this is okay. (The alternative is to litter the code with ifdefs for non-replxx and a replxx paths.) [0] https://github.com/ClickHouse/ClickHouse/pull/33201 --- base/base/CMakeLists.txt | 5 +---- contrib/replxx-cmake/CMakeLists.txt | 7 ------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index adffb91625c..175a4836e64 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -17,15 +17,12 @@ set (SRCS sleep.cpp terminalColors.cpp errnoToString.cpp + ReplxxLineReader.cpp StringRef.cpp safeExit.cpp throwError.cpp ) -if (ENABLE_REPLXX) - list (APPEND SRCS ReplxxLineReader.cpp) -endif () - if (USE_DEBUG_HELPERS) get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES) # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc. diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index 8487ad520bc..c7cf6eb7687 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -1,10 +1,3 @@ -option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_REPLXX) - message (STATUS "Not using replxx") - return() -endif() - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") set(SRCS From 12871a43e1090f2a48f1070989286857f7183723 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 10:10:06 +0200 Subject: [PATCH 118/136] Cosmetics --- contrib/abseil-cpp-cmake/CMakeLists.txt | 3 -- contrib/amqpcpp-cmake/CMakeLists.txt | 1 + contrib/arrow-cmake/CMakeLists.txt | 2 +- contrib/bzip2-cmake/CMakeLists.txt | 6 +--- contrib/cassandra-cmake/CMakeLists.txt | 1 + contrib/cppkafka-cmake/CMakeLists.txt | 2 +- contrib/fastops-cmake/CMakeLists.txt | 2 +- contrib/fmtlib-cmake/CMakeLists.txt | 32 ++++++++++--------- contrib/h3-cmake/CMakeLists.txt | 34 ++++++++++----------- contrib/hive-metastore-cmake/CMakeLists.txt | 2 +- contrib/libcpuid-cmake/CMakeLists.txt | 2 +- contrib/libgsasl-cmake/CMakeLists.txt | 2 +- contrib/libuv-cmake/CMakeLists.txt | 1 + contrib/minizip-ng-cmake/CMakeLists.txt | 2 +- contrib/nanodbc-cmake/CMakeLists.txt | 4 +-- contrib/thrift-cmake/CMakeLists.txt | 2 +- 16 files changed, 48 insertions(+), 50 deletions(-) diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index 4fb02327d17..4c31ecfc599 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -1,7 +1,4 @@ set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") -if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt") - message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive") -endif() set(BUILD_TESTING OFF) set(ABSL_PROPAGATE_CXX_STD ON) add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp") diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index 6e655d3c255..e5c17c234e9 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -5,6 +5,7 @@ if (NOT ENABLE_AMQPCPP) return() endif() +# can be removed once libuv build on MacOS with GCC is possible if (NOT TARGET ch_contrib::uv) message(STATUS "Not using AMQP-CPP because libuv is disabled") return() diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index a4574493440..74bbb300fa5 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -20,7 +20,7 @@ endif() option (ENABLE_PARQUET "Enable parquet" ${ENABLE_PARQUET_DEFAULT}) if (NOT ENABLE_PARQUET) - message(STATUS "Building without Parquet support") + message(STATUS "Not using parquet") return() endif() diff --git a/contrib/bzip2-cmake/CMakeLists.txt b/contrib/bzip2-cmake/CMakeLists.txt index 2e01a624000..693d4c1663c 100644 --- a/contrib/bzip2-cmake/CMakeLists.txt +++ b/contrib/bzip2-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES}) if (NOT ENABLE_BZIP2) - message (STATUS "bzip2 compression disabled") + message (STATUS "Not using bzip2") return() endif() @@ -26,8 +26,4 @@ configure_file ( add_library(_bzip2 ${SRCS}) add_library(ch_contrib::bzip2 ALIAS _bzip2) -# To avoid -Wreserved-id-macro we use SYSTEM: -# -# clickhouse/contrib/bzip2/bzlib.h:23:9: error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] -# #define _BZLIB_H target_include_directories(_bzip2 SYSTEM BEFORE PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}") diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 986ac438bb2..59ff908b63a 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -5,6 +5,7 @@ if (NOT ENABLE_CASSANDRA) return() endif() +# can be removed once libuv build on MacOS with GCC is possible if (NOT TARGET ch_contrib::uv) message(STATUS "Not using cassandra because libuv is disabled") return() diff --git a/contrib/cppkafka-cmake/CMakeLists.txt b/contrib/cppkafka-cmake/CMakeLists.txt index 87bf2356a80..fa1c52180e8 100644 --- a/contrib/cppkafka-cmake/CMakeLists.txt +++ b/contrib/cppkafka-cmake/CMakeLists.txt @@ -1,5 +1,5 @@ if (NOT ENABLE_KAFKA) - message(STATUS "Not using librdkafka (skip cppkafka)") + message(STATUS "Not using kafka") return() endif() diff --git a/contrib/fastops-cmake/CMakeLists.txt b/contrib/fastops-cmake/CMakeLists.txt index 17d6a7f5fcb..e9aa4803583 100644 --- a/contrib/fastops-cmake/CMakeLists.txt +++ b/contrib/fastops-cmake/CMakeLists.txt @@ -5,7 +5,7 @@ elseif(ENABLE_FASTOPS) endif() if(NOT ENABLE_FASTOPS) - message(STATUS "Not using fast vectorized mathematical functions library by Mikhail Parakhin") + message(STATUS "Not using fastops") return() endif() diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt index fecec5f3e43..fe399ddc6e1 100644 --- a/contrib/fmtlib-cmake/CMakeLists.txt +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -1,22 +1,24 @@ +set(FMT_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/fmtlib") + set (SRCS # NOTE: do not build module for now: # ../fmtlib/src/fmt.cc - ../fmtlib/src/format.cc - ../fmtlib/src/os.cc + ${FMT_SOURCE_DIR}/src/format.cc + ${FMT_SOURCE_DIR}/src/os.cc - ../fmtlib/include/fmt/args.h - ../fmtlib/include/fmt/chrono.h - ../fmtlib/include/fmt/color.h - ../fmtlib/include/fmt/compile.h - ../fmtlib/include/fmt/core.h - ../fmtlib/include/fmt/format.h - ../fmtlib/include/fmt/format-inl.h - ../fmtlib/include/fmt/locale.h - ../fmtlib/include/fmt/os.h - ../fmtlib/include/fmt/ostream.h - ../fmtlib/include/fmt/printf.h - ../fmtlib/include/fmt/ranges.h - ../fmtlib/include/fmt/xchar.h + ${FMT_SOURCE_DIR}/include/fmt/args.h + ${FMT_SOURCE_DIR}/include/fmt/chrono.h + ${FMT_SOURCE_DIR}/include/fmt/color.h + ${FMT_SOURCE_DIR}/include/fmt/compile.h + ${FMT_SOURCE_DIR}/include/fmt/core.h + ${FMT_SOURCE_DIR}/include/fmt/format.h + ${FMT_SOURCE_DIR}/include/fmt/format-inl.h + ${FMT_SOURCE_DIR}/include/fmt/locale.h + ${FMT_SOURCE_DIR}/include/fmt/os.h + ${FMT_SOURCE_DIR}/include/fmt/ostream.h + ${FMT_SOURCE_DIR}/include/fmt/printf.h + ${FMT_SOURCE_DIR}/include/fmt/ranges.h + ${FMT_SOURCE_DIR}/include/fmt/xchar.h ) add_library(_fmt ${SRCS}) diff --git a/contrib/h3-cmake/CMakeLists.txt b/contrib/h3-cmake/CMakeLists.txt index 984d1b1ae7c..869550224e6 100644 --- a/contrib/h3-cmake/CMakeLists.txt +++ b/contrib/h3-cmake/CMakeLists.txt @@ -9,23 +9,23 @@ set(H3_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib") set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib") set(SRCS -"${H3_SOURCE_DIR}/lib/algos.c" -"${H3_SOURCE_DIR}/lib/coordijk.c" -"${H3_SOURCE_DIR}/lib/bbox.c" -"${H3_SOURCE_DIR}/lib/polygon.c" -"${H3_SOURCE_DIR}/lib/h3Index.c" -"${H3_SOURCE_DIR}/lib/vec2d.c" -"${H3_SOURCE_DIR}/lib/vec3d.c" -"${H3_SOURCE_DIR}/lib/vertex.c" -"${H3_SOURCE_DIR}/lib/linkedGeo.c" -"${H3_SOURCE_DIR}/lib/localij.c" -"${H3_SOURCE_DIR}/lib/latLng.c" -"${H3_SOURCE_DIR}/lib/directedEdge.c" -"${H3_SOURCE_DIR}/lib/mathExtensions.c" -"${H3_SOURCE_DIR}/lib/iterators.c" -"${H3_SOURCE_DIR}/lib/vertexGraph.c" -"${H3_SOURCE_DIR}/lib/faceijk.c" -"${H3_SOURCE_DIR}/lib/baseCells.c" + "${H3_SOURCE_DIR}/lib/algos.c" + "${H3_SOURCE_DIR}/lib/coordijk.c" + "${H3_SOURCE_DIR}/lib/bbox.c" + "${H3_SOURCE_DIR}/lib/polygon.c" + "${H3_SOURCE_DIR}/lib/h3Index.c" + "${H3_SOURCE_DIR}/lib/vec2d.c" + "${H3_SOURCE_DIR}/lib/vec3d.c" + "${H3_SOURCE_DIR}/lib/vertex.c" + "${H3_SOURCE_DIR}/lib/linkedGeo.c" + "${H3_SOURCE_DIR}/lib/localij.c" + "${H3_SOURCE_DIR}/lib/latLng.c" + "${H3_SOURCE_DIR}/lib/directedEdge.c" + "${H3_SOURCE_DIR}/lib/mathExtensions.c" + "${H3_SOURCE_DIR}/lib/iterators.c" + "${H3_SOURCE_DIR}/lib/vertexGraph.c" + "${H3_SOURCE_DIR}/lib/faceijk.c" + "${H3_SOURCE_DIR}/lib/baseCells.c" ) configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h") diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index 9069d46cea7..a5e16c739af 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -5,7 +5,7 @@ elseif(ENABLE_HIVE) endif() if (NOT ENABLE_HIVE) - message("Hive disabled") + message(STATUS "Not using hive") return() endif() diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index 1940b39b6aa..95f653c7ea2 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -6,7 +6,7 @@ elseif(ENABLE_CPUID) endif() if (NOT ENABLE_CPUID) - message("Not using cpuid") + message(STATUS "Not using cpuid") return() endif() diff --git a/contrib/libgsasl-cmake/CMakeLists.txt b/contrib/libgsasl-cmake/CMakeLists.txt index 4bb4ca9dc33..3cf087c2f4c 100644 --- a/contrib/libgsasl-cmake/CMakeLists.txt +++ b/contrib/libgsasl-cmake/CMakeLists.txt @@ -1,7 +1,7 @@ option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES}) if (NOT ENABLE_GSASL_LIBRARY) - message(STATUS "Not using gsasl library") + message(STATUS "Not using gsasl") return() endif() diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt index 45f6d8e2083..1a7714e47ce 100644 --- a/contrib/libuv-cmake/CMakeLists.txt +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -1,3 +1,4 @@ +# once fixed, please remove similar places in CMakeLists of libuv users (search "ch_contrib::uv") if (OS_DARWIN AND COMPILER_GCC) message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082") return() diff --git a/contrib/minizip-ng-cmake/CMakeLists.txt b/contrib/minizip-ng-cmake/CMakeLists.txt index 4aabbd3c9fb..043f0fc68f9 100644 --- a/contrib/minizip-ng-cmake/CMakeLists.txt +++ b/contrib/minizip-ng-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ option(ENABLE_MINIZIP "Enable minizip-ng the zip manipulation library" ${ENABLE_LIBRARIES}) if (NOT ENABLE_MINIZIP) - message (STATUS "minizip-ng disabled") + message (STATUS "Not using minizip-ng") return() endif() diff --git a/contrib/nanodbc-cmake/CMakeLists.txt b/contrib/nanodbc-cmake/CMakeLists.txt index 9ed6c9525b6..7aacf5bed7e 100644 --- a/contrib/nanodbc-cmake/CMakeLists.txt +++ b/contrib/nanodbc-cmake/CMakeLists.txt @@ -2,12 +2,12 @@ if (NOT ENABLE_ODBC) return () endif () -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc") - if (NOT TARGET ch_contrib::unixodbc) message(FATAL_ERROR "Configuration error: unixodbc is not a target") endif() +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc") + set (SRCS "${LIBRARY_DIR}/nanodbc/nanodbc.cpp" ) diff --git a/contrib/thrift-cmake/CMakeLists.txt b/contrib/thrift-cmake/CMakeLists.txt index 2a62a6fe7ab..6f94c1ebdc0 100644 --- a/contrib/thrift-cmake/CMakeLists.txt +++ b/contrib/thrift-cmake/CMakeLists.txt @@ -1,7 +1,7 @@ option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES}) if (NOT ENABLE_THRIFT) - message (STATUS "thrift disabled") + message (STATUS "Not using thrift") return() endif() From a4e037c728e3f8fbbc0e038d1173dbac4c9bde67 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 10:24:12 +0200 Subject: [PATCH 119/136] Remove unused M_LIBRARY link --- contrib/brotli-cmake/CMakeLists.txt | 3 --- contrib/h3-cmake/CMakeLists.txt | 3 --- contrib/libxml2-cmake/CMakeLists.txt | 3 --- contrib/s2geometry-cmake/CMakeLists.txt | 4 ---- 4 files changed, 13 deletions(-) diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt index c81a6bf9076..b89e81ecda1 100644 --- a/contrib/brotli-cmake/CMakeLists.txt +++ b/contrib/brotli-cmake/CMakeLists.txt @@ -45,7 +45,4 @@ add_library(ch_contrib::brotli ALIAS _brotli) target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include") -if(M_LIBRARY) - target_link_libraries(_brotli PRIVATE ${M_LIBRARY}) -endif() target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1) diff --git a/contrib/h3-cmake/CMakeLists.txt b/contrib/h3-cmake/CMakeLists.txt index 869550224e6..c0c2162bd26 100644 --- a/contrib/h3-cmake/CMakeLists.txt +++ b/contrib/h3-cmake/CMakeLists.txt @@ -34,8 +34,5 @@ add_library(_h3 ${SRCS}) target_include_directories(_h3 SYSTEM PUBLIC "${H3_SOURCE_DIR}/include") target_include_directories(_h3 SYSTEM PUBLIC "${H3_BINARY_DIR}/include") target_compile_definitions(_h3 PRIVATE H3_HAVE_VLA) -if(M_LIBRARY) - target_link_libraries(_h3 PRIVATE ${M_LIBRARY}) -endif() add_library(ch_contrib::h3 ALIAS _h3) diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt index e9c4641c161..a84936f8e3a 100644 --- a/contrib/libxml2-cmake/CMakeLists.txt +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -53,9 +53,6 @@ set(SRCS add_library(_libxml2 ${SRCS}) target_link_libraries(_libxml2 PRIVATE ch_contrib::zlib) -if(M_LIBRARY) - target_link_libraries(_libxml2 PRIVATE ${M_LIBRARY}) -endif() target_include_directories(_libxml2 BEFORE PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include") target_include_directories(_libxml2 BEFORE PUBLIC "${LIBXML2_SOURCE_DIR}/include") diff --git a/contrib/s2geometry-cmake/CMakeLists.txt b/contrib/s2geometry-cmake/CMakeLists.txt index 49c80e45b18..102ceb0db3c 100644 --- a/contrib/s2geometry-cmake/CMakeLists.txt +++ b/contrib/s2geometry-cmake/CMakeLists.txt @@ -149,7 +149,3 @@ target_link_libraries(_s2 PRIVATE target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/") target_include_directories(_s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}") - -if(M_LIBRARY) - target_link_libraries(_s2 PRIVATE ${M_LIBRARY}) -endif() From 933f98a900a5dfe4ad06774133f81281b73780b5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 10:47:11 +0200 Subject: [PATCH 120/136] Removed warning flags in contribs warnings are disabled for all contribs in contrib/CMakeLists.txt already --- contrib/amqpcpp-cmake/CMakeLists.txt | 15 --------------- contrib/avro-cmake/CMakeLists.txt | 8 -------- contrib/azure-cmake/CMakeLists.txt | 14 -------------- contrib/capnproto-cmake/CMakeLists.txt | 12 ++++-------- contrib/icu-cmake/CMakeLists.txt | 4 ---- contrib/jemalloc-cmake/CMakeLists.txt | 1 - contrib/libcpuid-cmake/CMakeLists.txt | 3 --- contrib/replxx-cmake/CMakeLists.txt | 5 ----- contrib/unixodbc-cmake/CMakeLists.txt | 10 +--------- 9 files changed, 5 insertions(+), 67 deletions(-) diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index e5c17c234e9..6f6a0188e6f 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -38,21 +38,6 @@ set (SRCS add_library(_amqp-cpp ${SRCS}) -target_compile_options (_amqp-cpp - PRIVATE - -Wno-old-style-cast - -Wno-inconsistent-missing-destructor-override - -Wno-deprecated - -Wno-unused-parameter - -Wno-shadow - -Wno-tautological-type-limit-compare - -Wno-extra-semi -# NOTE: disable all warnings at last because the warning: - # "conversion function converting 'XXX' to itself will never be used" - # doesn't have it's own diagnostic flag yet. - -w -) - target_include_directories (_amqp-cpp SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include" "${LIBRARY_DIR}") target_link_libraries (_amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv) add_library (ch_contrib::amqp_cpp ALIAS _amqp-cpp) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index c5bda41782d..25474650d0e 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -60,14 +60,6 @@ target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy) -if (COMPILER_GCC) - set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) -elseif (COMPILER_CLANG) - set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) -endif () - -target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS}) - # create a symlink to include headers with set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") ADD_CUSTOM_TARGET(avro_symlink_headers ALL diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index 031d8dc9a0b..19f2940cbf0 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -52,20 +52,6 @@ include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake") add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC}) -if (COMPILER_CLANG) - target_compile_options(_azure_sdk PRIVATE - -Wno-deprecated-copy-dtor - -Wno-extra-semi - -Wno-suggest-destructor-override - -Wno-inconsistent-missing-destructor-override - -Wno-error=unknown-warning-option - ) - - if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13) - target_compile_options(_azure_sdk PRIVATE -Wno-reserved-identifier) - endif() -endif() - # Originally, on Windows azure-core is built with bcrypt and crypt32 by default if (TARGET OpenSSL::SSL) target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL) diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt index 297b847cd58..e76268592ee 100644 --- a/contrib/capnproto-cmake/CMakeLists.txt +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -81,16 +81,12 @@ set (CAPNPC_SRCS add_library(_capnpc ${CAPNPC_SRCS}) target_link_libraries(_capnpc PUBLIC _capnp) -# The library has substandard code -if (COMPILER_GCC) - set (SUPPRESS_WARNINGS -w) -elseif (COMPILER_CLANG) - set (SUPPRESS_WARNINGS -w) +if (COMPILER_CLANG) set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t) endif () -target_compile_options(_kj PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) -target_compile_options(_capnp PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) -target_compile_options(_capnpc PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) +target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS}) +target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS}) +target_compile_options(_capnpc PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS}) add_library(ch_contrib::capnp ALIAS _capnpc) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index 9c34228e2a0..ce82155218c 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -481,10 +481,6 @@ target_include_directories(_icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/") target_compile_definitions(_icuuc PRIVATE -DU_COMMON_IMPLEMENTATION) target_compile_definitions(_icui18n PRIVATE -DU_I18N_IMPLEMENTATION) -if (COMPILER_CLANG) - target_compile_options(_icudata PRIVATE -Wno-unused-command-line-argument) -endif () - add_library(_icu INTERFACE) target_link_libraries(_icu INTERFACE _icui18n _icuuc _icudata) add_library(ch_contrib::icu ALIAS _icu) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index c59b4da890b..fdb0fd0e8af 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -180,7 +180,6 @@ if (USE_UNWIND) target_link_libraries (_jemalloc PRIVATE unwind) endif () -target_compile_options(_jemalloc PRIVATE -Wno-redundant-decls) # for RTLD_NEXT target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE) diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index 95f653c7ea2..fd5af925c57 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -27,8 +27,5 @@ add_library (_cpuid ${SRCS}) target_include_directories (_cpuid SYSTEM PUBLIC "${LIBRARY_DIR}") target_compile_definitions (_cpuid PRIVATE VERSION="v0.4.1") -if (COMPILER_CLANG) - target_compile_options (_cpuid PRIVATE -Wno-reserved-id-macro) -endif () add_library(ch_contrib::cpuid ALIAS _cpuid) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index c7cf6eb7687..95a19875621 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -15,9 +15,4 @@ set(SRCS add_library (_replxx ${SRCS}) target_include_directories(_replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include") - -if (COMPILER_CLANG) - target_compile_options(_replxx PRIVATE -Wno-documentation) -endif () - add_library(ch_contrib::replxx ALIAS _replxx) diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt index b594ead3ba0..3317654cd67 100644 --- a/contrib/unixodbc-cmake/CMakeLists.txt +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -294,14 +294,6 @@ target_include_directories (_unixodbc "${LIBRARY_DIR}/include" ) target_compile_definitions (_unixodbc PRIVATE -DHAVE_CONFIG_H) -target_compile_options (_unixodbc - PRIVATE - -Wno-dangling-else - -Wno-parentheses - -Wno-misleading-indentation - -Wno-unknown-warning-option - -Wno-reserved-id-macro - -O2 -) +target_compile_options (_unixodbc PRIVATE -O2) # intended? add_library (ch_contrib::unixodbc ALIAS _unixodbc) From 393b97763a7879d5df91023b16e296a6a5c2492e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 11:18:56 +0200 Subject: [PATCH 121/136] Make SSL a mandatory dependency for now - SSL is a dependency of too many libs + unit tests (via poco crypto which requires SSL) - optional SSL is desirable but right now, turning off SSL (via -DENABLE_LIBRARIES=0 or =DENABLE_SSL=0) breaks the build - therefore make SSL mandatory for now + add a TODO comment --- contrib/boringssl-cmake/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt index 180fb3874c1..faee2dfddb3 100644 --- a/contrib/boringssl-cmake/CMakeLists.txt +++ b/contrib/boringssl-cmake/CMakeLists.txt @@ -1,7 +1,12 @@ # Needed for: # - securely connecting to an external server, e.g. clickhouse-client --host ... --secure # - lots of thirdparty libraries -option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES}) + +# Actually, so many 3rd party libraries + unit tests need SSL that we cannot disable it +# without breaking the build ... +option(ENABLE_SSL "Enable ssl" ON) # breaks if OFF +# TODO: Making SSL dependent on ENABLE_LIBRARIES is desirable but needs fixing dependent libs + tests. +# option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES}) if(NOT ENABLE_SSL) message(STATUS "Not using openssl") From 600512cc08622b672a0876d14d3b61a6f011d6a3 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 31 May 2022 09:15:59 +0200 Subject: [PATCH 122/136] Replace exceptions thrown for programming errors by asserts --- src/Functions/MatchImpl.h | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 96ce0ca2eb0..78ce2627c35 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -18,8 +18,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; - extern const int LOGICAL_ERROR; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace impl @@ -112,16 +110,14 @@ struct MatchImpl const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, const String & needle, - const ColumnPtr & start_pos_, + [[maybe_unused]] const ColumnPtr & start_pos_, PaddedPODArray & res) { const size_t haystack_size = haystack_offsets.size(); - if (haystack_size != res.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name); + assert(haystack_size == res.size()); - if (start_pos_ != nullptr) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name); + assert(start_pos_ == nullptr); if (haystack_offsets.empty()) return; @@ -274,8 +270,7 @@ struct MatchImpl { const size_t haystack_size = haystack.size() / N; - if (haystack_size != res.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks and results", name); + assert(haystack_size == res.size()); if (haystack.empty()) return; @@ -433,16 +428,15 @@ struct MatchImpl const ColumnString::Offsets & haystack_offsets, const ColumnString::Chars & needle_data, const ColumnString::Offsets & needle_offset, - const ColumnPtr & start_pos_, + [[maybe_unused]] const ColumnPtr & start_pos_, PaddedPODArray & res) { const size_t haystack_size = haystack_offsets.size(); - if (haystack_size != needle_offset.size() || haystack_size != res.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name); + assert(haystack_size == needle_offset.size()); + assert(haystack_size == res.size()); - if (start_pos_ != nullptr) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name); + assert(start_pos_ == nullptr); if (haystack_offsets.empty()) return; @@ -547,16 +541,15 @@ struct MatchImpl size_t N, const ColumnString::Chars & needle_data, const ColumnString::Offsets & needle_offset, - const ColumnPtr & start_pos_, + [[maybe_unused]] const ColumnPtr & start_pos_, PaddedPODArray & res) { const size_t haystack_size = haystack.size()/N; - if (haystack_size != needle_offset.size() || haystack_size != res.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Function '{}' unexpectedly received a different number of haystacks, needles and results", name); + assert(haystack_size == needle_offset.size()); + assert(haystack_size == res.size()); - if (start_pos_ != nullptr) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name); + assert(start_pos_ == nullptr); if (haystack.empty()) return; From 81318e07d642def5090753c34082e859f2a42a65 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 31 May 2022 09:29:04 +0200 Subject: [PATCH 123/136] Try to fix performance test results --- ...h_pattern_caching.xml => re2_regex_caching.xml} | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) rename tests/performance/{like_and_match_pattern_caching.xml => re2_regex_caching.xml} (88%) diff --git a/tests/performance/like_and_match_pattern_caching.xml b/tests/performance/re2_regex_caching.xml similarity index 88% rename from tests/performance/like_and_match_pattern_caching.xml rename to tests/performance/re2_regex_caching.xml index c0a8ec9442e..6edc83097ba 100644 --- a/tests/performance/like_and_match_pattern_caching.xml +++ b/tests/performance/re2_regex_caching.xml @@ -5,15 +5,15 @@ numbers - numbers_mt(2000000) + numbers_mt(1500000) needle_like - simple patterns, all unique + '%' || toString(number) || '_' - simple patterns, low distinctness (10 patterns) + '%' || toString(number % 10) || '_' @@ -40,23 +40,27 @@ select toString(number) as haystack, like(haystack, '%x_') from(select * from {numbers}) + format Null select toString(number) as haystack, match(haystack, '.*x.') from(select * from {numbers}) + format Null select toString(number) as haystack, {needle_like} as needle, like(haystack, needle) - from (select * from {numbers}); + from (select * from {numbers}) + format Null select toString(number) as haystack, {needle_match} as needle, match(haystack, needle) - from (select * from {numbers}); + from (select * from {numbers}) + format Null From ded1398565ab3bafb049f1f40d6461204b86f15a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 1 Jun 2022 10:26:43 +0000 Subject: [PATCH 124/136] Fix intersect with const string --- src/Common/ColumnsHashing.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index c3a087c0a6e..e921f4fbf9a 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -6,9 +6,11 @@ #include #include #include +#include "Columns/IColumn.h" #include #include +#include #include #include @@ -83,8 +85,11 @@ struct HashMethodString HashMethodString(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const HashMethodContextPtr &) { - const IColumn & column = *key_columns[0]; - const ColumnString & column_string = assert_cast(column); + const IColumn * column = key_columns[0]; + if (isColumnConst(*column)) + column = &assert_cast(*column).getDataColumn(); + + const ColumnString & column_string = assert_cast(*column); offsets = column_string.getOffsets().data(); chars = column_string.getChars().data(); } From 6c31d06b2ecb3e9a3a0afa8e257a27abc7b0629a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 1 Jun 2022 11:17:56 +0000 Subject: [PATCH 125/136] Add test for const string intersect --- tests/queries/0_stateless/02316_const_string_intersact.reference | 1 + tests/queries/0_stateless/02316_const_string_intersact.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/02316_const_string_intersact.reference create mode 100644 tests/queries/0_stateless/02316_const_string_intersact.sql diff --git a/tests/queries/0_stateless/02316_const_string_intersact.reference b/tests/queries/0_stateless/02316_const_string_intersact.reference new file mode 100644 index 00000000000..957124d5fdd --- /dev/null +++ b/tests/queries/0_stateless/02316_const_string_intersact.reference @@ -0,0 +1 @@ +Play ClickHouse diff --git a/tests/queries/0_stateless/02316_const_string_intersact.sql b/tests/queries/0_stateless/02316_const_string_intersact.sql new file mode 100644 index 00000000000..ace3c8d03c5 --- /dev/null +++ b/tests/queries/0_stateless/02316_const_string_intersact.sql @@ -0,0 +1 @@ +SELECT 'Play ClickHouse' InterSect SELECT 'Play ClickHouse' From 5a1b873f7bcbd247a5809f6794d0517efdc34dbc Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 1 Jun 2022 13:54:53 +0200 Subject: [PATCH 126/136] No need to checkout submodules/contribs recursively Also verified locally by building from a freshly cloned ClickHouse and "flat" checkout of submodules without recursion --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abe263834ed..a6a09afc489 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # Check that submodules are present if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md") - message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") + message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init") endif () # Take care to add prlimit in command line before ccache, or else ccache thinks that From 6a5f5997cae8479d19bc21e4254c3e66f8dcf58c Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 1 Jun 2022 12:55:47 +0000 Subject: [PATCH 127/136] Add test 02315_pmj_union_ubsan_35857 --- .../02315_pmj_union_ubsan_35857.reference | 2 ++ .../02315_pmj_union_ubsan_35857.sql | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/queries/0_stateless/02315_pmj_union_ubsan_35857.reference create mode 100644 tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql diff --git a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.reference b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.reference new file mode 100644 index 00000000000..96e34d5a44c --- /dev/null +++ b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.reference @@ -0,0 +1,2 @@ +\N +\N diff --git a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql new file mode 100644 index 00000000000..38f1d2e1b4e --- /dev/null +++ b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql @@ -0,0 +1,22 @@ +SET join_algorithm = 'partial_merge'; + +SELECT NULL +FROM +( + SELECT + NULL, + 1 AS a, + 0 :: Nullable(UInt8) AS c + UNION ALL + SELECT + NULL, + 65536, + NULL +) AS js1 +ALL LEFT JOIN +( + SELECT 2 :: Nullable(UInt8) AS a +) AS js2 +USING (a) +ORDER BY c +; From 3a824ef9a45d7231b9243e21cb6be8f5712edb0d Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Wed, 1 Jun 2022 16:00:30 +0200 Subject: [PATCH 128/136] Add no-backward-compatibility-check to 02315_pmj_union_ubsan_35857 --- tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql index 38f1d2e1b4e..47b47101a79 100644 --- a/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql +++ b/tests/queries/0_stateless/02315_pmj_union_ubsan_35857.sql @@ -1,3 +1,5 @@ +-- Tags: no-backward-compatibility-check + SET join_algorithm = 'partial_merge'; SELECT NULL From 2626a496167d42fa5953bbce51b8386ef11961ee Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 May 2022 20:16:42 +0300 Subject: [PATCH 129/136] FR: Expose what triggered the merge in system.part_log #26255 --- src/Interpreters/PartLog.cpp | 31 ++++++++++++++++ src/Interpreters/PartLog.h | 15 ++++++++ src/Storages/MergeTree/MergeTreeData.cpp | 4 ++ .../02293_part_log_has_merge_reason.reference | 1 + .../02293_part_log_has_merge_reason.sql | 37 +++++++++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 tests/queries/0_stateless/02293_part_log_has_merge_reason.reference create mode 100644 tests/queries/0_stateless/02293_part_log_has_merge_reason.sql diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 6d57f6b7045..274fc7384ab 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -16,6 +16,25 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) { + switch (merge_type) + { + case MergeType::REGULAR: + return REGULAR_MERGE; + case MergeType::TTL_DELETE: + return TTL_DELETE_MERGE; + case MergeType::TTL_RECOMPRESS: + return TTL_RECOMPRESS_MERGE; + } + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast(merge_type)); +} + NamesAndTypesList PartLogElement::getNamesAndTypes() { auto event_type_datatype = std::make_shared( @@ -30,11 +49,22 @@ NamesAndTypesList PartLogElement::getNamesAndTypes() } ); + auto merge_reason_datatype = std::make_shared( + DataTypeEnum8::Values + { + {"NotAMerge", static_cast(NOT_A_MERGE)}, + {"RegularMerge", static_cast(REGULAR_MERGE)}, + {"TTLDeleteMerge", static_cast(TTL_DELETE_MERGE)}, + {"TTLRecompressMerge", static_cast(TTL_RECOMPRESS_MERGE)}, + } + ); + ColumnsWithTypeAndName columns_with_type_and_name; return { {"query_id", std::make_shared()}, {"event_type", std::move(event_type_datatype)}, + {"merge_reason", std::move(merge_reason_datatype)}, {"event_date", std::make_shared()}, {"event_time", std::make_shared()}, @@ -72,6 +102,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); + columns[i++]->insert(merge_reason); columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); diff --git a/src/Interpreters/PartLog.h b/src/Interpreters/PartLog.h index 470dce09fa0..16a7e37ee9d 100644 --- a/src/Interpreters/PartLog.h +++ b/src/Interpreters/PartLog.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -21,9 +22,22 @@ struct PartLogElement MOVE_PART = 6, }; + enum MergeReasonType + { + /// merge_reason is relevant only for event_type = 'MERGE_PARTS', in other cases it is NOT_A_MERGE + NOT_A_MERGE = 1, + /// Just regular merge + REGULAR_MERGE = 2, + /// Merge assigned to delete some data from parts (with TTLMergeSelector) + TTL_DELETE_MERGE = 3, + /// Merge with recompression + TTL_RECOMPRESS_MERGE = 4, + }; + String query_id; Type event_type = NEW_PART; + MergeReasonType merge_reason = NOT_A_MERGE; time_t event_time = 0; Decimal64 event_time_microseconds = 0; @@ -57,6 +71,7 @@ struct PartLogElement static std::string name() { return "PartLog"; } + static MergeReasonType getMergeReasonType(MergeType merge_type); static NamesAndTypesList getNamesAndTypes(); static NamesAndAliases getNamesAndAliases() { return {}; } void appendToBlock(MutableColumns & columns) const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4eb4049be60..c4c99e66873 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6176,6 +6176,10 @@ try part_log_elem.event_type = type; + if (part_log_elem.event_type == PartLogElement::MERGE_PARTS) + if (merge_entry) + part_log_elem.merge_reason = PartLogElement::getMergeReasonType((*merge_entry)->merge_type); + part_log_elem.error = static_cast(execution_status.code); part_log_elem.exception = execution_status.message; diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql new file mode 100644 index 00000000000..db1f4c26af4 --- /dev/null +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS t_part_log_has_merge_type_table; + +CREATE TABLE t_part_log_has_merge_type_table +( + event_time DateTime, + UserID UInt64, + Comment String +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL event_time + toIntervalMonth(3) +SETTINGS min_bytes_for_wide_part = 0, merge_with_ttl_timeout = 1; + +INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1'); +INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); + +OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL; + +SYSTEM FLUSH LOGS; + +SELECT count(*) +FROM +( + SELECT + metadata_modification_time, + event_time + FROM system.tables AS l + INNER JOIN system.part_log AS r + ON l.name = r.table + WHERE (l.database = currentDatabase()) AND + (l.name = 't_part_log_has_merge_type_table') AND + (r.event_type = 'MergeParts') AND + (r.merge_reason = 'TTLDeleteMerge') +) +WHERE (metadata_modification_time <= event_time); + +DROP TABLE t_part_log_has_merge_type_table; From 16dc3ed97d29a17d34d9b8090ecbb070ebab3424 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 4 May 2022 20:16:42 +0300 Subject: [PATCH 130/136] FR: Expose what triggered the merge in system.part_log #26255 --- docs/en/operations/system-tables/part_log.md | 6 +++++ src/Interpreters/PartLog.cpp | 9 ++++--- .../02293_part_log_has_merge_reason.reference | 2 +- .../02293_part_log_has_merge_reason.sql | 26 +++++++------------ 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md index 00eaca23862..1b567367c97 100644 --- a/docs/en/operations/system-tables/part_log.md +++ b/docs/en/operations/system-tables/part_log.md @@ -14,6 +14,11 @@ The `system.part_log` table contains the following columns: - `REMOVE_PART` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition). - `MUTATE_PART` — Mutating of a data part. - `MOVE_PART` — Moving the data part from the one disk to another one. +- `merge_reason` ([Enum8](../../sql-reference/data-types/enum.md)) — The reason for the event with type `MERGE_PARTS`. Can have one of the following values: + - `NOT_A_MERGE` — The current event has the type other than `MERGE_PARTS`. + - `REGULAR_MERGE` — Some regular merge. + - `TTL_DELETE_MERGE` — Cleaning up expired data. + - `TTL_RECOMPRESS_MERGE` — Recompressing data part with the. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — Event date. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Event time. - `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Event time with microseconds precision. @@ -46,6 +51,7 @@ Row 1: ────── query_id: 983ad9c7-28d5-4ae1-844e-603116b7de31 event_type: NewPart +merge_reason: NotAMerge event_date: 2021-02-02 event_time: 2021-02-02 11:14:28 event_time_microseconds: 2021-02-02 11:14:28.861919 diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 274fc7384ab..13b74f3d00a 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -21,14 +21,15 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } -PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) { +PartLogElement::MergeReasonType PartLogElement::getMergeReasonType(MergeType merge_type) +{ switch (merge_type) { - case MergeType::REGULAR: + case MergeType::Regular: return REGULAR_MERGE; - case MergeType::TTL_DELETE: + case MergeType::TTLDelete: return TTL_DELETE_MERGE; - case MergeType::TTL_RECOMPRESS: + case MergeType::TTLRecompress: return TTL_RECOMPRESS_MERGE; } diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference index d00491fd7e5..220107cf15b 100644 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.reference @@ -1 +1 @@ -1 +MergeParts TTLDeleteMerge diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql index db1f4c26af4..7ef86354e71 100644 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql @@ -8,30 +8,22 @@ CREATE TABLE t_part_log_has_merge_type_table ) ENGINE = MergeTree() ORDER BY tuple() -TTL event_time + toIntervalMonth(3) -SETTINGS min_bytes_for_wide_part = 0, merge_with_ttl_timeout = 1; +SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true; INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1'); INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +ALTER TABLE t_part_log_has_merge_type_table + MODIFY TTL event_time + INTERVAL 3 MONTH; + OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL; SYSTEM FLUSH LOGS; -SELECT count(*) -FROM -( - SELECT - metadata_modification_time, - event_time - FROM system.tables AS l - INNER JOIN system.part_log AS r - ON l.name = r.table - WHERE (l.database = currentDatabase()) AND - (l.name = 't_part_log_has_merge_type_table') AND - (r.event_type = 'MergeParts') AND - (r.merge_reason = 'TTLDeleteMerge') -) -WHERE (metadata_modification_time <= event_time); +SELECT + event_type, + merge_reason +FROM system.part_log +WHERE (table = 't_part_log_has_merge_type_table') AND (merge_reason = 'TTLDeleteMerge'); DROP TABLE t_part_log_has_merge_type_table; From b3b3d7a45950d04d630e9930b7da2c331f604e4a Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 1 Jun 2022 16:57:28 +0200 Subject: [PATCH 131/136] Fix test --- .../02293_part_log_has_merge_reason.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql index 7ef86354e71..002bc1f37dd 100644 --- a/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql +++ b/tests/queries/0_stateless/02293_part_log_has_merge_reason.sql @@ -8,14 +8,12 @@ CREATE TABLE t_part_log_has_merge_type_table ) ENGINE = MergeTree() ORDER BY tuple() +TTL event_time + INTERVAL 3 MONTH SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true; INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1'); INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); -ALTER TABLE t_part_log_has_merge_type_table - MODIFY TTL event_time + INTERVAL 3 MONTH; - OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL; SYSTEM FLUSH LOGS; @@ -23,7 +21,13 @@ SYSTEM FLUSH LOGS; SELECT event_type, merge_reason -FROM system.part_log -WHERE (table = 't_part_log_has_merge_type_table') AND (merge_reason = 'TTLDeleteMerge'); +FROM + system.part_log +WHERE + table = 't_part_log_has_merge_type_table' + AND + merge_reason = 'TTLDeleteMerge' + AND + database = currentDatabase(); DROP TABLE t_part_log_has_merge_type_table; From 663261673381d392a6a00753c165de84f5629827 Mon Sep 17 00:00:00 2001 From: lthaooo Date: Thu, 2 Jun 2022 03:09:53 +0800 Subject: [PATCH 132/136] Fix TTL merge scheduling bug (#36387) --- src/Storages/MergeTree/BackgroundJobsAssignee.cpp | 3 ++- src/Storages/MergeTree/BackgroundJobsAssignee.h | 2 +- src/Storages/MergeTree/MergeList.h | 5 +++++ src/Storages/StorageMergeTree.cpp | 8 ++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/BackgroundJobsAssignee.cpp b/src/Storages/MergeTree/BackgroundJobsAssignee.cpp index 4dc15d6e794..81445f40ed6 100644 --- a/src/Storages/MergeTree/BackgroundJobsAssignee.cpp +++ b/src/Storages/MergeTree/BackgroundJobsAssignee.cpp @@ -50,10 +50,11 @@ void BackgroundJobsAssignee::postpone() } -void BackgroundJobsAssignee::scheduleMergeMutateTask(ExecutableTaskPtr merge_task) +bool BackgroundJobsAssignee::scheduleMergeMutateTask(ExecutableTaskPtr merge_task) { bool res = getContext()->getMergeMutateExecutor()->trySchedule(merge_task); res ? trigger() : postpone(); + return res; } diff --git a/src/Storages/MergeTree/BackgroundJobsAssignee.h b/src/Storages/MergeTree/BackgroundJobsAssignee.h index e6c5845c657..db93b5f710b 100644 --- a/src/Storages/MergeTree/BackgroundJobsAssignee.h +++ b/src/Storages/MergeTree/BackgroundJobsAssignee.h @@ -66,7 +66,7 @@ public: void postpone(); void finish(); - void scheduleMergeMutateTask(ExecutableTaskPtr merge_task); + bool scheduleMergeMutateTask(ExecutableTaskPtr merge_task); void scheduleFetchTask(ExecutableTaskPtr fetch_task); void scheduleMoveTask(ExecutableTaskPtr move_task); void scheduleCommonTask(ExecutableTaskPtr common_task, bool need_trigger); diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index a944779ad44..ac1db503d9b 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -197,6 +197,11 @@ public: ++merges_with_ttl_counter; } + void cancelMergeWithTTL() + { + --merges_with_ttl_counter; + } + size_t getMergesWithTTLCount() const { return merges_with_ttl_counter; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 2cb62801fd5..81b61909228 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1168,8 +1168,12 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign { auto task = std::make_shared(*this, metadata_snapshot, false, Names{}, merge_entry, share_lock, common_assignee_trigger); task->setCurrentTransaction(std::move(transaction_for_merge), std::move(txn)); - assignee.scheduleMergeMutateTask(task); - return true; + bool scheduled = assignee.scheduleMergeMutateTask(task); + /// The problem that we already booked a slot for TTL merge, but a merge list entry will be created only in a prepare method + /// in MergePlainMergeTreeTask. So, this slot will never be freed. + if (!scheduled && isTTLMergeType(merge_entry->future_part->merge_type)) + getContext()->getMergeList().cancelMergeWithTTL(); + return scheduled; } if (mutate_entry) { From ec6e413f0bae58b2dd0cebb5babd1182a0f9d60e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Wed, 1 Jun 2022 23:00:49 +0200 Subject: [PATCH 133/136] Fixed runtime check for AVX512F --- src/Common/CpuId.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/CpuId.h b/src/Common/CpuId.h index 5037c687943..167fa22faf6 100644 --- a/src/Common/CpuId.h +++ b/src/Common/CpuId.h @@ -221,7 +221,7 @@ bool haveAVX512F() noexcept && (our_xgetbv(0) & 6u) == 6u // XMM state and YMM state are enabled by OS && ((our_xgetbv(0) >> 5) & 7u) == 7u // ZMM state is enabled by OS && CpuInfo(0x0).registers.eax >= 0x7 // leaf 7 is present - && ((CpuInfo(0x7).registers.ebx >> 16) & 1u); // AVX512F bit + && ((CpuInfo(0x7, 0).registers.ebx >> 16) & 1u); // AVX512F bit #else return false; #endif From 5fcf8401562bc67cf0a4ed6fd197c5283fe07c34 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 2 Jun 2022 08:43:44 +0300 Subject: [PATCH 134/136] Typo. --- src/IO/S3Common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 98471f5b81f..327730d9740 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -45,7 +45,7 @@ public: const String & force_region, const RemoteHostFilter & remote_host_filter, unsigned int s3_max_redirects, - bool enable_s3_requestrs_logging); + bool enable_s3_requests_logging); private: ClientFactory(); From a857bc2ccff315aeb16f1a4106789164cab8034e Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 2 Jun 2022 08:46:41 +0300 Subject: [PATCH 135/136] Update S3Common.cpp --- src/IO/S3Common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index c283afb21e4..fc4ab707026 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -765,9 +765,9 @@ namespace S3 const String & force_region, const RemoteHostFilter & remote_host_filter, unsigned int s3_max_redirects, - bool enable_s3_requestrs_logging) + bool enable_s3_requests_logging) { - return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requestrs_logging); + return PocoHTTPClientConfiguration(force_region, remote_host_filter, s3_max_redirects, enable_s3_requests_logging); } URI::URI(const Poco::URI & uri_) From eef6a5ec9684b057708bdb23e690e7da5fffd8fc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 2 Jun 2022 09:41:12 +0300 Subject: [PATCH 136/136] Revert "Remove resursive submodules" --- .github/workflows/master.yml | 64 +++++++++++++++--------------- .github/workflows/pull_request.yml | 64 +++++++++++++++--------------- .gitmodules | 6 +-- contrib/arrow | 2 +- contrib/brotli | 2 +- contrib/cppkafka | 2 +- contrib/msgpack-c | 2 +- contrib/rapidjson | 2 +- contrib/snappy | 2 +- utils/check-style/check-style | 3 -- 10 files changed, 73 insertions(+), 76 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e0954aab236..c890488ea80 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -215,8 +215,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -259,8 +259,8 @@ jobs: fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -305,8 +305,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -350,8 +350,8 @@ jobs: # uses: actions/checkout@v2 # - name: Build # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + # git -C "$GITHUB_WORKSPACE" submodule sync --recursive + # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -395,8 +395,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -440,8 +440,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -485,8 +485,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -530,8 +530,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -575,8 +575,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -623,8 +623,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -668,8 +668,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -715,8 +715,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -762,8 +762,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -809,8 +809,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -856,8 +856,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -903,8 +903,8 @@ jobs: fetch-depth: 0 # otherwise we will have no info about contributors - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f6e9880d088..76a26d685c5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -277,8 +277,8 @@ jobs: fetch-depth: 0 # for performance artifact - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -322,8 +322,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -367,8 +367,8 @@ jobs: # uses: actions/checkout@v2 # - name: Build # run: | - # git -C "$GITHUB_WORKSPACE" submodule sync - # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + # git -C "$GITHUB_WORKSPACE" submodule sync --recursive + # git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 # sudo rm -fr "$TEMP_PATH" # mkdir -p "$TEMP_PATH" # cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -414,8 +414,8 @@ jobs: fetch-depth: 0 # for performance artifact - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -459,8 +459,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -504,8 +504,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -549,8 +549,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -594,8 +594,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -639,8 +639,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -687,8 +687,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -732,8 +732,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -777,8 +777,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -822,8 +822,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -867,8 +867,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -912,8 +912,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" @@ -957,8 +957,8 @@ jobs: uses: actions/checkout@v2 - name: Build run: | - git -C "$GITHUB_WORKSPACE" submodule sync - git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10 + git -C "$GITHUB_WORKSPACE" submodule sync --recursive + git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10 sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" diff --git a/.gitmodules b/.gitmodules index aa68aa218b5..55fd684fddb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -79,10 +79,10 @@ url = https://github.com/ClickHouse/snappy.git [submodule "contrib/cppkafka"] path = contrib/cppkafka - url = https://github.com/ClickHouse/cppkafka.git + url = https://github.com/mfontanini/cppkafka.git [submodule "contrib/brotli"] path = contrib/brotli - url = https://github.com/ClickHouse/brotli.git + url = https://github.com/google/brotli.git [submodule "contrib/h3"] path = contrib/h3 url = https://github.com/ClickHouse/h3 @@ -144,7 +144,7 @@ ignore = untracked [submodule "contrib/msgpack-c"] path = contrib/msgpack-c - url = https://github.com/ClickHouse/msgpack-c + url = https://github.com/msgpack/msgpack-c [submodule "contrib/libcpuid"] path = contrib/libcpuid url = https://github.com/ClickHouse/libcpuid.git diff --git a/contrib/arrow b/contrib/arrow index 6f274b737c6..efdcd015cfd 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 6f274b737c66a6c39bab0d3bdf6cf7d139ef06f5 +Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5 diff --git a/contrib/brotli b/contrib/brotli index 5bd78768449..63be8a99401 160000 --- a/contrib/brotli +++ b/contrib/brotli @@ -1 +1 @@ -Subproject commit 5bd78768449751a78d4b4c646b0612917986f5b1 +Subproject commit 63be8a99401992075c23e99f7c84de1c653e39e2 diff --git a/contrib/cppkafka b/contrib/cppkafka index 64bd67db12b..5a119f689f8 160000 --- a/contrib/cppkafka +++ b/contrib/cppkafka @@ -1 +1 @@ -Subproject commit 64bd67db12b9c705e9127439a5b05b351d9df7da +Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1 diff --git a/contrib/msgpack-c b/contrib/msgpack-c index 790b3fe58eb..46684265d50 160000 --- a/contrib/msgpack-c +++ b/contrib/msgpack-c @@ -1 +1 @@ -Subproject commit 790b3fe58ebded7a8bd130782ef28bec5784c248 +Subproject commit 46684265d50b5d1b062d4c5c428ba08462844b1d diff --git a/contrib/rapidjson b/contrib/rapidjson index b571bd5c1a3..c4ef90ccdbc 160000 --- a/contrib/rapidjson +++ b/contrib/rapidjson @@ -1 +1 @@ -Subproject commit b571bd5c1a3b1fc931d77ae36932537a3c9018c3 +Subproject commit c4ef90ccdbc21d5d5a628d08316bfd301e32d6fa diff --git a/contrib/snappy b/contrib/snappy index 3786173af20..fb057edfed8 160000 --- a/contrib/snappy +++ b/contrib/snappy @@ -1 +1 @@ -Subproject commit 3786173af204d21da97180977ad6ab4321138b3d +Subproject commit fb057edfed820212076239fd32cb2ff23e9016bf diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 406b36e9251..84ce7ae5742 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -340,6 +340,3 @@ fi # Forbid files that differ only by character case find $ROOT_PATH | sort -f | uniq -i -c | awk '{ if ($1 > 1) print }' - -# Forbid recursive submodules -find $ROOT_PATH/contrib -name '.gitmodules' -size +0 | xargs cat | grep -P '.' && echo "Recursive submodules are forbidden."