From d1369165084dabdbdfe72378df8c84c4066235dd Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 17 Mar 2022 10:49:14 +0100 Subject: [PATCH 01/40] Remove testmode option --- src/Client/ClientBase.cpp | 22 +++---------------- src/Client/TestHint.cpp | 5 +---- src/Client/TestHint.h | 8 +++---- tests/clickhouse-test | 2 +- ...825_protobuf_format_no_length_delimiter.sh | 2 +- ..._block_size_rows_for_materialized_views.sh | 4 ++-- .../01280_ssd_complex_key_dictionary.sh | 4 ++-- ...006_client_test_hint_no_such_error_name.sh | 2 +- .../02234_clickhouse_local_test_mode.sh | 3 +-- ..._parallel_processing_on_replicas_part_1.sh | 4 ++-- 10 files changed, 18 insertions(+), 38 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index c575cd37a5f..be63b96f654 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1487,24 +1487,12 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( bool ClientBase::executeMultiQuery(const String & all_queries_text) { - // It makes sense not to base any control flow on this, so that it is - // the same in tests and in normal usage. The only difference is that in - // normal mode we ignore the test hints. - const bool test_mode = config().has("testmode"); - if (test_mode) - { - /// disable logs if expects errors - TestHint test_hint(test_mode, all_queries_text); - if (test_hint.clientError() || test_hint.serverError()) - processTextAsSingleQuery("SET send_logs_level = 'fatal'"); - } - bool echo_query = echo_queries; /// Test tags are started with "--" so they are interpreted as comments anyway. /// But if the echo is enabled we have to remove the test tags from `all_queries_text` /// because we don't want test tags to be echoed. - size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0; + size_t test_tags_length = getTestTagsLength(all_queries_text); /// Several queries separated by ';'. /// INSERT data is ended by the end of line, not ';'. @@ -1541,7 +1529,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) // Try to find test hint for syntax error. We don't know where // the query ends because we failed to parse it, so we consume // the entire line. - TestHint hint(test_mode, String(this_query_begin, this_query_end - this_query_begin)); + TestHint hint(String(this_query_begin, this_query_end - this_query_begin)); if (hint.serverError()) { // Syntax errors are considered as client errors @@ -1579,7 +1567,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) // Look for the hint in the text of query + insert data + trailing // comments, e.g. insert into t format CSV 'a' -- { serverError 123 }. // Use the updated query boundaries we just calculated. - TestHint test_hint(test_mode, full_query); + TestHint test_hint(full_query); // Echo all queries if asked; makes for a more readable reference file. echo_query = test_hint.echoQueries().value_or(echo_query); @@ -2182,8 +2170,6 @@ void ClientBase::init(int argc, char ** argv) ("suggestion_limit", po::value()->default_value(10000), "Suggestion limit for how many databases, tables and columns to fetch.") - ("testmode,T", "enable test hints in comments") - ("format,f", po::value(), "default output format") ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command") ("highlight", po::value()->default_value(true), "enable or disable basic syntax highlight in interactive command line") @@ -2289,8 +2275,6 @@ void ClientBase::init(int argc, char ** argv) config().setBool("interactive", true); if (options.count("pager")) config().setString("pager", options["pager"].as()); - if (options.count("testmode")) - config().setBool("testmode", true); if (options.count("log-level")) Poco::Logger::root().setLevel(options["log-level"].as()); diff --git a/src/Client/TestHint.cpp b/src/Client/TestHint.cpp index 2f3be2a5350..f6d1e5d73c3 100644 --- a/src/Client/TestHint.cpp +++ b/src/Client/TestHint.cpp @@ -32,12 +32,9 @@ int parseErrorCode(DB::ReadBufferFromString & in) namespace DB { -TestHint::TestHint(bool enabled_, const String & query_) +TestHint::TestHint(const String & query_) : query(query_) { - if (!enabled_) - return; - // Don't parse error hints in leading comments, because it feels weird. // Leading 'echo' hint is OK. bool is_leading_hint = true; diff --git a/src/Client/TestHint.h b/src/Client/TestHint.h index 377637d0db8..7fa4e86c025 100644 --- a/src/Client/TestHint.h +++ b/src/Client/TestHint.h @@ -7,7 +7,7 @@ namespace DB { -/// Checks expected server and client error codes in --testmode. +/// Checks expected server and client error codes. /// /// The following comment hints are supported: /// @@ -25,12 +25,12 @@ namespace DB /// /// Examples: /// -/// - echo 'select / -- { clientError 62 }' | clickhouse-client --testmode -nm +/// - echo 'select / -- { clientError 62 }' | clickhouse-client -nm /// // Here the client parses the query but it is incorrect, so it expects /// SYNTAX_ERROR (62). /// -/// - echo 'select foo -- { serverError 47 }' | clickhouse-client --testmode -nm +/// - echo 'select foo -- { serverError 47 }' | clickhouse-client -nm /// /// But here the query is correct, but there is no such column "foo", so it /// is UNKNOWN_IDENTIFIER server error. @@ -43,7 +43,7 @@ namespace DB class TestHint { public: - TestHint(bool enabled_, const String & query_); + TestHint(const String & query_); int serverError() const { return server_error; } int clientError() const { return client_error; } diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 121a283d0e4..9c2d599e9cd 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -657,7 +657,7 @@ class TestCase: pattern = '{test} > {stdout} 2> {stderr}' if self.ext == '.sql': - pattern = "{client} --send_logs_level={logs_level} --testmode --multiquery {options} < " + pattern + pattern = "{client} --send_logs_level={logs_level} --multiquery {options} < " + pattern command = pattern.format(**params) diff --git a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh index a16345c4bb1..a1bbdc318d5 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh @@ -43,7 +43,7 @@ $CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_no_length_delimiter_protobuf rm "$BINARY_FILE_PATH" # The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter. -$CLICKHOUSE_CLIENT --multiquery --testmode > /dev/null < /dev/null < /dev/null 2>&1 # fails echo "Should throw 1" -execute_insert --testmode +execute_insert echo "Should throw 2" -execute_insert --testmode --min_insert_block_size_rows=1 --min_insert_block_size_rows_for_materialized_views=$((1<<20)) +execute_insert --min_insert_block_size_rows=1 --min_insert_block_size_rows_for_materialized_views=$((1<<20)) # passes echo "Should pass 1" diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh index d5cae099f36..0de8b3a1a25 100755 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh @@ -41,7 +41,7 @@ $CLICKHOUSE_CLIENT -n --query=" LIFETIME(MIN 1000 MAX 2000) LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d'));" -$CLICKHOUSE_CLIENT --testmode -nq "SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 43 }" +$CLICKHOUSE_CLIENT -nq "SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 43 }" $CLICKHOUSE_CLIENT -n --query=" SELECT 'TEST_SMALL'; @@ -65,7 +65,7 @@ $CLICKHOUSE_CLIENT -n --query=" SELECT dictGetInt32('01280_db.ssd_dict', 'b', tuple('10', toInt32(-20))); SELECT dictGetString('01280_db.ssd_dict', 'c', tuple('10', toInt32(-20)));" -$CLICKHOUSE_CLIENT --testmode -nq "SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple(toInt32(3))); -- { serverError 53 }" +$CLICKHOUSE_CLIENT -nq "SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple(toInt32(3))); -- { serverError 53 }" $CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; DROP TABLE IF EXISTS 01280_db.keys_table; diff --git a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh index b846136ae58..972ff3ba73f 100755 --- a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh +++ b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh @@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT --testmode -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' +$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh index 6abe1e30334..f736751726d 100755 --- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh +++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh @@ -6,5 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL --query="SELECT n" 2>&1 | grep -q "Code: 47. DB::Exception: Missing columns:" && echo 'OK' || echo 'FAIL' ||: -$CLICKHOUSE_LOCAL --testmode --query="SELECT n -- { serverError 47 }" - +$CLICKHOUSE_LOCAL --query="SELECT n -- { serverError 47 }" diff --git a/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh b/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh index 276fc0274c2..58ce66056af 100755 --- a/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh +++ b/tests/queries/1_stateful/00168_parallel_processing_on_replicas_part_1.sh @@ -68,8 +68,8 @@ do TESTNAME_RESULT="/tmp/result_$TESTNAME" NEW_TESTNAME_RESULT="/tmp/result_dist_$TESTNAME" - $CLICKHOUSE_CLIENT $SETTINGS -nm --testmode < $TESTPATH > $TESTNAME_RESULT - $CLICKHOUSE_CLIENT $SETTINGS -nm --testmode < $NEW_TESTNAME > $NEW_TESTNAME_RESULT + $CLICKHOUSE_CLIENT $SETTINGS -nm < $TESTPATH > $TESTNAME_RESULT + $CLICKHOUSE_CLIENT $SETTINGS -nm < $NEW_TESTNAME > $NEW_TESTNAME_RESULT expected=$(cat $TESTNAME_RESULT | md5sum) actual=$(cat $NEW_TESTNAME_RESULT | md5sum) From 18ab49e788f3ed8a0c20634e31ba652129c0d24e Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:59:50 +0100 Subject: [PATCH 02/40] Check all logs for crashes, logical errors, etc in backward compatibility check --- docker/test/stress/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 3cef5b008db..e56afcbfd7a 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -348,13 +348,13 @@ then rm -f /test_output/tmp # OOM - zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv # Logical errors echo "Check for Logical errors in server log:" - zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_logical_errors.txt \ + zgrep -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log* > /test_output/bc_check_logical_errors.txt \ && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv @@ -362,13 +362,13 @@ then [ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt # Crash - zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv # It also checks for crash without stacktrace (printed by watchdog) echo "Check for Fatal message in server log:" - zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /test_output/bc_check_fatal_messages.txt \ + zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* > /test_output/bc_check_fatal_messages.txt \ && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv From 3bfd911ce2af3166efb7533a3a7fc33d577c70fd Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Sat, 26 Mar 2022 16:11:45 -0300 Subject: [PATCH 03/40] test for crash _join_with_nullable_lowcardinality --- ...th_nullable_lowcardinality_crash.reference | 2 ++ ...oin_with_nullable_lowcardinality_crash.sql | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.reference create mode 100644 tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.sql diff --git a/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.reference b/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.reference new file mode 100644 index 00000000000..12c61d9c54e --- /dev/null +++ b/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.reference @@ -0,0 +1,2 @@ +usa + diff --git a/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.sql b/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.sql new file mode 100644 index 00000000000..abc2ee41402 --- /dev/null +++ b/tests/queries/0_stateless/02245_join_with_nullable_lowcardinality_crash.sql @@ -0,0 +1,20 @@ +drop table if exists with_nullable; +drop table if exists without_nullable; + +CREATE TABLE with_nullable +( timestamp UInt32, + country LowCardinality(Nullable(String)) ) ENGINE = Memory; + +CREATE TABLE without_nullable +( timestamp UInt32, + country LowCardinality(String)) ENGINE = Memory; + +insert into with_nullable values(0,'f'),(0,'usa'); +insert into without_nullable values(0,'usa'),(0,'us2a'); + +select if(t0.country is null ,t2.country,t0.country) "country" +from without_nullable t0 right outer join with_nullable t2 on t0.country=t2.country; + +drop table with_nullable; +drop table without_nullable; + From bbfe8a2ca7bcd52aee0f138b59db4ad96b0b623f Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 28 Mar 2022 15:28:17 +0000 Subject: [PATCH 04/40] fix possible loss of subcolumns in type Object --- src/DataTypes/DataTypeTuple.cpp | 2 +- .../0_stateless/01825_type_json_9.reference | 1 + tests/queries/0_stateless/01825_type_json_9.sql | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01825_type_json_9.reference create mode 100644 tests/queries/0_stateless/01825_type_json_9.sql diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index a5e9868cf89..abf53a4baf1 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -206,7 +206,7 @@ bool DataTypeTuple::equals(const IDataType & rhs) const return false; for (size_t i = 0; i < size; ++i) - if (!elems[i]->equals(*rhs_tuple.elems[i])) + if (!elems[i]->equals(*rhs_tuple.elems[i]) || names[i] != rhs_tuple.names[i]) return false; return true; diff --git a/tests/queries/0_stateless/01825_type_json_9.reference b/tests/queries/0_stateless/01825_type_json_9.reference new file mode 100644 index 00000000000..a426b09a100 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_9.reference @@ -0,0 +1 @@ +Tuple(foo Int8, k1 Int8, k2 Int8) diff --git a/tests/queries/0_stateless/01825_type_json_9.sql b/tests/queries/0_stateless/01825_type_json_9.sql new file mode 100644 index 00000000000..8fa4b335578 --- /dev/null +++ b/tests/queries/0_stateless/01825_type_json_9.sql @@ -0,0 +1,16 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS t_json; + +SET allow_experimental_object_type = 1; + +CREATE TABLE t_json(id UInt64, obj JSON) ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_json format JSONEachRow {"id": 1, "obj": {"foo": 1, "k1": 2}}; +INSERT INTO t_json format JSONEachRow {"id": 2, "obj": {"foo": 1, "k2": 2}}; + +OPTIMIZE TABLE t_json FINAL; + +SELECT any(toTypeName(obj)) from t_json; + +DROP TABLE IF EXISTS t_json; From 6cbdc6af005f87e8b638c7c3f862cf1aea464a22 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 28 Mar 2022 18:44:53 +0000 Subject: [PATCH 05/40] remove obsolete parameter --- .../DataTypeLowCardinalityHelpers.cpp | 2 +- src/DataTypes/DataTypeTuple.cpp | 21 +++++++------------ src/DataTypes/DataTypeTuple.h | 6 +----- src/Functions/FunctionsConversion.h | 3 +-- src/Functions/tuple.cpp | 21 ++----------------- .../02008_tuple_to_name_value_pairs.sql | 2 +- 6 files changed, 13 insertions(+), 42 deletions(-) diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index 41ba81814d0..21ab25b6da3 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -36,7 +36,7 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type) element = recursiveRemoveLowCardinality(element); if (tuple_type->haveExplicitNames()) - return std::make_shared(elements, tuple_type->getElementNames(), tuple_type->serializeNames()); + return std::make_shared(elements, tuple_type->getElementNames()); else return std::make_shared(elements); } diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index abf53a4baf1..908e0184b8d 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -64,8 +64,8 @@ static std::optional checkTupleNames(const Strings & names) return {}; } -DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_, bool serialize_names_) - : elems(elems_), names(names_), have_explicit_names(true), serialize_names(serialize_names_) +DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_) + : elems(elems_), names(names_), have_explicit_names(true) { size_t size = elems.size(); if (names.size() != size) @@ -75,11 +75,6 @@ DataTypeTuple::DataTypeTuple(const DataTypes & elems_, const Strings & names_, b throw std::move(*exception); } -bool DataTypeTuple::canBeCreatedWithNames(const Strings & names) -{ - return checkTupleNames(names) == std::nullopt; -} - std::string DataTypeTuple::doGetName() const { size_t size = elems.size(); @@ -91,7 +86,7 @@ std::string DataTypeTuple::doGetName() const if (i != 0) s << ", "; - if (have_explicit_names && serialize_names) + if (have_explicit_names) s << backQuoteIfNeed(names[i]) << ' '; s << elems[i]->getName(); @@ -265,31 +260,29 @@ size_t DataTypeTuple::getSizeOfValueInMemory() const SerializationPtr DataTypeTuple::doGetDefaultSerialization() const { SerializationTuple::ElementSerializations serializations(elems.size()); - bool use_explicit_names = have_explicit_names && serialize_names; for (size_t i = 0; i < elems.size(); ++i) { - String elem_name = use_explicit_names ? names[i] : toString(i + 1); + String elem_name = have_explicit_names ? names[i] : toString(i + 1); auto serialization = elems[i]->getDefaultSerialization(); serializations[i] = std::make_shared(serialization, elem_name); } - return std::make_shared(std::move(serializations), use_explicit_names); + return std::make_shared(std::move(serializations), have_explicit_names); } SerializationPtr DataTypeTuple::getSerialization(const SerializationInfo & info) const { SerializationTuple::ElementSerializations serializations(elems.size()); const auto & info_tuple = assert_cast(info); - bool use_explicit_names = have_explicit_names && serialize_names; for (size_t i = 0; i < elems.size(); ++i) { - String elem_name = use_explicit_names ? names[i] : toString(i + 1); + String elem_name = have_explicit_names ? names[i] : toString(i + 1); auto serialization = elems[i]->getSerialization(*info_tuple.getElementInfo(i)); serializations[i] = std::make_shared(serialization, elem_name); } - return std::make_shared(std::move(serializations), use_explicit_names); + return std::make_shared(std::move(serializations), have_explicit_names); } MutableSerializationInfoPtr DataTypeTuple::createSerializationInfo(const SerializationInfo::Settings & settings) const diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h index db122aae5df..009a2284a0a 100644 --- a/src/DataTypes/DataTypeTuple.h +++ b/src/DataTypes/DataTypeTuple.h @@ -22,14 +22,11 @@ private: DataTypes elems; Strings names; bool have_explicit_names; - bool serialize_names = true; public: static constexpr bool is_parametric = true; explicit DataTypeTuple(const DataTypes & elems); - DataTypeTuple(const DataTypes & elems, const Strings & names, bool serialize_names_ = true); - - static bool canBeCreatedWithNames(const Strings & names); + DataTypeTuple(const DataTypes & elems, const Strings & names); TypeIndex getTypeId() const override { return TypeIndex::Tuple; } std::string doGetName() const override; @@ -66,7 +63,6 @@ public: String getNameByPosition(size_t i) const; bool haveExplicitNames() const { return have_explicit_names; } - bool serializeNames() const { return serialize_names; } }; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e098378f51a..587efa9f217 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -2957,8 +2957,7 @@ private: /// For named tuples allow conversions for tuples with /// different sets of elements. If element exists in @to_type /// and doesn't exist in @to_type it will be filled by default values. - if (from_type->haveExplicitNames() && from_type->serializeNames() - && to_type->haveExplicitNames() && to_type->serializeNames()) + if (from_type->haveExplicitNames() && to_type->haveExplicitNames()) { const auto & from_names = from_type->getElementNames(); std::unordered_map from_positions; diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 8e8b18e335d..6d5c53c0770 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -54,29 +54,12 @@ public: bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.empty()) throw Exception("Function " + getName() + " requires at least one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - DataTypes types; - Strings names; - - for (const auto & argument : arguments) - { - types.emplace_back(argument.type); - names.emplace_back(argument.name); - } - - /// Create named tuple if possible. We don't print tuple element names - /// because they are bad anyway -- aliases are not used, e.g. tuple(1 a) - /// will have element name '1' and not 'a'. If we ever change this, and - /// add the ability to access tuple elements by name, like tuple(1 a).a, - /// we should probably enable printing for better discoverability. - if (DataTypeTuple::canBeCreatedWithNames(names)) - return std::make_shared(types, names, false /*print names*/); - - return std::make_shared(types); + return std::make_shared(arguments); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override diff --git a/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql b/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql index 9204975b579..59987a86590 100644 --- a/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql +++ b/tests/queries/0_stateless/02008_tuple_to_name_value_pairs.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS test02008; CREATE TABLE test02008 ( col Tuple( a Tuple(key1 int, key2 int), - b Tuple(key1 int, key3 int) + b Tuple(key1 int, key2 int) ) ) ENGINE=Memory(); INSERT INTO test02008 VALUES (tuple(tuple(1, 2), tuple(3, 4))); From 8edf6e74487a59b6e21cb679509dd8fd95ca1d59 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:15:19 +0200 Subject: [PATCH 06/40] Mark test 02242_optimize_to_subcolumns_no_storage as backward incompatible for version 22.3.2.1 --- .../0_stateless/02242_optimize_to_subcolumns_no_storage.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql index e6e4663c5aa..8f8485eb58f 100644 --- a/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql +++ b/tests/queries/0_stateless/02242_optimize_to_subcolumns_no_storage.sql @@ -1,3 +1,4 @@ +-- Tags: no-backward-compatibility-check:22.3.2.1 SET optimize_functions_to_subcolumns = 1; SELECT count(*) FROM numbers(2) AS n1, numbers(3) AS n2, numbers(4) AS n3 WHERE (n1.number = n2.number) AND (n2.number = n3.number); From 8c05a3dffc662b2d638c0bbcbb4993b3e175149e Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:00:32 +0200 Subject: [PATCH 07/40] Update 01825_type_json_parallel_insert.sql --- tests/queries/0_stateless/01825_type_json_parallel_insert.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01825_type_json_parallel_insert.sql b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql index f54004a6630..93d1eecfbd7 100644 --- a/tests/queries/0_stateless/01825_type_json_parallel_insert.sql +++ b/tests/queries/0_stateless/01825_type_json_parallel_insert.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-backward-compatibility-check:22.3.2.1 DROP TABLE IF EXISTS t_json_parallel; SET allow_experimental_object_type = 1, max_insert_threads = 20, max_threads = 20; From 738966b6b6bf1683787906c692308e8f870bc040 Mon Sep 17 00:00:00 2001 From: shuchaome Date: Thu, 31 Mar 2022 00:03:45 +0800 Subject: [PATCH 08/40] fix filebuffer pos in RemoteReadBuffer --- src/Storages/Cache/ExternalDataSourceCache.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 18607c16ffa..17966d49c74 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -94,6 +94,8 @@ bool RemoteReadBuffer::nextImpl() return status; } + //file_buffer::pos should increase correspondingly when RemoteReadBuffer is consumed, otherwise start_offset will be incorrect. + local_file_holder->file_buffer->position() = local_file_holder->file_buffer->buffer().begin() + BufferBase::offset(); auto start_offset = local_file_holder->file_buffer->getPosition(); auto end_offset = start_offset + local_file_holder->file_buffer->internalBuffer().size(); local_file_holder->file_cache_controller->value().waitMoreData(start_offset, end_offset); From d166bb51153f630b9581902f8120ee0247b9d792 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Thu, 31 Mar 2022 13:47:15 +0200 Subject: [PATCH 09/40] Update 02245_format_string_stack_overflow.sql --- tests/queries/0_stateless/02245_format_string_stack_overflow.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02245_format_string_stack_overflow.sql b/tests/queries/0_stateless/02245_format_string_stack_overflow.sql index 1ee3606d3a6..9376b12aa1e 100644 --- a/tests/queries/0_stateless/02245_format_string_stack_overflow.sql +++ b/tests/queries/0_stateless/02245_format_string_stack_overflow.sql @@ -1 +1,2 @@ +-- Tags: no-backward-compatibility-check:22.3.2.2 select format('{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}{0}', toString(number)) str from numbers(1); From ececee3817edefb84794e00a2cea93f7ef07d29b Mon Sep 17 00:00:00 2001 From: zzsmdfj Date: Fri, 1 Apr 2022 20:13:34 +0800 Subject: [PATCH 10/40] to #34966_fix_dateTime_deserialize --- src/IO/ReadHelpers.h | 18 +++++++++++++++--- .../02249_parse_date_time_basic.reference | 3 +++ .../02249_parse_date_time_basic.sql | 6 ++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02249_parse_date_time_basic.reference create mode 100644 tests/queries/0_stateless/02249_parse_date_time_basic.sql diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 9396e1d32f7..8296b8db4d7 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -851,6 +851,8 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons /// YYYY-MM-DD hh:mm:ss static constexpr auto DateTimeStringInputSize = 19; + ///YYYY-MM-DD + static constexpr auto DateStringInputSize = 10; bool optimistic_path_for_date_time_input = s + DateTimeStringInputSize <= buf.buffer().end(); if (optimistic_path_for_date_time_input) @@ -861,16 +863,26 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons UInt8 month = (s[5] - '0') * 10 + (s[6] - '0'); UInt8 day = (s[8] - '0') * 10 + (s[9] - '0'); - UInt8 hour = (s[11] - '0') * 10 + (s[12] - '0'); - UInt8 minute = (s[14] - '0') * 10 + (s[15] - '0'); - UInt8 second = (s[17] - '0') * 10 + (s[18] - '0'); + UInt8 hour = 0; + UInt8 minute = 0; + UInt8 second = 0; + ///simply determine whether it is YYYY-MM-DD hh:mm:ss or YYYY-MM-DD by the content of the tenth character in an optimistic scenario + if (s[10] == ' ') + { + hour = (s[11] - '0') * 10 + (s[12] - '0'); + minute = (s[14] - '0') * 10 + (s[15] - '0'); + second = (s[17] - '0') * 10 + (s[18] - '0'); + } if (unlikely(year == 0)) datetime = 0; else datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); + if (s[10] == ' ') buf.position() += DateTimeStringInputSize; + else + buf.position() += DateStringInputSize; return ReturnType(true); } else diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.reference b/tests/queries/0_stateless/02249_parse_date_time_basic.reference new file mode 100644 index 00000000000..d67e0ae15e0 --- /dev/null +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.reference @@ -0,0 +1,3 @@ +2022-03-31 00:00:00 1 +2022-04-01 17:10:24 2 +2022-03-31 10:18:56 3 diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.sql b/tests/queries/0_stateless/02249_parse_date_time_basic.sql new file mode 100644 index 00000000000..dd2306d99aa --- /dev/null +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.sql @@ -0,0 +1,6 @@ +drop table if exists t; +CREATE TABLE t (a DateTime, b String, c String, d String, e Int32) ENGINE = Memory; +INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31','','','',1); +INSERT INTO t(a, b, c, d ,e) VALUES (1648804224,'','','',2); +INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31 10:18:56','','','',3); +select a, e from t; \ No newline at end of file From 3cae0c74d9fd18e76a2d02b1178aa7f533600900 Mon Sep 17 00:00:00 2001 From: zzsmdfj Date: Fri, 1 Apr 2022 20:16:07 +0800 Subject: [PATCH 11/40] to #34966_fix_dateTime_deserialize --- src/IO/ReadHelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 8296b8db4d7..48c291d8fcc 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -880,7 +880,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); if (s[10] == ' ') - buf.position() += DateTimeStringInputSize; + buf.position() += DateTimeStringInputSize; else buf.position() += DateStringInputSize; return ReturnType(true); From 79c75d8a712a3a49807164a0c618d04cfadbbb9c Mon Sep 17 00:00:00 2001 From: zzsmdfj Date: Fri, 1 Apr 2022 20:18:28 +0800 Subject: [PATCH 12/40] to #34966_fix_dateTime_deserialize --- tests/queries/0_stateless/02249_parse_date_time_basic.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.sql b/tests/queries/0_stateless/02249_parse_date_time_basic.sql index dd2306d99aa..2cea41874d5 100644 --- a/tests/queries/0_stateless/02249_parse_date_time_basic.sql +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.sql @@ -3,4 +3,4 @@ CREATE TABLE t (a DateTime, b String, c String, d String, e Int32) ENGINE = Memo INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31','','','',1); INSERT INTO t(a, b, c, d ,e) VALUES (1648804224,'','','',2); INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31 10:18:56','','','',3); -select a, e from t; \ No newline at end of file +select a, e from t; From 860b1a1b1bd8a1dec6d1176396400937d0346977 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Sat, 2 Apr 2022 14:04:04 +0200 Subject: [PATCH 13/40] Update 02248_nullable_custom_types_to_string.sql --- .../0_stateless/02248_nullable_custom_types_to_string.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql index 313f703fd03..b6032f7741b 100644 --- a/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql +++ b/tests/queries/0_stateless/02248_nullable_custom_types_to_string.sql @@ -1,3 +1,4 @@ +-- Tags: no-backward-compatibility-check:22.3.2.2 select toString(toNullable(true)); select toString(CAST(NULL, 'Nullable(Bool)')); select toString(toNullable(toIPv4('0.0.0.0'))); From 2a8e47927789d3d8b3d87794bfce0e22bb94aae9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Apr 2022 15:56:01 +0200 Subject: [PATCH 14/40] ExecutableUserDefinedFunction prevent function execution during query analysis --- src/Common/ProfileEvents.cpp | 2 ++ src/Common/ShellCommand.cpp | 6 ++++++ src/Interpreters/UserDefinedExecutableFunctionFactory.cpp | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 074ec02394b..3f55970f3aa 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -112,6 +112,8 @@ M(CompileExpressionsMicroseconds, "Total time spent for compilation of expressions to LLVM code.") \ M(CompileExpressionsBytes, "Number of bytes used for expressions compilation.") \ \ + M(ExecuteShellCommand, "Number of shell command executions.") \ + \ M(ExternalSortWritePart, "") \ M(ExternalSortMerge, "") \ M(ExternalAggregationWritePart, "") \ diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index 0093d72e766..229807c868e 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -29,6 +29,11 @@ namespace }; } +namespace ProfileEvents +{ + extern const int ExecuteShellCommand; +} + namespace DB { @@ -158,6 +163,7 @@ std::unique_ptr ShellCommand::executeImpl( const Config & config) { logCommand(filename, argv); + ProfileEvents::increment(ProfileEvents::ExecuteShellCommand); #if !defined(USE_MUSL) /** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs, diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index 6d7dee7a4c7..d3a38f42e21 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -57,6 +57,10 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + /// Do not start user defined script during query analysis. Because user script startup could be heavy. + if (input_rows_count == 0) + return result_type->createColumn(); + auto coordinator = executable_function->getCoordinator(); const auto & coordinator_configuration = coordinator->getConfiguration(); const auto & configuration = executable_function->getConfiguration(); From 482c8f667cee0bbc713ada47f9e129037988a72d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Apr 2022 16:10:19 +0200 Subject: [PATCH 15/40] Added tests --- ...table_user_defined_function_short_circuit.reference | 1 + ..._executable_user_defined_function_short_circuit.sql | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.reference create mode 100644 tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.sql diff --git a/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.reference b/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.sql b/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.sql new file mode 100644 index 00000000000..a475ba33740 --- /dev/null +++ b/tests/queries/0_stateless/02252_executable_user_defined_function_short_circuit.sql @@ -0,0 +1,10 @@ +SELECT number FROM numbers(10) WHERE number > 15 and test_function(number, number) == 4; + +SYSTEM FLUSH LOGS; + +SELECT ProfileEvents['ExecuteShellCommand'] FROM system.query_log WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query == 'SELECT number FROM numbers(10) WHERE number > 15 and test_function(number, number) == 4;' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute + LIMIT 1; From a46495de5c6f0ec2d44ef68666944be62eb4712f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 4 Apr 2022 16:22:16 +0200 Subject: [PATCH 16/40] JIT ProfileEvents added test --- .../02252_jit_profile_events.reference | 4 +++ .../0_stateless/02252_jit_profile_events.sql | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/queries/0_stateless/02252_jit_profile_events.reference create mode 100644 tests/queries/0_stateless/02252_jit_profile_events.sql diff --git a/tests/queries/0_stateless/02252_jit_profile_events.reference b/tests/queries/0_stateless/02252_jit_profile_events.reference new file mode 100644 index 00000000000..12d82114f75 --- /dev/null +++ b/tests/queries/0_stateless/02252_jit_profile_events.reference @@ -0,0 +1,4 @@ +0 +1 +0 1 2 +1 diff --git a/tests/queries/0_stateless/02252_jit_profile_events.sql b/tests/queries/0_stateless/02252_jit_profile_events.sql new file mode 100644 index 00000000000..e4c9d9d8791 --- /dev/null +++ b/tests/queries/0_stateless/02252_jit_profile_events.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest + +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +SYSTEM DROP COMPILED EXPRESSION CACHE; + +SELECT number + number + number FROM numbers(1); + +SYSTEM FLUSH LOGS; + +SELECT ProfileEvents['CompileFunction'] FROM system.query_log WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query == 'SELECT number + number + number FROM numbers(1);' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute + LIMIT 1; + +SET compile_aggregate_expressions = 1; +SET min_count_to_compile_aggregate_expression = 0; + +SELECT sum(number), sum(number + 1), sum(number + 2) FROM numbers(1) GROUP BY number; + +SYSTEM FLUSH LOGS; + +SELECT ProfileEvents['CompileFunction'] FROM system.query_log WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query == 'SELECT sum(number), sum(number), sum(number) FROM numbers(1) GROUP BY number;' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute + LIMIT 1; \ No newline at end of file From cf71b18472fbaf6a26b36f2062ae410e0f6bb01d Mon Sep 17 00:00:00 2001 From: Meena Renganathan Date: Mon, 4 Apr 2022 07:23:31 -0700 Subject: [PATCH 17/40] Modified the code to fix the getenv() call issue idenitified in the clang-tidy --- .clang-tidy | 2 ++ programs/client/Client.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 0400b500e5c..5e5fae57dba 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -164,6 +164,8 @@ Checks: '-*, clang-analyzer-unix.cstring.NullArg, boost-use-to-string, + + alpha.security.cert.env.InvalidPtr, ' WarningsAsErrors: '*' diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c2094b3b00d..becbc471d08 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -164,9 +164,9 @@ void Client::initialize(Poco::Util::Application & self) configReadClient(config(), home_path); const char * env_user = getenv("CLICKHOUSE_USER"); - const char * env_password = getenv("CLICKHOUSE_PASSWORD"); if (env_user) config().setString("user", env_user); + const char * env_password = getenv("CLICKHOUSE_PASSWORD"); if (env_password) config().setString("password", env_password); From 53c7376e37d206ebc954f3aeca7f98c565e4c53a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Mon, 4 Apr 2022 16:05:31 +0200 Subject: [PATCH 18/40] Add some metrics to engine Kafka --- src/Common/CurrentMetrics.cpp | 8 ++++ src/Common/ProfileEvents.cpp | 19 +++++++++ src/Storages/Kafka/KafkaSource.cpp | 30 ++++++++++++- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 42 +++++++++++++++++++ .../Kafka/ReadBufferFromKafkaConsumer.h | 7 ++++ src/Storages/Kafka/StorageKafka.cpp | 31 ++++++++++++++ .../Kafka/WriteBufferToKafkaProducer.cpp | 17 ++++++++ .../Kafka/WriteBufferToKafkaProducer.h | 9 ++++ 8 files changed, 162 insertions(+), 1 deletion(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index a741f1f1bfc..d49fc02084f 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -81,6 +81,14 @@ M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \ M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \ M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \ + M(KafkaConsumers, "Number of active Kafka consumers") \ + M(KafkaConsumersWithAssignment, "Number of active Kafka consumers which have some partitions assigned.") \ + M(KafkaProducers, "Number of active Kafka producer created") \ + M(KafkaLibrdkafkaThreads, "Number of active librdkafka threads") \ + M(KafkaBackgroundReads, "Number of background reads currently working (populating materialized views from Kafka)") \ + M(KafkaDirectReads, "Number of direct selects from Kafka currently executing") \ + M(KafkaWrites, "Number of currently running inserts to Kafka") \ + M(KafkaAssignedPartitions, "Number of partitions Kafka tables currently assigned to") \ namespace CurrentMetrics { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 074ec02394b..a963c024ab1 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -295,6 +295,25 @@ M(MergeTreeMetadataCacheHit, "Number of times the read of meta file was done from MergeTree metadata cache") \ M(MergeTreeMetadataCacheMiss, "Number of times the read of meta file was not done from MergeTree metadata cache") \ \ + M(KafkaRebalanceRevocations, "Number of partition revocations (the first stage of consumer group rebalance)") \ + M(KafkaRebalanceAssignments, "Number of partition assignments (the final stage of consumer group rebalance)") \ + M(KafkaRebalanceErrors, "Number of failed consumer group rebalances") \ + M(KafkaMessagesPolled, "Number of Kafka messages polled from librdkafka to ClickHouse") \ + M(KafkaMessagesRead, "Number of Kafka messages already processed by ClickHouse") \ + M(KafkaMessagesFailed, "Number of Kafka messages ClickHouse failed to parse") \ + M(KafkaRowsRead, "Number of rows parsed from Kafka messages") \ + M(KafkaRowsRejected, "Number of parsed rows which were later rejected (due to rebalances / errors or similar reasons). Those rows will be consumed again after the rebalance.") \ + M(KafkaDirectReads, "Number of direct selects from Kafka tables since server start") \ + M(KafkaBackgroundReads, "Number of background reads populating materialized views from Kafka since server start") \ + M(KafkaCommits, "Number of successful commits of consumed offsets to Kafka (normally should be the same as KafkaBackgroundReads)") \ + M(KafkaCommitFailures, "Number of failed commits of consumed offsets to Kafka (usually is a sign of some data duplication)") \ + M(KafkaConsumerErrors, "Number of errors reported by librdkafka during polls") \ + M(KafkaWrites, "Number of writes (inserts) to Kafka tables ") \ + M(KafkaRowsWritten, "Number of rows inserted into Kafka tables") \ + M(KafkaProducerFlushes, "Number of explicit flushes to Kafka producer") \ + M(KafkaMessagesProduced, "Number of messages produced to Kafka") \ + M(KafkaProducerErrors, "Number of errors during producing the messages to Kafka") \ + \ M(ScalarSubqueriesGlobalCacheHit, "Number of times a read from a scalar subquery was done using the global cache") \ M(ScalarSubqueriesLocalCacheHit, "Number of times a read from a scalar subquery was done using the local cache") \ M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely") diff --git a/src/Storages/Kafka/KafkaSource.cpp b/src/Storages/Kafka/KafkaSource.cpp index 99130f615f5..60047af8774 100644 --- a/src/Storages/Kafka/KafkaSource.cpp +++ b/src/Storages/Kafka/KafkaSource.cpp @@ -6,6 +6,16 @@ #include #include +#include + +namespace ProfileEvents +{ + extern const Event KafkaMessagesRead; + extern const Event KafkaMessagesFailed; + extern const Event KafkaRowsRead; + extern const Event KafkaRowsRejected; +} + namespace DB { namespace ErrorCodes @@ -85,6 +95,8 @@ Chunk KafkaSource::generateImpl() auto on_error = [&](const MutableColumns & result_columns, Exception & e) { + ProfileEvents::increment(ProfileEvents::KafkaMessagesFailed); + if (put_error_to_stream) { exception_message = e.message(); @@ -117,7 +129,11 @@ Chunk KafkaSource::generateImpl() size_t new_rows = 0; exception_message.reset(); if (buffer->poll()) + { + // poll provide one message at a time to the input_format + ProfileEvents::increment(ProfileEvents::KafkaMessagesRead); new_rows = executor.execute(); + } if (new_rows) { @@ -128,6 +144,8 @@ Chunk KafkaSource::generateImpl() if (buffer->isStalled()) throw Exception("Polled messages became unusable", ErrorCodes::LOGICAL_ERROR); + ProfileEvents::increment(ProfileEvents::KafkaRowsRead, new_rows); + buffer->storeLastReadMessageOffset(); auto topic = buffer->currentTopic(); @@ -212,8 +230,18 @@ Chunk KafkaSource::generateImpl() } } - if (buffer->polledDataUnusable() || total_rows == 0) + if (total_rows == 0) + { return {}; + } + else if (buffer->polledDataUnusable()) + { + // the rows were counted already before by KafkaRowsRead, + // so let's count the rows we ignore separately + // (they will be retried after the rebalance) + ProfileEvents::increment(ProfileEvents::KafkaRowsRejected, total_rows); + return {}; + } /// MATERIALIZED columns can be added here, but I think // they are not needed here: diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index ebfeaed8346..5ff90164064 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -10,6 +10,26 @@ #include #include +#include +#include + +namespace CurrentMetrics +{ + extern const Metric KafkaAssignedPartitions; + extern const Metric KafkaConsumersWithAssignment; +} + +namespace ProfileEvents +{ + extern const Event KafkaRebalanceRevocations; + extern const Event KafkaRebalanceAssignments; + extern const Event KafkaRebalanceErrors; + extern const Event KafkaMessagesPolled; + extern const Event KafkaCommitFailures; + extern const Event KafkaCommits; + extern const Event KafkaConsumerErrors; +} + namespace DB { @@ -45,6 +65,9 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // called (synchronously, during poll) when we enter the consumer group consumer->set_assignment_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { + CurrentMetrics::add(CurrentMetrics::KafkaAssignedPartitions, topic_partitions.size()); + ProfileEvents::increment(ProfileEvents::KafkaRebalanceAssignments); + if (topic_partitions.empty()) { LOG_INFO(log, "Got empty assignment: Not enough partitions in the topic for all consumers?"); @@ -52,6 +75,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( else { LOG_TRACE(log, "Topics/partitions assigned: {}", topic_partitions); + CurrentMetrics::add(CurrentMetrics::KafkaConsumersWithAssignment, 1); } assignment = topic_partitions; @@ -60,10 +84,18 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( // called (synchronously, during poll) when we leave the consumer group consumer->set_revocation_callback([this](const cppkafka::TopicPartitionList & topic_partitions) { + CurrentMetrics::sub(CurrentMetrics::KafkaAssignedPartitions, topic_partitions.size()); + ProfileEvents::increment(ProfileEvents::KafkaRebalanceRevocations); + // Rebalance is happening now, and now we have a chance to finish the work // with topics/partitions we were working with before rebalance LOG_TRACE(log, "Rebalance initiated. Revoking partitions: {}", topic_partitions); + if (!topic_partitions.empty()) + { + CurrentMetrics::sub(CurrentMetrics::KafkaConsumersWithAssignment, 1); + } + // we can not flush data to target from that point (it is pulled, not pushed) // so the best we can now it to // 1) repeat last commit in sync mode (async could be still in queue, we need to be sure is is properly committed before rebalance) @@ -91,6 +123,7 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( consumer->set_rebalance_error_callback([this](cppkafka::Error err) { LOG_ERROR(log, "Rebalance error: {}", err); + ProfileEvents::increment(ProfileEvents::KafkaRebalanceErrors); }); } @@ -229,8 +262,14 @@ void ReadBufferFromKafkaConsumer::commit() if (!committed) { // TODO: insert atomicity / transactions is needed here (possibility to rollback, on 2 phase commits) + ProfileEvents::increment(ProfileEvents::KafkaCommitFailures); throw Exception("All commit attempts failed. Last block was already written to target table(s), but was not committed to Kafka.", ErrorCodes::CANNOT_COMMIT_OFFSET); } + else + { + ProfileEvents::increment(ProfileEvents::KafkaCommits); + } + } else { @@ -423,6 +462,8 @@ bool ReadBufferFromKafkaConsumer::poll() return false; } + ProfileEvents::increment(ProfileEvents::KafkaMessagesPolled, messages.size()); + stalled_status = NOT_STALLED; allowed = true; return true; @@ -436,6 +477,7 @@ size_t ReadBufferFromKafkaConsumer::filterMessageErrors() { if (auto error = message.get_error()) { + ProfileEvents::increment(ProfileEvents::KafkaConsumerErrors); LOG_ERROR(log, "Consumer error: {}", error); return true; } diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 4e9bf2e55c2..f390d1c1330 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -5,6 +5,12 @@ #include #include +#include + +namespace CurrentMetrics +{ + extern const Metric KafkaConsumers; +} namespace Poco { @@ -67,6 +73,7 @@ public: private: using Messages = std::vector; + CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaConsumers}; enum StalledStatus { diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 4c7465d587d..722c55e6c93 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -41,6 +41,26 @@ #include +#include +#include + + +namespace CurrentMetrics +{ + extern const Metric KafkaLibrdkafkaThreads; + extern const Metric KafkaBackgroundReads; + extern const Metric KafkaDirectReads; + extern const Metric KafkaWrites; +} + +namespace ProfileEvents +{ + extern const Event KafkaDirectReads; + extern const Event KafkaBackgroundReads; + extern const Event KafkaWrites; +} + + namespace DB { @@ -58,6 +78,7 @@ struct StorageKafkaInterceptors static rd_kafka_resp_err_t rdKafkaOnThreadStart(rd_kafka_t *, rd_kafka_thread_type_t thread_type, const char *, void * ctx) { StorageKafka * self = reinterpret_cast(ctx); + CurrentMetrics::add(CurrentMetrics::KafkaLibrdkafkaThreads, 1); const auto & storage_id = self->getStorageID(); const auto & table = storage_id.getTableName(); @@ -89,6 +110,7 @@ struct StorageKafkaInterceptors static rd_kafka_resp_err_t rdKafkaOnThreadExit(rd_kafka_t *, rd_kafka_thread_type_t, const char *, void * ctx) { StorageKafka * self = reinterpret_cast(ctx); + CurrentMetrics::sub(CurrentMetrics::KafkaLibrdkafkaThreads, 1); std::lock_guard lock(self->thread_statuses_mutex); const auto it = std::find_if(self->thread_statuses.begin(), self->thread_statuses.end(), [](const auto & thread_status_ptr) @@ -279,6 +301,9 @@ Pipe StorageKafka::read( if (mv_attached) throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "Cannot read from StorageKafka with attached materialized views"); + CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaDirectReads}; + ProfileEvents::increment(ProfileEvents::KafkaDirectReads); + /// Always use all consumers at once, otherwise SELECT may not read messages from all partitions. Pipes pipes; pipes.reserve(num_created_consumers); @@ -304,6 +329,9 @@ SinkToStoragePtr StorageKafka::write(const ASTPtr &, const StorageMetadataPtr & auto modified_context = Context::createCopy(local_context); modified_context->applySettingsChanges(settings_adjustments); + CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaWrites}; + ProfileEvents::increment(ProfileEvents::KafkaWrites); + if (topics.size() > 1) throw Exception("Can't write to Kafka table with multiple topics!", ErrorCodes::NOT_IMPLEMENTED); return std::make_shared(*this, metadata_snapshot, modified_context); @@ -615,6 +643,9 @@ bool StorageKafka::streamToViews() if (!table) throw Exception("Engine table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::LOGICAL_ERROR); + CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaBackgroundReads}; + ProfileEvents::increment(ProfileEvents::KafkaBackgroundReads); + auto storage_snapshot = getStorageSnapshot(getInMemoryMetadataPtr()); // Create an INSERT query for streaming data diff --git a/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp b/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp index 748ea02ac6d..28877864e16 100644 --- a/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp +++ b/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp @@ -3,6 +3,16 @@ #include "Columns/ColumnString.h" #include "Columns/ColumnsNumber.h" +#include + +namespace ProfileEvents +{ + extern const Event KafkaRowsWritten; + extern const Event KafkaProducerFlushes; + extern const Event KafkaMessagesProduced; + extern const Event KafkaProducerErrors; +} + namespace DB { WriteBufferToKafkaProducer::WriteBufferToKafkaProducer( @@ -53,6 +63,8 @@ WriteBufferToKafkaProducer::~WriteBufferToKafkaProducer() void WriteBufferToKafkaProducer::countRow(const Columns & columns, size_t current_row) { + ProfileEvents::increment(ProfileEvents::KafkaRowsWritten); + if (++rows % max_rows == 0) { const std::string & last_chunk = chunks.back(); @@ -103,8 +115,10 @@ void WriteBufferToKafkaProducer::countRow(const Columns & columns, size_t curren producer->poll(timeout); continue; } + ProfileEvents::increment(ProfileEvents::KafkaProducerErrors); throw; } + ProfileEvents::increment(ProfileEvents::KafkaMessagesProduced); break; } @@ -126,9 +140,12 @@ void WriteBufferToKafkaProducer::flush() { if (e.get_error() == RD_KAFKA_RESP_ERR__TIMED_OUT) continue; + + ProfileEvents::increment(ProfileEvents::KafkaProducerErrors); throw; } + ProfileEvents::increment(ProfileEvents::KafkaProducerFlushes); break; } } diff --git a/src/Storages/Kafka/WriteBufferToKafkaProducer.h b/src/Storages/Kafka/WriteBufferToKafkaProducer.h index 15881b7a8e5..64b06571f0a 100644 --- a/src/Storages/Kafka/WriteBufferToKafkaProducer.h +++ b/src/Storages/Kafka/WriteBufferToKafkaProducer.h @@ -7,6 +7,14 @@ #include +#include + +namespace CurrentMetrics +{ + extern const Metric KafkaProducers; +} + + namespace DB { class Block; @@ -32,6 +40,7 @@ private: void nextImpl() override; void addChunk(); void reinitializeChunks(); + CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaProducers}; ProducerPtr producer; const std::string topic; From 0340932d57c538a1c56c5e5761872805ea78e8ac Mon Sep 17 00:00:00 2001 From: LAL2211 Date: Mon, 4 Apr 2022 14:35:21 -0400 Subject: [PATCH 19/40] updated hard coded/ default credentials --- tests/integration/helpers/config_cluster.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/helpers/config_cluster.py diff --git a/tests/integration/helpers/config_cluster.py b/tests/integration/helpers/config_cluster.py new file mode 100644 index 00000000000..e69de29bb2d From 3ecdad9d4baa0fc4cf3934fc3f170b34e04b5fb4 Mon Sep 17 00:00:00 2001 From: LAL2211 Date: Mon, 4 Apr 2022 14:49:30 -0400 Subject: [PATCH 20/40] updated --- tests/integration/helpers/cluster.py | 56 +++++++++++---------- tests/integration/helpers/config_cluster.py | 35 +++++++++++++ 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index d0b5e892f5b..3a9107d821e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -41,6 +41,8 @@ import docker from .client import Client from .hdfs_api import HDFSApi +from .config_cluster import * + HELPERS_DIR = p.dirname(__file__) CLICKHOUSE_ROOT_DIR = p.join(p.dirname(__file__), "../../..") LOCAL_DOCKER_COMPOSE_DIR = p.join( @@ -1657,8 +1659,8 @@ class ClickHouseCluster: while time.time() - start < timeout: try: conn = pymysql.connect( - user="root", - password="clickhouse", + user=mysql_user, + password=mysql_pass, host=self.mysql_ip, port=self.mysql_port, ) @@ -1679,8 +1681,8 @@ class ClickHouseCluster: while time.time() - start < timeout: try: conn = pymysql.connect( - user="root", - password="clickhouse", + user=mysql8_user, + password=mysql8_pass, host=self.mysql8_ip, port=self.mysql8_port, ) @@ -1704,8 +1706,8 @@ class ClickHouseCluster: try: for ip in [self.mysql2_ip, self.mysql3_ip, self.mysql4_ip]: conn = pymysql.connect( - user="root", - password="clickhouse", + user=mysql_user, + password=mysql_pass, host=ip, port=self.mysql_port, ) @@ -1728,9 +1730,9 @@ class ClickHouseCluster: self.postgres_conn = psycopg2.connect( host=self.postgres_ip, port=self.postgres_port, - database="postgres", - user="postgres", - password="mysecretpassword", + database=pg_db, + user=pg_user, + password=pg_pass, ) self.postgres_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres_conn.autocommit = True @@ -1752,9 +1754,9 @@ class ClickHouseCluster: self.postgres2_conn = psycopg2.connect( host=self.postgres2_ip, port=self.postgres_port, - database="postgres", - user="postgres", - password="mysecretpassword", + database=pg_db, + user=pg_user, + password=pg_pass, ) self.postgres2_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres2_conn.autocommit = True @@ -1768,9 +1770,9 @@ class ClickHouseCluster: self.postgres3_conn = psycopg2.connect( host=self.postgres3_ip, port=self.postgres_port, - database="postgres", - user="postgres", - password="mysecretpassword", + database=pg_db, + user=pg_user, + password=pg_pass, ) self.postgres3_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres3_conn.autocommit = True @@ -1784,9 +1786,9 @@ class ClickHouseCluster: self.postgres4_conn = psycopg2.connect( host=self.postgres4_ip, port=self.postgres_port, - database="postgres", - user="postgres", - password="mysecretpassword", + database=pg_db, + user=pg_user, + password=pg_pass, ) self.postgres4_conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) self.postgres4_conn.autocommit = True @@ -1938,7 +1940,7 @@ class ClickHouseCluster: def wait_mongo_to_start(self, timeout=30, secure=False): connection_str = "mongodb://{user}:{password}@{host}:{port}".format( - host="localhost", port=self.mongo_port, user="root", password="clickhouse" + host="localhost", port=self.mongo_port, user=mongo_user, password=mongo_pass ) if secure: connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" @@ -1962,8 +1964,8 @@ class ClickHouseCluster: ) minio_client = Minio( f"{self.minio_ip}:{self.minio_port}", - access_key="minio", - secret_key="minio123", + access_key=minio_access_key, + secret_key=minio_secret_key, secure=secure, http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"), ) # disable SSL check as we test ClickHouse and not Python library @@ -3481,16 +3483,16 @@ class ClickHouseInstance: "MySQL": { "DSN": "mysql_odbc", "Driver": "/usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so", - "Database": "clickhouse", - "Uid": "root", - "Pwd": "clickhouse", + "Database": odbc_mysql_db, + "Uid": odbc_mysql_uid, + "Pwd": odbc_mysql_pass, "Server": self.cluster.mysql_host, }, "PostgreSQL": { "DSN": "postgresql_odbc", - "Database": "postgres", - "UserName": "postgres", - "Password": "mysecretpassword", + "Database": odbc_psql_db, + "UserName": odbc_psql_user, + "Password": odbc_psql_pass, "Port": str(self.cluster.postgres_port), "Servername": self.cluster.postgres_host, "Protocol": "9.3", diff --git a/tests/integration/helpers/config_cluster.py b/tests/integration/helpers/config_cluster.py index e69de29bb2d..cb4bc6286ff 100644 --- a/tests/integration/helpers/config_cluster.py +++ b/tests/integration/helpers/config_cluster.py @@ -0,0 +1,35 @@ + +# MYSQL CREDENTIALS +mysql_user = 'root' +mysql_pass = 'clickhouse' + + +# MYSQL8 CREDENTIALS +mysql8_user = 'root' +mysql8_pass = 'clickhouse' + +# POSTGRES CREDENTIALS +pg_user = 'postgres' +pg_pass = 'mysecretpassword' +pg_db = 'postgres' + + +# MINIO CREDENTIALS +minio_access_key="minio" +minio_secret_key="minio123" + +# MONGODB CREDENTIALS +mongo_user = 'root' +mongo_pass = 'clickhouse' + +# ODBC CREDENTIALS +odbc_mysql_uid = 'root' +odbc_mysql_pass = 'clickhouse' +odbc_mysql_db = 'clickhouse' + +odbc_psql_db = 'postgres' +odbc_psql_user = 'postgres' +odbc_psql_pass = 'mysecretpassword' + + + From e78ff3ea7b0759ba80f9c509204a2e93462076f3 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Mon, 4 Apr 2022 21:20:18 +0000 Subject: [PATCH 21/40] fix: docker/test/integration/mysql_js_client/Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-DEBIAN9-PYTHON27-341379 - https://snyk.io/vuln/SNYK-DEBIAN9-PYTHON35-1063181 - https://snyk.io/vuln/SNYK-DEBIAN9-PYTHON35-340072 - https://snyk.io/vuln/SNYK-DEBIAN9-PYTHON35-453739 - https://snyk.io/vuln/SNYK-DEBIAN9-PYTHON35-584435 --- docker/test/integration/mysql_js_client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index b1397b40d38..14e0a20e87f 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -1,7 +1,7 @@ # docker build -t clickhouse/mysql-js-client . # MySQL JavaScript client docker container -FROM node:8 +FROM node:16.14.2 RUN npm install mysql From 4d7618585b7a5235e820663556932e004ee601ba Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 5 Apr 2022 00:10:33 +0200 Subject: [PATCH 22/40] Fix WORKDIR issue after upgrading npm --- docker/test/integration/mysql_js_client/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index 14e0a20e87f..4c9df10ace1 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -3,6 +3,8 @@ FROM node:16.14.2 +WORKDIR /usr/app + RUN npm install mysql -COPY ./test.js test.js +COPY ./test.js ./test.js From f408c86a13812f00c5ebf121879ea9ce09a23872 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 5 Apr 2022 00:42:29 +0200 Subject: [PATCH 23/40] Use FATAL logs level as default for clickhouse-test --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index de36fc3da27..f925fddcd1a 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1140,7 +1140,7 @@ def run_tests_array(all_tests_with_params): sys.stdout.flush() -server_logs_level = "warning" +server_logs_level = "fatal" def check_server_started(args): From 163664fad776eb2fd5613dda2fffd390d585b458 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 31 Mar 2022 15:55:56 +0800 Subject: [PATCH 24/40] Improve minmax_count_projection --- src/Storages/MergeTree/MergeTreeData.cpp | 260 +++++++++++------- src/Storages/MergeTree/MergeTreeData.h | 1 + .../01710_minmax_count_projection.reference | 2 + .../01710_minmax_count_projection.sql | 12 + 4 files changed, 181 insertions(+), 94 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 533f44ac9cf..a902895fcf0 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4641,6 +4641,7 @@ static void selectBestProjection( Block MergeTreeData::getMinMaxCountProjectionBlock( const StorageMetadataPtr & metadata_snapshot, const Names & required_columns, + bool has_filter, const SelectQueryInfo & query_info, const DataPartsVector & parts, DataPartsVector & normal_parts, @@ -4655,13 +4656,12 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( auto block = metadata_snapshot->minmax_count_projection->sample_block.cloneEmpty(); bool need_primary_key_max_column = false; const auto & primary_key_max_column_name = metadata_snapshot->minmax_count_projection->primary_key_max_column_name; + NameSet required_columns_set(required_columns.begin(), required_columns.end()); if (!primary_key_max_column_name.empty()) - { - need_primary_key_max_column = std::any_of( - required_columns.begin(), required_columns.end(), [&](const auto & name) { return primary_key_max_column_name == name; }); - } + need_primary_key_max_column = required_columns_set.contains(primary_key_max_column_name); auto partition_minmax_count_columns = block.mutateColumns(); + auto partition_minmax_count_column_names = block.getNames(); auto insert = [](ColumnAggregateFunction & column, const Field & value) { auto func = column.getAggregateFunction(); @@ -4670,51 +4670,77 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( size_t align_of_state = func->alignOfData(); auto * place = arena.alignedAlloc(size_of_state, align_of_state); func->create(place); - auto value_column = func->getReturnType()->createColumnConst(1, value)->convertToFullColumnIfConst(); - const auto * value_column_ptr = value_column.get(); - func->add(place, &value_column_ptr, 0, &arena); + if (const AggregateFunctionCount * agg_count = typeid_cast(func.get())) + agg_count->set(place, value.get()); + else + { + auto value_column = func->getReturnType()->createColumnConst(1, value)->convertToFullColumnIfConst(); + const auto * value_column_ptr = value_column.get(); + func->add(place, &value_column_ptr, 0, &arena); + } column.insertFrom(place); }; - ASTPtr expression_ast; - Block virtual_columns_block = getBlockWithVirtualPartColumns(parts, false /* one_part */, true /* ignore_empty */); - if (virtual_columns_block.rows() == 0) - return {}; - std::optional partition_pruner; std::optional minmax_idx_condition; DataTypes minmax_columns_types; - if (metadata_snapshot->hasPartitionKey()) - { - const auto & partition_key = metadata_snapshot->getPartitionKey(); - auto minmax_columns_names = getMinMaxColumnsNames(partition_key); - minmax_columns_types = getMinMaxColumnsTypes(partition_key); + size_t rows = parts.size(); + ColumnPtr part_name_column; - minmax_idx_condition.emplace( - query_info, - query_context, - minmax_columns_names, - getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(query_context))); - partition_pruner.emplace(metadata_snapshot, query_info, query_context, false /* strict */); + Block virtual_columns_block; + auto virtual_block = getSampleBlockWithVirtualColumns(); + bool has_virtual_column = std::any_of(required_columns.begin(), required_columns.end(), [&](const auto & name) { return virtual_block.has(name); }); + if (has_virtual_column || has_filter) + { + virtual_columns_block = getBlockWithVirtualPartColumns(parts, false /* one_part */, true /* ignore_empty */); + if (virtual_columns_block.rows() == 0) + return {}; } - // Generate valid expressions for filtering - VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, query_context, virtual_columns_block, expression_ast); - if (expression_ast) - VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, query_context, expression_ast); + if (has_filter) + { + if (metadata_snapshot->hasPartitionKey()) + { + const auto & partition_key = metadata_snapshot->getPartitionKey(); + auto minmax_columns_names = getMinMaxColumnsNames(partition_key); + minmax_columns_types = getMinMaxColumnsTypes(partition_key); + + minmax_idx_condition.emplace( + query_info, + query_context, + minmax_columns_names, + getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(query_context))); + partition_pruner.emplace(metadata_snapshot, query_info, query_context, false /* strict */); + } + + // Generate valid expressions for filtering + ASTPtr expression_ast; + VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, query_context, virtual_columns_block, expression_ast); + if (expression_ast) + VirtualColumnUtils::filterBlockWithQuery(query_info.query, virtual_columns_block, query_context, expression_ast); + + rows = virtual_columns_block.rows(); + part_name_column = virtual_columns_block.getByName("_part").column; + } - size_t rows = virtual_columns_block.rows(); - const ColumnString & part_name_column = typeid_cast(*virtual_columns_block.getByName("_part").column); - size_t part_idx = 0; auto filter_column = ColumnUInt8::create(); auto & filter_column_data = filter_column->getData(); - for (size_t row = 0; row < rows; ++row) + + DataPartsVector real_parts; + real_parts.reserve(rows); + for (size_t row = 0, part_idx = 0; row < rows; ++row, ++part_idx) { - while (parts[part_idx]->name != part_name_column.getDataAt(row)) - ++part_idx; + if (has_filter) + { + while (parts[part_idx]->name != part_name_column->getDataAt(row)) + ++part_idx; + } const auto & part = parts[part_idx]; + if (part->isEmpty()) + continue; + if (!part->minmax_idx->initialized) throw Exception("Found a non-empty part with uninitialized minmax_idx. It's a bug", ErrorCodes::LOGICAL_ERROR); @@ -4743,49 +4769,14 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( continue; } + real_parts.push_back(part); filter_column_data.back() = 1; - size_t pos = 0; - for (size_t i : metadata_snapshot->minmax_count_projection->partition_value_indices) - { - if (i >= part->partition.value.size()) - throw Exception("Partition value index is out of boundary. It's a bug", ErrorCodes::LOGICAL_ERROR); - partition_minmax_count_columns[pos++]->insert(part->partition.value[i]); - } - - size_t minmax_idx_size = part->minmax_idx->hyperrectangle.size(); - for (size_t i = 0; i < minmax_idx_size; ++i) - { - auto & min_column = assert_cast(*partition_minmax_count_columns[pos++]); - auto & max_column = assert_cast(*partition_minmax_count_columns[pos++]); - const auto & range = part->minmax_idx->hyperrectangle[i]; - insert(min_column, range.left); - insert(max_column, range.right); - } - - if (!primary_key_max_column_name.empty()) - { - const auto & primary_key_column = *part->index[0]; - auto & min_column = assert_cast(*partition_minmax_count_columns[pos++]); - auto & max_column = assert_cast(*partition_minmax_count_columns[pos++]); - insert(min_column, primary_key_column[0]); - insert(max_column, primary_key_column[primary_key_column.size() - 1]); - } - - { - auto & column = assert_cast(*partition_minmax_count_columns.back()); - auto func = column.getAggregateFunction(); - Arena & arena = column.createOrGetArena(); - size_t size_of_state = func->sizeOfData(); - size_t align_of_state = func->alignOfData(); - auto * place = arena.alignedAlloc(size_of_state, align_of_state); - func->create(place); - const AggregateFunctionCount & agg_count = assert_cast(*func); - agg_count.set(place, part->rows_count); - column.insertFrom(place); - } } - block.setColumns(std::move(partition_minmax_count_columns)); + if (real_parts.empty()) + return {}; + + size_t minmax_idx_size = real_parts.front()->minmax_idx->hyperrectangle.size(); FilterDescription filter(*filter_column); for (size_t i = 0; i < virtual_columns_block.columns(); ++i) { @@ -4793,8 +4784,77 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( column = column->filter(*filter.data, -1); } - if (block.rows() == 0) - return {}; + size_t pos = 0; + for (size_t i : metadata_snapshot->minmax_count_projection->partition_value_indices) + { + if (required_columns_set.contains(partition_minmax_count_column_names[pos])) + for (const auto & part : real_parts) + partition_minmax_count_columns[pos]->insert(part->partition.value[i]); + ++pos; + } + + for (size_t i = 0; i < minmax_idx_size; ++i) + { + if (required_columns_set.contains(partition_minmax_count_column_names[pos])) + { + for (const auto & part : real_parts) + { + const auto & range = part->minmax_idx->hyperrectangle[i]; + auto & min_column = assert_cast(*partition_minmax_count_columns[pos]); + insert(min_column, range.left); + } + } + ++pos; + + if (required_columns_set.contains(partition_minmax_count_column_names[pos])) + { + for (const auto & part : real_parts) + { + const auto & range = part->minmax_idx->hyperrectangle[i]; + auto & max_column = assert_cast(*partition_minmax_count_columns[pos]); + insert(max_column, range.right); + } + } + ++pos; + } + + if (!primary_key_max_column_name.empty()) + { + if (required_columns_set.contains(partition_minmax_count_column_names[pos])) + { + for (const auto & part : real_parts) + { + const auto & primary_key_column = *part->index[0]; + auto & min_column = assert_cast(*partition_minmax_count_columns[pos]); + insert(min_column, primary_key_column[0]); + } + } + ++pos; + + if (required_columns_set.contains(partition_minmax_count_column_names[pos])) + { + for (const auto & part : real_parts) + { + const auto & primary_key_column = *part->index[0]; + auto & max_column = assert_cast(*partition_minmax_count_columns[pos]); + insert(max_column, primary_key_column[primary_key_column.size() - 1]); + } + } + ++pos; + } + + bool has_count + = std::any_of(required_columns.begin(), required_columns.end(), [&](const auto & name) { return startsWith(name, "count"); }); + if (has_count) + { + for (const auto & part : real_parts) + { + auto & column = assert_cast(*partition_minmax_count_columns.back()); + insert(column, part->rows_count); + } + } + + block.setColumns(std::move(partition_minmax_count_columns)); Block res; for (const auto & name : required_columns) @@ -4803,6 +4863,11 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( res.insert(virtual_columns_block.getByName(name)); else if (block.has(name)) res.insert(block.getByName(name)); + else if (startsWith(name, "count")) // special case to match count(...) variants + { + const auto & column = block.getByName("count()"); + res.insert({column.column, column.type, name}); + } else throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -4974,7 +5039,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg }; auto virtual_block = getSampleBlockWithVirtualColumns(); - auto add_projection_candidate = [&](const ProjectionDescription & projection) + auto add_projection_candidate = [&](const ProjectionDescription & projection, bool normalize_count_not_null = false) { ProjectionCandidate candidate{}; candidate.desc = &projection; @@ -5001,22 +5066,28 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (projection.type == ProjectionDescription::Type::Aggregate && analysis_result.need_aggregate && can_use_aggregate_projection) { - bool match = true; Block aggregates; // Let's first check if all aggregates are provided by current projection for (const auto & aggregate : select.getQueryAnalyzer()->aggregates()) { - const auto * column = sample_block.findByName(aggregate.column_name); - if (column) - aggregates.insert(*column); - else + if (const auto * column = sample_block.findByName(aggregate.column_name)) { - match = false; - break; + aggregates.insert(*column); + continue; } - } - if (!match) + + if (normalize_count_not_null && dynamic_cast(aggregate.function.get())) + { + const auto * count_column = sample_block.findByName("count()"); + if (!count_column) + throw Exception(ErrorCodes::LOGICAL_ERROR, "count_column is missing when normalize_count_not_null == true. It is a bug"); + aggregates.insert({count_column->column, count_column->type, aggregate.column_name}); + continue; + } + + // No match return; + } // Check if all aggregation keys can be either provided by some action, or by current // projection directly. Reshape the `before_aggregation` action DAG so that it only @@ -5069,11 +5140,11 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg ProjectionCandidate * selected_candidate = nullptr; size_t min_sum_marks = std::numeric_limits::max(); if (metadata_snapshot->minmax_count_projection) - add_projection_candidate(*metadata_snapshot->minmax_count_projection); - std::optional minmax_conut_projection_candidate; + add_projection_candidate(*metadata_snapshot->minmax_count_projection, true); + std::optional minmax_count_projection_candidate; if (!candidates.empty()) { - minmax_conut_projection_candidate.emplace(std::move(candidates.front())); + minmax_count_projection_candidate.emplace(std::move(candidates.front())); candidates.clear(); } MergeTreeDataSelectExecutor reader(*this); @@ -5086,21 +5157,22 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg auto parts = getDataPartsVector(); // If minmax_count_projection is a valid candidate, check its completeness. - if (minmax_conut_projection_candidate) + if (minmax_count_projection_candidate) { DataPartsVector normal_parts; query_info.minmax_count_projection_block = getMinMaxCountProjectionBlock( metadata_snapshot, - minmax_conut_projection_candidate->required_columns, + minmax_count_projection_candidate->required_columns, + analysis_result.prewhere_info || analysis_result.before_where, query_info, parts, normal_parts, max_added_blocks.get(), query_context); - if (query_info.minmax_count_projection_block && minmax_conut_projection_candidate->prewhere_info) + if (query_info.minmax_count_projection_block && minmax_count_projection_candidate->prewhere_info) { - const auto & prewhere_info = minmax_conut_projection_candidate->prewhere_info; + const auto & prewhere_info = minmax_count_projection_candidate->prewhere_info; if (prewhere_info->alias_actions) ExpressionActions(prewhere_info->alias_actions, actions_settings).execute(query_info.minmax_count_projection_block); @@ -5119,7 +5191,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (normal_parts.empty()) { - selected_candidate = &*minmax_conut_projection_candidate; + selected_candidate = &*minmax_count_projection_candidate; selected_candidate->complete = true; min_sum_marks = query_info.minmax_count_projection_block.rows(); } @@ -5143,7 +5215,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (!normal_result_ptr->error()) { - selected_candidate = &*minmax_conut_projection_candidate; + selected_candidate = &*minmax_count_projection_candidate; selected_candidate->merge_tree_normal_select_result_ptr = normal_result_ptr; min_sum_marks = query_info.minmax_count_projection_block.rows() + normal_result_ptr->marks(); } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 1cbcd4282d0..2fec580f876 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -393,6 +393,7 @@ public: Block getMinMaxCountProjectionBlock( const StorageMetadataPtr & metadata_snapshot, const Names & required_columns, + bool has_filter, const SelectQueryInfo & query_info, const DataPartsVector & parts, DataPartsVector & normal_parts, diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index b13738a66de..bb35f3cfbd3 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -17,3 +17,5 @@ 0 2021-10-24 10:00:00 0 +1000 +1000 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index 0792fe331bb..dd360b65016 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -59,3 +59,15 @@ SELECT min(dt) FROM d PREWHERE ((0.9998999834060669 AND 1023) AND 255) <= ceil(j SELECT count('') AND NULL FROM d PREWHERE ceil(j) <= NULL; drop table d; + +-- count variant optimization + +drop table if exists test; +create table test (id Int64, d Int64, projection dummy(select * order by id)) engine MergeTree order by id; +insert into test select number, number from numbers(1e3); + +select count(if(d=4, d, 1)) from test settings force_optimize_projection = 1; +select count(d/3) from test settings force_optimize_projection = 1; +select count(if(d=4, Null, 1)) from test settings force_optimize_projection = 1; -- { serverError 584 } + +drop table test; From f87b25f2d705b5df4503b73178a7d9f6bb1dc4ac Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 31 Mar 2022 20:46:46 +0800 Subject: [PATCH 25/40] Fix tests --- ...2_last_granula_adjust_LOGICAL_ERROR.reference | 16 ++++++++-------- ...2052_last_granula_adjust_LOGICAL_ERROR.sql.j2 | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.reference b/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.reference index d7d3ee8f362..72d9eb2928a 100644 --- a/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.reference +++ b/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.reference @@ -1,8 +1,8 @@ -1 -1 -10 -10 -100 -100 -10000 -10000 +0 00000 +0 00000 +9 99999 +9 99999 +99 9999999999 +99 9999999999 +9999 99999999999999999999 +9999 99999999999999999999 diff --git a/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.sql.j2 b/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.sql.j2 index 465aa22beb3..53d970496b2 100644 --- a/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.sql.j2 +++ b/tests/queries/0_stateless/02052_last_granula_adjust_LOGICAL_ERROR.sql.j2 @@ -11,8 +11,8 @@ settings as select number, repeat(toString(number), 5) from numbers({{ rows_in_table }}); -- avoid any optimizations with ignore(*) -select count(ignore(*)) from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=1, max_threads=1; -select count(ignore(*)) from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=0, max_threads=1; -- { serverError CANNOT_READ_ALL_DATA } +select * apply max from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=1, max_threads=1; +select * apply max from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=0, max_threads=1; -- { serverError CANNOT_READ_ALL_DATA } drop table data_02052_{{ rows_in_table }}_wide{{ wide }}; {% endfor %} From 35a8bb2a9bb90c5706537361c19d29ee113e7611 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 1 Apr 2022 17:18:55 +0800 Subject: [PATCH 26/40] add comment --- src/Storages/MergeTree/MergeTreeData.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 2fec580f876..44736fe2cc5 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -383,6 +383,8 @@ public: /// Build a block of minmax and count values of a MergeTree table. These values are extracted /// from minmax_indices, the first expression of primary key, and part rows. /// + /// has_filter - if query has no filter, bypass partition pruning completely + /// /// query_info - used to filter unneeded parts /// /// parts - part set to filter From 5bc09550d877f5086741ddecfa99928a8c72f879 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 5 Apr 2022 15:55:17 +0800 Subject: [PATCH 27/40] Fix tests --- src/Storages/MergeTree/MergeTreeData.cpp | 36 ++++++++++++------- .../01710_minmax_count_projection.reference | 1 + .../01710_minmax_count_projection.sql | 2 ++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a902895fcf0..a2ae344994f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4681,12 +4681,6 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( column.insertFrom(place); }; - std::optional partition_pruner; - std::optional minmax_idx_condition; - DataTypes minmax_columns_types; - size_t rows = parts.size(); - ColumnPtr part_name_column; - Block virtual_columns_block; auto virtual_block = getSampleBlockWithVirtualColumns(); bool has_virtual_column = std::any_of(required_columns.begin(), required_columns.end(), [&](const auto & name) { return virtual_block.has(name); }); @@ -4697,6 +4691,11 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( return {}; } + size_t rows = parts.size(); + ColumnPtr part_name_column; + std::optional partition_pruner; + std::optional minmax_idx_condition; + DataTypes minmax_columns_types; if (has_filter) { if (metadata_snapshot->hasPartitionKey()) @@ -4730,7 +4729,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( real_parts.reserve(rows); for (size_t row = 0, part_idx = 0; row < rows; ++row, ++part_idx) { - if (has_filter) + if (part_name_column) { while (parts[part_idx]->name != part_name_column->getDataAt(row)) ++part_idx; @@ -4776,7 +4775,6 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( if (real_parts.empty()) return {}; - size_t minmax_idx_size = real_parts.front()->minmax_idx->hyperrectangle.size(); FilterDescription filter(*filter_column); for (size_t i = 0; i < virtual_columns_block.columns(); ++i) { @@ -4793,6 +4791,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( ++pos; } + size_t minmax_idx_size = real_parts.front()->minmax_idx->hyperrectangle.size(); for (size_t i = 0; i < minmax_idx_size; ++i) { if (required_columns_set.contains(partition_minmax_count_column_names[pos])) @@ -5039,7 +5038,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg }; auto virtual_block = getSampleBlockWithVirtualColumns(); - auto add_projection_candidate = [&](const ProjectionDescription & projection, bool normalize_count_not_null = false) + auto add_projection_candidate = [&](const ProjectionDescription & projection, bool minmax_count_projection = false) { ProjectionCandidate candidate{}; candidate.desc = &projection; @@ -5076,11 +5075,13 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg continue; } - if (normalize_count_not_null && dynamic_cast(aggregate.function.get())) + // We can treat every count_not_null_column as count() when selecting minmax_count_projection + if (minmax_count_projection && dynamic_cast(aggregate.function.get())) { const auto * count_column = sample_block.findByName("count()"); if (!count_column) - throw Exception(ErrorCodes::LOGICAL_ERROR, "count_column is missing when normalize_count_not_null == true. It is a bug"); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "`count()` column is missing when minmax_count_projection == true. It is a bug"); aggregates.insert({count_column->column, count_column->type, aggregate.column_name}); continue; } @@ -5107,9 +5108,20 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg candidate.before_aggregation->reorderAggregationKeysForProjection(key_name_pos_map); candidate.before_aggregation->addAggregatesViaProjection(aggregates); + // minmax_count_projections only have aggregation actions + if (minmax_count_projection) + candidate.required_columns = {required_columns.begin(), required_columns.end()}; + if (rewrite_before_where(candidate, projection, required_columns, sample_block_for_keys, aggregates)) { - candidate.required_columns = {required_columns.begin(), required_columns.end()}; + if (minmax_count_projection) + { + candidate.before_where = nullptr; + candidate.prewhere_info = nullptr; + } + else + candidate.required_columns = {required_columns.begin(), required_columns.end()}; + for (const auto & aggregate : aggregates) candidate.required_columns.push_back(aggregate.name); candidates.push_back(std::move(candidate)); diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index bb35f3cfbd3..259d320a38a 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -9,6 +9,7 @@ 1 9999 3 2021-10-25 10:00:00 2021-10-27 10:00:00 3 +2021-10-25 10:00:00 2021-10-27 10:00:00 3 1 1 1 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index dd360b65016..a6c04725583 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -50,6 +50,8 @@ drop table if exists d; create table d (dt DateTime, j int) engine MergeTree partition by (toDate(dt), ceiling(j), toDate(dt), CEILING(j)) order by tuple(); insert into d values ('2021-10-24 10:00:00', 10), ('2021-10-25 10:00:00', 10), ('2021-10-26 10:00:00', 10), ('2021-10-27 10:00:00', 10); select min(dt), max(dt), count() from d where toDate(dt) >= '2021-10-25'; +-- fuzz crash +select min(dt), max(dt), count(toDate(dt) >= '2021-10-25') from d where toDate(dt) >= '2021-10-25'; select count() from d group by toDate(dt); -- fuzz crash From 6e1d8374394b993811c56cfe962ae29f13efad8e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 5 Apr 2022 11:24:33 +0200 Subject: [PATCH 28/40] Fixed style check --- src/Common/ShellCommand.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ShellCommand.cpp b/src/Common/ShellCommand.cpp index 229807c868e..f24add7acf0 100644 --- a/src/Common/ShellCommand.cpp +++ b/src/Common/ShellCommand.cpp @@ -31,7 +31,7 @@ namespace namespace ProfileEvents { - extern const int ExecuteShellCommand; + extern const Event ExecuteShellCommand; } namespace DB From fd1c8103a0e675c8828fef70909426a9242e4ba7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 5 Apr 2022 11:47:45 +0200 Subject: [PATCH 29/40] Fixed tests --- tests/queries/0_stateless/02252_jit_profile_events.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02252_jit_profile_events.sql b/tests/queries/0_stateless/02252_jit_profile_events.sql index e4c9d9d8791..561e25505bc 100644 --- a/tests/queries/0_stateless/02252_jit_profile_events.sql +++ b/tests/queries/0_stateless/02252_jit_profile_events.sql @@ -26,6 +26,6 @@ SYSTEM FLUSH LOGS; SELECT ProfileEvents['CompileFunction'] FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryFinish' - AND query == 'SELECT sum(number), sum(number), sum(number) FROM numbers(1) GROUP BY number;' + AND query == 'SELECT sum(number), sum(number + 1), sum(number + 2) FROM numbers(1) GROUP BY number;' AND event_date >= yesterday() AND event_time > now() - interval 10 minute - LIMIT 1; \ No newline at end of file + LIMIT 1; From 4bfac4ec9990de8cea1218fa875f8cb946ad986d Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 5 Apr 2022 12:17:34 +0200 Subject: [PATCH 30/40] Remove more testmode mentions --- tests/integration/helpers/client.py | 2 +- tests/queries/0_stateless/01691_parser_data_type_exponential.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index af49408abee..41c5608081d 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -55,7 +55,7 @@ class Client: command = self.command[:] if stdin is None: - command += ["--multiquery", "--testmode"] + command += ["--multiquery"] stdin = sql else: command += ["--query", sql] diff --git a/tests/queries/0_stateless/01691_parser_data_type_exponential.sh b/tests/queries/0_stateless/01691_parser_data_type_exponential.sh index 2b1d34982a2..f8004f9350d 100755 --- a/tests/queries/0_stateless/01691_parser_data_type_exponential.sh +++ b/tests/queries/0_stateless/01691_parser_data_type_exponential.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh # Check that DataType parser does not have exponential complexity in the case found by fuzzer. -for _ in {1..10}; do ${CLICKHOUSE_CLIENT} -n --testmode --query "SELECT CAST(1 AS A2222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222220000000000000000000000000000000000000000000000000000000000000000000000000000002260637443813394204 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpio22222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggre222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 22222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 2222222222222eFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222222222222222222222222200000000000000000000178859639454016722222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpio22222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222222222222222222222222200000000000000000000178859639454016722222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 22222222222222222222222222222222222222222222222222222222222222222222222222222222222222222, 222222222222222ggregateFuncpion(groupBitmap222222222222222222222222222222222222222222222222222222222222222222222222000000000000000000001788596394540167623 222222222222222222ggregateFu22222222222222222222222222 222222222, UInt33)); -- { clientError 62 }"; done +for _ in {1..10}; do ${CLICKHOUSE_CLIENT} -n --query "SELECT CAST(1 AS A2222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222220000000000000000000000000000000000000000000000000000000000000000000000000000002260637443813394204 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpio22222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggre222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 22222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 2222222222222eFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222222222222222222222222200000000000000000000178859639454016722222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpio22222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 00000000000000000000000000000000000000000000000000000000000000000000000000000001841416382, 222222222222222ggregateFuncpion(groupBitmap22222222222222222222222222222222222222222222222222222222222222222222222200000000000000000000178859639454016722222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmapp, 222222222222222ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateF222222222222222222222222222222222222222222222222222222222teFuncpion(groupBitmap, 222222222222223ggregateFuncpion(groupBitmap2222222222222222222222222222222222222222222222222222 222222222222222222ggregateFuncpion(groupBitmap, 22222222222222222222222222222222222222222222222222222222222222222222222222222222222222222, 222222222222222ggregateFuncpion(groupBitmap222222222222222222222222222222222222222222222222222222222222222222222222000000000000000000001788596394540167623 222222222222222222ggregateFu22222222222222222222222222 222222222, UInt33)); -- { clientError 62 }"; done From 3a91b17044e58bd5bdd126c5fa983b628886874f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Violette?= Date: Tue, 5 Apr 2022 16:10:06 +0200 Subject: [PATCH 31/40] Update Contentsquare company case --- docs/en/introduction/adopters.md | 2 +- docs/ja/introduction/adopters.md | 2 +- .../clickhouse-community-meetup-in-paris-on-october-2-2018.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index ad199ce452e..20d6b20feb6 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -43,7 +43,7 @@ toc_title: Adopters | Citymobil | Taxi | Analytics | — | — | [Blog Post in Russian, March 2020](https://habr.com/en/company/citymobil/blog/490660/) | | Cloudflare | CDN | Traffic analysis | 36 servers | — | [Blog post, May 2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [Blog post, March 2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | Comcast | Media | CDN Traffic Analysis | — | — | [ApacheCon 2019 Talk](https://www.youtube.com/watch?v=e9TZ6gFDjNg) | -| ContentSquare | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Contentsquare | Web analytics | Main product | — | — | [Blog post in French, November 2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | Corunet | Analytics | Main product | — | — | [Slides in English, April 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | | CraiditX 氪信 | Finance AI | Analysis | — | — | [Slides in English, November 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | | Crazypanda | Games | | — | — | Live session on ClickHouse meetup | diff --git a/docs/ja/introduction/adopters.md b/docs/ja/introduction/adopters.md index 6f878bf1dfe..3372bb74f12 100644 --- a/docs/ja/introduction/adopters.md +++ b/docs/ja/introduction/adopters.md @@ -27,7 +27,7 @@ toc_title: "\u30A2\u30C0\u30D7\u30BF\u30FC" | Cisco | ネットワーク | トラフィック分析 | — | — | [ライトニングトーク2019](https://youtu.be/-hI1vDR2oPY?t=5057) | | Citadel Securities | 金融 | — | — | — | [2019年の貢献](https://github.com/ClickHouse/ClickHouse/pull/4774) | | シティモービル | タクシー | 分析 | — | — | [ロシア語でのブログ投稿,月2020](https://habr.com/en/company/citymobil/blog/490660/) | -| ContentSquare | ウェブ分析 | 主な製品 | — | — | [フランス語でのブログ投稿,November2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | +| Contentsquare | ウェブ分析 | 主な製品 | — | — | [フランス語でのブログ投稿,November2018](http://souslecapot.net/2018/11/21/patrick-chatain-vp-engineering-chez-contentsquare-penser-davantage-amelioration-continue-que-revolution-constante/) | | Cloudflare | CDN | トラフィック分析 | 36台のサーバー | — | [ブログ投稿,月2017](https://blog.cloudflare.com/how-cloudflare-analyzes-1m-dns-queries-per-second/), [ブログ投稿,月2018](https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/) | | コルネット | 分析 | 主な製品 | — | — | [2019年英語スライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup21/predictive_models.pdf) | | CraiditX 氪信 | ファイナンスAI | 分析 | — | — | [2019年のスライド](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup33/udf.pptx) | diff --git a/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md b/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md index a8c5c2a92dd..f94d2de411c 100644 --- a/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md +++ b/website/blog/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018.md @@ -7,7 +7,7 @@ tags: ['meetup', 'Paris', 'France', 'events'] Agenda of Paris ClickHouse Meetup was full of use cases, mostly from France-based companies which are actively using ClickHouse. Slides for all talks are [available on the GitHub](https://github.com/clickhouse/clickhouse-presentations/tree/master/meetup18). -Christophe Kalenzaga and Vianney Foucault, engineers from ContentSquare, company that provided the meetup venue: +Christophe Kalenzaga and Vianney Foucault, engineers from Contentsquare, company that provided the meetup venue: ![Christophe Kalenzaga and Vianney Foucault](https://blog-images.clickhouse.com/en/2018/clickhouse-community-meetup-in-paris-on-october-2-2018/1.jpg) Matthieu Jacquet from Storetail (Criteo): From 6eff1d2b02983738fbb30786c8fb5de7c9df43be Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 5 Apr 2022 17:30:03 +0200 Subject: [PATCH 32/40] Fixed tests --- tests/queries/0_stateless/02252_jit_profile_events.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02252_jit_profile_events.sql b/tests/queries/0_stateless/02252_jit_profile_events.sql index 561e25505bc..ddb95d4fa37 100644 --- a/tests/queries/0_stateless/02252_jit_profile_events.sql +++ b/tests/queries/0_stateless/02252_jit_profile_events.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-ubsan, no-cpu-aarch64 SET compile_expressions = 1; SET min_count_to_compile_expression = 0; From 3c0c1a11761f8d491ed5342c75ba24cc563db91b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Apr 2022 18:35:23 +0200 Subject: [PATCH 33/40] Add a comment #35919 --- programs/client/Client.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index b632849484c..cae74df97da 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -163,9 +163,23 @@ void Client::initialize(Poco::Util::Application & self) configReadClient(config(), home_path); + /** getenv is thread-safe in Linux glibc and in all sane libc implementations. + * But the standard does not guarantee that subsequent calls will not rewrite the value by returned pointer. + * + * man getenv: + * + * As typically implemented, getenv() returns a pointer to a string within the environment list. + * The caller must take care not to modify this string, since that would change the environment of + * the process. + * + * The implementation of getenv() is not required to be reentrant. The string pointed to by the return value of getenv() + * may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3). + */ + const char * env_user = getenv("CLICKHOUSE_USER"); if (env_user) config().setString("user", env_user); + const char * env_password = getenv("CLICKHOUSE_PASSWORD"); if (env_password) config().setString("password", env_password); From acaeaf28422d2e9a21148c99d3c7d8421b7ab37a Mon Sep 17 00:00:00 2001 From: LAL2211 Date: Tue, 5 Apr 2022 12:46:54 -0400 Subject: [PATCH 34/40] black check formatted --- tests/integration/helpers/config_cluster.py | 38 +++++++++------------ 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/tests/integration/helpers/config_cluster.py b/tests/integration/helpers/config_cluster.py index cb4bc6286ff..1912d9be06b 100644 --- a/tests/integration/helpers/config_cluster.py +++ b/tests/integration/helpers/config_cluster.py @@ -1,35 +1,31 @@ - # MYSQL CREDENTIALS -mysql_user = 'root' -mysql_pass = 'clickhouse' +mysql_user = "root" +mysql_pass = "clickhouse" # MYSQL8 CREDENTIALS -mysql8_user = 'root' -mysql8_pass = 'clickhouse' +mysql8_user = "root" +mysql8_pass = "clickhouse" # POSTGRES CREDENTIALS -pg_user = 'postgres' -pg_pass = 'mysecretpassword' -pg_db = 'postgres' +pg_user = "postgres" +pg_pass = "mysecretpassword" +pg_db = "postgres" # MINIO CREDENTIALS -minio_access_key="minio" -minio_secret_key="minio123" +minio_access_key = "minio" +minio_secret_key = "minio123" # MONGODB CREDENTIALS -mongo_user = 'root' -mongo_pass = 'clickhouse' +mongo_user = "root" +mongo_pass = "clickhouse" # ODBC CREDENTIALS -odbc_mysql_uid = 'root' -odbc_mysql_pass = 'clickhouse' -odbc_mysql_db = 'clickhouse' - -odbc_psql_db = 'postgres' -odbc_psql_user = 'postgres' -odbc_psql_pass = 'mysecretpassword' - - +odbc_mysql_uid = "root" +odbc_mysql_pass = "clickhouse" +odbc_mysql_db = "clickhouse" +odbc_psql_db = "postgres" +odbc_psql_user = "postgres" +odbc_psql_pass = "mysecretpassword" From 3412be9d4d6f13001734d4e34dabedbf1692d9ba Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 5 Apr 2022 20:25:05 +0200 Subject: [PATCH 35/40] Change KafkaDirectReads to KafkaConsumersInUse --- src/Common/CurrentMetrics.cpp | 2 +- src/Storages/Kafka/StorageKafka.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index d49fc02084f..4df1c4eaec8 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -86,7 +86,7 @@ M(KafkaProducers, "Number of active Kafka producer created") \ M(KafkaLibrdkafkaThreads, "Number of active librdkafka threads") \ M(KafkaBackgroundReads, "Number of background reads currently working (populating materialized views from Kafka)") \ - M(KafkaDirectReads, "Number of direct selects from Kafka currently executing") \ + M(KafkaConsumersInUse, "Number of consumers which are currently used by direct or background reads") \ M(KafkaWrites, "Number of currently running inserts to Kafka") \ M(KafkaAssignedPartitions, "Number of partitions Kafka tables currently assigned to") \ diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 722c55e6c93..32c6fd1a655 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -49,7 +49,7 @@ namespace CurrentMetrics { extern const Metric KafkaLibrdkafkaThreads; extern const Metric KafkaBackgroundReads; - extern const Metric KafkaDirectReads; + extern const Metric KafkaConsumersInUse; extern const Metric KafkaWrites; } @@ -301,7 +301,6 @@ Pipe StorageKafka::read( if (mv_attached) throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "Cannot read from StorageKafka with attached materialized views"); - CurrentMetrics::Increment metric_increment{CurrentMetrics::KafkaDirectReads}; ProfileEvents::increment(ProfileEvents::KafkaDirectReads); /// Always use all consumers at once, otherwise SELECT may not read messages from all partitions. @@ -386,6 +385,7 @@ void StorageKafka::pushReadBuffer(ConsumerBufferPtr buffer) std::lock_guard lock(mutex); buffers.push_back(buffer); semaphore.set(); + CurrentMetrics::sub(CurrentMetrics::KafkaConsumersInUse, 1); } @@ -410,6 +410,7 @@ ConsumerBufferPtr StorageKafka::popReadBuffer(std::chrono::milliseconds timeout) std::lock_guard lock(mutex); auto buffer = buffers.back(); buffers.pop_back(); + CurrentMetrics::add(CurrentMetrics::KafkaConsumersInUse, 1); return buffer; } From ea9ce3ea18cbe80ce63c2b02867dc11e39ac0bd8 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 5 Apr 2022 19:50:16 -0400 Subject: [PATCH 36/40] 'T' is added as delimiter, tests added --- src/IO/ReadHelpers.h | 5 +++-- .../0_stateless/02249_parse_date_time_basic.reference | 8 +++++--- tests/queries/0_stateless/02249_parse_date_time_basic.sql | 6 +++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 48c291d8fcc..e68da3a1c7d 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -867,7 +867,8 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons UInt8 minute = 0; UInt8 second = 0; ///simply determine whether it is YYYY-MM-DD hh:mm:ss or YYYY-MM-DD by the content of the tenth character in an optimistic scenario - if (s[10] == ' ') + bool dt_long = (s[10] == ' ' || s[10] == 'T'); + if (dt_long) { hour = (s[11] - '0') * 10 + (s[12] - '0'); minute = (s[14] - '0') * 10 + (s[15] - '0'); @@ -879,7 +880,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons else datetime = date_lut.makeDateTime(year, month, day, hour, minute, second); - if (s[10] == ' ') + if (dt_long) buf.position() += DateTimeStringInputSize; else buf.position() += DateStringInputSize; diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.reference b/tests/queries/0_stateless/02249_parse_date_time_basic.reference index d67e0ae15e0..027c72d802f 100644 --- a/tests/queries/0_stateless/02249_parse_date_time_basic.reference +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.reference @@ -1,3 +1,5 @@ -2022-03-31 00:00:00 1 -2022-04-01 17:10:24 2 -2022-03-31 10:18:56 3 +2022-03-31T04:00:00Z 1 +2022-04-01T09:10:24Z 2 +2022-03-31T14:18:56Z 3 +2022-03-31T14:18:56Z 4 +2022-04-01T09:10:24Z 5 diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.sql b/tests/queries/0_stateless/02249_parse_date_time_basic.sql index 2cea41874d5..cb443bbdd8e 100644 --- a/tests/queries/0_stateless/02249_parse_date_time_basic.sql +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.sql @@ -1,6 +1,10 @@ +SET date_time_output_format='iso'; drop table if exists t; CREATE TABLE t (a DateTime, b String, c String, d String, e Int32) ENGINE = Memory; INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31','','','',1); INSERT INTO t(a, b, c, d ,e) VALUES (1648804224,'','','',2); INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31 10:18:56','','','',3); -select a, e from t; +INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31T10:18:56','','','',4); +INSERT INTO t(a, b, c, d ,e) VALUES ('1648804224','','','',5); +select a, e from t order by e; +drop table if exists t; From d59d4eda4f3ffc4c45ea543fd5acaab64243bd72 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 5 Apr 2022 15:35:59 +0200 Subject: [PATCH 37/40] Fix tests --- docker/test/fuzzer/run-fuzzer.sh | 1 - src/Client/ClientBase.cpp | 7 +++++++ tests/clickhouse-test | 2 +- .../0_stateless/00921_datetime64_compatibility_long.sh | 2 +- tests/queries/0_stateless/02221_parallel_replicas_bug.sh | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 74711f476f8..32799a669eb 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -226,7 +226,6 @@ quit --receive_data_timeout_ms=10000 \ --stacktrace \ --query-fuzzer-runs=1000 \ - --testmode \ --queries-file $(ls -1 ch/tests/queries/0_stateless/*.sql | sort -R) \ $NEW_TESTS_OPT \ > >(tail -n 100000 > fuzzer.log) \ diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 24dba19a72c..e1d2b673571 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1499,6 +1499,13 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) /// Test tags are started with "--" so they are interpreted as comments anyway. /// But if the echo is enabled we have to remove the test tags from `all_queries_text` /// because we don't want test tags to be echoed. + { + /// disable logs if expects errors + TestHint test_hint(all_queries_text); + if (test_hint.clientError() || test_hint.serverError()) + processTextAsSingleQuery("SET send_logs_level = 'fatal'"); + } + size_t test_tags_length = getTestTagsLength(all_queries_text); /// Several queries separated by ';'. diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f925fddcd1a..de36fc3da27 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1140,7 +1140,7 @@ def run_tests_array(all_tests_with_params): sys.stdout.flush() -server_logs_level = "fatal" +server_logs_level = "warning" def check_server_started(args): diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh index 6d2cd0a176b..d310a2c3612 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh @@ -13,5 +13,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # ${CURDIR}/00921_datetime64_compatibility.python python3 "${CURDIR}"/00921_datetime64_compatibility_long.python \ - | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ + | ${CLICKHOUSE_CLIENT} --ignore-error -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ | grep -v -e 'Received exception .*$' -e '^(query: ' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh index b4ac6817a54..cce32bf8272 100755 --- a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh +++ b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh @@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 -nmT < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null +${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 -nm < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null From c7d72b92dac1d3650c61bdfd3d6a4286dbc97819 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy Date: Tue, 5 Apr 2022 20:32:52 -0400 Subject: [PATCH 38/40] explicit timezone added to test --- .../0_stateless/02249_parse_date_time_basic.reference | 6 +++--- tests/queries/0_stateless/02249_parse_date_time_basic.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.reference b/tests/queries/0_stateless/02249_parse_date_time_basic.reference index 027c72d802f..eb030a8fd3d 100644 --- a/tests/queries/0_stateless/02249_parse_date_time_basic.reference +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.reference @@ -1,5 +1,5 @@ -2022-03-31T04:00:00Z 1 +2022-03-31T00:00:00Z 1 2022-04-01T09:10:24Z 2 -2022-03-31T14:18:56Z 3 -2022-03-31T14:18:56Z 4 +2022-03-31T10:18:56Z 3 +2022-03-31T10:18:56Z 4 2022-04-01T09:10:24Z 5 diff --git a/tests/queries/0_stateless/02249_parse_date_time_basic.sql b/tests/queries/0_stateless/02249_parse_date_time_basic.sql index cb443bbdd8e..7146462fb74 100644 --- a/tests/queries/0_stateless/02249_parse_date_time_basic.sql +++ b/tests/queries/0_stateless/02249_parse_date_time_basic.sql @@ -1,6 +1,6 @@ SET date_time_output_format='iso'; drop table if exists t; -CREATE TABLE t (a DateTime, b String, c String, d String, e Int32) ENGINE = Memory; +CREATE TABLE t (a DateTime('UTC'), b String, c String, d String, e Int32) ENGINE = Memory; INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31','','','',1); INSERT INTO t(a, b, c, d ,e) VALUES (1648804224,'','','',2); INSERT INTO t(a, b, c, d ,e) VALUES ('2022-03-31 10:18:56','','','',3); From a2ce366c3420a2806eb6974e3cb45020115deb39 Mon Sep 17 00:00:00 2001 From: fenglv Date: Wed, 6 Apr 2022 04:49:43 +0000 Subject: [PATCH 39/40] parallel reading files for FileLog Engine --- src/Storages/FileLog/StorageFileLog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 32ca936f039..4fb19b12bab 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -723,6 +723,7 @@ bool StorageFileLog::streamToViews() size_t rows = 0; { block_io.pipeline.complete(std::move(input)); + block_io.pipeline.setNumThreads(max_streams_number); block_io.pipeline.setProgressCallback([&](const Progress & progress) { rows += progress.read_rows.load(); }); CompletedPipelineExecutor executor(block_io.pipeline); executor.execute(); From 84eef61d17ee430f2703f24e17c0ab8bb05715b9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 6 Apr 2022 06:39:56 +0000 Subject: [PATCH 40/40] Pull under reader mutex --- src/Storages/StorageURL.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index a435ab1a654..7bdb070bfdf 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -224,14 +224,12 @@ namespace } Chunk chunk; + std::lock_guard lock(reader_mutex); if (reader->pull(chunk)) return chunk; - { - std::lock_guard lock(reader_mutex); - pipeline->reset(); - reader.reset(); - } + pipeline->reset(); + reader.reset(); } }