diff --git a/README.md b/README.md index 0ded3035b55..aab4cb9f63c 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,15 @@ ClickHouse is an open-source column-oriented database management system that all * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster. * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. +* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. ## Upcoming Events +* [ClickHouse Online Meetup (in Russian)](https://events.yandex.ru/events/click-house-onlajn-vs-03-04-2020) on April 3, 2020. +* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date. * [Talks on Saint HighLoad++ in St. Petersburg](https://www.highload.ru/spb/2020/abstracts/6647) on TBD date. * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date. -* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020. diff --git a/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp b/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp index e577df472c8..60712636562 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionNull.cpp @@ -51,6 +51,10 @@ public: if (!has_nullable_types) throw Exception("Aggregate function combinator 'Null' requires at least one argument to be Nullable", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (nested_function) + if (auto adapter = nested_function->getOwnNullAdapter(nested_function, arguments, params)) + return adapter; + /// Special case for 'count' function. It could be called with Nullable arguments /// - that means - count number of calls, when all arguments are not NULL. if (nested_function && nested_function->getName() == "count") @@ -71,9 +75,9 @@ public: else { if (return_type_is_nullable) - return std::make_shared>(nested_function, arguments, params); + return std::make_shared>(nested_function, arguments, params); else - return std::make_shared>(nested_function, arguments, params); + return std::make_shared>(nested_function, arguments, params); } } }; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionNull.h b/dbms/src/AggregateFunctions/AggregateFunctionNull.h index a5000f30cd5..a0fe96b6f62 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionNull.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionNull.h @@ -204,12 +204,12 @@ public: }; -template -class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase> +template +class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase> { public: AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) - : AggregateFunctionNullBase>(std::move(nested_function_), arguments, params), + : AggregateFunctionNullBase>(std::move(nested_function_), arguments, params), number_of_arguments(arguments.size()) { if (number_of_arguments == 1) @@ -233,7 +233,7 @@ public: if (is_nullable[i]) { const ColumnNullable & nullable_col = assert_cast(*columns[i]); - if (nullable_col.isNullAt(row_num)) + if (null_is_skipped && nullable_col.isNullAt(row_num)) { /// If at least one column has a null value in the current row, /// we don't process this row. diff --git a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index e8668b6172e..3a1d2adee4a 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -11,7 +11,7 @@ #include #include -#include +#include namespace DB { @@ -186,6 +186,14 @@ private: { return event_idx + 1; } + else if (strict_order && first_event && events_timestamp[event_idx - 1] == -1) + { + for (size_t event = 0; event < events_timestamp.size(); ++event) + { + if (events_timestamp[event] == -1) + return event; + } + } else if (events_timestamp[event_idx - 1] >= 0 && timestamp <= events_timestamp[event_idx - 1] + window) { events_timestamp[event_idx] = events_timestamp[event_idx - 1]; @@ -232,6 +240,11 @@ public: return std::make_shared(); } + AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params) const override + { + return std::make_shared>(nested_function, arguments, params); + } + void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override { bool has_event = false; diff --git a/dbms/src/AggregateFunctions/IAggregateFunction.h b/dbms/src/AggregateFunctions/IAggregateFunction.h index d7ccd4c206a..48d31793a5a 100644 --- a/dbms/src/AggregateFunctions/IAggregateFunction.h +++ b/dbms/src/AggregateFunctions/IAggregateFunction.h @@ -31,6 +31,8 @@ using DataTypes = std::vector; using AggregateDataPtr = char *; using ConstAggregateDataPtr = const char *; +class IAggregateFunction; +using AggregateFunctionPtr = std::shared_ptr; /** Aggregate functions interface. * Instances of classes with this interface do not contain the data itself for aggregation, @@ -149,6 +151,17 @@ public: virtual void addBatchArray( size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena) const = 0; + /** By default all NULLs are skipped during aggregation. + * If it returns nullptr, the default one will be used. + * If an aggregate function wants to use something instead of the default one, it overrides this function and returns its own null adapter. + * nested_function is a smart pointer to this aggregate function itself. + * arguments and params are for nested_function. + */ + virtual AggregateFunctionPtr getOwnNullAdapter(const AggregateFunctionPtr & /*nested_function*/, const DataTypes & /*arguments*/, const Array & /*params*/) const + { + return nullptr; + } + const DataTypes & getArgumentTypes() const { return argument_types; } const Array & getParameters() const { return parameters; } @@ -244,6 +257,4 @@ public: }; -using AggregateFunctionPtr = std::shared_ptr; - } diff --git a/dbms/src/Common/QueryProfiler.cpp b/dbms/src/Common/QueryProfiler.cpp index 4bf5009d51c..ac2987a3795 100644 --- a/dbms/src/Common/QueryProfiler.cpp +++ b/dbms/src/Common/QueryProfiler.cpp @@ -40,7 +40,7 @@ namespace if (overrun_count) { /// But pass with some frequency to avoid drop of all traces. - if (write_trace_iteration % overrun_count == 0) + if (write_trace_iteration % (overrun_count + 1) == 0) { ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, overrun_count); } diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 8fe87eb9f3b..eaee356264d 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -421,6 +421,18 @@ void DDLWorker::processTasks() { processTask(task, zookeeper); } + catch (const Coordination::Exception & e) + { + if (server_startup && e.code == Coordination::ZNONODE) + { + LOG_WARNING(log, "ZooKeeper NONODE error during startup. Ignoring entry " << + task.entry_name << " (" << task.entry.query << ") : " << getCurrentExceptionMessage(true)); + } + else + { + throw; + } + } catch (...) { LOG_WARNING(log, "An error occurred while processing task " << task.entry_name << " (" << task.entry.query << ") : " diff --git a/dbms/src/Interpreters/SubqueryForSet.cpp b/dbms/src/Interpreters/SubqueryForSet.cpp index 47de516d154..98f670e5c64 100644 --- a/dbms/src/Interpreters/SubqueryForSet.cpp +++ b/dbms/src/Interpreters/SubqueryForSet.cpp @@ -53,7 +53,11 @@ bool SubqueryForSet::insertJoinedBlock(Block & block) void SubqueryForSet::setTotals() { if (join && source) - join->setTotals(source->getTotals()); + { + Block totals = source->getTotals(); + renameColumns(totals); + join->setTotals(totals); + } } } diff --git a/dbms/src/Interpreters/SubqueryForSet.h b/dbms/src/Interpreters/SubqueryForSet.h index 3463f708a46..4c99e34a2fc 100644 --- a/dbms/src/Interpreters/SubqueryForSet.h +++ b/dbms/src/Interpreters/SubqueryForSet.h @@ -42,6 +42,7 @@ struct SubqueryForSet private: NamesWithAliases joined_block_aliases; /// Rename column from joined block from this list. + /// Rename source right table column names into qualified column names if they conflicts with left table ones. void renameColumns(Block & block); }; diff --git a/dbms/src/Processors/Transforms/ExpressionTransform.cpp b/dbms/src/Processors/Transforms/ExpressionTransform.cpp index cbc83692c4b..bf52a13f08a 100644 --- a/dbms/src/Processors/Transforms/ExpressionTransform.cpp +++ b/dbms/src/Processors/Transforms/ExpressionTransform.cpp @@ -26,7 +26,7 @@ void ExpressionTransform::transform(Chunk & chunk) { initialized = true; - if (expression->resultIsAlwaysEmpty()) + if (expression->resultIsAlwaysEmpty() && !on_totals) { stopReading(); chunk.clear(); diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp index b5d288197b1..017cfc7cf82 100644 --- a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp @@ -25,7 +25,7 @@ void InflatingExpressionTransform::transform(Chunk & chunk) { initialized = true; - if (expression->resultIsAlwaysEmpty()) + if (expression->resultIsAlwaysEmpty() && !on_totals) { stopReading(); chunk.clear(); diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index e0f8cc5813a..6645b41376a 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -71,7 +71,7 @@ Pipes StorageMySQL::read( { check(column_names_); String query = transformQueryForExternalDatabase( - *query_info_.query, getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, context_); + query_info_, getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, remote_table_name, context_); Block sample_block; for (const String & column_name : column_names_) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 90c0d3418e6..336fef069d0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3336,7 +3336,7 @@ void StorageReplicatedMergeTree::alter( } else if (rc == Coordination::ZBADVERSION) { - if (dynamic_cast(*results[0]).error) + if (results[0]->error) throw Exception("Metadata on replica is not up to date with common metadata in Zookeeper. Cannot alter", ErrorCodes::CANNOT_ASSIGN_ALTER); continue; diff --git a/dbms/src/Storages/StorageXDBC.cpp b/dbms/src/Storages/StorageXDBC.cpp index 932d7799368..cedd2abf38f 100644 --- a/dbms/src/Storages/StorageXDBC.cpp +++ b/dbms/src/Storages/StorageXDBC.cpp @@ -74,7 +74,7 @@ std::function StorageXDBC::getReadPOSTDataCallback(const N QueryProcessingStage::Enum & /*processed_stage*/, size_t /*max_block_size*/) const { - String query = transformQueryForExternalDatabase(*query_info.query, + String query = transformQueryForExternalDatabase(query_info, getColumns().getOrdinary(), bridge_helper->getIdentifierQuotingStyle(), remote_database_name, diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index abf0b7a6361..385e47201a5 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -49,7 +49,10 @@ static void check(const std::string & query, const std::string & expected, const { ParserSelectQuery parser; ASTPtr ast = parseQuery(parser, query, 1000); - std::string transformed_query = transformQueryForExternalDatabase(*ast, columns, IdentifierQuotingStyle::DoubleQuotes, "test", "table", context); + SelectQueryInfo query_info; + query_info.syntax_analyzer_result = SyntaxAnalyzer(context).analyzeSelect(ast, columns); + query_info.query = ast; + std::string transformed_query = transformQueryForExternalDatabase(query_info, columns, IdentifierQuotingStyle::DoubleQuotes, "test", "table", context); EXPECT_EQ(transformed_query, expected); } diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index 5ba7deadb33..839175c9b72 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -134,16 +134,15 @@ bool isCompatible(const IAST & node) String transformQueryForExternalDatabase( - const IAST & query, + const SelectQueryInfo & query_info, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, const String & table, const Context & context) { - auto clone_query = query.clone(); - auto syntax_result = SyntaxAnalyzer(context).analyzeSelect(clone_query, available_columns); - const Names used_columns = syntax_result->requiredSourceColumns(); + auto clone_query = query_info.query->clone(); + const Names used_columns = query_info.syntax_analyzer_result->requiredSourceColumns(); auto select = std::make_shared(); diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.h b/dbms/src/Storages/transformQueryForExternalDatabase.h index 9b838da5127..09de53e36d3 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.h +++ b/dbms/src/Storages/transformQueryForExternalDatabase.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -23,7 +24,7 @@ class Context; * Compatible expressions are comparisons of identifiers, constants, and logical operations on them. */ String transformQueryForExternalDatabase( - const IAST & query, + const SelectQueryInfo & query_info, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, diff --git a/dbms/tests/integration/test_mysql_database_engine/test.py b/dbms/tests/integration/test_mysql_database_engine/test.py index 9e776b65f32..86e0b9df5fd 100644 --- a/dbms/tests/integration/test_mysql_database_engine/test.py +++ b/dbms/tests/integration/test_mysql_database_engine/test.py @@ -99,3 +99,24 @@ def test_clickhouse_dml_for_mysql_database(started_cluster): assert clickhouse_node.query("SELECT count() FROM `test_database`.`test_table`").rstrip() == '10000' mysql_node.query("DROP DATABASE test_database") + + +def test_clickhouse_join_for_mysql_database(started_cluster): + with contextlib.closing(MySQLNodeInstance('root', 'clickhouse', '127.0.0.1', port=3308)) as mysql_node: + mysql_node.query("CREATE DATABASE IF NOT EXISTS test DEFAULT CHARACTER SET 'utf8'") + mysql_node.query("CREATE TABLE test.t1_mysql_local (" + "pays VARCHAR(55) DEFAULT 'FRA' NOT NULL," + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco CHAR(3) DEFAULT '' NOT NULL" + ")") + mysql_node.query("CREATE TABLE test.t2_mysql_local (" + "service VARCHAR(5) DEFAULT '' NOT NULL," + "opco VARCHAR(5) DEFAULT ''" + ")") + clickhouse_node.query("CREATE TABLE default.t1_remote_mysql AS mysql('mysql1:3306','test','t1_mysql_local','root','clickhouse')") + clickhouse_node.query("CREATE TABLE default.t2_remote_mysql AS mysql('mysql1:3306','test','t2_mysql_local','root','clickhouse')") + assert clickhouse_node.query("SELECT s.pays " + "FROM default.t1_remote_mysql AS s " + "LEFT JOIN default.t1_remote_mysql AS s_ref " + "ON (s_ref.opco = s.opco AND s_ref.service = s.service)") == '' + mysql_node.query("DROP DATABASE test") diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference index 1e572be797c..492135567ea 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.reference @@ -36,3 +36,24 @@ [4, 2] [5, 2] [6, 1] +[7, 1] +[1, 2] +[2, 2] +[3, 0] +[4, 0] +[1, 2] +[2, 1] +[3, 0] +[4, 0] +[1, 0] +[2, 0] +[3, 1] +[4, 0] +[1, 0] +[2, 0] +[3, 1] +[4, 2] +[1, 0] +[2, 0] +[3, 1] +[4, 1] diff --git a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql index a8a2f522be9..5a1610256ac 100644 --- a/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql +++ b/dbms/tests/queries/0_stateless/00632_aggregation_window_funnel.sql @@ -63,4 +63,19 @@ insert into funnel_test_strict_order values (1, 6, 'c') (2, 6, 'c') (3, 6, 'b') select user, windowFunnel(86400)(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; select user, windowFunnel(86400, 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; select user, windowFunnel(86400, 'strict', 'strict_order')(dt, event='a', event='b', event='c') as s from funnel_test_strict_order group by user order by user format JSONCompactEachRow; +insert into funnel_test_strict_order values (1, 7, 'a') (2, 7, 'c') (3, 7, 'b'); +select user, windowFunnel(10, 'strict_order')(dt, event = 'a', event = 'b', event = 'c') as s from funnel_test_strict_order where user = 7 group by user format JSONCompactEachRow; drop table funnel_test_strict_order; + +drop table if exists funnel_test_non_null; +create table funnel_test_non_null (`dt` DateTime, `u` int, `a` Nullable(String), `b` Nullable(String)) engine = MergeTree() partition by dt order by u; +insert into funnel_test_non_null values (1, 1, 'a1', 'b1') (2, 1, 'a2', 'b2'); +insert into funnel_test_non_null values (1, 2, 'a1', null) (2, 2, 'a2', null); +insert into funnel_test_non_null values (1, 3, null, null); +insert into funnel_test_non_null values (1, 4, null, 'b1') (2, 4, 'a2', null) (3, 4, null, 'b3'); +select u, windowFunnel(86400)(dt, a = 'a1', a = 'a2') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a = 'a1', b = 'b2') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a is null and b is null) as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400)(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +select u, windowFunnel(86400, 'strict_order')(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; +drop table funnel_test_non_null; diff --git a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index c6e16fc5148..28e7bd0b94e 100755 --- a/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -46,15 +46,12 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO test.indices_mutaions1 VALUES $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" -sleep 1 +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 CLEAR INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" -wait_for_mutation "indices_mutaions1" "0000000000" "test" -wait_for_mutation "indices_mutaions2" "0000000000" "test" +$CLICKHOUSE_CLIENT --query="ALTER TABLE test.indices_mutaions1 MATERIALIZE INDEX idx IN PARTITION 1;" --replication_alter_partitions_sync=2 --mutations_sync=2 $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2;" $CLICKHOUSE_CLIENT --query="SELECT count() FROM test.indices_mutaions2 WHERE i64 = 2 FORMAT JSON;" | grep "rows_read" diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index 1fbccf47c72..37ed463f59b 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -97,7 +97,7 @@ done echo "Equal number of columns" # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_1 MODIFY COLUMN value0 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index 0ec6f01137f..114008ded26 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -96,7 +96,7 @@ done # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 5d181727301..bacc742d16a 100755 --- a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -101,7 +101,7 @@ wait echo "Finishing alters" # This alter will finish all previous, but replica 1 maybe still not up-to-date -while [[ $($CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do +while [[ $(timeout 30 $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1) ]]; do sleep 1 done diff --git a/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference new file mode 100644 index 00000000000..77db8015b0e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.reference @@ -0,0 +1,10 @@ + +0 + +0 + +0 + +0 + +0 0 0 diff --git a/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql new file mode 100644 index 00000000000..77e8848c957 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01107_join_right_table_totals.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS t; +CREATE TABLE t (item_id UInt64, price_sold Float32, date Date) ENGINE MergeTree ORDER BY item_id; + +SELECT item_id +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +FULL JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) r +USING (item_id); + +SELECT id +FROM (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) l +FULL JOIN (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) r +USING (id); + +SELECT item_id +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +INNER JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) r +USING (item_id); + +SELECT id +FROM (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) l +INNER JOIN (SELECT item_id AS id FROM t GROUP BY id WITH TOTALS) r +USING (id); + +SELECT id, yago, recent +FROM ( + SELECT item_id AS id, SUM(price_sold) AS recent + FROM t WHERE (date BETWEEN '2019-12-16' AND '2020-03-08') + GROUP BY id WITH TOTALS +) ll +FULL JOIN +( + SELECT item_id AS id, SUM(price_sold) AS yago + FROM t WHERE (date BETWEEN '2018-12-17' AND '2019-03-10') + GROUP BY id WITH TOTALS +) rr +USING (id); + +DROP TABLE t; diff --git a/docs/en/query_language/functions/ym_dict_functions.md b/docs/en/query_language/functions/ym_dict_functions.md index 0d556b0057c..344c850cc33 100644 --- a/docs/en/query_language/functions/ym_dict_functions.md +++ b/docs/en/query_language/functions/ym_dict_functions.md @@ -100,6 +100,30 @@ Example: `regionToCountry(toUInt32(213)) = 225` converts Moscow (213) to Russia Converts a region to a continent. In every other way, this function is the same as ‘regionToCity’. Example: `regionToContinent(toUInt32(213)) = 10001` converts Moscow (213) to Eurasia (10001). +### regionToTopContinent (#regiontotopcontinent) + +Finds the highest continent in the hierarchy for the region. + +**Syntax** + +```sql +regionToTopContinent(id[, geobase]); +``` + +**Parameters** + +- `id` — Region ID from the Yandex geobase. [UInt32](../../data_types/int_uint.md). +- `geobase` — Dictionary key. See [Multiple Geobases](#multiple-geobases). [String](../../data_types/string.md). Optional. + + +**Returned value** + +- Identifier of the top level continent (the latter when you climb the hierarchy of regions). +- 0, if there is none. + +Type: `UInt32`. + + ### regionToPopulation(id\[, geobase\]) {#regiontopopulationid-geobase} Gets the population for a region. diff --git a/docs/ru/query_language/functions/ym_dict_functions.md b/docs/ru/query_language/functions/ym_dict_functions.md index 0f08831b8dc..d5e11658a4f 100644 --- a/docs/ru/query_language/functions/ym_dict_functions.md +++ b/docs/ru/query_language/functions/ym_dict_functions.md @@ -101,6 +101,28 @@ LIMIT 15 Переводит регион в континент. В остальном, аналогично функции regionToCity. Пример: `regionToContinent(toUInt32(213)) = 10001` - преобразовали Москву (213) в Евразию (10001). +### regionToTopContinent (#regiontotopcontinent) + +Находит для региона верхний в иерархии континент. + +**Синтаксис** + +```sql +regionToTopContinent(id[, geobase]); +``` + +**Параметры** + +- `id` — Идентификатор региона из геобазы Яндекса. [UInt32](../../data_types/int_uint.md). +- `geobase` — Ключ словаря. Смотрите [Множественные геобазы](#multiple-geobases). [String](../../data_types/string.md). Опциональный параметр. + +**Возвращаемое значение** + +- Идентификатор континента верхнего уровня (последний при подъеме по иерархии регионов). +- 0, если его нет. + +Тип: `UInt32`. + ### regionToPopulation(id\[, geobase\]) {#regiontopopulationid-geobase} Получает население для региона. diff --git a/docs/ru/security_changelog.md b/docs/ru/security_changelog.md index d0becdbde9d..c8f66bf0475 100644 --- a/docs/ru/security_changelog.md +++ b/docs/ru/security_changelog.md @@ -65,4 +65,5 @@ unixODBC позволял указать путь для подключения Некорректная конфигурация в deb пакете могла привести к неавторизованному доступу к базе данных. Обнаружено благодаря: the UK’s National Cyber Security Centre (NCSC) -[Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) + +{## [Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) ##} diff --git a/docs/tools/README.md b/docs/tools/README.md index 56ca016ad9e..11e04de4b6a 100644 --- a/docs/tools/README.md +++ b/docs/tools/README.md @@ -38,5 +38,5 @@ The easiest way to see the result is to use `--livereload=8888` argument of buil At the moment there’s no easy way to do just that, but you can consider: -- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk). +- To hit the “Watch” button on top of GitHub web interface to know as early as possible, even during pull request. Alternative to this is `#github-activity` channel of [public ClickHouse Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng). - Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for https://clickhouse.tech. diff --git a/docs/tools/build.py b/docs/tools/build.py index 741e9343923..e841c6a212e 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -1,17 +1,15 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import argparse import datetime import http.server import logging +import multiprocessing import os import shutil import socketserver import subprocess import sys -import threading import time import jinja2 @@ -218,54 +216,64 @@ def build_single_page_version(lang, args, cfg): os.path.join(site_temp, 'single'), single_page_output_path ) + + logging.info(f'Re-building single page for {lang} pdf/test') + with util.temp_dir() as test_dir: + single_page_pdf = os.path.abspath( + os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf') + ) + extra['single_page'] = False + cfg.load_dict({ + 'docs_dir': docs_temp_lang, + 'site_dir': test_dir, + 'extra': extra, + 'nav': [ + {cfg.data.get('site_name'): 'single.md'} + ] + }) + mkdocs_build.build(cfg) - if not args.skip_pdf: - with util.temp_dir() as test_dir: - single_page_pdf = os.path.abspath( - os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf') - ) - extra['single_page'] = False - cfg.load_dict({ - 'docs_dir': docs_temp_lang, - 'site_dir': test_dir, - 'extra': extra, - 'nav': [ - {cfg.data.get('site_name'): 'single.md'} - ] - }) - mkdocs_build.build(cfg) - - css_in = ' '.join(website.get_css_in(args)) - js_in = ' '.join(website.get_js_in(args)) - subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) - subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + css_in = ' '.join(website.get_css_in(args)) + js_in = ' '.join(website.get_js_in(args)) + subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) + subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + if not args.skip_pdf: port_for_pdf = util.get_free_port() - with socketserver.TCPServer( - ('', port_for_pdf), http.server.SimpleHTTPRequestHandler - ) as httpd: - logging.info(f"serving for pdf at port {port_for_pdf}") - thread = threading.Thread(target=httpd.serve_forever) - with util.cd(test_dir): - thread.start() - create_pdf_command = [ - 'wkhtmltopdf', - '--print-media-type', - '--no-stop-slow-scripts', - '--log-level', 'warn', - f'http://localhost:{port_for_pdf}/single/', single_page_pdf - ] - try: - if args.save_raw_single_page: - shutil.copytree(test_dir, args.save_raw_single_page) - logging.info(' '.join(create_pdf_command)) - subprocess.check_call(' '.join(create_pdf_command), shell=True) - finally: - httpd.shutdown() - thread.join(timeout=5.0) + httpd = socketserver.TCPServer( + ('', port_for_pdf), http.server.SimpleHTTPRequestHandler + ) + logging.info(f"Serving for {lang} pdf at port {port_for_pdf}") + process = multiprocessing.Process(target=httpd.serve_forever) + with util.cd(test_dir): + process.start() + create_pdf_command = [ + 'wkhtmltopdf', + '--print-media-type', + '--disable-javascript', + # TODO: restore '--log-level', 'warn', + f'http://localhost:{port_for_pdf}/single/', single_page_pdf + ] + try: + if args.save_raw_single_page: + shutil.copytree(test_dir, args.save_raw_single_page) + logging.info(' '.join(create_pdf_command)) + subprocess.check_call(' '.join(create_pdf_command), shell=True) + finally: + logging.info(f'Stop serving for {lang} pdf at port {port_for_pdf}') + process.kill() + while True: + time.sleep(0.25) + try: + process.close() + break + except ValueError: + logging.info(f'Waiting for {lang} httpd at port {port_for_pdf} to stop') - if not args.version_prefix: # maybe enable in future - test.test_single_page( - os.path.join(test_dir, 'single', 'index.html'), lang) + + if not args.version_prefix: # maybe enable in future + logging.info(f'Running tests for {lang}') + test.test_single_page( + os.path.join(test_dir, 'single', 'index.html'), lang) logging.info(f'Finished building single page version for {lang}') diff --git a/docs/tools/concatenate.py b/docs/tools/concatenate.py index cfbc2acbb4c..65efe9bb5eb 100755 --- a/docs/tools/concatenate.py +++ b/docs/tools/concatenate.py @@ -14,9 +14,8 @@ def concatenate(lang, docs_path, single_page_file): files_to_concatenate = [(l[l.index(':') + 1:]).strip(" '\n") for l in cfg_file if '.md' in l and 'single_page' not in l] - logging.info( - str(len(files_to_concatenate)) + - ' files will be concatenated into single md-file.') + files_count = len(files_to_concatenate) + logging.info(f'{files_count} files will be concatenated into single md-file for {lang}.') logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) for path in files_to_concatenate: diff --git a/docs/tools/mdx_clickhouse.py b/docs/tools/mdx_clickhouse.py index f024c129462..50fe6ca4f15 100755 --- a/docs/tools/mdx_clickhouse.py +++ b/docs/tools/mdx_clickhouse.py @@ -11,6 +11,7 @@ import macros.plugin import slugify as slugify_impl + class ClickHouseLinkMixin(object): def handleMatch(self, m, data): diff --git a/docs/tools/util.py b/docs/tools/util.py index 3dc58807612..a746a50db1d 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -44,16 +44,19 @@ def get_free_port(): def run_function_in_parallel(func, args_list, threads=False): - processes = [] - exit_code = 0 for task in args_list: - cls = threading.Thread if threads else multiprocessing.Process - processes.append(cls(target=func, args=task)) - processes[-1].start() - for process in processes: - process.join() - if not threads: - if process.exitcode and not exit_code: - exit_code = process.exitcode - if exit_code: - sys.exit(exit_code) + func(*task) +# TODO: back to parallel +# processes = [] +# exit_code = 0 +# for task in args_list: +# cls = threading.Thread if threads else multiprocessing.Process +# processes.append(cls(target=func, args=task)) +# processes[-1].start() +# for process in processes: +# process.join() +# if not threads: +# if process.exitcode and not exit_code: +# exit_code = process.exitcode +# if exit_code: +# sys.exit(exit_code) diff --git a/docs/tools/website.py b/docs/tools/website.py index df2db26dd8a..a994982ce96 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -125,8 +125,7 @@ def minify_website(args): '--compilation_level', 'SIMPLE', '--dependency_mode', 'NONE', '--third_party', '--use_types_for_optimization', - '--isolation_mode', 'IIFE', - '--create_source_map', '%outname%.map' + '--isolation_mode', 'IIFE' ] logging.info(closure_args) if closure.run(*closure_args): diff --git a/website/css/bootstrap.css b/website/css/bootstrap.css index 8eac957a512..dcfed4306e6 100644 --- a/website/css/bootstrap.css +++ b/website/css/bootstrap.css @@ -7978,15 +7978,14 @@ button.bg-dark:focus { z-index: 1030; } -@supports ((position: -webkit-sticky) or (position: sticky)) { - .sticky-top { - position: -webkit-sticky; - position: sticky; - top: 0; - z-index: 1020; - } +.sticky-top { + position: -webkit-sticky; + position: sticky; + top: 0; + z-index: 1020; } + .sr-only { position: absolute; width: 1px; @@ -10221,4 +10220,4 @@ a.text-dark:hover, a.text-dark:focus { border-color: #dee2e6; } } -/*# sourceMappingURL=bootstrap.css.map */ \ No newline at end of file +/*# sourceMappingURL=bootstrap.css.map */