diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 3044050d12b..7ed2aab66bb 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -148,7 +148,7 @@ function run_tests TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") # the grep is to filter out set -x output and keep only time output - { time "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue + { time "$script_dir/perf.py" --host=localhost --port=9001 --host=localhost --port=9002 "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue # The test completed with zero status, so we treat stderr as warnings mv "$test_name-err.log" "$test_name-warn.log" diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index dc516d7029e..55d93f89c6e 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -23,8 +23,8 @@ report_stage_end('start') parser = argparse.ArgumentParser(description='Run performance test.') # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.") -parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.") +parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.") +parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.") parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') args = parser.parse_args() diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 2d2025de58b..82e2f3a6373 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -62,12 +62,14 @@ CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \ ln -s /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/; \ + ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/; \ ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/; \ + ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/; \ ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/; \ ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/; \ diff --git a/docs/en/interfaces/third-party/client_libraries.md b/docs/en/interfaces/third-party/client_libraries.md index 1f5c0e31933..f408e4e0193 100644 --- a/docs/en/interfaces/third-party/client_libraries.md +++ b/docs/en/interfaces/third-party/client_libraries.md @@ -34,6 +34,7 @@ toc_title: Client Libraries - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/es/interfaces/third-party/client_libraries.md b/docs/es/interfaces/third-party/client_libraries.md index 28c3a9529f4..31e9afcac91 100644 --- a/docs/es/interfaces/third-party/client_libraries.md +++ b/docs/es/interfaces/third-party/client_libraries.md @@ -36,6 +36,7 @@ toc_title: Bibliotecas de clientes - [Cualquier evento-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Rubí - [Haga clic en Casa (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [Sistema abierto.](https://github.com/hannesmuehleisen/clickhouse-r) - [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/fa/interfaces/third-party/client_libraries.md b/docs/fa/interfaces/third-party/client_libraries.md index 9d61ab4bd77..dff3dc48717 100644 --- a/docs/fa/interfaces/third-party/client_libraries.md +++ b/docs/fa/interfaces/third-party/client_libraries.md @@ -37,6 +37,7 @@ toc_title: "\u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0647\u0627\u06CC - [هرفنت-کلیکهاوس](https://metacpan.org/release/AnyEvent-ClickHouse) - روبی - [تاتر (روبی)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [کلیک تحقیق](https://github.com/hannesmuehleisen/clickhouse-r) - [خانه روستایی](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/fr/interfaces/third-party/client_libraries.md b/docs/fr/interfaces/third-party/client_libraries.md index 940887e4010..e4be3a53646 100644 --- a/docs/fr/interfaces/third-party/client_libraries.md +++ b/docs/fr/interfaces/third-party/client_libraries.md @@ -36,6 +36,7 @@ toc_title: "Biblioth\xE8ques Clientes" - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Rubis - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/ja/interfaces/third-party/client_libraries.md b/docs/ja/interfaces/third-party/client_libraries.md index bcd0ff895d5..b561e504f7c 100644 --- a/docs/ja/interfaces/third-party/client_libraries.md +++ b/docs/ja/interfaces/third-party/client_libraries.md @@ -36,6 +36,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - [クリックハウス(ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [クリックハウス-r](https://github.com/hannesmuehleisen/clickhouse-r) - [Rクリックハウス](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/ru/interfaces/third-party/client_libraries.md b/docs/ru/interfaces/third-party/client_libraries.md index 3ec3901666a..27fb3f67512 100644 --- a/docs/ru/interfaces/third-party/client_libraries.md +++ b/docs/ru/interfaces/third-party/client_libraries.md @@ -29,6 +29,7 @@ - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) diff --git a/docs/zh/interfaces/third-party/client_libraries.md b/docs/zh/interfaces/third-party/client_libraries.md index bf74c490092..4814ca5cf9a 100644 --- a/docs/zh/interfaces/third-party/client_libraries.md +++ b/docs/zh/interfaces/third-party/client_libraries.md @@ -28,6 +28,7 @@ - [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse) - Ruby - [ClickHouse (Ruby)](https://github.com/shlima/click_house) + - [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord) - R - [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r) - [RClickhouse](https://github.com/IMSMWU/RClickhouse) diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index 4c0323fcf6a..12d289deaed 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -275,15 +275,20 @@ void GroupingAggregatedTransform::work() { if (!single_level_chunks.empty()) { - auto & header = getOutputs().front().getHeader(); + auto & header = getInputs().front().getHeader(); /// Take header from input port. Output header is empty. auto block = header.cloneWithColumns(single_level_chunks.back().detachColumns()); single_level_chunks.pop_back(); auto blocks = params->aggregator.convertBlockToTwoLevel(block); for (auto & cur_block : blocks) { + if (!cur_block) + continue; + Int32 bucket = cur_block.info.bucket_num; - chunks_map[bucket].emplace_back(Chunk(cur_block.getColumns(), cur_block.rows())); + auto chunk_info = std::make_shared(); + chunk_info->bucket_num = bucket; + chunks_map[bucket].emplace_back(Chunk(cur_block.getColumns(), cur_block.rows(), std::move(chunk_info))); } } } diff --git a/tests/config/access_management.xml b/tests/config/access_management.xml new file mode 100644 index 00000000000..7e799cb7b10 --- /dev/null +++ b/tests/config/access_management.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/config/clusters.xml b/tests/config/clusters.xml new file mode 100644 index 00000000000..c0babf0ff89 --- /dev/null +++ b/tests/config/clusters.xml @@ -0,0 +1,20 @@ + + + + + + shard_0 + localhost + 9000 + + + + + shard_1 + localhost + 9000 + + + + + \ No newline at end of file diff --git a/tests/config/decimals_dictionary.xml b/tests/config/decimals_dictionary.xml index ff465b91b85..f728fa774a7 100644 --- a/tests/config/decimals_dictionary.xml +++ b/tests/config/decimals_dictionary.xml @@ -7,7 +7,7 @@ 9000 default - test_00950 + system decimals
@@ -45,7 +45,7 @@ 9000 default - test_00950 + system decimals
@@ -83,7 +83,7 @@ 9000 default - test_00950 + system decimals
@@ -121,7 +121,7 @@ 9000 default - test_00950 + system decimals
@@ -162,7 +162,7 @@ 9000 default - test_00950 + system decimals
diff --git a/tests/config/ints_dictionary.xml b/tests/config/ints_dictionary.xml index 5cf8419ad77..a22dab8933c 100644 --- a/tests/config/ints_dictionary.xml +++ b/tests/config/ints_dictionary.xml @@ -7,7 +7,7 @@ 9000 default - test_00950 + system ints
@@ -70,7 +70,7 @@ 9000 default - test_00950 + system ints
@@ -133,7 +133,7 @@ 9000 default - test_00950 + system ints
@@ -196,7 +196,7 @@ 9000 default - test_00950 + system ints
@@ -259,7 +259,7 @@ 9000 default - test_00950 + system ints
@@ -325,7 +325,7 @@ 9000 default - test_00950 + system ints
diff --git a/tests/config/strings_dictionary.xml b/tests/config/strings_dictionary.xml index 88fad6ae2d7..c5643eecb68 100644 --- a/tests/config/strings_dictionary.xml +++ b/tests/config/strings_dictionary.xml @@ -7,7 +7,7 @@ 9000 default - test_00950 + system strings
@@ -35,7 +35,7 @@ 9000 default - test_00950 + system strings
@@ -63,7 +63,7 @@ 9000 default - test_00950 + system strings
@@ -91,7 +91,7 @@ 9000 default - test_00950 + system strings
@@ -122,7 +122,7 @@ 9000 default - test_00950 + system strings
@@ -153,7 +153,7 @@ 9000 default - test_00950 + system strings
@@ -184,7 +184,7 @@ 9000 default - test_00950 + system strings
diff --git a/tests/queries/0_stateless/00950_dict_get.sql b/tests/queries/0_stateless/00950_dict_get.sql index 73fddce91c7..93bb8f9b813 100644 --- a/tests/queries/0_stateless/00950_dict_get.sql +++ b/tests/queries/0_stateless/00950_dict_get.sql @@ -1,6 +1,5 @@ --- Must use `test_00950` database and these tables - they're configured in tests/*_dictionary.xml -create database if not exists test_00950; -use test_00950; +-- Must use `system` database and these tables - they're configured in tests/*_dictionary.xml +use system; drop table if exists ints; drop table if exists strings; drop table if exists decimals; @@ -270,7 +269,14 @@ select 'dictGetOrDefault', 'complex_cache_decimals' as dict_name, tuple(toUInt64 dictGetOrDefault(dict_name, 'd64', k, toDecimal64(42, 6)), dictGetOrDefault(dict_name, 'd128', k, toDecimal128(42, 1)); -drop table ints; -drop table strings; -drop table decimals; -drop database test_00950; +-- +-- Keep the tables, so that the dictionaries can be reloaded correctly and +-- SYSTEM RELOAD DICTIONARIES doesn't break. +-- We could also: +-- * drop the dictionaries -- not possible, they are configured in a .xml; +-- * switch dictionaries to DDL syntax so that they can be dropped -- tedious, +-- because there are a couple dozens of them, and also we need to have some +-- .xml dictionaries in tests so that we test backward compatibility with this +-- format; +-- * unload dictionaries -- no command for that. +-- diff --git a/tests/queries/0_stateless/01048_exists_query.sql b/tests/queries/0_stateless/01048_exists_query.sql index 6228b57fe3f..9a4c0558b60 100644 --- a/tests/queries/0_stateless/01048_exists_query.sql +++ b/tests/queries/0_stateless/01048_exists_query.sql @@ -1,44 +1,44 @@ -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; -DROP DATABASE IF EXISTS database_for_dict; -CREATE DATABASE database_for_dict Engine = Ordinary; +DROP DATABASE IF EXISTS db_01048; +CREATE DATABASE db_01048 Engine = Ordinary; -DROP TABLE IF EXISTS database_for_dict.t; -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +DROP TABLE IF EXISTS db_01048.t_01048; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; -CREATE TABLE database_for_dict.t (x UInt8) ENGINE = Memory; -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +CREATE TABLE db_01048.t_01048 (x UInt8) ENGINE = Memory; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; -DROP TABLE database_for_dict.t; -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +DROP TABLE db_01048.t_01048; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; -DROP DICTIONARY IF EXISTS t; -CREATE TEMPORARY TABLE t (x UInt8); -EXISTS t; -- Does not work for temporary tables. Maybe have to fix. -EXISTS TABLE t; -EXISTS DICTIONARY t; +DROP DICTIONARY IF EXISTS t_01048; +CREATE TEMPORARY TABLE t_01048 (x UInt8); +EXISTS t_01048; -- Does not work for temporary tables. Maybe have to fix. +EXISTS TABLE t_01048; +EXISTS DICTIONARY t_01048; -CREATE DICTIONARY database_for_dict.t (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000); -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -- Dictionaries are tables as well. But not all tables are dictionaries. -EXISTS DICTIONARY database_for_dict.t; +CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000); +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; -- Dictionaries are tables as well. But not all tables are dictionaries. +EXISTS DICTIONARY db_01048.t_01048; -- But dictionary-tables cannot be dropped as usual tables. -DROP TABLE database_for_dict.t; -- { serverError 60 } -DROP DICTIONARY database_for_dict.t; -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +DROP TABLE db_01048.t_01048; -- { serverError 60 } +DROP DICTIONARY db_01048.t_01048; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; -DROP DATABASE database_for_dict; -EXISTS database_for_dict.t; -EXISTS TABLE database_for_dict.t; -EXISTS DICTIONARY database_for_dict.t; +DROP DATABASE db_01048; +EXISTS db_01048.t_01048; +EXISTS TABLE db_01048.t_01048; +EXISTS DICTIONARY db_01048.t_01048; diff --git a/tests/queries/0_stateless/01073_attach_if_not_exists.sql b/tests/queries/0_stateless/01073_attach_if_not_exists.sql index 1c7877a7c73..8bb52556ccc 100644 --- a/tests/queries/0_stateless/01073_attach_if_not_exists.sql +++ b/tests/queries/0_stateless/01073_attach_if_not_exists.sql @@ -1,7 +1,7 @@ -CREATE TABLE t (a Int) ENGINE = Log; -ATTACH TABLE t; -- { serverError 57 } -ATTACH TABLE IF NOT EXISTS t; -DETACH TABLE t; -ATTACH TABLE IF NOT EXISTS t; -EXISTS TABLE t; -DROP TABLE t; +CREATE TABLE aine (a Int) ENGINE = Log; +ATTACH TABLE aine; -- { serverError 57 } +ATTACH TABLE IF NOT EXISTS aine; +DETACH TABLE aine; +ATTACH TABLE IF NOT EXISTS aine; +EXISTS TABLE aine; +DROP TABLE aine; diff --git a/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.reference b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.reference new file mode 100644 index 00000000000..ac13b3f193e --- /dev/null +++ b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.reference @@ -0,0 +1,10 @@ +0 2 +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +8 1 +9 1 diff --git a/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql new file mode 100644 index 00000000000..31f09b35bf3 --- /dev/null +++ b/tests/queries/0_stateless/01231_distributed_aggregation_memory_efficient_mix_levels.sql @@ -0,0 +1,25 @@ +set send_logs_level = 'error'; + +create database if not exists shard_0; +create database if not exists shard_1; + +drop table if exists shard_0.shard_01231_distributed_aggregation_memory_efficient; +drop table if exists shard_1.shard_01231_distributed_aggregation_memory_efficient; +drop table if exists ma_dist; + +create table shard_0.shard_01231_distributed_aggregation_memory_efficient (x UInt64) engine = MergeTree order by x; +create table shard_1.shard_01231_distributed_aggregation_memory_efficient (x UInt64) engine = MergeTree order by x; + +insert into shard_0.shard_01231_distributed_aggregation_memory_efficient select * from numbers(1); +insert into shard_1.shard_01231_distributed_aggregation_memory_efficient select * from numbers(10); + +create table ma_dist (x UInt64) ENGINE = Distributed(test_cluster_two_shards_different_databases, '', 'shard_01231_distributed_aggregation_memory_efficient'); + +set distributed_aggregation_memory_efficient = 1; +set group_by_two_level_threshold = 2; +set max_bytes_before_external_group_by = 16; + +select x, count() from ma_dist group by x order by x; + +drop table if exists shard_0.shard_01231_distributed_aggregation_memory_efficient; +drop table if exists shard_1.shard_01231_distributed_aggregation_memory_efficient;