From 3d89f0e9df325b6830153735263b5efaa829cf0a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 21 Jul 2020 15:41:14 +0300 Subject: [PATCH 01/26] Perf test: bind server to one NUMA node --- docker/test/performance-comparison/compare.sh | 12 ++++++++++-- tests/performance/decimal_aggregates.xml | 2 +- tests/performance/jit_large_requests.xml | 4 +--- tests/performance/string_sort.xml | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 378e87f443b..4bde4f945a6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,12 +55,18 @@ function restart set -m # Spawn servers in their own process groups - left/clickhouse-server --config-file=left/config/config.xml -- --path left/db --user_files_path left/db/user_files &>> left-server-log.log & + numactl --membind=0 --cpunodebind=0 --localalloc \ + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - right/clickhouse-server --config-file=right/config/config.xml -- --path right/db --user_files_path right/db/user_files &>> right-server-log.log & + numactl --membind=0 --cpunodebind=0 --localalloc \ + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid @@ -909,6 +915,8 @@ case "$stage" in time configure ;& "restart") + numactl --hardware ||: + lscpu ||: time restart ;& "run_tests") diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 0c8df88c73c..142d9388404 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -1,6 +1,6 @@ - 30G + 35G CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory diff --git a/tests/performance/jit_large_requests.xml b/tests/performance/jit_large_requests.xml index 805b7f2edb1..6aed7bea544 100644 --- a/tests/performance/jit_large_requests.xml +++ b/tests/performance/jit_large_requests.xml @@ -1,6 +1,4 @@ - - CREATE TABLE jit_test ( a UInt64, @@ -43,7 +41,7 @@ SETTINGS compile_expressions = 0; - + SELECT COUNT() FROM diff --git a/tests/performance/string_sort.xml b/tests/performance/string_sort.xml index ce5a54e2680..5d859398ece 100644 --- a/tests/performance/string_sort.xml +++ b/tests/performance/string_sort.xml @@ -43,7 +43,7 @@ - + From 9f49bf2d82154db6417585505bfab0897d3d1d4e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 22 Jul 2020 01:09:54 +0300 Subject: [PATCH 02/26] fixup --- docker/test/performance-comparison/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 912a8bd12cd..df666af8e8e 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ libc6-dbg \ moreutils \ ncdu \ + numactl \ p7zip-full \ parallel \ psmisc \ From 2b7c0167cb0fda79b59eecf752a56611eae36ab0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 22 Jul 2020 21:02:56 +0300 Subject: [PATCH 03/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 4bde4f945a6..caaff129cfb 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function restart set -m # Spawn servers in their own process groups - numactl --membind=0 --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --localalloc \ left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --membind=0 --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --localalloc \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & From 15cd448afaf98c57f5300674e451eebc5b1f427e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 23 Jul 2020 17:59:32 +0300 Subject: [PATCH 04/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index caaff129cfb..84aa31fb76b 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function restart set -m # Spawn servers in their own process groups - numactl --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --membind=0 \ left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --membind=0 \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & From 0e205075394aec860a959c5c4125bf01f04a1c86 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 30 Jul 2020 17:26:47 +0300 Subject: [PATCH 05/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 84aa31fb76b..387e259823f 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --membind=0 \ + numactl --cpunodebind=1 --membind=1 \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & From 42e5f8ec44b14e65eb3c8efc097a39356b9bf750 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 31 Jul 2020 03:49:36 +0300 Subject: [PATCH 06/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 387e259823f..29576be2903 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -952,7 +952,7 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. Start in a subshell, so that # it doesn't interfere with the watchdog through `wait`. - ( get_profiles || restart || get_profiles ||: ) + ( get_profiles || restart && get_profiles ||: ) # Kill the whole process group, because somehow when the subshell is killed, # the sleep inside remains alive and orphaned. From ff3f378e8adfe21cfdcb412f6e014cc42a4eec79 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 6 Aug 2020 00:58:56 +0300 Subject: [PATCH 07/26] try to rebuild package for 2b7c0167cb0fda79b59eecf752a56611eae36ab0 --- docker/test/performance-comparison/compare.sh | 6 +- .../test/performance-comparison/perf.py.orig | 190 ++++++++++++++++++ 2 files changed, 193 insertions(+), 3 deletions(-) create mode 100755 docker/test/performance-comparison/perf.py.orig diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 29576be2903..caaff129cfb 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function restart set -m # Spawn servers in their own process groups - numactl --cpunodebind=0 --membind=0 \ + numactl --cpunodebind=0 --localalloc \ left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --cpunodebind=1 --membind=1 \ + numactl --cpunodebind=0 --localalloc \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & @@ -952,7 +952,7 @@ case "$stage" in # to collect the logs. Prefer not to restart, because addresses might change # and we won't be able to process trace_log data. Start in a subshell, so that # it doesn't interfere with the watchdog through `wait`. - ( get_profiles || restart && get_profiles ||: ) + ( get_profiles || restart || get_profiles ||: ) # Kill the whole process group, because somehow when the subshell is killed, # the sleep inside remains alive and orphaned. diff --git a/docker/test/performance-comparison/perf.py.orig b/docker/test/performance-comparison/perf.py.orig new file mode 100755 index 00000000000..c25a3041a67 --- /dev/null +++ b/docker/test/performance-comparison/perf.py.orig @@ -0,0 +1,190 @@ +#!/usr/bin/python3 + +import os +import sys +import itertools +import clickhouse_driver +import xml.etree.ElementTree as et +import argparse +import pprint +import string +import time +import traceback + +stage_start_seconds = time.perf_counter() + +def report_stage_end(stage_name): + global stage_start_seconds + print('{}\t{}'.format(stage_name, time.perf_counter() - stage_start_seconds)) + stage_start_seconds = time.perf_counter() + +report_stage_end('start') + +parser = argparse.ArgumentParser(description='Run performance test.') +# Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. +parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') +parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.") +parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.") +parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') +parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') +args = parser.parse_args() + +tree = et.parse(args.file[0]) +root = tree.getroot() + +# Skip long tests +for tag in root.findall('.//tag'): + if tag.text == 'long': + print('skipped\tTest is tagged as long.') + sys.exit(0) + +# Check main metric +main_metric_element = root.find('main_metric/*') +if main_metric_element is not None and main_metric_element.tag != 'min_time': + raise Exception('Only the min_time main metric is supported. This test uses \'{}\''.format(main_metric_element.tag)) + +# FIXME another way to detect infinite tests. They should have an appropriate main_metric but sometimes they don't. +infinite_sign = root.find('.//average_speed_not_changing_for_ms') +if infinite_sign is not None: + raise Exception('Looks like the test is infinite (sign 1)') + +# Open connections +servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] +connections = [clickhouse_driver.Client(**server) for server in servers] + +for s in servers: + print('server\t{}\t{}'.format(s['host'], s['port'])) + +report_stage_end('connect') + +# Process query parameters +subst_elems = root.findall('substitutions/substitution') +available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } +for e in subst_elems: + available_parameters[e.find('name').text] = [v.text for v in e.findall('values/value')] + +# Take care to keep the order of queries -- sometimes we have DROP IF EXISTS +# followed by CREATE in create queries section, so the order matters. +def substitute_parameters(query_templates): + result = [] + for q in query_templates: + keys = set(n for _, n, _, _ in string.Formatter().parse(q) if n) + values = [available_parameters[k] for k in keys] + result.extend([ + q.format(**dict(zip(keys, values_combo))) + for values_combo in itertools.product(*values)]) + return result + +report_stage_end('substitute') + +# Run drop queries, ignoring errors. Do this before all other activity, because +# clickhouse_driver disconnects on error (this is not configurable), and the new +# connection loses the changes in settings. +drop_query_templates = [q.text for q in root.findall('drop_query')] +drop_queries = substitute_parameters(drop_query_templates) +for c in connections: + for q in drop_queries: + try: + c.execute(q) + except: + traceback.print_exc() + pass + +report_stage_end('drop1') + +# Apply settings +settings = root.findall('settings/*') +for c in connections: + for s in settings: + c.execute("set {} = '{}'".format(s.tag, s.text)) + +report_stage_end('settings') + +# Check tables that should exist. If they don't exist, just skip this test. +tables = [e.text for e in root.findall('preconditions/table_exists')] +for t in tables: + for c in connections: + try: + res = c.execute("show create table {}".format(t)) + except: + print('skipped\t' + traceback.format_exception_only(*sys.exc_info()[:2])[-1]) + traceback.print_exc() + sys.exit(0) + +report_stage_end('preconditions') + +# Run create queries +create_query_templates = [q.text for q in root.findall('create_query')] +create_queries = substitute_parameters(create_query_templates) +for c in connections: + for q in create_queries: + c.execute(q) + +# Run fill queries +fill_query_templates = [q.text for q in root.findall('fill_query')] +fill_queries = substitute_parameters(fill_query_templates) +for c in connections: + for q in fill_queries: + c.execute(q) + +report_stage_end('fill') + +# Run test queries +def tsv_escape(s): + return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') + +test_query_templates = [q.text for q in root.findall('query')] +test_queries = substitute_parameters(test_query_templates) + +report_stage_end('substitute2') + +for q in test_queries: + # Prewarm: run once on both servers. Helps to bring the data into memory, + # precompile the queries, etc. +<<<<<<< HEAD + for conn_index, c in enumerate(connections): + res = c.execute(q, query_id = 'prewarm {} {}'.format(0, q)) + print('prewarm\t' + tsv_escape(q) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) +======= + try: + for conn_index, c in enumerate(connections): + prewarm_id = f'{query_prefix}.prewarm0' + res = c.execute(q, query_id = prewarm_id) + print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') + except KeyboardInterrupt: + raise + except: + # If prewarm fails for some query -- skip it, and try to test the others. + # This might happen if the new test introduces some function that the + # old server doesn't support. Still, report it as an error. + # FIXME the driver reconnects on error and we lose settings, so this might + # lead to further errors or unexpected behavior. + print(traceback.format_exc(), file=sys.stderr) + continue +>>>>>>> 4b1bb43543... Merge pull request #11076 from ClickHouse/aku/join-error-messages + + # Now, perform measured runs. + # Track the time spent by the client to process this query, so that we can notice + # out the queries that take long to process on the client side, e.g. by sending + # excessive data. + start_seconds = time.perf_counter() + server_seconds = 0 + for run in range(0, args.runs): + for conn_index, c in enumerate(connections): + res = c.execute(q) + print('query\t' + tsv_escape(q) + '\t' + str(run) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) + server_seconds += c.last_query.elapsed + + client_seconds = time.perf_counter() - start_seconds + print('client-time\t{}\t{}\t{}'.format(tsv_escape(q), client_seconds, server_seconds)) + +report_stage_end('benchmark') + +# Run drop queries +drop_query_templates = [q.text for q in root.findall('drop_query')] +drop_queries = substitute_parameters(drop_query_templates) +for c in connections: + for q in drop_queries: + c.execute(q) + +report_stage_end('drop2') From 5d31442fa88271072a5478b95f4903977cef4fb1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 26 Aug 2020 03:26:54 +0300 Subject: [PATCH 08/26] Delete perf.py.orig --- .../test/performance-comparison/perf.py.orig | 190 ------------------ 1 file changed, 190 deletions(-) delete mode 100755 docker/test/performance-comparison/perf.py.orig diff --git a/docker/test/performance-comparison/perf.py.orig b/docker/test/performance-comparison/perf.py.orig deleted file mode 100755 index c25a3041a67..00000000000 --- a/docker/test/performance-comparison/perf.py.orig +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/python3 - -import os -import sys -import itertools -import clickhouse_driver -import xml.etree.ElementTree as et -import argparse -import pprint -import string -import time -import traceback - -stage_start_seconds = time.perf_counter() - -def report_stage_end(stage_name): - global stage_start_seconds - print('{}\t{}'.format(stage_name, time.perf_counter() - stage_start_seconds)) - stage_start_seconds = time.perf_counter() - -report_stage_end('start') - -parser = argparse.ArgumentParser(description='Run performance test.') -# Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. -parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.") -parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.") -parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') -parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') -args = parser.parse_args() - -tree = et.parse(args.file[0]) -root = tree.getroot() - -# Skip long tests -for tag in root.findall('.//tag'): - if tag.text == 'long': - print('skipped\tTest is tagged as long.') - sys.exit(0) - -# Check main metric -main_metric_element = root.find('main_metric/*') -if main_metric_element is not None and main_metric_element.tag != 'min_time': - raise Exception('Only the min_time main metric is supported. This test uses \'{}\''.format(main_metric_element.tag)) - -# FIXME another way to detect infinite tests. They should have an appropriate main_metric but sometimes they don't. -infinite_sign = root.find('.//average_speed_not_changing_for_ms') -if infinite_sign is not None: - raise Exception('Looks like the test is infinite (sign 1)') - -# Open connections -servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] -connections = [clickhouse_driver.Client(**server) for server in servers] - -for s in servers: - print('server\t{}\t{}'.format(s['host'], s['port'])) - -report_stage_end('connect') - -# Process query parameters -subst_elems = root.findall('substitutions/substitution') -available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } -for e in subst_elems: - available_parameters[e.find('name').text] = [v.text for v in e.findall('values/value')] - -# Take care to keep the order of queries -- sometimes we have DROP IF EXISTS -# followed by CREATE in create queries section, so the order matters. -def substitute_parameters(query_templates): - result = [] - for q in query_templates: - keys = set(n for _, n, _, _ in string.Formatter().parse(q) if n) - values = [available_parameters[k] for k in keys] - result.extend([ - q.format(**dict(zip(keys, values_combo))) - for values_combo in itertools.product(*values)]) - return result - -report_stage_end('substitute') - -# Run drop queries, ignoring errors. Do this before all other activity, because -# clickhouse_driver disconnects on error (this is not configurable), and the new -# connection loses the changes in settings. -drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates) -for c in connections: - for q in drop_queries: - try: - c.execute(q) - except: - traceback.print_exc() - pass - -report_stage_end('drop1') - -# Apply settings -settings = root.findall('settings/*') -for c in connections: - for s in settings: - c.execute("set {} = '{}'".format(s.tag, s.text)) - -report_stage_end('settings') - -# Check tables that should exist. If they don't exist, just skip this test. -tables = [e.text for e in root.findall('preconditions/table_exists')] -for t in tables: - for c in connections: - try: - res = c.execute("show create table {}".format(t)) - except: - print('skipped\t' + traceback.format_exception_only(*sys.exc_info()[:2])[-1]) - traceback.print_exc() - sys.exit(0) - -report_stage_end('preconditions') - -# Run create queries -create_query_templates = [q.text for q in root.findall('create_query')] -create_queries = substitute_parameters(create_query_templates) -for c in connections: - for q in create_queries: - c.execute(q) - -# Run fill queries -fill_query_templates = [q.text for q in root.findall('fill_query')] -fill_queries = substitute_parameters(fill_query_templates) -for c in connections: - for q in fill_queries: - c.execute(q) - -report_stage_end('fill') - -# Run test queries -def tsv_escape(s): - return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') - -test_query_templates = [q.text for q in root.findall('query')] -test_queries = substitute_parameters(test_query_templates) - -report_stage_end('substitute2') - -for q in test_queries: - # Prewarm: run once on both servers. Helps to bring the data into memory, - # precompile the queries, etc. -<<<<<<< HEAD - for conn_index, c in enumerate(connections): - res = c.execute(q, query_id = 'prewarm {} {}'.format(0, q)) - print('prewarm\t' + tsv_escape(q) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) -======= - try: - for conn_index, c in enumerate(connections): - prewarm_id = f'{query_prefix}.prewarm0' - res = c.execute(q, query_id = prewarm_id) - print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') - except KeyboardInterrupt: - raise - except: - # If prewarm fails for some query -- skip it, and try to test the others. - # This might happen if the new test introduces some function that the - # old server doesn't support. Still, report it as an error. - # FIXME the driver reconnects on error and we lose settings, so this might - # lead to further errors or unexpected behavior. - print(traceback.format_exc(), file=sys.stderr) - continue ->>>>>>> 4b1bb43543... Merge pull request #11076 from ClickHouse/aku/join-error-messages - - # Now, perform measured runs. - # Track the time spent by the client to process this query, so that we can notice - # out the queries that take long to process on the client side, e.g. by sending - # excessive data. - start_seconds = time.perf_counter() - server_seconds = 0 - for run in range(0, args.runs): - for conn_index, c in enumerate(connections): - res = c.execute(q) - print('query\t' + tsv_escape(q) + '\t' + str(run) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) - server_seconds += c.last_query.elapsed - - client_seconds = time.perf_counter() - start_seconds - print('client-time\t{}\t{}\t{}'.format(tsv_escape(q), client_seconds, server_seconds)) - -report_stage_end('benchmark') - -# Run drop queries -drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates) -for c in connections: - for q in drop_queries: - c.execute(q) - -report_stage_end('drop2') From ab19bb25fd8c286713580649ad1c183d493ea5dc Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 22 Sep 2020 14:31:33 +0300 Subject: [PATCH 09/26] disable percpu arenas --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index dd7f9f3e2bb..563d41301b1 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT EXTERNAL_JEMALLOC_LIBRARY_FOUND OR NOT EXTERNAL_JEMALLOC_LIBRARY_WORKS) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") endif() From 833c07f1f76b71527262362545fef3973fad686c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 11:31:18 +0300 Subject: [PATCH 10/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 2fd5641b9fd..7851cf8e81d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -7,6 +7,10 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# https://github.com/jemalloc/jemalloc/wiki/Getting-Started +export MALLOC_CONF="percpu_arena:disabled" +echo "$MALLOC_CONF" > /etc/malloc.conf ||: + function wait_for_server # port, pid { for _ in {1..60} From bb51aade56d478588a8db60315c30870208aeec3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:02:22 +0300 Subject: [PATCH 11/26] Update docker/test/performance-comparison/compare.sh Co-authored-by: Azat Khuzhin --- docker/test/performance-comparison/compare.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7851cf8e81d..35bb5890488 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,8 +8,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="percpu_arena:disabled" -echo "$MALLOC_CONF" > /etc/malloc.conf ||: +ln -s "percpu_arena:disabled" > /etc/malloc.conf function wait_for_server # port, pid { From d96c89972cc9e3ff7e8baca7a3899c1aeffed891 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:04:13 +0300 Subject: [PATCH 12/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 35bb5890488..ed89e6f875c 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,7 +8,8 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -ln -s "percpu_arena:disabled" > /etc/malloc.conf +export MALLOC_CONF="percpu_arena:disabled" +ln -s "percpu_arena:disabled" /etc/malloc.conf function wait_for_server # port, pid { From 26abe8cb30819eea1c1f3383cc99620ab2e9da9a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 12:24:30 +0300 Subject: [PATCH 13/26] Update compare.sh --- docker/test/performance-comparison/compare.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index ed89e6f875c..7708d11d81f 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,8 +8,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="percpu_arena:disabled" -ln -s "percpu_arena:disabled" /etc/malloc.conf +export MALLOC_CONF="confirm_conf:true" function wait_for_server # port, pid { @@ -83,18 +82,16 @@ function restart set -m # Spawn servers in their own process groups - numactl --cpunodebind=0 --localalloc \ - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid From 425150e78308941c5e7a7c097ac66c84721008df Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 25 Sep 2020 13:19:37 +0300 Subject: [PATCH 14/26] bind to different nodes --- docker/test/performance-comparison/compare.sh | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7708d11d81f..98040b037f6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -7,9 +7,6 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="confirm_conf:true" - function wait_for_server # port, pid { for _ in {1..60} @@ -80,24 +77,31 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed - set -m # Spawn servers in their own process groups + # https://github.com/jemalloc/jemalloc/wiki/Getting-Started + export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + set -m # Spawn servers in their own process groups + + numactl --cpunodebind=1 --localalloc \ + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + numactl --cpunodebind=0 --localalloc \ + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid set +m + unset MALLOC_CONF + wait_for_server 9001 $left_pid echo left ok From bde19bf240571ca6d05450412a838ae4bea2f782 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 28 Sep 2020 15:26:51 +0300 Subject: [PATCH 15/26] restart the build From 824d5b093c453506cb64f2abc67ce3034a184da6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 14:32:49 +0300 Subject: [PATCH 16/26] bind everything to node 0 --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- docker/test/performance-comparison/Dockerfile | 9 ++++++++- docker/test/performance-comparison/compare.sh | 18 ++++++++++-------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 563d41301b1..dd7f9f3e2bb 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT EXTERNAL_JEMALLOC_LIBRARY_FOUND OR NOT EXTERNAL_JEMALLOC_LIBRARY_WORKS) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") endif() diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index a4f8af2f388..99f2f9b2b4b 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -37,7 +37,14 @@ RUN apt-get update \ COPY * / -CMD /entrypoint.sh +# Bind everything to node 0 early. We have to bind both servers and the tmpfs +# on which the database is stored. How to do it through Yandex Sandbox API is +# unclear, but by default tmpfs uses 'process allocation policy', not sure +# which process but hopefully the one that writes to it, so just bind the +# downloader script as well. +# We could also try to remount it with proper options in Sandbox task. +# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt +CMD ['numactl', '--cpunodebind=0', '--localalloc', '/entrypoint.sh'] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 0134d03aea1..2f03ecc9ad7 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -77,23 +77,25 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed + # Disable percpu arenas because they segfault when the process is bound to + # a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939 + # + # About the jemalloc settings: # https://github.com/jemalloc/jemalloc/wiki/Getting-Started export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" set -m # Spawn servers in their own process groups - numactl --cpunodebind=1 --localalloc \ - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid From aa543a2d3d51f16c6d043a36bb9f4249ba35cd05 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 17:40:24 +0300 Subject: [PATCH 17/26] quotes --- docker/test/performance-comparison/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 99f2f9b2b4b..535f7de9e29 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -44,7 +44,7 @@ COPY * / # downloader script as well. # We could also try to remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ['numactl', '--cpunodebind=0', '--localalloc', '/entrypoint.sh'] +CMD ["numactl", "--cpunodebind=0", "--localalloc", "/entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison From 74f8e41b55a2504c3419cb33e7c2429a2c1e116a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 1 Oct 2020 13:56:56 +0300 Subject: [PATCH 18/26] calculate on all nodes --- docker/test/performance-comparison/compare.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 2f03ecc9ad7..08b18758874 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -462,7 +462,10 @@ wait unset IFS ) -parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log +# The comparison script might be bound to one NUMA node for better test +# stability, and the calculation runs out of memory because of this. Use +# all nodes. +numactl --all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make From 2241ea9f33e770d019abfbfedba9e1ae03dd609f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 16 Oct 2020 17:44:59 +0300 Subject: [PATCH 19/26] not sure what's going on... --- docker/test/performance-comparison/compare.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 08b18758874..15d23226a0d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -465,6 +465,8 @@ unset IFS # The comparison script might be bound to one NUMA node for better test # stability, and the calculation runs out of memory because of this. Use # all nodes. +numactl --show +numactl --all numactl --show numactl --all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log clickhouse-local --query " From 38a2dd55ce895da119089ddc28c6f716a182d11a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 19 Oct 2020 17:31:02 +0300 Subject: [PATCH 20/26] fix numactl parameters --- docker/test/performance-comparison/Dockerfile | 2 +- docker/test/performance-comparison/compare.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 535f7de9e29..3b43b68319b 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -44,7 +44,7 @@ COPY * / # downloader script as well. # We could also try to remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ["numactl", "--cpunodebind=0", "--localalloc", "/entrypoint.sh"] +CMD ["numactl", "--cpunodebind=0", "--membind=0", "/entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 15d23226a0d..34ddbc37eec 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -466,8 +466,8 @@ unset IFS # stability, and the calculation runs out of memory because of this. Use # all nodes. numactl --show -numactl --all numactl --show -numactl --all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log +numactl --cpunodebind=all --membind=all numactl --show +numactl --cpunodebind=all --membind=all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make From 764f19820b6745cebb7defd47df31fe740260910 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 19 Oct 2020 17:39:52 +0300 Subject: [PATCH 21/26] max threads --- .../config/users.d/perf-comparison-tweaks-users.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml index cee7dc3ff16..ce7a6ae094a 100644 --- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml +++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml @@ -14,6 +14,9 @@ we might also add time check to perf.py script. --> 300 + + + 20 From 242ec7e56c72c44cd26eacf22eb4a902e8b7c0b2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 20 Oct 2020 15:12:06 +0300 Subject: [PATCH 22/26] make some tests faster --- docker/test/performance-comparison/compare.sh | 1 + tests/performance/constant_column_search.xml | 3 ++- tests/performance/trim_urls.xml | 2 +- tests/performance/website.xml | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 34ddbc37eec..769a4f8f735 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1090,6 +1090,7 @@ case "$stage" in "restart") numactl --hardware ||: lscpu ||: + sudo dmidecode -t 4 ||: time restart ;& "run_tests") diff --git a/tests/performance/constant_column_search.xml b/tests/performance/constant_column_search.xml index 62baab14e3c..2f90844cb4a 100644 --- a/tests/performance/constant_column_search.xml +++ b/tests/performance/constant_column_search.xml @@ -5,6 +5,7 @@ hits_100m_single + hits_10m_single @@ -36,7 +37,7 @@ - + diff --git a/tests/performance/trim_urls.xml b/tests/performance/trim_urls.xml index 276a12bc570..075984682f9 100644 --- a/tests/performance/trim_urls.xml +++ b/tests/performance/trim_urls.xml @@ -20,5 +20,5 @@ - SELECT count() FROM hits_100m_single WHERE NOT ignore({func}URL)) + SELECT ignore({func}URL)) FROM hits_100m_single LIMIT 50000000 FORMAT Null diff --git a/tests/performance/website.xml b/tests/performance/website.xml index 65b3d79b5f1..2127a71c55c 100644 --- a/tests/performance/website.xml +++ b/tests/performance/website.xml @@ -37,7 +37,7 @@ SELECT UserID, count() FROM {table} GROUP BY UserID ORDER BY count() DESC LIMIT 10 SELECT UserID, SearchPhrase, count() FROM {table} GROUP BY UserID, SearchPhrase ORDER BY count() DESC LIMIT 10 SELECT UserID, SearchPhrase, count() FROM {table} GROUP BY UserID, SearchPhrase LIMIT 10 -SELECT UserID, toMinute(EventTime) AS m, SearchPhrase, count() FROM {table} GROUP BY UserID, m, SearchPhrase ORDER BY count() DESC LIMIT 10 +SELECT UserID, toMinute(EventTime) AS m, SearchPhrase, count() FROM hits_10m_single GROUP BY UserID, m, SearchPhrase ORDER BY count() DESC LIMIT 10 SELECT count() FROM hits_100m_single WHERE UserID = 12345678901234567890 SELECT count() FROM hits_100m_single WHERE URL LIKE '%metrika%' SELECT SearchPhrase, any(URL), count() AS c FROM hits_100m_single WHERE URL LIKE '%metrika%' AND SearchPhrase != '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10 @@ -52,8 +52,8 @@ SELECT SearchEngineID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_100m_single WHERE SearchPhrase != '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10 SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM {table} WHERE SearchPhrase != '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM hits_10m_single GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10 -SELECT URL, count() AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10 -SELECT 1, URL, count() AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10 +SELECT URL, count() AS c FROM hits_10m_single GROUP BY URL ORDER BY c DESC LIMIT 10 +SELECT 1, URL, count() AS c FROM hits_10m_single GROUP BY 1, URL ORDER BY c DESC LIMIT 10 SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM hits_100m_single GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10 SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10 SETTINGS max_threads = 1 SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 34 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10 From c10dada3d42ff0526c9ffbaa7c4beeb4467cde73 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 21 Oct 2020 18:06:42 +0300 Subject: [PATCH 23/26] try node 1 --- docker/test/performance-comparison/Dockerfile | 4 ++-- docker/test/performance-comparison/compare.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 3b43b68319b..1b4a75f7f5d 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -37,14 +37,14 @@ RUN apt-get update \ COPY * / -# Bind everything to node 0 early. We have to bind both servers and the tmpfs +# Bind everything to node 1 early. We have to bind both servers and the tmpfs # on which the database is stored. How to do it through Yandex Sandbox API is # unclear, but by default tmpfs uses 'process allocation policy', not sure # which process but hopefully the one that writes to it, so just bind the # downloader script as well. # We could also try to remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ["numactl", "--cpunodebind=0", "--membind=0", "/entrypoint.sh"] +CMD ["numactl", "--cpunodebind=1", "--membind=1", "/entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 769a4f8f735..8a567d7a11a 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1090,7 +1090,7 @@ case "$stage" in "restart") numactl --hardware ||: lscpu ||: - sudo dmidecode -t 4 ||: + dmidecode -t 4 ||: time restart ;& "run_tests") From a678f0322999afd4a3c054531220339086f25c6a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 21 Oct 2020 18:21:54 +0300 Subject: [PATCH 24/26] fixup --- docker/test/performance-comparison/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 1b4a75f7f5d..1a904cf73c2 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -37,14 +37,15 @@ RUN apt-get update \ COPY * / -# Bind everything to node 1 early. We have to bind both servers and the tmpfs -# on which the database is stored. How to do it through Yandex Sandbox API is -# unclear, but by default tmpfs uses 'process allocation policy', not sure -# which process but hopefully the one that writes to it, so just bind the -# downloader script as well. +# Bind everything to NUMA node #1, if there's more than one. We avoid node #0, +# because it is more susceptible to system interruptions. We have to bind both +# servers and the tmpfs on which the database is stored. How to do it through +# Yandex Sandbox API is unclear, but by default tmpfs uses 'process allocation +# policy', not sure which process but hopefully the one that writes to it, so +# just bind the downloader script as well. # We could also try to remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ["numactl", "--cpunodebind=1", "--membind=1", "/entrypoint.sh"] +CMD ["bash", "-c", "node=$(numactl --hardware | grep -q 'available: 1 nodes' && echo 0 || echo 1); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison From ac7af67ede43e8a1923e1476d91e03c65ce1a16d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 22 Oct 2020 15:52:43 +0300 Subject: [PATCH 25/26] fixup --- docker/test/performance-comparison/Dockerfile | 18 ++++++++++-------- docker/test/performance-comparison/compare.sh | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 1a904cf73c2..92c3da4d059 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ bash \ curl \ + dmidecode \ g++ \ gdb \ git \ @@ -37,15 +38,16 @@ RUN apt-get update \ COPY * / -# Bind everything to NUMA node #1, if there's more than one. We avoid node #0, -# because it is more susceptible to system interruptions. We have to bind both -# servers and the tmpfs on which the database is stored. How to do it through -# Yandex Sandbox API is unclear, but by default tmpfs uses 'process allocation -# policy', not sure which process but hopefully the one that writes to it, so -# just bind the downloader script as well. -# We could also try to remount it with proper options in Sandbox task. +# Bind everything to one NUMA node, if there's more than one. Theoretically the +# node #0 should be less stable because of system interruptions. We bind +# randomly to node 1 or 0 to gather some statistics on that. We have to bind +# both servers and the tmpfs on which the database is stored. How to do it +# through Yandex Sandbox API is unclear, but by default tmpfs uses +# 'process allocation policy', not sure which process but hopefully the one that +# writes to it, so just bind the downloader script as well. We could also try to +# remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ["bash", "-c", "node=$(numactl --hardware | grep -q 'available: 1 nodes' && echo 0 || echo 1); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] +CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\(.*\)nodes.*$/\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 8a567d7a11a..258bc0a95f7 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -1088,6 +1088,7 @@ case "$stage" in time configure ;& "restart") + numactl --show ||: numactl --hardware ||: lscpu ||: dmidecode -t 4 ||: From 0275e9c0104768c229a04004499f8e708a49240e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 22 Oct 2020 19:42:31 +0300 Subject: [PATCH 26/26] work around docker weirdness --- docker/test/performance-comparison/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 92c3da4d059..76cadc3ce11 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -47,7 +47,9 @@ COPY * / # writes to it, so just bind the downloader script as well. We could also try to # remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\(.*\)nodes.*$/\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] +# Double-escaped backslashes are a tribute to the engineering wonder of docker -- +# it gives '/bin/sh: 1: [bash,: not found' otherwise. +CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison