diff --git a/contrib/aws b/contrib/aws index fb5c604525f..f7d9ce39f41 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit fb5c604525f5151d75a856462653e7e38b559b79 +Subproject commit f7d9ce39f41323300044567be007c233338bb94a diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 0d1bdc2a88a..4b566ef2158 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -18,8 +18,7 @@ ccache --zero-stats ||: ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||: rm -f CMakeCache.txt cmake .. -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS -ninja -ccache --show-stats ||: +ninja clickhouse-bundle mv ./programs/clickhouse* /output mv ./src/unit_tests_dbms /output find . -name '*.so' -print -exec mv '{}' /output \; @@ -47,3 +46,4 @@ then rm -r /output/* mv "$COMBINED_OUTPUT.tgz" /output fi +ccache --show-stats ||: diff --git a/docker/packager/packager b/docker/packager/packager index 8a5bdda60e8..ccb01a4df92 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -120,6 +120,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ result.append("CCACHE_BASEDIR=/build") result.append("CCACHE_NOHASHDIR=true") result.append("CCACHE_COMPILERCHECK=content") + result.append("CCACHE_MAXSIZE=15G") # result.append("CCACHE_UMASK=777") if distcc_hosts: diff --git a/docker/test/integration/compose/docker_compose_minio.yml b/docker/test/integration/compose/docker_compose_minio.yml index c52c45b9d69..eefbe4abff5 100644 --- a/docker/test/integration/compose/docker_compose_minio.yml +++ b/docker/test/integration/compose/docker_compose_minio.yml @@ -43,7 +43,10 @@ services: # Empty container to run proxy resolver. resolver: - image: python:3 + build: + context: ../../../docker/test/integration/ + dockerfile: resolver/Dockerfile + network: host ports: - "4083:8080" tty: true diff --git a/docker/test/integration/resolver/Dockerfile b/docker/test/integration/resolver/Dockerfile new file mode 100644 index 00000000000..37118b7a555 --- /dev/null +++ b/docker/test/integration/resolver/Dockerfile @@ -0,0 +1,4 @@ +# Helper docker container to run python bottle apps + +FROM python:3 +RUN python -m pip install bottle \ No newline at end of file diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 209b36f59af..e63ba6122c8 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -104,13 +104,12 @@ function run_tests # allows the tests to pass even when we add new functions and tests for # them, that are not supported in the old revision. test_prefix=left/performance - elif [ "$PR_TO_TEST" != "" ] && [ "$PR_TO_TEST" != "0" ] - then + else # For PRs, use newer test files so we can test these changes. test_prefix=right/performance - # If some tests were changed in the PR, we may want to run only these - # ones. The list of changed tests in changed-test.txt is prepared in + # If only the perf tests were changed in the PR, we will run only these + # tests. The list of changed tests in changed-test.txt is prepared in # entrypoint.sh from git diffs, because it has the cloned repo. Used # to use rsync for that but it was really ugly and not always correct # (e.g. when the reference SHA is really old and has some other diff --git a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml index e41ab8eb75d..5dcc3c51eca 100644 --- a/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml +++ b/docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml @@ -19,6 +19,5 @@ 1000 - 0 1000000000 diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml index ce1416ac9dc..6e3e3df5d39 100644 --- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml +++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml @@ -5,6 +5,7 @@ 0 1 1 + 1 diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index ef62c8981e9..5afaf725c50 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -83,10 +83,17 @@ if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi if [ "$PR_TO_TEST" != "0" ] then - # Prepare the list of tests changed in the PR for use by compare.sh. Compare to - # merge base, because master might be far in the future and have unrelated test - # changes. - git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST" master)" -- tests/performance | tee changed-tests.txt + # If the PR only changes the tests and nothing else, prepare a list of these + # tests for use by compare.sh. Compare to merge base, because master might be + # far in the future and have unrelated test changes. + base=$(git -C ch merge-base "$SHA_TO_TEST" master) + git -C ch diff --name-only "$SHA_TO_TEST" "$base" | tee changed-tests.txt + if grep -vq '^tests/performance' changed-tests.txt + then + # Have some other changes besides the tests, so truncate the test list, + # meaning, run all tests. + : > changed-tests.txt + fi fi # Set python output encoding so that we can print queries with Russian letters. @@ -124,5 +131,5 @@ done dmesg -T > dmesg.log -7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze +7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze cp compare.log /output diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index ac506d046b1..308d4760b48 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -100,11 +100,20 @@ for c in connections: report_stage_end('drop1') -# Apply settings +# Apply settings. +# If there are errors, report them and continue -- maybe a new test uses a setting +# that is not in master, but the queries can still run. If we have multiple +# settings and one of them throws an exception, all previous settings for this +# connection will be reset, because the driver reconnects on error (not +# configurable). So the end result is uncertain, but hopefully we'll be able to +# run at least some queries. settings = root.findall('settings/*') for c in connections: for s in settings: - c.execute("set {} = '{}'".format(s.tag, s.text)) + try: + c.execute("set {} = '{}'".format(s.tag, s.text)) + except: + print(traceback.format_exc(), file=sys.stderr) report_stage_end('settings') diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 3a6774037c1..3776c9b513f 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -137,7 +137,7 @@ Official Yandex builds currently use GCC because it generates machine code of sl To install GCC on Ubuntu run: `sudo apt install gcc g++` -Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9. +Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-9. Mac OS X build is supported only for Clang. Just run `brew install llvm` @@ -245,7 +245,7 @@ The Code Style Guide: https://clickhouse.tech/docs/en/development/style/ Writing tests: https://clickhouse.tech/docs/en/development/tests/ -List of tasks: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +List of tasks: https://github.com/ClickHouse/ClickHouse/contribute ## Test Data {#test-data} diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md index 423f7d1ef33..72316284e3b 100644 --- a/docs/en/operations/backup.md +++ b/docs/en/operations/backup.md @@ -31,6 +31,7 @@ For smaller volumes of data, a simple `INSERT INTO ... SELECT ...` to remote tab ## Manipulations with Parts {#manipulations-with-parts} ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that doesn’t require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)). +Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ...` For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). diff --git a/docs/en/operations/system-tables.md b/docs/en/operations/system-tables.md index d3d58834e60..7b76f737824 100644 --- a/docs/en/operations/system-tables.md +++ b/docs/en/operations/system-tables.md @@ -18,9 +18,11 @@ System tables: - Available only for reading data. - Can't be dropped or altered, but can be detached. -Most of system tables store their data in RAM. ClickHouse server creates such system tables at the start. +Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start. -The [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) system tables store data in a storage filesystem. You can alter them or remove from a disk manually. If you remove one of that tables from a disk, the ClickHouse server creates the table again at the time of the next recording. A storage period for these tables is not limited, and ClickHouse server doesn't delete their data automatically. You need to organize removing of outdated logs by yourself. For example, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. +Unlike other system tables, the system tables [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) are served by [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one. + +By default, table growth is unlimited. To control a size of a table, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables. ### Sources of System Metrics {#system-tables-sources-of-system-metrics} @@ -636,9 +638,9 @@ You can change settings of queries logging in the [query_log](server-configurati You can disable queries logging by setting [log_queries = 0](settings/settings.md#settings-log-queries). We don't recommend to turn off logging because information in this table is important for solving issues. -The flushing period of logs is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing logs, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query. +The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query. -ClickHouse doesn't delete logs from the table automatically. See [Introduction](#system-tables-introduction) for more details. +ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details. The `system.query_log` table registers two kinds of queries: @@ -766,68 +768,117 @@ Settings.Values: ['0','random','1','10000000000'] ## system.query_thread_log {#system_tables-query_thread_log} -The table contains information about each query execution thread. +Contains information about threads which execute queries, for example, thread name, thread start time, duration of query processing. -ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. +To start logging: -To enable query logging, set the [log\_query\_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section. +1. Configure parameters in the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) section. +2. Set [log_query_threads](settings/settings.md#settings-log-query-threads) to 1. + +The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query. + +ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details. Columns: -- `event_date` (Date) — the date when the thread has finished execution of the query. -- `event_time` (DateTime) — the date and time when the thread has finished execution of the query. -- `query_start_time` (DateTime) — Start time of query execution. -- `query_duration_ms` (UInt64) — Duration of query execution. -- `read_rows` (UInt64) — Number of read rows. -- `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. -- `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. -- `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread. -- `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread. -- `thread_name` (String) — Name of the thread. -- `thread_number` (UInt32) — Internal thread ID. -- `os_thread_id` (Int32) — OS thread ID. -- `master_thread_id` (UInt64) — OS initial ID of initial thread. -- `query` (String) — Query string. -- `is_initial_query` (UInt8) — Query type. Possible values: +- `event_date` ([Date](../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query. +- `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time of query execution. +- `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution. +- `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read rows. +- `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read bytes. +- `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. +- `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. +- `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread. +- `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread. +- `thread_name` ([String](../sql-reference/data-types/string.md)) — Name of the thread. +- `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Internal thread ID. +- `thread_id` ([Int32](../sql-reference/data-types/int-uint.md)) — thread ID. +- `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread. +- `query` ([String](../sql-reference/data-types/string.md)) — Query string. +- `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values: - 1 — Query was initiated by the client. - 0 — Query was initiated by another query for distributed query execution. -- `user` (String) — Name of the user who initiated the current query. -- `query_id` (String) — ID of the query. -- `address` (IPv6) — IP address that was used to make the query. -- `port` (UInt16) — The client port that was used to make the query. -- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` (String) — ID of the initial query (for distributed query execution). -- `initial_address` (IPv6) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The client port that was used to make the parent query. -- `interface` (UInt8) — Interface that the query was initiated from. Possible values: +- `user` ([String](../sql-reference/data-types/string.md)) — Name of the user who initiated the current query. +- `query_id` ([String](../sql-reference/data-types/string.md)) — ID of the query. +- `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query. +- `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query. +- `initial_user` ([String](../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution). +- `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from. +- `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query. +- `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. -- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. -- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. -- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: +- `os_user` ([String](../sql-reference/data-types/string.md)) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). +- `client_hostname` ([String](../sql-reference/data-types/string.md)) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. +- `client_name` ([String](../sql-reference/data-types/string.md)) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. +- `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. +- `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP method that initiated the query. Possible values: - 0 — The query was launched from the TCP interface. - 1 — `GET` method was used. - 2 — `POST` method was used. -- `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request. -- `quota_key` (String) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`). -- `revision` (UInt32) — ClickHouse revision. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column. +- `http_user_agent` ([String](../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request. +- `quota_key` ([String](../sql-reference/data-types/string.md)) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`). +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse revision. +- `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events). +- `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column. -By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. +**Example** -When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. +``` sql + SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical +``` -!!! note "Note" - The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself. +``` text +Row 1: +────── +event_date: 2020-05-13 +event_time: 2020-05-13 14:02:28 +query_start_time: 2020-05-13 14:02:28 +query_duration_ms: 0 +read_rows: 1 +read_bytes: 1 +written_rows: 0 +written_bytes: 0 +memory_usage: 0 +peak_memory_usage: 0 +thread_name: QueryPipelineEx +thread_id: 28952 +master_thread_id: 28924 +query: SELECT 1 +is_initial_query: 1 +user: default +query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +address: ::ffff:127.0.0.1 +port: 57720 +initial_user: default +initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +initial_address: ::ffff:127.0.0.1 +initial_port: 57720 +interface: 1 +os_user: bayonet +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client +client_revision: 54434 +client_version_major: 20 +client_version_minor: 4 +client_version_patch: 1 +http_method: 0 +http_user_agent: +quota_key: +revision: 54434 +ProfileEvents.Names: ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds'] +ProfileEvents.Values: [1,97,81,5,81] +... +``` -You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `partition_by` parameter). +**See Also** + +- [system.query_log](#system_tables-query_log) — Description of the `query_log` system table which contains common information about queries execution. ## system.trace\_log {#system_tables-trace_log} diff --git a/docs/es/development/developer-instruction.md b/docs/es/development/developer-instruction.md index 9c56abe33a1..8f21e851b89 100644 --- a/docs/es/development/developer-instruction.md +++ b/docs/es/development/developer-instruction.md @@ -141,7 +141,7 @@ Las compilaciones oficiales de Yandex actualmente usan GCC porque genera código Para instalar GCC en Ubuntu, ejecute: `sudo apt install gcc g++` -Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse .tech/docs/en/development/build/\#install-gcc-9. +Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse.tech/docs/es/development/build/#install-gcc-9. La compilación de Mac OS X solo es compatible con Clang. Sólo tiene que ejecutar `brew install llvm` @@ -249,7 +249,7 @@ La Guía de estilo de código: https://clickhouse.tech/docs/en/development/style Pruebas de escritura: https://clickhouse.tech/docs/en/development/tests/ -Lista de tareas: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +Lista de tareas: https://github.com/ClickHouse/ClickHouse/contribute # Datos de prueba {#test-data} diff --git a/docs/fa/development/developer-instruction.md b/docs/fa/development/developer-instruction.md index cbc4734cec8..287960aff04 100644 --- a/docs/fa/development/developer-instruction.md +++ b/docs/fa/development/developer-instruction.md @@ -143,7 +143,7 @@ toc_title: "\u062F\u0633\u062A\u0648\u0631\u0627\u0644\u0639\u0645\u0644 \u062A\ برای نصب شورای همکاری خلیج فارس در اوبونتو اجرای: `sudo apt install gcc g++` -بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.فناوری / اسناد / ارتباطات / توسعه/ساختن / \#نصب شورای همکاری خلیج فارس-9. +بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.tech/docs/fa/development/build/#install-gcc-9. سیستم عامل مک ایکس ساخت فقط برای صدای جرنگ جرنگ پشتیبانی می شود. فقط فرار کن `brew install llvm` @@ -251,7 +251,7 @@ KDevelop و QTCreator دیگر از جایگزین های بسیار خوبی ا تست نوشتن: https://clickhouse.فناوری / اسناد/توسعه/تست/ -فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/contribute # داده های تست {#test-data} diff --git a/docs/fr/development/developer-instruction.md b/docs/fr/development/developer-instruction.md index 414cfc1d339..a20066fa3f7 100644 --- a/docs/fr/development/developer-instruction.md +++ b/docs/fr/development/developer-instruction.md @@ -141,7 +141,7 @@ Les builds officiels de Yandex utilisent actuellement GCC car ils génèrent du Pour installer GCC sur Ubuntu Exécutez: `sudo apt install gcc g++` -Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/développement/construction/\#install-gcc-9. +Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/development/build/#install-gcc-9. Mac OS X build est pris en charge uniquement pour Clang. Il suffit d'exécuter `brew install llvm` @@ -249,7 +249,7 @@ Le code Style Guide: https://clickhouse.tech/docs/fr/développement/style/ Rédaction de tests: https://clickhouse.tech/docs/fr/développement/tests/ -Liste des tâches: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +Liste des tâches: https://github.com/ClickHouse/ClickHouse/contribute # Des Données De Test {#test-data} diff --git a/docs/ja/development/developer-instruction.md b/docs/ja/development/developer-instruction.md index d65b25bd98c..6441e77185f 100644 --- a/docs/ja/development/developer-instruction.md +++ b/docs/ja/development/developer-instruction.md @@ -141,7 +141,7 @@ ClickHouseのビルドには、バージョン9以降のGCCとClangバージョ UBUNTUにGCCをインストールするには: `sudo apt install gcc g++` -Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/en/development/build/\#install-gcc-9. +Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/ja/development/build/#install-gcc-9. Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm` @@ -249,7 +249,7 @@ KDevelopとQTCreatorは、ClickHouseを開発するためのIDEの他の優れ 筆記試験：https://clickhouse.tech/docs/en/development/tests/ -タスクのリスト：https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +タスクのリスト：https://github.com/ClickHouse/ClickHouse/contribute # テストデータ {#test-data} diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 11ac3a73f6e..775b156dff5 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -135,7 +135,7 @@ ClickHouse использует для сборки некоторое коли Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`. -Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9 +Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/ru/development/build/#install-gcc-9. Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm` @@ -244,7 +244,7 @@ Mac OS X: Разработка тестов: https://clickhouse.tech/docs/ru/development/tests/ -Список задач: https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_ru.md +Список задач: https://github.com/ClickHouse/ClickHouse/contribute # Тестовые данные {#testovye-dannye} diff --git a/docs/ru/operations/system-tables.md b/docs/ru/operations/system-tables.md index 38971d6ee99..6e57e7a63f3 100644 --- a/docs/ru/operations/system-tables.md +++ b/docs/ru/operations/system-tables.md @@ -593,15 +593,9 @@ CurrentMetric_ReplicatedChecks: 0 Можно отключить логгирование настройкой [log_queries = 0](settings/settings.md#settings-log-queries). По-возможности, не отключайте логгирование, поскольку информация из таблицы важна при решении проблем. -Период сброса логов в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs). +Период сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs). -ClickHouse не удаляет логи из таблица автоматически. Смотрите [Введение](#system-tables-introduction). - -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`). - - - -Если таблицу удалить вручную, она создается заново автоматически «на лету». При этом все логи на момент удаления таблицы будут убраны. +ClickHouse не удаляет данные из таблица автоматически. Смотрите [Введение](#system-tables-introduction). Таблица `system.query_log` содержит информацию о двух видах запросов: @@ -729,71 +723,116 @@ Settings.Values: ['0','random','1','10000000000'] ## system.query_thread_log {#system_tables-query_thread_log} -Содержит информацию о каждом потоке выполняемых запросов. +Содержит информацию о потоках, которые выполняют запросы, например, имя потока, время его запуска, продолжительность обработки запроса. -ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. +Чтобы начать логирование: -Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md#settings). +1. Настройте параметры [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) в конфигурации сервера. +2. Установите значение [log_query_threads](settings/settings.md#settings-log-query-threads) равным 1. + +Интервал сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в разделе настроек сервера [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Чтобы принудительно записать логи из буфера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs). + +ClickHouse не удаляет данные из таблицы автоматически. Подробности в разделе [Введение](#system-tables-introduction). Столбцы: -- `event_date` (Date) — дата завершения выполнения запроса потоком. -- `event_time` (DateTime) — дата и время завершения выполнения запроса потоком. -- `query_start_time` (DateTime) — время начала обработки запроса. -- `query_duration_ms` (UInt64) — длительность обработки запроса в миллисекундах. -- `read_rows` (UInt64) — количество прочитанных строк. -- `read_bytes` (UInt64) — количество прочитанных байтов. -- `written_rows` (UInt64) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. -- `written_bytes` (UInt64) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. -- `memory_usage` (Int64) — разница между выделенной и освобождённой памятью в контексте потока. -- `peak_memory_usage` (Int64) — максимальная разница между выделенной и освобождённой памятью в контексте потока. -- `thread_name` (String) — Имя потока. -- `thread_id` (UInt64) — tid (ID потока операционной системы). -- `master_thread_id` (UInt64) — tid (ID потока операционной системы) главного потока. -- `query` (String) — текст запроса. -- `is_initial_query` (UInt8) — вид запроса. Возможные значения: +- `event_date` ([Date](../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком. +- `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — время начала обработки запроса. +- `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах. +- `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных строк. +- `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных байтов. +- `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. +- `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. +- `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — разница между выделенной и освобождённой памятью в контексте потока. +- `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — максимальная разница между выделенной и освобождённой памятью в контексте потока. +- `thread_name` ([String](../sql-reference/data-types/string.md)) — Имя потока. +- `thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы). +- `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы) главного потока. +- `query` ([String](../sql-reference/data-types/string.md)) — текст запроса. +- `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — вид запроса. Возможные значения: - 1 — запрос был инициирован клиентом. - 0 — запрос был инициирован другим запросом при распределенном запросе. -- `user` (String) — пользователь, запустивший текущий запрос. -- `query_id` (String) — ID запроса. -- `address` (IPv6) — IP адрес, с которого пришел запрос. -- `port` (UInt16) — порт, с которого пришел запрос. -- `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов). -- `initial_query_id` (String) — ID родительского запроса. -- `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос. -- `initial_port` (UInt16) — порт, пришел родительский запрос. -- `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения: +- `user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос. +- `query_id` ([String](../sql-reference/data-types/string.md)) — ID запроса. +- `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос. +- `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, с которого пришел запрос. +- `initial_user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов). +- `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID родительского запроса. +- `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос. +- `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, пришел родительский запрос. +- `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — интерфейс, с которого ушёл запрос. Возможные значения: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md). -- `client_hostname` (String) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. -- `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. -- `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения: +- `os_user` ([String](../sql-reference/data-types/string.md)) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md). +- `client_hostname` ([String](../sql-reference/data-types/string.md)) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_name` ([String](../sql-reference/data-types/string.md)) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP метод, инициировавший запрос. Возможные значения: - 0 — запрос запущен с интерфейса TCP. - 1 — `GET`. - 2 — `POST`. -- `http_user_agent` (String) — HTTP заголовок `UserAgent`. -- `quota_key` (String) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). -- `revision` (UInt32) — ревизия ClickHouse. -- `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events -- `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. +- `http_user_agent` ([String](../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`. +- `quota_key` ([String](../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. +- `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events). +- `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. -По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. +**Пример** -Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены. +``` sql + SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical +``` -!!! note "Примечание" - Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. +``` text +Row 1: +────── +event_date: 2020-05-13 +event_time: 2020-05-13 14:02:28 +query_start_time: 2020-05-13 14:02:28 +query_duration_ms: 0 +read_rows: 1 +read_bytes: 1 +written_rows: 0 +written_bytes: 0 +memory_usage: 0 +peak_memory_usage: 0 +thread_name: QueryPipelineEx +thread_id: 28952 +master_thread_id: 28924 +query: SELECT 1 +is_initial_query: 1 +user: default +query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +address: ::ffff:127.0.0.1 +port: 57720 +initial_user: default +initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +initial_address: ::ffff:127.0.0.1 +initial_port: 57720 +interface: 1 +os_user: bayonet +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client +client_revision: 54434 +client_version_major: 20 +client_version_minor: 4 +client_version_patch: 1 +http_method: 0 +http_user_agent: +quota_key: +revision: 54434 +ProfileEvents.Names: ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds'] +ProfileEvents.Values: [1,97,81,5,81] +... +``` -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (параметр `partition_by`). +**Смотрите также** -## system.query_thread_log {#system_tables-query_thread_log} - -Содержит информацию о каждом потоке исполнения запроса. +- [system.query_log](#system_tables-query_log) — описание системной таблицы `query_log`, которая содержит общую информацию о выполненных запросах. ## system.trace\_log {#system_tables-trace_log} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 3534fc4e48a..1f09eb28d2e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -32,7 +32,7 @@ ClickHouse поддерживает иерархические словари с ClickHouse поддерживает свойство [hierarchical](external-dicts-dict-structure.md#hierarchical-dict-attr) для атрибутов [внешнего словаря](index.md). Это свойство позволяет конфигурировать словари, подобные описанному выше. -С помощью функции [dictGetHierarchy](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md#dictgethierarchy) можно получить цепочку предков элемента. +С помощью функции [dictGetHierarchy](../../../sql-reference/functions/ext-dict-functions.md#dictgethierarchy) можно получить цепочку предков элемента. Структура словаря для нашего примера может выглядеть следующим образом: diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 9256fab5e0c..368da949dc8 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -2,7 +2,7 @@ Словари можно размещать в памяти множеством способов. -Рекомендуем [flat](#flat), [hashed](#hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна. +Рекомендуем [flat](#flat), [hashed](#dicts-external_dicts_dict_layout-hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна. Размещение с кэшированием не рекомендуется использовать из-за потенциально низкой производительности и сложностей в подборе оптимальных параметров. Читайте об этом подробнее в разделе «[cache](#cache)». @@ -34,7 +34,7 @@ ``` -Соответствущий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query): +Соответствущий [DDL-запрос](../../statements/create.md#create-dictionary-query): ``` sql CREATE DICTIONARY (...) @@ -46,7 +46,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings ## Способы размещения словарей в памяти {#sposoby-razmeshcheniia-slovarei-v-pamiati} - [flat](#flat) -- [hashed](#hashed) +- [hashed](#dicts-external_dicts_dict_layout-hashed) - [sparse\_hashed](#dicts-external_dicts_dict_layout-sparse_hashed) - [cache](#cache) - [direct](#direct) @@ -80,7 +80,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings LAYOUT(FLAT()) ``` -### hashed {#hashed} +### hashed {#dicts-external_dicts_dict_layout-hashed} Словарь полностью хранится в оперативной памяти в виде хэш-таблиц. Словарь может содержать произвольное количество элементов с произвольными идентификаторами. На практике, количество ключей может достигать десятков миллионов элементов. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 4190e8e1015..e5b20f3960c 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -19,7 +19,7 @@ ``` -Аналогичный [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query): +Аналогичный [DDL-запрос](../../statements/create.md#create-dictionary-query): ``` sql CREATE DICTIONARY dict_name (...) @@ -150,7 +150,7 @@ SOURCE(HTTP( )) ``` -Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) в конфигурации сервера. +Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl) в конфигурации сервера. Поля настройки: @@ -531,7 +531,7 @@ SOURCE(CLICKHOUSE( Поля настройки: -- `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) и прописать её в дальнейших настройках. +- `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../engines/table-engines/special/distributed.md) и прописать её в дальнейших настройках. - `port` — порт сервера ClickHouse. - `user` — имя пользователя ClickHouse. - `password` — пароль пользователя ClickHouse. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 27702959eac..4c3b4eb22e4 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -154,7 +154,7 @@ CREATE DICTIONARY somename ( | Тег | Описание | Обязательный | |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | `name` | Имя столбца. | Да | -| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md) не поддерживается. | Да | +| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/data-types/nullable.md) не поддерживается. | Да | | `null_value` | Значение по умолчанию для несуществующего элемента.
В примере это пустая строка. Нельзя указать значение `NULL`. | Да | | `expression` | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.
Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.

Значение по умолчанию: нет выражения. | Нет | | `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).

Default value: `false`. | No | @@ -162,6 +162,6 @@ CREATE DICTIONARY somename ( ## Смотрите также {#smotrite-takzhe} -- [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md). +- [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md). [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 9eb6c8d8d86..a7d3394864b 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -24,7 +24,7 @@ XML-конфигурация словаря имеет следующую стр ``` -Соответствующий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query) имеет следующий вид: +Соответствующий [DDL-запрос](../../statements/create.md#create-dictionary-query) имеет следующий вид: ``` sql CREATE DICTIONARY dict_name diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 7442a5dd3be..80f717dfe93 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -5,11 +5,11 @@ ClickHouse: - Полностью или частично хранит словари в оперативной памяти. - Периодически обновляет их и динамически подгружает отсутствующие значения. -- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../../sql-reference/statements/create.md#create-dictionary-query). +- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../statements/create.md#create-dictionary-query). -Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). +Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config). -Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). +Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load). Системная таблица [system.dictionaries](../../../operations/system-tables.md#system_tables-dictionaries) содержит информацию о словарях, сконфигурированных на сервере. Для каждого словаря там можно найти: @@ -41,10 +41,10 @@ ClickHouse: В одном файле можно [сконфигурировать](external-dicts-dict.md) произвольное количество словарей. -Если вы создаёте внешние словари [DDL-запросами](../../../sql-reference/statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера. +Если вы создаёте внешние словари [DDL-запросами](../../statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера. !!! attention "Внимание" - Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)). Эта функциональность не связана с внешними словарями. + Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/functions/other-functions.md)). Эта функциональность не связана с внешними словарями. ## Смотрите также {#ext-dicts-see-also} @@ -53,6 +53,6 @@ ClickHouse: - [Обновление словарей](external-dicts-dict-lifetime.md) - [Источники внешних словарей](external-dicts-dict-sources.md) - [Ключ и поля словаря](external-dicts-dict-structure.md) -- [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md#ext_dict_functions) +- [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) diff --git a/docs/ru/sql-reference/index.md b/docs/ru/sql-reference/index.md index a13e3774b86..ea611e75995 100644 --- a/docs/ru/sql-reference/index.md +++ b/docs/ru/sql-reference/index.md @@ -10,7 +10,7 @@ toc_title: hidden - [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create.md) -- [ALTER](statements/alter.md) +- [ALTER](statements/alter.md#query_language_queries_alter) - [Прочие виды запросов](statements/misc.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) diff --git a/docs/tr/development/developer-instruction.md b/docs/tr/development/developer-instruction.md index a65c6666288..0ca5f9cdd63 100644 --- a/docs/tr/development/developer-instruction.md +++ b/docs/tr/development/developer-instruction.md @@ -141,7 +141,7 @@ Resmi Yandex şu anda GCC'Yİ kullanıyor çünkü biraz daha iyi performansa sa Ubuntu run GCC yüklemek için: `sudo apt install gcc g++` -Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9. +Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech/docs/tr/development/build/#install-gcc-9. Mac OS X build sadece Clang için desteklenir. Sadece koş `brew install llvm` @@ -249,7 +249,7 @@ Kod stili Kılavuzu: https://clickhouse.tech / doscs / TR / development / style/ Yazma testleri: https://clickhouse.teknoloji / doscs / TR / geliştirme / testler/ -Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md +Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/contribute # Test Verileri {#test-data} diff --git a/docs/zh/development/developer-instruction.md b/docs/zh/development/developer-instruction.md index 6911a0e4dc9..b40e6db3af1 100644 --- a/docs/zh/development/developer-instruction.md +++ b/docs/zh/development/developer-instruction.md @@ -129,7 +129,7 @@ Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性 在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++` -请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/en/development/build/\#install-gcc-9 。 +请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/zh/development/build/#an-zhuang-gcc-9 。 在Mac OS X上安装GCC，请执行：`brew install gcc` @@ -234,7 +234,7 @@ ClickHouse的架构描述可以在此处查看：https://clickhouse.tech/docs/en 编写测试用例：https://clickhouse.tech/docs/en/development/tests/ -任务列表：https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_en.md +任务列表：https://github.com/ClickHouse/ClickHouse/contribute # 测试数据 {#ce-shi-shu-ju} diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index c8f942013ea..830581beba7 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -313,7 +313,7 @@ ORDER BY level ASC └───────┴───┘ ``` -## 保留 {#retention} +## Retention {#retention} 该函数将一组条件作为参数，类型为1到32个参数 `UInt8` 表示事件是否满足特定条件。 任何条件都可以指定为参数（如 [WHERE](../../sql-reference/statements/select/where.md#select-where)). diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index baf8270d1bf..b36a2ff8194 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -207,7 +207,7 @@ if (TARGET clickhouse-server AND TARGET copy-headers) endif () if (ENABLE_TESTS AND USE_GTEST) - set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor) - add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS}) + set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms) + add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS}) add_dependencies(clickhouse-bundle clickhouse-tests) endif() diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp index 8fffbec4fab..4ac5e735fd5 100644 --- a/programs/client/Suggest.cpp +++ b/programs/client/Suggest.cpp @@ -114,6 +114,8 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo << " UNION ALL " "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str << " UNION ALL " + "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str + << " UNION ALL " "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index e587e134075..d70806303d2 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -125,6 +125,7 @@ namespace ErrorCodes extern const int FAILED_TO_GETPWUID; extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; extern const int NETWORK_ERROR; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; } @@ -210,6 +211,52 @@ void Server::defineOptions(Poco::Util::OptionSet & options) BaseDaemon::defineOptions(options); } + +/// Check that there is no user-level settings at the top level in config. +/// This is a common source of mistake (user don't know where to write user-level setting). +void checkForUserSettingsAtTopLevel(const Poco::Util::AbstractConfiguration & config, const std::string & path) +{ + if (config.getBool("skip_check_for_incorrect_settings", false)) + return; + + Settings settings; + for (const auto & setting : settings) + { + std::string name = setting.getName().toString(); + if (config.has(name)) + { + throw Exception(fmt::format("A setting '{}' appeared at top level in config {}." + " But it is user-level setting that should be located in users.xml inside section for specific profile." + " You can add it to if you want to change default value of this setting." + " You can also disable the check - specify 1" + " in the main configuration file.", + name, path), + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + } + } +} + +void checkForUsersNotInMainConfig( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_path, + const std::string & users_config_path, + Poco::Logger * log) +{ + if (config.getBool("skip_check_for_incorrect_settings", false)) + return; + + if (config.has("users") || config.has("profiles") || config.has("quotas")) + { + /// We cannot throw exception here, because we have support for obsolete 'conf.d' directory + /// (that does not correspond to config.d or users.d) but substitute configuration to both of them. + + LOG_ERROR(log, "The , and elements should be located in users config file: {} not in main config {}." + " Also note that you should place configuration changes to the appropriate *.d directory like 'users.d'.", + users_config_path, config_path); + } +} + + int Server::main(const std::vector & /*args*/) { Poco::Logger * log = &logger(); @@ -269,6 +316,8 @@ int Server::main(const std::vector & /*args*/) config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } + checkForUserSettingsAtTopLevel(config(), config_path); + const auto memory_amount = getMemoryAmount(); #if defined(OS_LINUX) @@ -473,13 +522,16 @@ int Server::main(const std::vector & /*args*/) SensitiveDataMasker::setInstance(std::make_unique(config(), "query_masking_rules")); } - auto main_config_reloader = std::make_unique(config_path, + auto main_config_reloader = std::make_unique( + config_path, include_from_path, config().getString("path", ""), std::move(main_config_zk_node_cache), main_config_zk_changed_event, [&](ConfigurationPtr config) { + checkForUserSettingsAtTopLevel(*config, config_path); + // FIXME logging-related things need synchronization -- see the 'Logger * log' saved // in a lot of places. For now, disable updating log configuration without server restart. //setTextLog(global_context->getTextLog()); @@ -508,12 +560,21 @@ int Server::main(const std::vector & /*args*/) if (Poco::File(config_dir + users_config_path).exists()) users_config_path = config_dir + users_config_path; } - auto users_config_reloader = std::make_unique(users_config_path, + + if (users_config_path != config_path) + checkForUsersNotInMainConfig(config(), config_path, users_config_path, log); + + auto users_config_reloader = std::make_unique( + users_config_path, include_from_path, config().getString("path", ""), zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), std::make_shared(), - [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, + [&](ConfigurationPtr config) + { + global_context->setUsersConfig(config); + checkForUserSettingsAtTopLevel(*config, users_config_path); + }, /* already_loaded = */ false); /// Reload config in SYSTEM RELOAD CONFIG query. diff --git a/programs/server/config.xml b/programs/server/config.xml index 21605edeb36..ba870d8a8ea 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1,6 +1,9 @@ diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8e4314ec7c5..a7af61c7712 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -12,10 +12,10 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; extern const int ACCESS_ENTITY_ALREADY_EXISTS; extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; + extern const int LOGICAL_ERROR; } @@ -403,7 +403,7 @@ void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String { throw Exception( "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type), - ErrorCodes::BAD_CAST); + ErrorCodes::LOGICAL_ERROR); } diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index ce4ef98e0cf..a11ca9032a5 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -15,7 +15,6 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int BAD_CAST; } /** @@ -381,7 +380,7 @@ public: auto * column = typeid_cast(&to); if (!column) throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to", - ErrorCodes::BAD_CAST); + ErrorCodes::LOGICAL_ERROR); this->data(place).predict(column->getData(), block, offset, limit, arguments, context); } diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 1870eee07b8..0087a41d437 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -150,6 +150,8 @@ public: virtual void addBatchSinglePlaceNotNull( size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const = 0; + virtual void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0; + /** In addition to addBatch, this method collects multiple rows of arguments into array "places" * as long as they are between offsets[i-1] and offsets[i]. This is used for arrayReduce and * -Array combinator. It might also be used generally to break data dependency when array @@ -214,6 +216,12 @@ public: static_cast(this)->add(place, columns, i, arena); } + void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override + { + for (size_t i = batch_begin; i < batch_end; ++i) + static_cast(this)->add(place, columns, i, arena); + } + void addBatchArray( size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena) const override diff --git a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp index 2a6b9e3b499..27772c143e8 100644 --- a/src/AggregateFunctions/parseAggregateFunctionParameters.cpp +++ b/src/AggregateFunctions/parseAggregateFunctionParameters.cpp @@ -27,8 +27,12 @@ Array getAggregateFunctionParametersArray(const ASTPtr & expression_list, const const auto * literal = parameters[i]->as(); if (!literal) { - throw Exception("Parameters to aggregate functions must be literals" + (error_context.empty() ? "" : " (in " + error_context +")"), - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}'{}", + parameters[i]->formatForErrorMessage(), + (error_context.empty() ? "" : " (in " + error_context +")")); } params_row[i] = literal->value; diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 40f73665ebe..002bc71f561 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -121,6 +121,7 @@ public: std::string getName() const override { return "AggregateFunction(" + func->getName() + ")"; } const char * getFamilyName() const override { return "AggregateFunction"; } + TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; } MutableColumnPtr predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const; diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 55935a91cde..a20165826bb 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -52,6 +52,7 @@ public: std::string getName() const override; const char * getFamilyName() const override { return "Array"; } + TypeIndex getDataType() const override { return TypeIndex::Array; } MutableColumnPtr cloneResized(size_t size) const override; size_t size() const override; Field operator[](size_t n) const override; diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 5fc96b14be8..02dfcc5b620 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -50,6 +50,11 @@ public: return "Const"; } + TypeIndex getDataType() const override + { + return data->getDataType(); + } + MutableColumnPtr cloneResized(size_t new_size) const override { return ColumnConst::create(data, new_size); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 1c238cc6458..3e6fb833b56 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -333,17 +333,6 @@ void ColumnDecimal::getExtremes(Field & min, Field & max) const max = NearestFieldType(cur_max, scale); } -TypeIndex columnDecimalDataType(const IColumn * column) -{ - if (checkColumn>(column)) - return TypeIndex::Decimal32; - else if (checkColumn>(column)) - return TypeIndex::Decimal64; - else if (checkColumn>(column)) - return TypeIndex::Decimal128; - return TypeIndex::Nothing; -} - template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 16c6a47c30a..37d85b05d4c 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -81,6 +81,7 @@ private: public: const char * getFamilyName() const override { return TypeName::get(); } + TypeIndex getDataType() const override { return TypeId::value; } bool isNumeric() const override { return false; } bool canBeInsideNullable() const override { return true; } @@ -197,6 +198,4 @@ ColumnPtr ColumnDecimal::indexImpl(const PaddedPODArray & indexes, size return res; } -TypeIndex columnDecimalDataType(const IColumn * column); - } diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 996a1f99ef1..6b7f1ecf793 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -43,6 +43,7 @@ private: public: std::string getName() const override { return "FixedString(" + std::to_string(n) + ")"; } const char * getFamilyName() const override { return "FixedString"; } + TypeIndex getDataType() const override { return TypeIndex::FixedString; } MutableColumnPtr cloneResized(size_t size) const override; diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h index 31cb8708a6e..267f3c7285a 100644 --- a/src/Columns/ColumnFunction.h +++ b/src/Columns/ColumnFunction.h @@ -29,6 +29,7 @@ private: public: const char * getFamilyName() const override { return "Function"; } + TypeIndex getDataType() const override { return TypeIndex::Function; } MutableColumnPtr cloneResized(size_t size) const override; diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 905d15f8167..1e6319a2cb1 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -39,6 +39,7 @@ public: std::string getName() const override { return "ColumnLowCardinality"; } const char * getFamilyName() const override { return "ColumnLowCardinality"; } + TypeIndex getDataType() const override { return TypeIndex::LowCardinality; } ColumnPtr convertToFullColumn() const { return getDictionary().getNestedColumn()->index(getIndexes(), 0); } ColumnPtr convertToFullColumnIfLowCardinality() const override { return convertToFullColumn(); } diff --git a/src/Columns/ColumnNothing.h b/src/Columns/ColumnNothing.h index 691143e2c15..c2738bb4cdc 100644 --- a/src/Columns/ColumnNothing.h +++ b/src/Columns/ColumnNothing.h @@ -21,6 +21,7 @@ private: public: const char * getFamilyName() const override { return "Nothing"; } MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); } + TypeIndex getDataType() const override { return TypeIndex::Nothing; } bool canBeInsideNullable() const override { return true; } diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 2cd8ff9f40f..a8f226ed37d 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -45,6 +45,7 @@ public: const char * getFamilyName() const override { return "Nullable"; } std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; } + TypeIndex getDataType() const override { return TypeIndex::Nullable; } MutableColumnPtr cloneResized(size_t size) const override; size_t size() const override { return nested_column->size(); } bool isNullAt(size_t n) const override { return assert_cast(*null_map).getData()[n] != 0;} diff --git a/src/Columns/ColumnSet.h b/src/Columns/ColumnSet.h index b30ba86fafe..316f8196e5a 100644 --- a/src/Columns/ColumnSet.h +++ b/src/Columns/ColumnSet.h @@ -25,6 +25,7 @@ private: public: const char * getFamilyName() const override { return "Set"; } + TypeIndex getDataType() const override { return TypeIndex::Set; } MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnSet::create(s_, data); } ConstSetPtr getData() const { return data; } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index a0b3d259b67..f067bce47bc 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -56,6 +56,7 @@ private: public: const char * getFamilyName() const override { return "String"; } + TypeIndex getDataType() const override { return TypeIndex::String; } size_t size() const override { diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 69b18e2fc0f..33c48a0cdd1 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -40,6 +40,7 @@ public: std::string getName() const override; const char * getFamilyName() const override { return "Tuple"; } + TypeIndex getDataType() const override { return TypeIndex::Tuple; } MutableColumnPtr cloneEmpty() const override; MutableColumnPtr cloneResized(size_t size) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 95efd0dedad..50f1dba4fdb 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -289,13 +289,6 @@ void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_dire equal_range = std::move(new_ranges); } - -template -const char * ColumnVector::getFamilyName() const -{ - return TypeName::get(); -} - template MutableColumnPtr ColumnVector::cloneResized(size_t size) const { @@ -517,33 +510,6 @@ void ColumnVector::getExtremes(Field & min, Field & max) const max = NearestFieldType(cur_max); } -TypeIndex columnVectorDataType(const IColumn * column) -{ - if (checkColumn>(column)) - return TypeIndex::UInt8; - else if (checkColumn>(column)) - return TypeIndex::UInt16; - else if (checkColumn>(column)) - return TypeIndex::UInt32; - else if (checkColumn>(column)) - return TypeIndex::UInt64; - else if (checkColumn>(column)) - return TypeIndex::Int8; - else if (checkColumn>(column)) - return TypeIndex::Int16; - else if (checkColumn>(column)) - return TypeIndex::Int32; - else if (checkColumn>(column)) - return TypeIndex::Int64; - else if (checkColumn>(column)) - return TypeIndex::Int128; - else if (checkColumn>(column)) - return TypeIndex::Float32; - else if (checkColumn>(column)) - return TypeIndex::Float64; - return TypeIndex::Nothing; -} - /// Explicit template instantiations - to avoid code bloat in headers. template class ColumnVector; template class ColumnVector; diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 5e934b42df0..b9b14f4b2a1 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -199,7 +199,8 @@ public: data.reserve(n); } - const char * getFamilyName() const override; + const char * getFamilyName() const override { return TypeName::get(); } + TypeIndex getDataType() const override { return TypeId::value; } MutableColumnPtr cloneResized(size_t size) const override; @@ -320,6 +321,4 @@ ColumnPtr ColumnVector::indexImpl(const PaddedPODArray & indexes, size_ return res; } -TypeIndex columnVectorDataType(const IColumn * column); - } diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 1d92ed1c3ab..c227ec97e3a 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -51,6 +51,9 @@ public: /// Name of a Column kind, without parameters (example: FixedString, Array). virtual const char * getFamilyName() const = 0; + /// Type of data that column contains. It's an underlying type: UInt16 for Date, UInt32 for DateTime, so on. + virtual TypeIndex getDataType() const = 0; + /** If column isn't constant, returns itself. * If column is constant, transforms constant to full column (if column type allows such transform) and return it. */ diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index af5d9878a3b..693ed18b87e 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -66,6 +66,7 @@ public: virtual UInt128 getHash() const = 0; const char * getFamilyName() const override { return "ColumnUnique"; } + TypeIndex getDataType() const override { return getNestedColumn()->getDataType(); } void insert(const Field &) override { diff --git a/src/Common/Arena.h b/src/Common/Arena.h index f1d42e53345..d203a92d4a3 100644 --- a/src/Common/Arena.h +++ b/src/Common/Arena.h @@ -150,7 +150,7 @@ public: return res; } - /// Get peice of memory with alignment + /// Get piece of memory with alignment char * alignedAlloc(size_t size, size_t alignment) { do diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index cb4c591041c..f29140a64ec 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -341,7 +341,6 @@ namespace ErrorCodes extern const int OUTPUT_IS_NOT_SORTED = 365; extern const int SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT = 366; extern const int TOO_MANY_FETCHES = 367; - extern const int BAD_CAST = 368; extern const int ALL_REPLICAS_ARE_STALE = 369; extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES = 370; extern const int INCONSISTENT_CLUSTER_DEFINITION = 371; @@ -398,7 +397,6 @@ namespace ErrorCodes extern const int CANNOT_GETTIMEOFDAY = 423; extern const int CANNOT_LINK = 424; extern const int SYSTEM_ERROR = 425; - extern const int NULL_POINTER_DEREFERENCE = 426; extern const int CANNOT_COMPILE_REGEXP = 427; extern const int UNKNOWN_LOG_LEVEL = 428; extern const int FAILED_TO_GETPWUID = 429; @@ -458,7 +456,6 @@ namespace ErrorCodes extern const int TOO_MANY_REDIRECTS = 483; extern const int INTERNAL_REDIS_ERROR = 484; extern const int SCALAR_ALREADY_EXISTS = 485; - extern const int UNKNOWN_SCALAR = 486; extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY = 487; extern const int UNKNOWN_DICTIONARY = 488; extern const int INCORRECT_DICTIONARY_DEFINITION = 489; diff --git a/src/Common/Exception.h b/src/Common/Exception.h index de63f35f463..45a1b3d6340 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -8,6 +8,8 @@ #include +#include + namespace Poco { class Logger; } @@ -20,8 +22,14 @@ public: Exception() = default; Exception(const std::string & msg, int code); - enum CreateFromPocoTag { CreateFromPoco }; - enum CreateFromSTDTag { CreateFromSTD }; + // Format message with fmt::format, like the logging functions. + template + Exception(int code, Fmt&&... fmt) + : Exception(fmt::format(std::forward(fmt)...), code) + {} + + struct CreateFromPocoTag {}; + struct CreateFromSTDTag {}; Exception(CreateFromPocoTag, const Poco::Exception & exc); Exception(CreateFromSTDTag, const std::exception & exc); diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index a75339a644d..8393ea85112 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -180,6 +180,25 @@ M(OSWriteBytes, "Number of bytes written to disks or block devices. Doesn't include bytes that are in page cache dirty pages. May not include data that was written by OS asynchronously.") \ M(OSReadChars, "Number of bytes read from filesystem, including page cache.") \ M(OSWriteChars, "Number of bytes written to filesystem, including page cache.") \ + \ + M(PerfCpuCycles, "Total cycles. Be wary of what happens during CPU frequency scaling.") \ + M(PerfInstructions, "Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts.") \ + M(PerfCacheReferences, "Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU.") \ + M(PerfCacheMisses, "Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in con‐junction with the PERFCOUNTHWCACHEREFERENCES event to calculate cache miss rates.") \ + M(PerfBranchInstructions, "Retired branch instructions. Prior to Linux 2.6.35, this used the wrong event on AMD processors.") \ + M(PerfBranchMisses, "Mispredicted branch instructions.") \ + M(PerfBusCycles, "Bus cycles, which can be different from total cycles.") \ + M(PerfStalledCyclesFrontend, "Stalled cycles during issue.") \ + M(PerfStalledCyclesBackend, "Stalled cycles during retirement.") \ + M(PerfRefCpuCycles, "Total cycles; not affected by CPU frequency scaling.") \ + \ + M(PerfCpuClock, "The CPU clock, a high-resolution per-CPU timer") \ + M(PerfTaskClock, "A clock count specific to the task that is running") \ + M(PerfContextSwitches, "Number of context switches") \ + M(PerfCpuMigrations, "Number of times the process has migrated to a new CPU") \ + M(PerfAlignmentFaults, "Number of alignment faults. These happen when unaligned memory accesses happen; the kernel can handle these but it reduces performance. This happens only on some architectures (never on x86).") \ + M(PerfEmulationFaults, "Number of emulation faults. The kernel sometimes traps on unimplemented instructions and emulates them for user space. This can negatively impact performance.") \ + \ M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \ \ M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \ diff --git a/src/Common/ThreadProfileEvents.cpp b/src/Common/ThreadProfileEvents.cpp index 42452bf590b..fdc27f7efa3 100644 --- a/src/Common/ThreadProfileEvents.cpp +++ b/src/Common/ThreadProfileEvents.cpp @@ -4,9 +4,22 @@ #include "TaskStatsInfoGetter.h" #include "ProcfsMetricsProvider.h" +#include "hasLinuxCapability.h" +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB { @@ -104,6 +117,404 @@ void TasksStatsCounters::incrementProfileEvents(const ::taskstats & prev, const profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.read_bytes, curr.read_bytes)); profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.write_bytes, curr.write_bytes)); } + +} + +#endif + +#if defined(__linux__) && !defined(ARCADIA_BUILD) + +namespace DB +{ + +thread_local PerfEventsCounters current_thread_counters; + +#define SOFTWARE_EVENT(PERF_NAME, LOCAL_NAME) \ + PerfEventInfo \ + { \ + .event_type = perf_type_id::PERF_TYPE_SOFTWARE, \ + .event_config = (PERF_NAME), \ + .profile_event = ProfileEvents::LOCAL_NAME, \ + .settings_name = #LOCAL_NAME \ + } + +#define HARDWARE_EVENT(PERF_NAME, LOCAL_NAME) \ + PerfEventInfo \ + { \ + .event_type = perf_type_id::PERF_TYPE_HARDWARE, \ + .event_config = (PERF_NAME), \ + .profile_event = ProfileEvents::LOCAL_NAME, \ + .settings_name = #LOCAL_NAME \ + } + +// descriptions' source: http://man7.org/linux/man-pages/man2/perf_event_open.2.html +static const PerfEventInfo raw_events_info[] = { + HARDWARE_EVENT(PERF_COUNT_HW_CPU_CYCLES, PerfCpuCycles), + HARDWARE_EVENT(PERF_COUNT_HW_INSTRUCTIONS, PerfInstructions), + HARDWARE_EVENT(PERF_COUNT_HW_CACHE_REFERENCES, PerfCacheReferences), + HARDWARE_EVENT(PERF_COUNT_HW_CACHE_MISSES, PerfCacheMisses), + HARDWARE_EVENT(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, PerfBranchInstructions), + HARDWARE_EVENT(PERF_COUNT_HW_BRANCH_MISSES, PerfBranchMisses), + HARDWARE_EVENT(PERF_COUNT_HW_BUS_CYCLES, PerfBusCycles), + HARDWARE_EVENT(PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, PerfStalledCyclesFrontend), + HARDWARE_EVENT(PERF_COUNT_HW_STALLED_CYCLES_BACKEND, PerfStalledCyclesBackend), + HARDWARE_EVENT(PERF_COUNT_HW_REF_CPU_CYCLES, PerfRefCpuCycles), + // `cpu-clock` is a bit broken according to this: https://stackoverflow.com/a/56967896 + SOFTWARE_EVENT(PERF_COUNT_SW_CPU_CLOCK, PerfCpuClock), + SOFTWARE_EVENT(PERF_COUNT_SW_TASK_CLOCK, PerfTaskClock), + SOFTWARE_EVENT(PERF_COUNT_SW_CONTEXT_SWITCHES, PerfContextSwitches), + SOFTWARE_EVENT(PERF_COUNT_SW_CPU_MIGRATIONS, PerfCpuMigrations), + SOFTWARE_EVENT(PERF_COUNT_SW_ALIGNMENT_FAULTS, PerfAlignmentFaults), + SOFTWARE_EVENT(PERF_COUNT_SW_EMULATION_FAULTS, PerfEmulationFaults) +}; + +#undef HARDWARE_EVENT +#undef SOFTWARE_EVENT + +// A map of event name -> event index, to parse event list in settings. +static std::unordered_map populateEventMap() +{ + std::unordered_map name_to_index; + name_to_index.reserve(NUMBER_OF_RAW_EVENTS); + + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + { + name_to_index.emplace(raw_events_info[i].settings_name, i); + } + + return name_to_index; +} + +static const auto event_name_to_index = populateEventMap(); + +static int openPerfEvent(perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, UInt64 flags) +{ + return static_cast(syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags)); +} + +static int openPerfEventDisabled(Int32 perf_event_paranoid, bool has_cap_sys_admin, UInt32 perf_event_type, UInt64 perf_event_config) +{ + perf_event_attr pe{}; + pe.type = perf_event_type; + pe.size = sizeof(struct perf_event_attr); + pe.config = perf_event_config; + // disable by default to add as little extra time as possible + pe.disabled = 1; + // can record kernel only when `perf_event_paranoid` <= 1 or have CAP_SYS_ADMIN + pe.exclude_kernel = perf_event_paranoid >= 2 && !has_cap_sys_admin; + pe.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; + + return openPerfEvent(&pe, /* measure the calling thread */ 0, /* on any cpu */ -1, -1, 0); +} + +static void enablePerfEvent(int event_fd) +{ + if (ioctl(event_fd, PERF_EVENT_IOC_ENABLE, 0)) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Can't enable perf event with file descriptor {}: '{}' ({})", + event_fd, strerror(errno), errno); + } +} + +static void disablePerfEvent(int event_fd) +{ + if (ioctl(event_fd, PERF_EVENT_IOC_DISABLE, 0)) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Can't disable perf event with file descriptor {}: '{}' ({})", + event_fd, strerror(errno), errno); + } +} + +static void releasePerfEvent(int event_fd) +{ + if (close(event_fd)) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Can't close perf event file descriptor {}: {} ({})", + event_fd, strerror(errno), errno); + } +} + +static bool validatePerfEventDescriptor(int & fd) +{ + if (fcntl(fd, F_GETFL) != -1) + return true; + + if (errno == EBADF) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Event descriptor {} was closed from the outside; reopening", fd); + } + else + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Error while checking availability of event descriptor {}: {} ({})", + fd, strerror(errno), errno); + + disablePerfEvent(fd); + releasePerfEvent(fd); + } + + fd = -1; + return false; +} + +bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_events_list) +{ + const auto valid_event_indices = eventIndicesFromString(needed_events_list); + + // find state changes (if there are any) + bool old_state[NUMBER_OF_RAW_EVENTS]; + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + old_state[i] = thread_events_descriptors_holder.descriptors[i] != -1; + + bool new_state[NUMBER_OF_RAW_EVENTS]; + std::fill_n(new_state, NUMBER_OF_RAW_EVENTS, false); + for (size_t opened_index : valid_event_indices) + new_state[opened_index] = true; + + std::vector events_to_open; + std::vector events_to_release; + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + { + bool old_one = old_state[i]; + bool new_one = new_state[i]; + + if (old_one == new_one) + { + if (old_one + && !validatePerfEventDescriptor( + thread_events_descriptors_holder.descriptors[i])) + { + events_to_open.push_back(i); + } + continue; + } + + if (new_one) + events_to_open.push_back(i); + else + events_to_release.push_back(i); + } + + // release unused descriptors + for (size_t i : events_to_release) + { + int & fd = thread_events_descriptors_holder.descriptors[i]; + disablePerfEvent(fd); + releasePerfEvent(fd); + fd = -1; + } + + if (events_to_open.empty()) + { + return true; + } + + // check permissions + // cat /proc/sys/kernel/perf_event_paranoid + // -1: Allow use of (almost) all events by all users + // >=0: Disallow raw tracepoint access by users without CAP_IOC_LOCK + // >=1: Disallow CPU event access by users without CAP_SYS_ADMIN + // >=2: Disallow kernel profiling by users without CAP_SYS_ADMIN + // >=3: Disallow all event access by users without CAP_SYS_ADMIN + Int32 perf_event_paranoid = 0; + std::ifstream paranoid_file("/proc/sys/kernel/perf_event_paranoid"); + paranoid_file >> perf_event_paranoid; + + bool has_cap_sys_admin = hasLinuxCapability(CAP_SYS_ADMIN); + if (perf_event_paranoid >= 3 && !has_cap_sys_admin) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Not enough permissions to record perf events: " + "perf_event_paranoid = {} and CAP_SYS_ADMIN = 0", + perf_event_paranoid); + return false; + } + + // Open descriptors for new events. + // Theoretically, we can run out of file descriptors. Threads go up to 10k, + // and there might be a dozen perf events per thread, so we're looking at + // 100k open files. In practice, this is not likely -- perf events are + // mostly used in performance tests or other kinds of testing, and the + // number of threads stays below hundred. + // We used to check the number of open files by enumerating /proc/self/fd, + // but listing all open files before opening more files is obviously + // quadratic, and quadraticity never ends well. + for (size_t i : events_to_open) + { + const PerfEventInfo & event_info = raw_events_info[i]; + int & fd = thread_events_descriptors_holder.descriptors[i]; + // disable by default to add as little extra time as possible + fd = openPerfEventDisabled(perf_event_paranoid, has_cap_sys_admin, event_info.event_type, event_info.event_config); + + if (fd == -1 && errno != ENOENT) + { + // ENOENT means that the event is not supported. Don't log it, because + // this is called for each thread and would be too verbose. Log other + // error codes because they might signify an error. + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Failed to open perf event {} (event_type={}, event_config={}): " + "'{}' ({})", event_info.settings_name, event_info.event_type, + event_info.event_config, strerror(errno), errno); + } + } + + return true; +} + +// Parse comma-separated list of event names. Empty means all available +// events. +std::vector PerfEventsCounters::eventIndicesFromString(const std::string & events_list) +{ + std::vector result; + result.reserve(NUMBER_OF_RAW_EVENTS); + + if (events_list.empty()) + { + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + { + result.push_back(i); + } + return result; + } + + std::istringstream iss(events_list); + std::string event_name; + while (std::getline(iss, event_name, ',')) + { + // Allow spaces at the beginning of the token, so that you can write + // 'a, b'. + event_name.erase(0, event_name.find_first_not_of(' ')); + + auto entry = event_name_to_index.find(event_name); + if (entry != event_name_to_index.end()) + { + result.push_back(entry->second); + } + else + { + LOG_ERROR(&Poco::Logger::get("PerfEvents"), + "Unknown perf event name '{}' specified in settings", event_name); + } + } + + return result; +} + +void PerfEventsCounters::initializeProfileEvents(const std::string & events_list) +{ + if (!processThreadLocalChanges(events_list)) + return; + + for (int fd : thread_events_descriptors_holder.descriptors) + { + if (fd == -1) + continue; + + // We don't reset the event, because the time_running and time_enabled + // can't be reset anyway and we have to calculate deltas. + enablePerfEvent(fd); + } +} + +void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile_events) +{ + // Disable all perf events. + for (auto fd : thread_events_descriptors_holder.descriptors) + { + if (fd == -1) + continue; + disablePerfEvent(fd); + } + + // Read the counter values. + PerfEventValue current_values[NUMBER_OF_RAW_EVENTS]; + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + { + int fd = thread_events_descriptors_holder.descriptors[i]; + if (fd == -1) + continue; + + constexpr ssize_t bytes_to_read = sizeof(current_values[0]); + const int bytes_read = read(fd, ¤t_values[i], bytes_to_read); + + if (bytes_read != bytes_to_read) + { + LOG_WARNING(&Poco::Logger::get("PerfEvents"), + "Can't read event value from file descriptor {}: '{}' ({})", + fd, strerror(errno), errno); + current_values[i] = {}; + } + } + + // actually process counters' values + for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i) + { + int fd = thread_events_descriptors_holder.descriptors[i]; + if (fd == -1) + continue; + + const PerfEventInfo & info = raw_events_info[i]; + const PerfEventValue & previous_value = previous_values[i]; + const PerfEventValue & current_value = current_values[i]; + + // Account for counter multiplexing. time_running and time_enabled are + // not reset by PERF_EVENT_IOC_RESET, so we don't use it and calculate + // deltas from old values. + const UInt64 delta = (current_value.value - previous_value.value) + * (current_value.time_enabled - previous_value.time_enabled) + / std::max(1.f, + float(current_value.time_running - previous_value.time_running)); + + profile_events.increment(info.profile_event, delta); + } + + // Store current counter values for the next profiling period. + memcpy(previous_values, current_values, sizeof(current_values)); +} + +void PerfEventsCounters::closeEventDescriptors() +{ + thread_events_descriptors_holder.releaseResources(); +} + +PerfDescriptorsHolder::PerfDescriptorsHolder() +{ + for (int & descriptor : descriptors) + descriptor = -1; +} + +PerfDescriptorsHolder::~PerfDescriptorsHolder() +{ + releaseResources(); +} + +void PerfDescriptorsHolder::releaseResources() +{ + for (int & descriptor : descriptors) + { + if (descriptor == -1) + continue; + + disablePerfEvent(descriptor); + releasePerfEvent(descriptor); + descriptor = -1; + } +} + +} + +#else + +namespace DB +{ + +// Not on Linux or in Arcadia: the functionality is disabled. +PerfEventsCounters current_thread_counters; + } #endif diff --git a/src/Common/ThreadProfileEvents.h b/src/Common/ThreadProfileEvents.h index 038e04c4955..b6281234214 100644 --- a/src/Common/ThreadProfileEvents.h +++ b/src/Common/ThreadProfileEvents.h @@ -5,6 +5,7 @@ #include #include #include +#include #if defined(__linux__) @@ -34,6 +35,24 @@ namespace ProfileEvents extern const Event OSWriteChars; extern const Event OSReadBytes; extern const Event OSWriteBytes; + + extern const Event PerfCpuCycles; + extern const Event PerfInstructions; + extern const Event PerfCacheReferences; + extern const Event PerfCacheMisses; + extern const Event PerfBranchInstructions; + extern const Event PerfBranchMisses; + extern const Event PerfBusCycles; + extern const Event PerfStalledCyclesFrontend; + extern const Event PerfStalledCyclesBackend; + extern const Event PerfRefCpuCycles; + + extern const Event PerfCpuClock; + extern const Event PerfTaskClock; + extern const Event PerfContextSwitches; + extern const Event PerfCpuMigrations; + extern const Event PerfAlignmentFaults; + extern const Event PerfEmulationFaults; #endif } @@ -116,6 +135,78 @@ struct RUsageCounters } }; +// thread_local is disabled in Arcadia, so we have to use a dummy implementation +// there. +#if defined(__linux__) && !defined(ARCADIA_BUILD) + +struct PerfEventInfo +{ + // see perf_event.h/perf_type_id enum + int event_type; + // see configs in perf_event.h + int event_config; + ProfileEvents::Event profile_event; + std::string settings_name; +}; + +struct PerfEventValue +{ + UInt64 value = 0; + UInt64 time_enabled = 0; + UInt64 time_running = 0; +}; + +static constexpr size_t NUMBER_OF_RAW_EVENTS = 16; + +struct PerfDescriptorsHolder : boost::noncopyable +{ + int descriptors[NUMBER_OF_RAW_EVENTS]{}; + + PerfDescriptorsHolder(); + + ~PerfDescriptorsHolder(); + + void releaseResources(); +}; + +struct PerfEventsCounters +{ + PerfDescriptorsHolder thread_events_descriptors_holder; + + // time_enabled and time_running can't be reset, so we have to store the + // data from the previous profiling period and calculate deltas to them, + // to be able to properly account for counter multiplexing. + PerfEventValue previous_values[NUMBER_OF_RAW_EVENTS]{}; + + + void initializeProfileEvents(const std::string & events_list); + void finalizeProfileEvents(ProfileEvents::Counters & profile_events); + void closeEventDescriptors(); + bool processThreadLocalChanges(const std::string & needed_events_list); + + + static std::vector eventIndicesFromString(const std::string & events_list); +}; + +// Perf event creation is moderately heavy, so we create them once per thread and +// then reuse. +extern thread_local PerfEventsCounters current_thread_counters; + +#else + +// Not on Linux, or in Arcadia: the functionality is disabled. +struct PerfEventsCounters +{ + void initializeProfileEvents(const std::string & /* events_list */) {} + void finalizeProfileEvents(ProfileEvents::Counters & /* profile_events */) {} + void closeEventDescriptors() {} +}; + +// thread_local is disabled in Arcadia, so we are going to use a static dummy. +extern PerfEventsCounters current_thread_counters; + +#endif + #if defined(__linux__) class TasksStatsCounters diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 682a4b0a412..ddb0b96df0e 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -57,36 +56,6 @@ ThreadStatus::~ThreadStatus() current_thread = nullptr; } -void ThreadStatus::initPerformanceCounters() -{ - performance_counters_finalized = false; - - /// Clear stats from previous query if a new query is started - /// TODO: make separate query_thread_performance_counters and thread_performance_counters - performance_counters.resetCounters(); - memory_tracker.resetCounters(); - memory_tracker.setDescription("(for thread)"); - - query_start_time_nanoseconds = getCurrentTimeNanoseconds(); - query_start_time = time(nullptr); - ++queries_started; - - *last_rusage = RUsageCounters::current(query_start_time_nanoseconds); - if (!taskstats) - { - try - { - taskstats = TasksStatsCounters::create(thread_id); - } - catch (...) - { - tryLogCurrentException(log); - } - } - if (taskstats) - taskstats->reset(); -} - void ThreadStatus::updatePerformanceCounters() { try diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 11cd7628a7d..d0952c3ab28 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -33,6 +33,7 @@ class QueryProfilerCpu; class QueryThreadLog; class TasksStatsCounters; struct RUsageCounters; +struct PerfEventsCounters; class TaskStatsInfoGetter; class InternalTextLogsQueue; using InternalTextLogsQueuePtr = std::shared_ptr; diff --git a/src/Common/ZooKeeper/ZooKeeperHolder.cpp b/src/Common/ZooKeeper/ZooKeeperHolder.cpp index 41a36a51082..ea8a2017e37 100644 --- a/src/Common/ZooKeeper/ZooKeeperHolder.cpp +++ b/src/Common/ZooKeeper/ZooKeeperHolder.cpp @@ -5,7 +5,7 @@ namespace DB { namespace ErrorCodes { - extern const int NULL_POINTER_DEREFERENCE; + extern const int LOGICAL_ERROR; } } @@ -57,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR); return zk_ptr.get(); } @@ -65,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR); return zk_ptr.get(); } ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR); return *zk_ptr; } const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const { if (zk_ptr == nullptr) - throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE); + throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR); return *zk_ptr; } diff --git a/src/Common/assert_cast.h b/src/Common/assert_cast.h index 7f9a19805bb..b70068b8e81 100644 --- a/src/Common/assert_cast.h +++ b/src/Common/assert_cast.h @@ -13,7 +13,7 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; + extern const int LOGICAL_ERROR; } } @@ -41,11 +41,11 @@ To assert_cast(From && from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); + throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } throw DB::Exception("Bad cast from type " + demangle(typeid(from).name()) + " to " + demangle(typeid(To).name()), - DB::ErrorCodes::BAD_CAST); + DB::ErrorCodes::LOGICAL_ERROR); #else return static_cast(from); #endif diff --git a/src/Common/typeid_cast.h b/src/Common/typeid_cast.h index 29ad2e520c0..f28271fb53b 100644 --- a/src/Common/typeid_cast.h +++ b/src/Common/typeid_cast.h @@ -15,7 +15,7 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; + extern const int LOGICAL_ERROR; } } @@ -34,11 +34,11 @@ std::enable_if_t, To> typeid_cast(From & from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); + throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } throw DB::Exception("Bad cast from type " + demangle(typeid(from).name()) + " to " + demangle(typeid(To).name()), - DB::ErrorCodes::BAD_CAST); + DB::ErrorCodes::LOGICAL_ERROR); } @@ -54,7 +54,7 @@ std::enable_if_t, To> typeid_cast(From * from) } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); + throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } } @@ -71,6 +71,6 @@ std::enable_if_t, To> typeid_cast(const std::shared_ptr } catch (const std::exception & e) { - throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST); + throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 2369e2ca232..6c7cf92a41d 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -166,6 +166,9 @@ void registerCodecDelta(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("Delta codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + size_t user_bytes_size = literal->value.safeGet(); if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) throw Exception("Delta value for delta codec can be 1, 2, 4 or 8, given " + toString(user_bytes_size), ErrorCodes::ILLEGAL_CODEC_PARAMETER); diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 95fa51d1bd0..19f2dc11e85 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -166,6 +166,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(ValueType)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; + const char * dest_start = dest; const UInt32 items_count = source_size / sizeof(ValueType); unalignedStore(dest, items_count); @@ -229,7 +230,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) writer.flush(); - return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + writer.count() / 8; + return (dest - dest_start) + (writer.count() + 7) / 8; } template @@ -237,7 +238,6 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) { static_assert(is_unsigned_v, "ValueType must be unsigned."); using UnsignedDeltaType = ValueType; - using SignedDeltaType = typename std::make_signed::type; const char * source_end = source + source_size; @@ -286,12 +286,13 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) if (write_spec.data_bits != 0) { const UInt8 sign = reader.readBit(); - SignedDeltaType signed_dd = static_cast(reader.readBits(write_spec.data_bits - 1) + 1); + double_delta = reader.readBits(write_spec.data_bits - 1) + 1; if (sign) { - signed_dd *= -1; + /// It's well defined for unsigned data types. + /// In constrast, it's undefined to do negation of the most negative signed number due to overflow. + double_delta = -double_delta; } - double_delta = static_cast(signed_dd); } const UnsignedDeltaType delta = double_delta + prev_delta; diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 5782da791a1..7ba128cfe4e 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -90,6 +90,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, if (source_size % sizeof(T) != 0) throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS); const char * source_end = source + source_size; + const char * dest_start = dest; const char * dest_end = dest + dest_size; const UInt32 items_count = source_size / sizeof(T); @@ -145,7 +146,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest, writer.flush(); - return sizeof(items_count) + sizeof(prev_value) + writer.count() / 8; + return (dest - dest_start) + (writer.count() + 7) / 8; } template diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index cf8f8e976ea..32c3958e65e 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int CANNOT_COMPRESS; extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; +extern const int ILLEGAL_CODEC_PARAMETER; } @@ -84,6 +85,9 @@ void registerCodecLZ4HC(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("LZ4HC codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + level = literal->value.safeGet(); } diff --git a/src/Compression/CompressionCodecZSTD.cpp b/src/Compression/CompressionCodecZSTD.cpp index a9dc5de59ad..f1030d87ddd 100644 --- a/src/Compression/CompressionCodecZSTD.cpp +++ b/src/Compression/CompressionCodecZSTD.cpp @@ -74,6 +74,9 @@ void registerCodecZSTD(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); + if (!literal) + throw Exception("ZSTD codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + level = literal->value.safeGet(); if (level > ZSTD_maxCLevel()) throw Exception("ZSTD codec can't have level more that " + toString(ZSTD_maxCLevel()) + ", given " + toString(level), ErrorCodes::ILLEGAL_CODEC_PARAMETER); diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index 64e6051b8d5..3c7766ba508 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -21,6 +21,8 @@ namespace ErrorCodes UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char * dest) const { + assert(source != nullptr && dest != nullptr); + dest[0] = getMethodByte(); UInt8 header_size = getHeaderSize(); /// Write data from header_size @@ -33,8 +35,9 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, char * dest) const { - UInt8 header_size = getHeaderSize(); + assert(source != nullptr && dest != nullptr); + UInt8 header_size = getHeaderSize(); if (source_size < header_size) throw Exception("Can't decompress data: the compressed data size (" + toString(source_size) + ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA); diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index b416b14efb4..f029d1b9c1d 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -220,7 +220,7 @@ template if (l_size != r_size) { - result = ::testing::AssertionFailure() << "size mismatch expected: " << l_size << " got:" << r_size; + result = ::testing::AssertionFailure() << "size mismatch" << " expected: " << l_size << " got:" << r_size; } if (l_size == 0 || r_size == 0) { @@ -403,11 +403,6 @@ CodecTestSequence generateSeq(Generator gen, const char* gen_name, B Begin = 0, { const T v = gen(static_cast(i)); -// if constexpr (debug_log_items) -// { -// std::cerr << "#" << i << " " << type_name() << "(" << sizeof(T) << " bytes) : " << v << std::endl; -// } - unalignedStore(write_pos, v); write_pos += sizeof(v); } @@ -483,6 +478,7 @@ void testTranscoding(Timer & timer, ICompressionCodec & codec, const CodecTestSe timer.start(); + assert(source_data.data() != nullptr); // Codec assumes that source buffer is not null. const UInt32 encoded_size = codec.compress(source_data.data(), source_data.size(), encoded.data()); timer.report("encoding"); @@ -800,7 +796,8 @@ std::vector generatePyramidOfSequences(const size_t sequences std::vector sequences; sequences.reserve(sequences_count); - sequences.push_back(makeSeq()); // sequence of size 0 + // Don't test against sequence of size 0, since it causes a nullptr source buffer as codec input and produces an error. + // sequences.push_back(makeSeq()); // sequence of size 0 for (size_t i = 1; i < sequences_count; ++i) { std::string name = generator_name + std::string(" from 0 to ") + std::to_string(i); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index e1d64a783d3..17d15a6643d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -28,6 +28,7 @@ class IColumn; /** Settings of query execution. + * These settings go to users.xml. */ struct Settings : public SettingsCollection { @@ -276,6 +277,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(SettingBool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ + M(SettingString, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ \ \ /** Limits during query execution are part of the settings. \ @@ -385,6 +388,7 @@ struct Settings : public SettingsCollection M(SettingBool, enable_debug_queries, false, "Enables debug queries such as AST.", 0) \ M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \ M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \ + M(SettingBool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \ M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \ M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ @@ -411,7 +415,7 @@ struct Settings : public SettingsCollection M(SettingBool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ - M(SettingBool, optimize_arithmetic_operations_in_agr_func, true, "Removing arithmetic operations from aggregation functions", 0) \ + M(SettingBool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \ M(SettingBool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \ @@ -441,6 +445,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \ M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \ + \ M(SettingBool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \ DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) diff --git a/src/Core/SortCursor.h b/src/Core/SortCursor.h index edf507f8a1d..4c90cc723bf 100644 --- a/src/Core/SortCursor.h +++ b/src/Core/SortCursor.h @@ -63,7 +63,7 @@ struct SortCursorImpl for (auto & column_desc : desc) { if (!column_desc.column_name.empty()) - throw Exception("SortDesctiption should contain column position if SortCursor was used without header.", + throw Exception("SortDescription should contain column position if SortCursor was used without header.", ErrorCodes::LOGICAL_ERROR); } reset(columns, {}); diff --git a/src/Core/SortDescription.h b/src/Core/SortDescription.h index 6cc957cac55..86e4bb573ed 100644 --- a/src/Core/SortDescription.h +++ b/src/Core/SortDescription.h @@ -59,6 +59,13 @@ struct SortColumnDescription { return !(*this == other); } + + std::string dump() const + { + std::stringstream ss; + ss << column_name << ":" << column_number << ":dir " << direction << "nulls " << nulls_direction; + return ss.str(); + } }; /// Description of the sorting rule for several columns. diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 3fb380eac0f..e94d761dc87 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -362,8 +362,11 @@ static DataTypePtr create(const ASTPtr & arguments) { const auto * literal = parameters[i]->as(); if (!literal) - throw Exception("Parameters to aggregate functions must be literals", - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}' for function '{}'", + parameters[i]->formatForErrorMessage(), function_name); params_row[i] = literal->value; } diff --git a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp index bf22845a5f6..2ddce184cce 100644 --- a/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp +++ b/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp @@ -82,8 +82,11 @@ static std::pair create(const ASTPtr & argum { const ASTLiteral * lit = parameters[i]->as(); if (!lit) - throw Exception("Parameters to aggregate functions must be literals", - ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS); + throw Exception( + ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS, + "Parameters to aggregate functions must be literals. " + "Got parameter '{}' for function '{}'", + parameters[i]->formatForErrorMessage(), function_name); params_row[i] = lit->value; } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 1b542c7a1ff..1886d0fc555 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -294,7 +294,7 @@ void DatabaseOnDisk::renameTable( { attachTable(table_name, table, table_data_relative_path); /// Better diagnostics. - throw Exception{Exception::CreateFromPoco, e}; + throw Exception{Exception::CreateFromPocoTag{}, e}; } /// Now table data are moved to new database, so we must add metadata and attach table to new database diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index 25c2da9915c..6e1b03a47bd 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -358,6 +358,9 @@ bool PointInPolygonWithGrid::contains(CoordinateType x, Coordina if (has_empty_bound) return false; + if (std::isnan(x) || std::isnan(y)) + return false; + CoordinateType float_row = (y + y_shift) * y_scale; CoordinateType float_col = (x + x_shift) * x_scale; diff --git a/src/Functions/URL/FunctionsURL.h b/src/Functions/URL/FunctionsURL.h index fa5e9246488..297b62ca256 100644 --- a/src/Functions/URL/FunctionsURL.h +++ b/src/Functions/URL/FunctionsURL.h @@ -21,6 +21,7 @@ namespace DB * queryString * fragment * queryStringAndFragment + * netloc * * Functions, removing parts from URL. * If URL has nothing like, then it is returned unchanged. diff --git a/src/Functions/URL/netloc.cpp b/src/Functions/URL/netloc.cpp new file mode 100644 index 00000000000..d8858c3364a --- /dev/null +++ b/src/Functions/URL/netloc.cpp @@ -0,0 +1,17 @@ +#include +#include +#include "netloc.h" + +namespace DB +{ + +struct NameNetloc { static constexpr auto name = "netloc"; }; +using FunctionNetloc = FunctionStringToString, NameNetloc>; + +void registerFunctionNetloc(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/src/Functions/URL/netloc.h b/src/Functions/URL/netloc.h new file mode 100644 index 00000000000..443ef7f9003 --- /dev/null +++ b/src/Functions/URL/netloc.h @@ -0,0 +1,129 @@ +#pragma once + +#include "FunctionsURL.h" +#include +#include "protocol.h" +#include +#include + + +namespace DB +{ + +struct ExtractNetloc +{ + /// We use the same as domain function + static size_t getReserveLengthForElement() { return 15; } + + static inline StringRef getNetworkLocation(const char * data, size_t size) + { + Pos pos = data; + Pos end = data + size; + + if (*pos == '/' && *(pos + 1) == '/') + { + pos += 2; + } + else + { + Pos scheme_end = data + std::min(size, 16UL); + for (++pos; pos < scheme_end; ++pos) + { + if (!isAlphaNumericASCII(*pos)) + { + switch (*pos) + { + case '.': + case '-': + case '+': + break; + case ' ': /// restricted symbols + case '\t': + case '<': + case '>': + case '%': + case '{': + case '}': + case '|': + case '\\': + case '^': + case '~': + case '[': + case ']': + case ';': + case '=': + case '&': + return StringRef{}; + default: + goto exloop; + } + } + } +exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos + 2) == '/') + pos += 3; + else + pos = data; + } + + bool has_identification = false; + Pos question_mark_pos = end; + Pos slash_pos = end; + auto start_of_host = pos; + for (; pos < end; ++pos) + { + switch (*pos) + { + case '/': + if (has_identification) + return StringRef(start_of_host, pos - start_of_host); + else + slash_pos = pos; + break; + case '?': + if (has_identification) + return StringRef(start_of_host, pos - start_of_host); + else + question_mark_pos = pos; + break; + case '#': + return StringRef(start_of_host, pos - start_of_host); + case '@': /// foo:bar@example.ru + has_identification = true; + break; + case ' ': /// restricted symbols in whole URL + case '\t': + case '<': + case '>': + case '%': + case '{': + case '}': + case '|': + case '\\': + case '^': + case '~': + case '[': + case ']': + case ';': + case '=': + case '&': + return StringRef(start_of_host, std::min(std::min(pos - 1, question_mark_pos), slash_pos) - start_of_host); + } + } + + if (has_identification) + return StringRef(start_of_host, pos - start_of_host); + else + return StringRef(start_of_host, std::min(std::min(pos, question_mark_pos), slash_pos) - start_of_host); + } + + static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size) + { + StringRef host = getNetworkLocation(data, size); + + res_data = host.data; + res_size = host.size; + } +}; + +} + diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp index 9ba5261f728..f3906c2723e 100644 --- a/src/Functions/URL/registerFunctionsURL.cpp +++ b/src/Functions/URL/registerFunctionsURL.cpp @@ -26,6 +26,7 @@ void registerFunctionCutFragment(FunctionFactory & factory); void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory); void registerFunctionCutURLParameter(FunctionFactory & factory); void registerFunctionDecodeURLComponent(FunctionFactory & factory); +void registerFunctionNetloc(FunctionFactory & factory); void registerFunctionsURL(FunctionFactory & factory) { @@ -52,6 +53,7 @@ void registerFunctionsURL(FunctionFactory & factory) registerFunctionCutQueryStringAndFragment(factory); registerFunctionCutURLParameter(factory); registerFunctionDecodeURLComponent(factory); + registerFunctionNetloc(factory); } } diff --git a/src/Functions/bitBoolMaskAnd.cpp b/src/Functions/bitBoolMaskAnd.cpp index 2c55e39506c..561caf316b2 100644 --- a/src/Functions/bitBoolMaskAnd.cpp +++ b/src/Functions/bitBoolMaskAnd.cpp @@ -7,7 +7,7 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; + extern const int BAD_ARGUMENTS; } /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h). @@ -23,8 +23,10 @@ namespace DB template static inline Result apply(A left, B right) { + // Should be a logical error, but this function is callable from SQL. + // Need to investigate this. if constexpr (!std::is_same_v || !std::is_same_v) - throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskAnd.", ErrorCodes::BAD_CAST); + throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskAnd.", ErrorCodes::BAD_ARGUMENTS); return static_cast( ((static_cast(left) & static_cast(right)) & 1) | ((((static_cast(left) >> 1) | (static_cast(right) >> 1)) & 1) << 1)); diff --git a/src/Functions/bitBoolMaskOr.cpp b/src/Functions/bitBoolMaskOr.cpp index 0b439165fca..a23be509f1a 100644 --- a/src/Functions/bitBoolMaskOr.cpp +++ b/src/Functions/bitBoolMaskOr.cpp @@ -7,7 +7,7 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; + extern const int BAD_ARGUMENTS; } /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h). @@ -24,7 +24,9 @@ namespace DB static inline Result apply(A left, B right) { if constexpr (!std::is_same_v || !std::is_same_v) - throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskOr.", ErrorCodes::BAD_CAST); + // Should be a logical error, but this function is callable from SQL. + // Need to investigate this. + throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskOr.", ErrorCodes::BAD_ARGUMENTS); return static_cast( ((static_cast(left) | static_cast(right)) & 1) | ((((static_cast(left) >> 1) & (static_cast(right) >> 1)) & 1) << 1)); diff --git a/src/Functions/bitSwapLastTwo.cpp b/src/Functions/bitSwapLastTwo.cpp index d6fa9a39ec3..9d942494258 100644 --- a/src/Functions/bitSwapLastTwo.cpp +++ b/src/Functions/bitSwapLastTwo.cpp @@ -7,7 +7,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int BAD_CAST; + extern const int BAD_ARGUMENTS; } /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h). @@ -21,7 +21,9 @@ namespace DB static inline ResultType NO_SANITIZE_UNDEFINED apply(A a) { if constexpr (!std::is_same_v) - throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitSwapLastTwo.", ErrorCodes::BAD_CAST); + // Should be a logical error, but this function is callable from SQL. + // Need to investigate this. + throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitSwapLastTwo.", ErrorCodes::BAD_ARGUMENTS); return static_cast( ((static_cast(a) & 1) << 1) | ((static_cast(a) >> 1) & 1)); } diff --git a/src/Functions/bitWrapperFunc.cpp b/src/Functions/bitWrapperFunc.cpp index 9f7276fbf98..d2d4b45781b 100644 --- a/src/Functions/bitWrapperFunc.cpp +++ b/src/Functions/bitWrapperFunc.cpp @@ -6,7 +6,7 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_CAST; + extern const int BAD_ARGUMENTS; } /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h). @@ -20,8 +20,10 @@ namespace DB static inline ResultType NO_SANITIZE_UNDEFINED apply(A a) { + // Should be a logical error, but this function is callable from SQL. + // Need to investigate this. if constexpr (!is_integral_v) - throw DB::Exception("It's a bug! Only integer types are supported by __bitWrapperFunc.", ErrorCodes::BAD_CAST); + throw DB::Exception("It's a bug! Only integer types are supported by __bitWrapperFunc.", ErrorCodes::BAD_ARGUMENTS); return a == 0 ? static_cast(0b10) : static_cast(0b1); } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 68d9b487062..dd1c0075ceb 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -423,6 +423,7 @@ SRCS( URL/extractURLParameters.cpp URL/firstSignificantSubdomain.cpp URL/fragment.cpp + URL/netloc.cpp URL/path.cpp URL/pathFull.cpp URL/port.cpp diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 1539b3c7025..2c75a137222 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -4,6 +4,7 @@ # include # include +# include # include # include @@ -60,6 +61,47 @@ public: private: Poco::Logger * log = &Poco::Logger::get("AWSClient"); }; + +class S3AuthSigner : public Aws::Client::AWSAuthV4Signer +{ +public: + S3AuthSigner( + const Aws::Client::ClientConfiguration & client_configuration, + const Aws::Auth::AWSCredentials & credentials, + const DB::HeaderCollection & headers_) + : Aws::Client::AWSAuthV4Signer( + std::make_shared(credentials), + "s3", + client_configuration.region, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false) + , headers(headers_) + { + } + + bool SignRequest(Aws::Http::HttpRequest & request, const char * region, bool sign_body) const override + { + auto result = Aws::Client::AWSAuthV4Signer::SignRequest(request, region, sign_body); + for (const auto & header : headers) + request.SetHeaderValue(header.name, header.value); + return result; + } + + bool PresignRequest( + Aws::Http::HttpRequest & request, + const char * region, + const char * serviceName, + long long expiration_time_sec) const override // NOLINT + { + auto result = Aws::Client::AWSAuthV4Signer::PresignRequest(request, region, serviceName, expiration_time_sec); + for (const auto & header : headers) + request.SetHeaderValue(header.name, header.value); + return result; + } + +private: + const DB::HeaderCollection headers; +}; } namespace DB @@ -139,6 +181,25 @@ namespace S3 ); } + std::shared_ptr ClientFactory::create( // NOLINT + const String & endpoint, + bool is_virtual_hosted_style, + const String & access_key_id, + const String & secret_access_key, + HeaderCollection headers) + { + Aws::Client::ClientConfiguration cfg; + if (!endpoint.empty()) + cfg.endpointOverride = endpoint; + + Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); + return std::make_shared( + std::make_shared(cfg, std::move(credentials), std::move(headers)), + std::move(cfg), // Client configuration. + is_virtual_hosted_style || cfg.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. + ); + } + URI::URI(const Poco::URI & uri_) { /// Case when bucket name represented in domain name of S3 URL. diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 84795a4b39a..7f8cba66aad 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -5,7 +5,7 @@ #if USE_AWS_S3 #include -#include +#include #include namespace Aws::S3 @@ -13,6 +13,12 @@ namespace Aws::S3 class S3Client; } +namespace DB +{ + struct HttpHeader; + using HeaderCollection = std::vector; +} + namespace DB::S3 { @@ -34,6 +40,14 @@ public: bool is_virtual_hosted_style, const String & access_key_id, const String & secret_access_key); + + std::shared_ptr create( + const String & endpoint, + bool is_virtual_hosted_style, + const String & access_key_id, + const String & secret_access_key, + HeaderCollection headers); + private: ClientFactory(); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index b5d9f30573e..512319375d5 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -512,7 +512,8 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (data.only_consts) arguments_present = false; else - throw Exception("Unknown identifier: " + child_column_name, ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception("Unknown identifier: " + child_column_name + " there are columns: " + data.getSampleBlock().dumpNames(), + ErrorCodes::UNKNOWN_IDENTIFIER); } } } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 96a9b1fc1df..4f557111a3a 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -530,63 +530,33 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl( } -bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & result, - ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) +void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl( + AggregatedDataWithoutKey & res, + size_t row_begin, + size_t row_end, + AggregateFunctionInstruction * aggregate_instructions, + Arena * arena) { - UInt64 num_rows = block.rows(); - return executeOnBlock(block.getColumns(), num_rows, result, key_columns, aggregate_columns, no_more_keys); + /// Adding values + for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst) + { + if (inst->offsets) + inst->batch_that->addBatchSinglePlaceFromInterval(inst->offsets[row_begin], inst->offsets[row_end - 1], res + inst->state_offset, inst->batch_arguments, arena); + else + inst->batch_that->addBatchSinglePlaceFromInterval(row_begin, row_end, res + inst->state_offset, inst->batch_arguments, arena); + } } -bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result, - ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) + +void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns, + AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) { - if (isCancelled()) - return true; - - /// `result` will destroy the states of aggregate functions in the destructor - result.aggregator = this; - - /// How to perform the aggregation? - if (result.empty()) - { - result.init(method_chosen); - result.keys_size = params.keys_size; - result.key_sizes = key_sizes; - LOG_TRACE(log, "Aggregation method: {}", result.getMethodName()); - } - - if (isCancelled()) - return true; - for (size_t i = 0; i < params.aggregates_size; ++i) aggregate_columns[i].resize(params.aggregates[i].arguments.size()); - /** Constant columns are not supported directly during aggregation. - * To make them work anyway, we materialize them. - */ - Columns materialized_columns; - - /// Remember the columns we will work with - for (size_t i = 0; i < params.keys_size; ++i) - { - materialized_columns.push_back(columns.at(params.keys[i])->convertToFullColumnIfConst()); - key_columns[i] = materialized_columns.back().get(); - - if (!result.isLowCardinality()) - { - auto column_no_lc = recursiveRemoveLowCardinality(key_columns[i]->getPtr()); - if (column_no_lc.get() != key_columns[i]) - { - materialized_columns.emplace_back(std::move(column_no_lc)); - key_columns[i] = materialized_columns.back().get(); - } - } - } - - AggregateFunctionInstructions aggregate_functions_instructions(params.aggregates_size + 1); + aggregate_functions_instructions.resize(params.aggregates_size + 1); aggregate_functions_instructions[params.aggregates_size].that = nullptr; - std::vector> nested_columns_holder; for (size_t i = 0; i < params.aggregates_size; ++i) { for (size_t j = 0; j < aggregate_columns[i].size(); ++j) @@ -627,6 +597,62 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData aggregate_functions_instructions[i].batch_that = that; } +} + + +bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & result, + ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) +{ + UInt64 num_rows = block.rows(); + return executeOnBlock(block.getColumns(), num_rows, result, key_columns, aggregate_columns, no_more_keys); +} + + +bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result, + ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys) +{ + if (isCancelled()) + return true; + + /// `result` will destroy the states of aggregate functions in the destructor + result.aggregator = this; + + /// How to perform the aggregation? + if (result.empty()) + { + result.init(method_chosen); + result.keys_size = params.keys_size; + result.key_sizes = key_sizes; + LOG_TRACE(log, "Aggregation method: {}", result.getMethodName()); + } + + if (isCancelled()) + return true; + + /** Constant columns are not supported directly during aggregation. + * To make them work anyway, we materialize them. + */ + Columns materialized_columns; + + /// Remember the columns we will work with + for (size_t i = 0; i < params.keys_size; ++i) + { + materialized_columns.push_back(columns.at(params.keys[i])->convertToFullColumnIfConst()); + key_columns[i] = materialized_columns.back().get(); + + if (!result.isLowCardinality()) + { + auto column_no_lc = recursiveRemoveLowCardinality(key_columns[i]->getPtr()); + if (column_no_lc.get() != key_columns[i]) + { + materialized_columns.emplace_back(std::move(column_no_lc)); + key_columns[i] = materialized_columns.back().get(); + } + } + } + NestedColumnsHolder nested_columns_holder; + AggregateFunctionInstructions aggregate_functions_instructions; + prepareAggregateInstructions(columns, aggregate_columns, materialized_columns, aggregate_functions_instructions, nested_columns_holder); if (isCancelled()) return true; @@ -1112,7 +1138,39 @@ Block Aggregator::prepareBlockAndFill( return res; } +void Aggregator::fillAggregateColumnsWithSingleKey( + AggregatedDataVariants & data_variants, + MutableColumns & final_aggregate_columns) +{ + AggregatedDataWithoutKey & data = data_variants.without_key; + for (size_t i = 0; i < params.aggregates_size; ++i) + { + ColumnAggregateFunction & column_aggregate_func = assert_cast(*final_aggregate_columns[i]); + for (auto & pool : data_variants.aggregates_pools) + { + column_aggregate_func.addArena(pool); + } + column_aggregate_func.getData().push_back(data + offsets_of_aggregate_states[i]); + } + data = nullptr; +} + +void Aggregator::createStatesAndFillKeyColumnsWithSingleKey( + AggregatedDataVariants & data_variants, + Columns & key_columns, + size_t key_row, + MutableColumns & final_key_columns) +{ + AggregateDataPtr place = data_variants.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states); + createAggregateStates(place); + data_variants.without_key = place; + + for (size_t i = 0; i < params.keys_size; ++i) + { + final_key_columns[i]->insertFrom(*key_columns[i].get(), key_row); + } +} Block Aggregator::prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const { diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 45addb976f1..a5d79ce46dc 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -1002,6 +1002,7 @@ protected: friend class MergingAndConvertingBlockInputStream; friend class ConvertingAggregatedToChunksTransform; friend class ConvertingAggregatedToChunksSource; + friend class AggregatingInOrderTransform; Params params; @@ -1033,12 +1034,13 @@ protected: }; using AggregateFunctionInstructions = std::vector; + using NestedColumnsHolder = std::vector>; Sizes offsets_of_aggregate_states; /// The offset to the n-th aggregate function in a row of aggregate functions. size_t total_size_of_aggregate_states = 0; /// The total size of the row from the aggregate functions. // add info to track alignment requirement - // If there are states whose alignmentment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) + // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn) size_t align_aggregate_states = 1; bool all_aggregates_has_trivial_destructor = false; @@ -1105,6 +1107,13 @@ protected: AggregateFunctionInstruction * aggregate_instructions, Arena * arena); + static void executeOnIntervalWithoutKeyImpl( + AggregatedDataWithoutKey & res, + size_t row_begin, + size_t row_end, + AggregateFunctionInstruction * aggregate_instructions, + Arena * arena); + template void writeToTemporaryFileImpl( AggregatedDataVariants & data_variants, @@ -1250,6 +1259,22 @@ protected: * - sets the variable no_more_keys to true. */ bool checkLimits(size_t result_size, bool & no_more_keys) const; + + void prepareAggregateInstructions( + Columns columns, + AggregateColumns & aggregate_columns, + Columns & materialized_columns, + AggregateFunctionInstructions & instructions, + NestedColumnsHolder & nested_columns_holder); + + void fillAggregateColumnsWithSingleKey( + AggregatedDataVariants & data_variants, + MutableColumns & final_aggregate_columns); + + void createStatesAndFillKeyColumnsWithSingleKey( + AggregatedDataVariants & data_variants, + Columns & key_columns, size_t key_row, + MutableColumns & final_key_columns); }; diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.h b/src/Interpreters/CollectJoinOnKeysVisitor.h index 8a1836a97ac..c9b106d21dd 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.h +++ b/src/Interpreters/CollectJoinOnKeysVisitor.h @@ -26,8 +26,8 @@ public: struct Data { TableJoin & analyzed_join; - const TableWithColumnNames & left_table; - const TableWithColumnNames & right_table; + const TableWithColumnNamesAndTypes & left_table; + const TableWithColumnNamesAndTypes & right_table; const Aliases & aliases; const bool is_asof{false}; ASTPtr asof_left_key{}; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 5e2f4ecadab..cbf00836103 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -101,7 +102,6 @@ namespace ErrorCodes extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; extern const int LOGICAL_ERROR; - extern const int UNKNOWN_SCALAR; extern const int AUTHENTICATION_FAILED; extern const int NOT_IMPLEMENTED; } @@ -351,6 +351,7 @@ struct ContextShared String format_schema_path; /// Path to a directory that contains schema files used by input formats. ActionLocksManagerPtr action_locks_manager; /// Set of storages' action lockers std::optional system_logs; /// Used to log queries and operations on parts + std::optional storage_s3_settings; /// Settings of S3 storage RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml @@ -821,7 +822,11 @@ const Block & Context::getScalar(const String & name) const { auto it = scalars.find(name); if (scalars.end() == it) - throw Exception("Scalar " + backQuoteIfNeed(name) + " doesn't exist (internal bug)", ErrorCodes::UNKNOWN_SCALAR); + { + // This should be a logical error, but it fails the sql_fuzz test too + // often, so 'bad arguments' for now. + throw Exception("Scalar " + backQuoteIfNeed(name) + " doesn't exist (internal bug)", ErrorCodes::BAD_ARGUMENTS); + } return it->second; } @@ -1764,6 +1769,11 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration LOG_ERROR(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: {}", e.message()); } } + + if (shared->storage_s3_settings) + { + shared->storage_s3_settings->loadFromConfig("s3", config); + } } @@ -1782,6 +1792,18 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const return *shared->merge_tree_settings; } +const StorageS3Settings & Context::getStorageS3Settings() const +{ + auto lock = getLock(); + + if (!shared->storage_s3_settings) + { + const auto & config = getConfigRef(); + shared->storage_s3_settings.emplace().loadFromConfig("s3", config); + } + + return *shared->storage_s3_settings; +} void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 864468c0663..1d46049fb92 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -81,6 +81,7 @@ class TextLog; class TraceLog; class MetricLog; struct MergeTreeSettings; +class StorageS3Settings; class IDatabase; class DDLWorker; class ITableFunction; @@ -531,6 +532,7 @@ public: std::shared_ptr getPartLog(const String & part_database); const MergeTreeSettings & getMergeTreeSettings() const; + const StorageS3Settings & getStorageS3Settings() const; /// Prevents DROP TABLE if its size is greater than max_size (50GB by default, max_size=0 turn off this check) void setMaxTableSizeToDrop(size_t max_size); diff --git a/src/Interpreters/DatabaseAndTableWithAlias.h b/src/Interpreters/DatabaseAndTableWithAlias.h index adb0829a54e..d4a1a582fdc 100644 --- a/src/Interpreters/DatabaseAndTableWithAlias.h +++ b/src/Interpreters/DatabaseAndTableWithAlias.h @@ -45,34 +45,6 @@ struct DatabaseAndTableWithAlias } }; -struct TableWithColumnNames -{ - DatabaseAndTableWithAlias table; - Names columns; - Names hidden_columns; /// Not general columns like MATERIALIZED and ALIAS. They are omitted in * and t.* results. - - TableWithColumnNames(const DatabaseAndTableWithAlias & table_, const Names & columns_) - : table(table_) - , columns(columns_) - { - columns_set.insert(columns.begin(), columns.end()); - } - - TableWithColumnNames(const DatabaseAndTableWithAlias table_, Names && columns_, Names && hidden_columns_) - : table(table_) - , columns(columns_) - , hidden_columns(hidden_columns_) - { - columns_set.insert(columns.begin(), columns.end()); - columns_set.insert(hidden_columns.begin(), hidden_columns.end()); - } - - bool hasColumn(const String & name) const { return columns_set.count(name); } - -private: - NameSet columns_set; -}; - struct TableWithColumnNamesAndTypes { DatabaseAndTableWithAlias table; @@ -96,21 +68,6 @@ struct TableWithColumnNamesAndTypes names.insert(col.name); } - TableWithColumnNames removeTypes() const - { - Names out_columns; - out_columns.reserve(columns.size()); - for (auto & col : columns) - out_columns.push_back(col.name); - - Names out_hidden_columns; - out_hidden_columns.reserve(hidden_columns.size()); - for (auto & col : hidden_columns) - out_hidden_columns.push_back(col.name); - - return TableWithColumnNames(table, std::move(out_columns), std::move(out_hidden_columns)); - } - private: NameSet names; }; @@ -118,7 +75,6 @@ private: std::vector getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database); std::optional getDatabaseAndTable(const ASTSelectQuery & select, size_t table_number); -using TablesWithColumnNames = std::vector; -using TablesWithColumnNamesAndTypes = std::vector; +using TablesWithColumns = std::vector; } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 4871d8d37aa..3171f84ec9c 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -25,7 +25,6 @@ namespace ErrorCodes extern const int DATABASE_NOT_EMPTY; extern const int DATABASE_ACCESS_DENIED; extern const int LOGICAL_ERROR; - extern const int NULL_POINTER_DEREFERENCE; } TemporaryTableHolder::TemporaryTableHolder(const Context & context_, @@ -385,38 +384,46 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, it->second = std::make_pair(std::move(database), std::move(table)); } +std::unique_ptr DatabaseCatalog::database_catalog; + DatabaseCatalog::DatabaseCatalog(Context * global_context_) : global_context(global_context_), log(&Poco::Logger::get("DatabaseCatalog")) { if (!global_context) - throw Exception("DatabaseCatalog is not initialized. It's a bug.", ErrorCodes::NULL_POINTER_DEREFERENCE); + throw Exception("DatabaseCatalog is not initialized. It's a bug.", ErrorCodes::LOGICAL_ERROR); } DatabaseCatalog & DatabaseCatalog::init(Context * global_context_) { - static DatabaseCatalog database_catalog(global_context_); - return database_catalog; + if (database_catalog) + { + throw Exception("Database catalog is initialized twice. This is a bug.", + ErrorCodes::LOGICAL_ERROR); + } + + database_catalog.reset(new DatabaseCatalog(global_context_)); + + return *database_catalog; } DatabaseCatalog & DatabaseCatalog::instance() { - return init(nullptr); + if (!database_catalog) + { + throw Exception("Database catalog is not initialized. This is a bug.", + ErrorCodes::LOGICAL_ERROR); + } + + return *database_catalog; } void DatabaseCatalog::shutdown() { - try + // The catalog might not be initialized yet by init(global_context). It can + // happen if some exception was thrown on first steps of startup. + if (database_catalog) { - instance().shutdownImpl(); - } - catch (const Exception & e) - { - /// If catalog was not initialized yet by init(global_context), instance() throws NULL_POINTER_DEREFERENCE. - /// It can happen if some exception was thrown on first steps of startup (e.g. command line arguments parsing). - /// Ignore it. - if (e.code() == ErrorCodes::NULL_POINTER_DEREFERENCE) - return; - throw; + database_catalog->shutdownImpl(); } } @@ -724,5 +731,3 @@ DDLGuard::~DDLGuard() } } - - diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index aefed0f372d..540568927cc 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -169,6 +169,11 @@ public: void enqueueDroppedTableCleanup(StorageID table_id, StoragePtr table, String dropped_metadata_path, bool ignore_delay = false); private: + // The global instance of database catalog. unique_ptr is to allow + // deferred initialization. Thought I'd use std::optional, but I can't + // make emplace(global_context_) compile with private constructor ¯\_(ツ)_/¯. + static std::unique_ptr database_catalog; + DatabaseCatalog(Context * global_context_); void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index b4988f00699..3010dfcfe12 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -726,7 +726,8 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, return true; } -bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain, bool only_types) +bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, + ManyExpressionActions & group_by_elements_actions) { const auto * select_query = getAggregatingQuery(); @@ -743,6 +744,20 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain getRootActions(ast, only_types, step.actions); } + if (optimize_aggregation_in_order) + { + auto all_columns = sourceWithJoinedColumns(); + for (auto & child : asts) + { + group_by_elements_actions.emplace_back(std::make_shared(all_columns, context)); + getRootActions(child, only_types, group_by_elements_actions.back()); + } +// std::cerr << "group_by_elements_actions\n"; +// for (const auto & elem : group_by_elements_actions) { +// std::cerr << elem->dumpActions() << "\n"; +// } + } + return true; } @@ -834,8 +849,11 @@ bool SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain order_by_elements_actions.emplace_back(std::make_shared(all_columns, context)); getRootActions(child, only_types, order_by_elements_actions.back()); } +// std::cerr << "order_by_elements_actions\n"; +// for (const auto & elem : order_by_elements_actions) { +// std::cerr << elem->dumpActions() << "\n"; +// } } - return true; } @@ -1115,7 +1133,12 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( if (need_aggregate) { - query_analyzer.appendGroupBy(chain, only_types || !first_stage); + /// TODO correct conditions + optimize_aggregation_in_order = + context.getSettingsRef().optimize_aggregation_in_order + && storage && query.groupBy(); + + query_analyzer.appendGroupBy(chain, only_types || !first_stage, optimize_aggregation_in_order, group_by_elements_actions); query_analyzer.appendAggregateFunctionsArguments(chain, only_types || !first_stage); before_aggregation = chain.getLastActions(); @@ -1128,13 +1151,13 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( } } - bool has_stream_with_non_joned_rows = (before_join && before_join->getTableJoinAlgo()->hasStreamWithNonJoinedRows()); + bool has_stream_with_non_joined_rows = (before_join && before_join->getTableJoinAlgo()->hasStreamWithNonJoinedRows()); optimize_read_in_order = settings.optimize_read_in_order && storage && query.orderBy() && !query_analyzer.hasAggregation() && !query.final() - && !has_stream_with_non_joned_rows; + && !has_stream_with_non_joined_rows; /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 1afb289430e..ed07ab3fe36 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -174,6 +174,7 @@ struct ExpressionAnalysisResult bool remove_where_filter = false; bool optimize_read_in_order = false; + bool optimize_aggregation_in_order = false; ExpressionActionsPtr before_join; /// including JOIN ExpressionActionsPtr before_where; @@ -195,6 +196,7 @@ struct ExpressionAnalysisResult ConstantFilterDescription where_constant_filter_description; /// Actions by every element of ORDER BY ManyExpressionActions order_by_elements_actions; + ManyExpressionActions group_by_elements_actions; ExpressionAnalysisResult() = default; @@ -303,7 +305,7 @@ private: /// Columns in `additional_required_columns` will not be removed (they can be used for e.g. sampling or FINAL modifier). bool appendPrewhere(ExpressionActionsChain & chain, bool only_types, const Names & additional_required_columns); bool appendWhere(ExpressionActionsChain & chain, bool only_types); - bool appendGroupBy(ExpressionActionsChain & chain, bool only_types); + bool appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order, ManyExpressionActions &); void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types); /// After aggregation: diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.cpp b/src/Interpreters/ExtractExpressionInfoVisitor.cpp index f0ca33b6b8b..5f7754d315a 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.cpp +++ b/src/Interpreters/ExtractExpressionInfoVisitor.cpp @@ -38,10 +38,10 @@ void ExpressionInfoMatcher::visit(const ASTIdentifier & identifier, const ASTPtr { for (size_t index = 0; index < data.tables.size(); ++index) { - const auto & columns = data.tables[index].columns; + const auto & table = data.tables[index]; // TODO: make sure no collision ever happens - if (std::find(columns.begin(), columns.end(), identifier.name) != columns.end()) + if (table.hasColumn(identifier.name)) { data.unique_reference_tables_pos.emplace(index); break; diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.h b/src/Interpreters/ExtractExpressionInfoVisitor.h index 65d23057e52..a412704edcc 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.h +++ b/src/Interpreters/ExtractExpressionInfoVisitor.h @@ -16,7 +16,7 @@ struct ExpressionInfoMatcher struct Data { const Context & context; - const std::vector & tables; + const TablesWithColumns & tables; bool is_array_join = false; bool is_stateful_function = false; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 9e2ad664765..d18649c4c17 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -107,7 +107,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, { if (nullable) { - JoinCommon::convertColumnToNullable(column); + JoinCommon::convertColumnToNullable(column, true); if (column.type->isNullable() && !negative_null_map.empty()) { MutableColumnPtr mutable_column = IColumn::mutate(std::move(column.column)); diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 26bb8e6261d..8f254b50400 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -125,12 +125,6 @@ std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & iden return tryChooseTable(identifier, tables, ambiguous); } -std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, - bool ambiguous) -{ - return tryChooseTable(identifier, tables, ambiguous); -} - std::optional IdentifierSemantic::chooseTable(const ASTIdentifier & identifier, const std::vector & tables, bool ambiguous) { @@ -196,19 +190,14 @@ IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const return ColumnMatch::NoMatch; } -IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, - const TableWithColumnNames & db_and_table) -{ - /// TODO: ColumnName match logic is disabled cause caller's code is not ready for it - return canReferColumnToTable(identifier, db_and_table.table); -} - IdentifierSemantic::ColumnMatch IdentifierSemantic::canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table) { ColumnMatch match = canReferColumnToTable(identifier, db_and_table.table); +#if 0 if (match == ColumnMatch::NoMatch && identifier.isShort() && db_and_table.hasColumn(identifier.shortName())) match = ColumnMatch::ColumnName; +#endif return match; } diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index 81019f65b1f..7e84e10a26f 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -41,7 +41,6 @@ struct IdentifierSemantic static std::optional extractNestedName(const ASTIdentifier & identifier, const String & table_name); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); - static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNames & db_and_table); static ColumnMatch canReferColumnToTable(const ASTIdentifier & identifier, const TableWithColumnNamesAndTypes & db_and_table); static void setColumnShortName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table); @@ -53,8 +52,6 @@ struct IdentifierSemantic static std::optional getMembership(const ASTIdentifier & identifier); static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, bool allow_ambiguous = false); - static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, - bool allow_ambiguous = false); static std::optional chooseTable(const ASTIdentifier &, const std::vector & tables, bool allow_ambiguous = false); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index c4a8e3041ac..7deed262eda 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -209,10 +209,7 @@ BlockIO InterpreterInsertQuery::execute() if (table->supportsParallelInsert() && settings.max_insert_threads > 1) out_streams_size = std::min(size_t(settings.max_insert_threads), res.pipeline.getNumStreams()); - if (out_streams_size == 1) - res.pipeline.addPipe({std::make_shared(res.pipeline.getHeader(), res.pipeline.getNumStreams())}); - else - res.pipeline.resize(out_streams_size); + res.pipeline.resize(out_streams_size); } else if (query.watch) { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 64a58e33231..1c67991a20a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -71,6 +71,8 @@ #include #include #include +#include +#include namespace DB @@ -601,6 +603,20 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, const Co return order_descr; } +static SortDescription getSortDescriptionFromGroupBy(const ASTSelectQuery & query) +{ + SortDescription order_descr; + order_descr.reserve(query.groupBy()->children.size()); + + for (const auto & elem : query.groupBy()->children) + { + String name = elem->getColumnName(); + order_descr.emplace_back(name, 1, 1); + } + + return order_descr; +} + static UInt64 getLimitUIntValue(const ASTPtr & node, const Context & context, const std::string & expr) { const auto & [field, type] = evaluateConstantExpression(node, context); @@ -739,7 +755,7 @@ void InterpreterSelectQuery::executeImpl(QueryPipeline & pipeline, const BlockIn if (!expressions.second_stage && !expressions.need_aggregate && !expressions.hasHaving()) { if (expressions.has_order_by) - executeOrder(pipeline, query_info.input_sorting_info); + executeOrder(pipeline, query_info.input_order_info); if (expressions.has_order_by && query.limitLength()) executeDistinct(pipeline, false, expressions.selected_columns); @@ -832,7 +848,11 @@ void InterpreterSelectQuery::executeImpl(QueryPipeline & pipeline, const BlockIn executeWhere(pipeline, expressions.before_where, expressions.remove_where_filter); if (expressions.need_aggregate) - executeAggregation(pipeline, expressions.before_aggregation, aggregate_overflow_row, aggregate_final); + { + executeAggregation(pipeline, expressions.before_aggregation, aggregate_overflow_row, aggregate_final, query_info.input_order_info); + /// We need to reset input order info, so that executeOrder can't use it + query_info.input_order_info.reset(); + } else { executeExpression(pipeline, expressions.before_order_and_select); @@ -898,7 +918,7 @@ void InterpreterSelectQuery::executeImpl(QueryPipeline & pipeline, const BlockIn if (!expressions.first_stage && !expressions.need_aggregate && !(query.group_by_with_totals && !aggregate_final)) executeMergeSorted(pipeline); else /// Otherwise, just sort. - executeOrder(pipeline, query_info.input_sorting_info); + executeOrder(pipeline, query_info.input_order_info); } /** Optimization - if there are several sources and there is LIMIT, then first apply the preliminary LIMIT, @@ -958,28 +978,16 @@ void InterpreterSelectQuery::executeFetchColumns( const Settings & settings = context->getSettingsRef(); /// Optimization for trivial query like SELECT count() FROM table. - auto check_trivial_count_query = [&]() -> std::optional + bool optimize_trivial_count = + syntax_analyzer_result->optimize_trivial_count && storage && + processing_stage == QueryProcessingStage::FetchColumns && + query_analyzer->hasAggregation() && (query_analyzer->aggregates().size() == 1) && + typeid_cast(query_analyzer->aggregates()[0].function.get()); + + if (optimize_trivial_count) { - if (!settings.optimize_trivial_count_query || !syntax_analyzer_result->maybe_optimize_trivial_count || !storage - || query.sampleSize() || query.sampleOffset() || query.final() || query.prewhere() || query.where() || query.groupBy() - || !query_analyzer->hasAggregation() || processing_stage != QueryProcessingStage::FetchColumns) - return {}; - - const AggregateDescriptions & aggregates = query_analyzer->aggregates(); - - if (aggregates.size() != 1) - return {}; - - const AggregateDescription & desc = aggregates[0]; - if (typeid_cast(desc.function.get())) - return desc; - - return {}; - }; - - if (auto desc = check_trivial_count_query()) - { - auto func = desc->function; + const auto & desc = query_analyzer->aggregates()[0]; + const auto & func = desc.function; std::optional num_rows = storage->totalRows(); if (num_rows) { @@ -998,13 +1006,13 @@ void InterpreterSelectQuery::executeFetchColumns( column->insertFrom(place); auto header = analysis_result.before_aggregation->getSampleBlock(); - size_t arguments_size = desc->argument_names.size(); + size_t arguments_size = desc.argument_names.size(); DataTypes argument_types(arguments_size); for (size_t j = 0; j < arguments_size; ++j) - argument_types[j] = header.getByName(desc->argument_names[j]).type; + argument_types[j] = header.getByName(desc.argument_names[j]).type; Block block_with_count{ - {std::move(column), std::make_shared(func, argument_types, desc->parameters), desc->column_name}}; + {std::move(column), std::make_shared(func, argument_types, desc.parameters), desc.column_name}}; auto istream = std::make_shared(block_with_count); pipeline.init(Pipe(std::make_shared(istream))); @@ -1274,15 +1282,21 @@ void InterpreterSelectQuery::executeFetchColumns( query_info.prewhere_info = prewhere_info; /// Create optimizer with prepared actions. - /// Maybe we will need to calc input_sorting_info later, e.g. while reading from StorageMerge. - if (analysis_result.optimize_read_in_order) + /// Maybe we will need to calc input_order_info later, e.g. while reading from StorageMerge. + if (analysis_result.optimize_read_in_order || analysis_result.optimize_aggregation_in_order) { - query_info.order_by_optimizer = std::make_shared( - analysis_result.order_by_elements_actions, - getSortDescription(query, *context), - query_info.syntax_analyzer_result); + if (analysis_result.optimize_read_in_order) + query_info.order_optimizer = std::make_shared( + analysis_result.order_by_elements_actions, + getSortDescription(query, *context), + query_info.syntax_analyzer_result); + else + query_info.order_optimizer = std::make_shared( + analysis_result.group_by_elements_actions, + getSortDescriptionFromGroupBy(query), + query_info.syntax_analyzer_result); - query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(storage); + query_info.input_order_info = query_info.order_optimizer->getInputOrder(storage); } Pipes pipes = storage->read(required_columns, query_info, *context, processing_stage, max_block_size, max_streams); @@ -1388,7 +1402,7 @@ void InterpreterSelectQuery::executeWhere(QueryPipeline & pipeline, const Expres } -void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final) +void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) { pipeline.addSimpleTransform([&](const Block & header) { @@ -1426,6 +1440,62 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const /// Forget about current totals and extremes. They will be calculated again after aggregation if needed. pipeline.dropTotalsAndExtremes(); + if (group_by_info && settings.optimize_aggregation_in_order) + { + auto & query = getSelectQuery(); + SortDescription group_by_descr = getSortDescriptionFromGroupBy(query); + bool need_finish_sorting = (group_by_info->order_key_prefix_descr.size() < group_by_descr.size()); + + if (need_finish_sorting) + { + /// TOO SLOW + } + else + { + if (pipeline.getNumStreams() > 1) + { + auto many_data = std::make_shared(pipeline.getNumStreams()); + size_t counter = 0; + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params, group_by_descr, settings.max_block_size, many_data, counter++); + }); + + for (auto & column_description : group_by_descr) + { + if (!column_description.column_name.empty()) + { + column_description.column_number = pipeline.getHeader().getPositionByName(column_description.column_name); + column_description.column_name.clear(); + } + } + + auto transform = std::make_shared( + pipeline.getHeader(), + pipeline.getNumStreams(), + group_by_descr, + settings.max_block_size); + + pipeline.addPipe({ std::move(transform) }); + } + else + { + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params, group_by_descr, settings.max_block_size); + }); + } + + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, transform_params); + }); + + pipeline.enableQuotaForCurrentStreams(); + return; + } + } + /// If there are several sources, then we perform parallel aggregation if (pipeline.getNumStreams() > 1) { @@ -1588,7 +1658,47 @@ void InterpreterSelectQuery::executeExpression(QueryPipeline & pipeline, const E } -void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSortingInfoPtr input_sorting_info) +void InterpreterSelectQuery::executeOrderOptimized(QueryPipeline & pipeline, InputOrderInfoPtr input_sorting_info, UInt64 limit, SortDescription & output_order_descr) +{ + const Settings & settings = context->getSettingsRef(); + + bool need_finish_sorting = (input_sorting_info->order_key_prefix_descr.size() < output_order_descr.size()); + if (pipeline.getNumStreams() > 1) + { + UInt64 limit_for_merging = (need_finish_sorting ? 0 : limit); + auto transform = std::make_shared( + pipeline.getHeader(), + pipeline.getNumStreams(), + input_sorting_info->order_key_prefix_descr, + settings.max_block_size, limit_for_merging); + + pipeline.addPipe({ std::move(transform) }); + } + + pipeline.enableQuotaForCurrentStreams(); + + if (need_finish_sorting) + { + pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr + { + if (stream_type != QueryPipeline::StreamType::Main) + return nullptr; + + return std::make_shared(header, output_order_descr, limit); + }); + + /// NOTE limits are not applied to the size of temporary sets in FinishSortingTransform + + pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr + { + return std::make_shared( + header, input_sorting_info->order_key_prefix_descr, + output_order_descr, settings.max_block_size, limit); + }); + } +} + +void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputOrderInfoPtr input_sorting_info) { auto & query = getSelectQuery(); SortDescription output_order_descr = getSortDescription(query, *context); @@ -1608,43 +1718,7 @@ void InterpreterSelectQuery::executeOrder(QueryPipeline & pipeline, InputSorting * and then merge them into one sorted stream. * At this stage we merge per-thread streams into one. */ - - bool need_finish_sorting = (input_sorting_info->order_key_prefix_descr.size() < output_order_descr.size()); - - if (pipeline.getNumStreams() > 1) - { - UInt64 limit_for_merging = (need_finish_sorting ? 0 : limit); - auto transform = std::make_shared( - pipeline.getHeader(), - pipeline.getNumStreams(), - input_sorting_info->order_key_prefix_descr, - settings.max_block_size, limit_for_merging); - - pipeline.addPipe({ std::move(transform) }); - } - - pipeline.enableQuotaForCurrentStreams(); - - if (need_finish_sorting) - { - pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) -> ProcessorPtr - { - if (stream_type != QueryPipeline::StreamType::Main) - return nullptr; - - return std::make_shared(header, output_order_descr, limit); - }); - - /// NOTE limits are not applied to the size of temporary sets in FinishSortingTransform - - pipeline.addSimpleTransform([&](const Block & header) -> ProcessorPtr - { - return std::make_shared( - header, input_sorting_info->order_key_prefix_descr, - output_order_descr, settings.max_block_size, limit); - }); - } - + executeOrderOptimized(pipeline, input_sorting_info, limit, output_order_descr); return; } @@ -1917,8 +1991,8 @@ void InterpreterSelectQuery::executeExtremes(QueryPipeline & pipeline) void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPipeline & pipeline, const SubqueriesForSets & subqueries_for_sets) { - if (query_info.input_sorting_info) - executeMergeSorted(pipeline, query_info.input_sorting_info->order_key_prefix_descr, 0); + if (query_info.input_order_info) + executeMergeSorted(pipeline, query_info.input_order_info->order_key_prefix_descr, 0); const Settings & settings = context->getSettingsRef(); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index ca7fb4c72ba..34d255e398e 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -113,12 +113,13 @@ private: const Names & columns_to_remove_after_prewhere); void executeWhere(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter); - void executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final); + void executeAggregation(QueryPipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info); void executeMergeAggregated(QueryPipeline & pipeline, bool overflow_row, bool final); void executeTotalsAndHaving(QueryPipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row, bool final); void executeHaving(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); static void executeExpression(QueryPipeline & pipeline, const ExpressionActionsPtr & expression); - void executeOrder(QueryPipeline & pipeline, InputSortingInfoPtr sorting_info); + void executeOrder(QueryPipeline & pipeline, InputOrderInfoPtr sorting_info); + void executeOrderOptimized(QueryPipeline & pipeline, InputOrderInfoPtr sorting_info, UInt64 limit, SortDescription & output_order_descr); void executeWithFill(QueryPipeline & pipeline); void executeMergeSorted(QueryPipeline & pipeline); void executePreLimit(QueryPipeline & pipeline, bool do_not_skip_offset); diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index b5d2c632135..fea0228e3fe 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -18,14 +18,17 @@ namespace ErrorCodes } PredicateExpressionsOptimizer::PredicateExpressionsOptimizer( - const Context & context_, const TablesWithColumnNames & tables_with_columns_, const Settings & settings_) - : context(context_), tables_with_columns(tables_with_columns_), settings(settings_) + const Context & context_, const TablesWithColumns & tables_with_columns_, const Settings & settings) + : enable_optimize_predicate_expression(settings.enable_optimize_predicate_expression) + , enable_optimize_predicate_expression_to_final_subquery(settings.enable_optimize_predicate_expression_to_final_subquery) + , context(context_) + , tables_with_columns(tables_with_columns_) { } bool PredicateExpressionsOptimizer::optimize(ASTSelectQuery & select_query) { - if (!settings.enable_optimize_predicate_expression) + if (!enable_optimize_predicate_expression) return false; if (select_query.having() && (!select_query.group_by_with_cube && !select_query.group_by_with_rollup && !select_query.group_by_with_totals)) @@ -133,7 +136,7 @@ bool PredicateExpressionsOptimizer::tryRewritePredicatesToTables(ASTs & tables_e break; /// Skip left and right table optimization is_rewrite_tables |= tryRewritePredicatesToTable(tables_element[table_pos], tables_predicates[table_pos], - tables_with_columns[table_pos].columns); + tables_with_columns[table_pos].columns.getNames()); if (table_element->table_join && isRight(table_element->table_join->as()->kind)) break; /// Skip left table optimization @@ -143,12 +146,12 @@ bool PredicateExpressionsOptimizer::tryRewritePredicatesToTables(ASTs & tables_e return is_rewrite_tables; } -bool PredicateExpressionsOptimizer::tryRewritePredicatesToTable(ASTPtr & table_element, const ASTs & table_predicates, const Names & table_column) const +bool PredicateExpressionsOptimizer::tryRewritePredicatesToTable(ASTPtr & table_element, const ASTs & table_predicates, Names && table_columns) const { if (!table_predicates.empty()) { - auto optimize_final = settings.enable_optimize_predicate_expression_to_final_subquery; - PredicateRewriteVisitor::Data data(context, table_predicates, table_column, optimize_final); + auto optimize_final = enable_optimize_predicate_expression_to_final_subquery; + PredicateRewriteVisitor::Data data(context, table_predicates, std::move(table_columns), optimize_final); PredicateRewriteVisitor(data).visit(table_element); return data.is_rewrite; diff --git a/src/Interpreters/PredicateExpressionsOptimizer.h b/src/Interpreters/PredicateExpressionsOptimizer.h index da6b98987a6..f555c68020e 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.h +++ b/src/Interpreters/PredicateExpressionsOptimizer.h @@ -18,34 +18,21 @@ struct Settings; class PredicateExpressionsOptimizer { public: - PredicateExpressionsOptimizer(const Context & context_, const TablesWithColumnNames & tables_with_columns_, const Settings & settings_); + PredicateExpressionsOptimizer(const Context & context_, const TablesWithColumns & tables_with_columns_, const Settings & settings_); bool optimize(ASTSelectQuery & select_query); private: - /// Extracts settings, mostly to show which are used and which are not. - struct ExtractedSettings - { - const bool enable_optimize_predicate_expression; - const bool enable_optimize_predicate_expression_to_final_subquery; - - template - ExtractedSettings(const T & settings_) - : enable_optimize_predicate_expression(settings_.enable_optimize_predicate_expression), - enable_optimize_predicate_expression_to_final_subquery(settings_.enable_optimize_predicate_expression_to_final_subquery) - {} - }; - + const bool enable_optimize_predicate_expression; + const bool enable_optimize_predicate_expression_to_final_subquery; const Context & context; - const std::vector & tables_with_columns; - - const ExtractedSettings settings; + const TablesWithColumns & tables_with_columns; std::vector extractTablesPredicates(const ASTPtr & where, const ASTPtr & prewhere); bool tryRewritePredicatesToTables(ASTs & tables_element, const std::vector & tables_predicates); - bool tryRewritePredicatesToTable(ASTPtr & table_element, const ASTs & table_predicates, const Names & table_column) const; + bool tryRewritePredicatesToTable(ASTPtr & table_element, const ASTs & table_predicates, Names && table_columns) const; bool tryMovePredicatesFromHavingToWhere(ASTSelectQuery & select_query); }; diff --git a/src/Interpreters/PredicateRewriteVisitor.cpp b/src/Interpreters/PredicateRewriteVisitor.cpp index a834e68172b..7fc45044a88 100644 --- a/src/Interpreters/PredicateRewriteVisitor.cpp +++ b/src/Interpreters/PredicateRewriteVisitor.cpp @@ -17,7 +17,7 @@ namespace DB { PredicateRewriteVisitorData::PredicateRewriteVisitorData( - const Context & context_, const ASTs & predicates_, const Names & column_names_, bool optimize_final_) + const Context & context_, const ASTs & predicates_, Names && column_names_, bool optimize_final_) : context(context_), predicates(predicates_), column_names(column_names_), optimize_final(optimize_final_) { } diff --git a/src/Interpreters/PredicateRewriteVisitor.h b/src/Interpreters/PredicateRewriteVisitor.h index cc1b6472a4c..fa25381f4b9 100644 --- a/src/Interpreters/PredicateRewriteVisitor.h +++ b/src/Interpreters/PredicateRewriteVisitor.h @@ -24,12 +24,12 @@ public: return true; } - PredicateRewriteVisitorData(const Context & context_, const ASTs & predicates_, const Names & column_names_, bool optimize_final_); + PredicateRewriteVisitorData(const Context & context_, const ASTs & predicates_, Names && column_names_, bool optimize_final_); private: const Context & context; const ASTs & predicates; - const Names & column_names; + const Names column_names; bool optimize_final; void visitFirstInternalSelect(ASTSelectQuery & select_query, ASTPtr &); diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index e10f8bb2ea7..879a0bcf88e 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -104,9 +104,7 @@ const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality std::optional AsofRowRefs::getTypeSize(const IColumn * asof_column, size_t & size) { - TypeIndex idx = columnVectorDataType(asof_column); - if (idx == TypeIndex::Nothing) - idx = columnDecimalDataType(asof_column); + TypeIndex idx = asof_column->getDataType(); switch (idx) { diff --git a/src/Interpreters/SyntaxAnalyzer.cpp b/src/Interpreters/SyntaxAnalyzer.cpp index 831379090ad..8f6d368e6ad 100644 --- a/src/Interpreters/SyntaxAnalyzer.cpp +++ b/src/Interpreters/SyntaxAnalyzer.cpp @@ -102,7 +102,7 @@ using CustomizeGlobalNotInVisitor = InDepthNodeVisitor & tables_with_columns) + const TablesWithColumns & tables_with_columns) { LogAST log; TranslateQualifiedNamesVisitor::Data visitor_data(source_columns_set, tables_with_columns); @@ -528,7 +528,7 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul /// Find the columns that are obtained by JOIN. void collectJoinedColumns(TableJoin & analyzed_join, const ASTSelectQuery & select_query, - const std::vector & tables, const Aliases & aliases) + const TablesWithColumns & tables, const Aliases & aliases) { const ASTTablesInSelectQueryElement * node = select_query.join(); if (!node) @@ -598,7 +598,7 @@ void SyntaxAnalyzerResult::collectSourceColumns(bool add_special) /// Calculate which columns are required to execute the expression. /// Then, delete all other columns from the list of available columns. /// After execution, columns will only contain the list of columns needed to read from the table. -void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) +void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, bool is_select) { /// We calculate required_source_columns with source_columns modifications and swap them on exit required_source_columns = source_columns; @@ -648,12 +648,11 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) required.insert(column_name_type.name); } - const auto * select_query = query->as(); - /// You need to read at least one column to find the number of rows. - if (select_query && required.empty()) + if (is_select && required.empty()) { - maybe_optimize_trivial_count = true; + optimize_trivial_count = true; + /// We will find a column with minimum . /// Because it is the column that is cheapest to read. struct ColumnSizeTuple @@ -662,12 +661,14 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) size_t type_size; size_t uncompressed_size; String name; + bool operator<(const ColumnSizeTuple & that) const { return std::tie(compressed_size, type_size, uncompressed_size) < std::tie(that.compressed_size, that.type_size, that.uncompressed_size); } }; + std::vector columns; if (storage) { @@ -681,6 +682,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) columns.emplace_back(ColumnSizeTuple{c->second.data_compressed, type_size, c->second.data_uncompressed, source_column.name}); } } + if (!columns.empty()) required.insert(std::min_element(columns.begin(), columns.end())->name); else @@ -760,6 +762,7 @@ void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query) required_source_columns.swap(source_columns); } + SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( ASTPtr & query, SyntaxAnalyzerResult && result, @@ -790,12 +793,6 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( if (remove_duplicates) renameDuplicatedColumns(select_query); - /// TODO: Remove unneeded conversion - std::vector tables_with_column_names; - tables_with_column_names.reserve(tables_with_columns.size()); - for (const auto & table : tables_with_columns) - tables_with_column_names.emplace_back(table.removeTypes()); - if (tables_with_columns.size() > 1) { result.analyzed_join->columns_from_joined_table = tables_with_columns[1].columns; @@ -803,7 +800,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( source_columns_set, tables_with_columns[1].table.getQualifiedNamePrefix()); } - translateQualifiedNames(query, *select_query, source_columns_set, tables_with_column_names); + translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); /// Optimizes logical expressions. LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); @@ -822,10 +819,11 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( { optimizeIf(query, result.aliases, settings.optimize_if_chain_to_miltiif); - optimizeArithmeticOperationsInAgr(query, settings.optimize_arithmetic_operations_in_agr_func); + /// Move arithmetic operations out of aggregation functions + optimizeArithmeticOperationsInAgr(query, settings.optimize_arithmetic_operations_in_aggregate_functions); /// Push the predicate expression down to the subqueries. - result.rewrite_subqueries = PredicateExpressionsOptimizer(context, tables_with_column_names, settings).optimize(*select_query); + result.rewrite_subqueries = PredicateExpressionsOptimizer(context, tables_with_columns, settings).optimize(*select_query); /// GROUP BY injective function elimination. optimizeGroupBy(select_query, source_columns_set, context); @@ -844,11 +842,18 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect( setJoinStrictness(*select_query, settings.join_default_strictness, settings.any_join_distinct_right_table_keys, result.analyzed_join->table_join); - collectJoinedColumns(*result.analyzed_join, *select_query, tables_with_column_names, result.aliases); + collectJoinedColumns(*result.analyzed_join, *select_query, tables_with_columns, result.aliases); } result.aggregates = getAggregates(query, *select_query); - result.collectUsedColumns(query); + result.collectUsedColumns(query, true); + + if (result.optimize_trivial_count) + result.optimize_trivial_count = settings.optimize_trivial_count_query && + !select_query->where() && !select_query->prewhere() && !select_query->groupBy() && !select_query->having() && + !select_query->sampleSize() && !select_query->sampleOffset() && !select_query->final() && + (tables_with_columns.size() < 2 || isLeft(result.analyzed_join->kind())); + return std::make_shared(result); } @@ -882,7 +887,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(ASTPtr & query, const NamesAndTy else assertNoAggregates(query, "in wrong place"); - result.collectUsedColumns(query); + result.collectUsedColumns(query, false); return std::make_shared(result); } diff --git a/src/Interpreters/SyntaxAnalyzer.h b/src/Interpreters/SyntaxAnalyzer.h index abacb25ac4d..175c2db295a 100644 --- a/src/Interpreters/SyntaxAnalyzer.h +++ b/src/Interpreters/SyntaxAnalyzer.h @@ -46,11 +46,11 @@ struct SyntaxAnalyzerResult /// Predicate optimizer overrides the sub queries bool rewrite_subqueries = false; + bool optimize_trivial_count = false; + /// Results of scalar sub queries Scalars scalars; - bool maybe_optimize_trivial_count = false; - SyntaxAnalyzerResult(const NamesAndTypesList & source_columns_, ConstStoragePtr storage_ = {}, bool add_special = true) : storage(storage_) , source_columns(source_columns_) @@ -59,7 +59,7 @@ struct SyntaxAnalyzerResult } void collectSourceColumns(bool add_special); - void collectUsedColumns(const ASTPtr & query); + void collectUsedColumns(const ASTPtr & query, bool is_select); Names requiredSourceColumns() const { return required_source_columns.getNames(); } const Scalars & getScalars() const { return scalars; } }; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index bde64919bc6..354c7568818 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -134,6 +134,54 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool setupState(thread_group_); } +void ThreadStatus::initPerformanceCounters() +{ + performance_counters_finalized = false; + + /// Clear stats from previous query if a new query is started + /// TODO: make separate query_thread_performance_counters and thread_performance_counters + performance_counters.resetCounters(); + memory_tracker.resetCounters(); + memory_tracker.setDescription("(for thread)"); + + query_start_time_nanoseconds = getCurrentTimeNanoseconds(); + query_start_time = time(nullptr); + ++queries_started; + + *last_rusage = RUsageCounters::current(query_start_time_nanoseconds); + + if (query_context) + { + const Settings & settings = query_context->getSettingsRef(); + if (settings.metrics_perf_events_enabled) + { + try + { + current_thread_counters.initializeProfileEvents( + settings.metrics_perf_events_list); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + } + + if (!taskstats) + { + try + { + taskstats = TasksStatsCounters::create(thread_id); + } + catch (...) + { + tryLogCurrentException(log); + } + } + if (taskstats) + taskstats->reset(); +} + void ThreadStatus::finalizePerformanceCounters() { if (performance_counters_finalized) @@ -142,6 +190,21 @@ void ThreadStatus::finalizePerformanceCounters() performance_counters_finalized = true; updatePerformanceCounters(); + bool close_perf_descriptors = true; + if (query_context) + close_perf_descriptors = !query_context->getSettingsRef().metrics_perf_events_enabled; + + try + { + current_thread_counters.finalizeProfileEvents(performance_counters); + if (close_perf_descriptors) + current_thread_counters.closeEventDescriptors(); + } + catch (...) + { + tryLogCurrentException(log); + } + try { if (global_context && query_context) diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 363e2e2ba64..908eb2fd57c 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -37,9 +37,10 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const auto nested2 = IdentifierSemantic::extractNestedName(identifier, table.alias); const String & short_name = identifier.shortName(); - const Names & column_names = tables[table_pos].columns; - for (const auto & known_name : column_names) + const auto & columns = tables[table_pos].columns; + for (const auto & column : columns) { + const String & known_name = column.name; if (short_name == known_name) return false; if (nested1 && *nested1 == known_name) @@ -48,9 +49,10 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const return false; } - const Names & hidden_names = tables[table_pos].hidden_columns; - for (const auto & known_name : hidden_names) + const auto & hidden_columns = tables[table_pos].hidden_columns; + for (const auto & column : hidden_columns) { + const String & known_name = column.name; if (short_name == known_name) return false; if (nested1 && *nested1 == known_name) @@ -59,7 +61,7 @@ bool TranslateQualifiedNamesMatcher::Data::unknownColumn(size_t table_pos, const return false; } - return !column_names.empty(); + return !columns.empty(); } bool TranslateQualifiedNamesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child) @@ -232,11 +234,11 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt bool first_table = true; for (const auto & table : tables_with_columns) { - for (const auto & column_name : table.columns) + for (const auto & column : table.columns) { - if (first_table || !data.join_using_columns.count(column_name)) + if (first_table || !data.join_using_columns.count(column.name)) { - addIdentifier(node.children, table.table, column_name, AsteriskSemantic::getAliases(*asterisk)); + addIdentifier(node.children, table.table, column.name, AsteriskSemantic::getAliases(*asterisk)); } } @@ -248,11 +250,11 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt bool first_table = true; for (const auto & table : tables_with_columns) { - for (const auto & column_name : table.columns) + for (const auto & column : table.columns) { - if (asterisk_pattern->isColumnMatching(column_name) && (first_table || !data.join_using_columns.count(column_name))) + if (asterisk_pattern->isColumnMatching(column.name) && (first_table || !data.join_using_columns.count(column.name))) { - addIdentifier(node.children, table.table, column_name, AsteriskSemantic::getAliases(*asterisk_pattern)); + addIdentifier(node.children, table.table, column.name, AsteriskSemantic::getAliases(*asterisk_pattern)); } } @@ -267,9 +269,9 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt { if (ident_db_and_name.satisfies(table.table, true)) { - for (const auto & column_name : table.columns) + for (const auto & column : table.columns) { - addIdentifier(node.children, table.table, column_name, AsteriskSemantic::getAliases(*qualified_asterisk)); + addIdentifier(node.children, table.table, column.name, AsteriskSemantic::getAliases(*qualified_asterisk)); } break; } diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.h b/src/Interpreters/TranslateQualifiedNamesVisitor.h index e8c320671bf..1ed4da57a93 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.h +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.h @@ -25,11 +25,11 @@ public: struct Data { const NameSet source_columns; - const std::vector & tables; + const TablesWithColumns & tables; std::unordered_set join_using_columns; bool has_columns; - Data(const NameSet & source_columns_, const std::vector & tables_, bool has_columns_ = true) + Data(const NameSet & source_columns_, const TablesWithColumns & tables_, bool has_columns_ = true) : source_columns(source_columns_) , tables(tables_) , has_columns(has_columns_) diff --git a/src/Interpreters/getTableExpressions.cpp b/src/Interpreters/getTableExpressions.cpp index 8467a98685d..6e3fd516e1c 100644 --- a/src/Interpreters/getTableExpressions.cpp +++ b/src/Interpreters/getTableExpressions.cpp @@ -115,10 +115,9 @@ NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression & table return getColumnsFromTableExpression(table_expression, context, materialized, aliases, virtuals); } -std::vector getDatabaseAndTablesWithColumns(const std::vector & table_expressions, - const Context & context) +TablesWithColumns getDatabaseAndTablesWithColumns(const std::vector & table_expressions, const Context & context) { - std::vector tables_with_columns; + TablesWithColumns tables_with_columns; if (!table_expressions.empty()) { @@ -146,15 +145,4 @@ std::vector getDatabaseAndTablesWithColumns(const return tables_with_columns; } -std::vector getDatabaseAndTablesWithColumnNames(const std::vector & table_expressions, - const Context & context) -{ - std::vector tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); - std::vector out; - out.reserve(tables_with_columns.size()); - for (auto & table : tables_with_columns) - out.emplace_back(table.removeTypes()); - return out; -} - } diff --git a/src/Interpreters/getTableExpressions.h b/src/Interpreters/getTableExpressions.h index 4e49a94bcd9..9254fb9d6a0 100644 --- a/src/Interpreters/getTableExpressions.h +++ b/src/Interpreters/getTableExpressions.h @@ -17,9 +17,6 @@ const ASTTableExpression * getTableExpression(const ASTSelectQuery & select, siz ASTPtr extractTableExpression(const ASTSelectQuery & select, size_t table_number); NamesAndTypesList getColumnsFromTableExpression(const ASTTableExpression & table_expression, const Context & context); -std::vector getDatabaseAndTablesWithColumns(const std::vector & table_expressions, - const Context & context); -std::vector getDatabaseAndTablesWithColumnNames(const std::vector & table_expressions, - const Context & context); +TablesWithColumns getDatabaseAndTablesWithColumns(const std::vector & table_expressions, const Context & context); } diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index e3ca9258892..6dd3a202d4d 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -16,8 +16,14 @@ namespace ErrorCodes namespace JoinCommon { -void convertColumnToNullable(ColumnWithTypeAndName & column) +void convertColumnToNullable(ColumnWithTypeAndName & column, bool low_card_nullability) { + if (low_card_nullability && column.type->lowCardinality()) + { + column.column = recursiveRemoveLowCardinality(column.column); + column.type = recursiveRemoveLowCardinality(column.type); + } + if (column.type->isNullable() || !column.type->canBeInsideNullable()) return; diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index b69a0a4a993..47fa082e700 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -13,7 +13,7 @@ using ColumnRawPtrs = std::vector; namespace JoinCommon { -void convertColumnToNullable(ColumnWithTypeAndName & column); +void convertColumnToNullable(ColumnWithTypeAndName & column, bool low_card_nullability = false); void convertColumnsToNullable(Block & block, size_t starting_pos = 0); void removeColumnNullability(ColumnWithTypeAndName & column); Columns materializeColumns(const Block & block, const Names & names); diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp index b281315f555..7a0d14dbc69 100644 --- a/src/Parsers/ASTColumnDeclaration.cpp +++ b/src/Parsers/ASTColumnDeclaration.cpp @@ -12,7 +12,9 @@ ASTPtr ASTColumnDeclaration::clone() const if (type) { - res->type = type; + // Type may be an ASTFunction (e.g. `create table t (a Decimal(9,0))`), + // so we have to clone it properly as well. + res->type = type->clone(); res->children.push_back(res->type); } diff --git a/src/Parsers/ASTExplainQuery.h b/src/Parsers/ASTExplainQuery.h index d921ff427ae..d7a40a2eb85 100644 --- a/src/Parsers/ASTExplainQuery.h +++ b/src/Parsers/ASTExplainQuery.h @@ -23,7 +23,13 @@ public: String getID(char delim) const override { return "Explain" + (delim + toString(kind)); } ExplainKind getKind() const { return kind; } - ASTPtr clone() const override { return std::make_shared(*this); } + ASTPtr clone() const override + { + auto res = std::make_shared(*this); + res->children.clear(); + res->children.push_back(children[0]->clone()); + return res; + } protected: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 720ca666023..70a2b339f28 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -59,11 +59,31 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::RESTART_REPLICA: case Type::SYNC_REPLICA: - case Type::FLUSH_DISTRIBUTED: if (!parseDatabaseAndTableName(pos, expected, res->database, res->table)) return false; break; + case Type::STOP_DISTRIBUTED_SENDS: + case Type::START_DISTRIBUTED_SENDS: + case Type::FLUSH_DISTRIBUTED: + { + String cluster_str; + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + res->cluster = cluster_str; + if (!parseDatabaseAndTableName(pos, expected, res->database, res->table)) + { + /// FLUSH DISTRIBUTED requires table + /// START/STOP DISTRIBUTED SENDS does not requires table + if (res->type == Type::FLUSH_DISTRIBUTED) + return false; + } + break; + } + case Type::STOP_MERGES: case Type::START_MERGES: case Type::STOP_TTL_MERGES: @@ -76,8 +96,6 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::START_REPLICATED_SENDS: case Type::STOP_REPLICATION_QUEUES: case Type::START_REPLICATION_QUEUES: - case Type::STOP_DISTRIBUTED_SENDS: - case Type::START_DISTRIBUTED_SENDS: parseDatabaseAndTableName(pos, expected, res->database, res->table); break; diff --git a/src/Processors/ConcatProcessor.cpp b/src/Processors/ConcatProcessor.cpp index 27338c7c879..f4648caf0f0 100644 --- a/src/Processors/ConcatProcessor.cpp +++ b/src/Processors/ConcatProcessor.cpp @@ -4,6 +4,11 @@ namespace DB { +ConcatProcessor::ConcatProcessor(const Block & header, size_t num_inputs) + : IProcessor(InputPorts(num_inputs, header), OutputPorts{header}), current_input(inputs.begin()) +{ +} + ConcatProcessor::Status ConcatProcessor::prepare() { auto & output = outputs.front(); diff --git a/src/Processors/ConcatProcessor.h b/src/Processors/ConcatProcessor.h index 4aa5099b38a..64f9712c69a 100644 --- a/src/Processors/ConcatProcessor.h +++ b/src/Processors/ConcatProcessor.h @@ -16,10 +16,7 @@ namespace DB class ConcatProcessor : public IProcessor { public: - ConcatProcessor(const Block & header, size_t num_inputs) - : IProcessor(InputPorts(num_inputs, header), OutputPorts{header}), current_input(inputs.begin()) - { - } + ConcatProcessor(const Block & header, size_t num_inputs); String getName() const override { return "Concat"; } diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 8017667909b..364e3282f00 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -638,7 +638,7 @@ private: } catch (const Poco::Exception & e) { - throw Exception(Exception::CreateFromPoco, e); + throw Exception(Exception::CreateFromPocoTag{}, e); } catch (const avro::Exception & e) { diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index b7c230cb6de..a9bd73d8026 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -158,11 +158,11 @@ public: static std::string statusToName(Status status); - /** Method 'prepare' is responsible for all cheap ("instantenous": O(1) of data volume, no wait) calculations. + /** Method 'prepare' is responsible for all cheap ("instantaneous": O(1) of data volume, no wait) calculations. * * It may access input and output ports, * indicate the need for work by another processor by returning NeedData or PortFull, - * or indicate the absense of work by returning Finished or Unneeded, + * or indicate the absence of work by returning Finished or Unneeded, * it may pull data from input ports and push data to output ports. * * The method is not thread-safe and must be called from a single thread in one moment of time, diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp new file mode 100644 index 00000000000..3cac1c9602c --- /dev/null +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -0,0 +1,244 @@ +#include +#include + +namespace DB +{ + +AggregatingInOrderTransform::AggregatingInOrderTransform( + Block header, AggregatingTransformParamsPtr params_, + const SortDescription & group_by_description_, size_t res_block_size_) + : AggregatingInOrderTransform(std::move(header), std::move(params_) + , group_by_description_, res_block_size_, std::make_unique(1), 0) +{ +} + +AggregatingInOrderTransform::AggregatingInOrderTransform( + Block header, AggregatingTransformParamsPtr params_, + const SortDescription & group_by_description_, size_t res_block_size_, + ManyAggregatedDataPtr many_data_, size_t current_variant) + : IProcessor({std::move(header)}, {params_->getCustomHeader(false)}) + , res_block_size(res_block_size_) + , params(std::move(params_)) + , group_by_description(group_by_description_) + , aggregate_columns(params->params.aggregates_size) + , many_data(std::move(many_data_)) + , variants(*many_data->variants[current_variant]) +{ + /// We won't finalize states in order to merge same states (generated due to multi-thread execution) in AggregatingSortedTransform + res_header = params->getCustomHeader(false); + + /// Replace column names to column position in description_sorted. + for (auto & column_description : group_by_description) + { + if (!column_description.column_name.empty()) + { + column_description.column_number = res_header.getPositionByName(column_description.column_name); + column_description.column_name.clear(); + } + } +} + +AggregatingInOrderTransform::~AggregatingInOrderTransform() = default; + +static bool less(const MutableColumns & lhs, const Columns & rhs, size_t i, size_t j, const SortDescription & descr) +{ + for (const auto & elem : descr) + { + size_t ind = elem.column_number; + int res = elem.direction * lhs[ind]->compareAt(i, j, *rhs[ind], elem.nulls_direction); + if (res < 0) + return true; + else if (res > 0) + return false; + } + return false; +} + + +void AggregatingInOrderTransform::consume(Chunk chunk) +{ + size_t rows = chunk.getNumRows(); + if (rows == 0) + return; + + if (!is_consume_started) + { + LOG_TRACE(log, "Aggregating in order"); + is_consume_started = true; + } + src_rows += rows; + src_bytes += chunk.bytes(); + + Columns materialized_columns; + Columns key_columns(params->params.keys_size); + for (size_t i = 0; i < params->params.keys_size; ++i) + { + materialized_columns.push_back(chunk.getColumns().at(params->params.keys[i])->convertToFullColumnIfConst()); + key_columns[i] = materialized_columns.back(); + } + + Aggregator::NestedColumnsHolder nested_columns_holder; + Aggregator::AggregateFunctionInstructions aggregate_function_instructions; + params->aggregator.prepareAggregateInstructions(chunk.getColumns(), aggregate_columns, materialized_columns, aggregate_function_instructions, nested_columns_holder); + + size_t key_end = 0; + size_t key_begin = 0; + /// If we don't have a block we create it and fill with first key + if (!cur_block_size) + { + res_key_columns.resize(params->params.keys_size); + res_aggregate_columns.resize(params->params.aggregates_size); + + for (size_t i = 0; i < params->params.keys_size; ++i) + { + res_key_columns[i] = res_header.safeGetByPosition(i).type->createColumn(); + } + for (size_t i = 0; i < params->params.aggregates_size; ++i) + { + res_aggregate_columns[i] = res_header.safeGetByPosition(i + params->params.keys_size).type->createColumn(); + } + params->aggregator.createStatesAndFillKeyColumnsWithSingleKey(variants, key_columns, key_begin, res_key_columns); + ++cur_block_size; + } + size_t mid = 0; + size_t high = 0; + size_t low = -1; + /// Will split block into segments with the same key + while (key_end != rows) + { + high = rows; + /// Find the first position of new (not current) key in current chunk + while (high - low > 1) + { + mid = (low + high) / 2; + if (!less(res_key_columns, key_columns, cur_block_size - 1, mid, group_by_description)) + low = mid; + else + high = mid; + } + key_end = high; + /// Add data to aggr. state if interval is not empty. Empty when haven't found current key in new block. + if (key_begin != key_end) + { + params->aggregator.executeOnIntervalWithoutKeyImpl(variants.without_key, key_begin, key_end, aggregate_function_instructions.data(), variants.aggregates_pool); + } + + low = key_begin = key_end; + /// We finalize last key aggregation state if a new key found. + if (key_begin != rows) + { + params->aggregator.fillAggregateColumnsWithSingleKey(variants, res_aggregate_columns); + /// If res_block_size is reached we have to stop consuming and generate the block. Save the extra rows into new chunk. + if (cur_block_size == res_block_size) + { + Columns source_columns = chunk.detachColumns(); + + for (auto & source_column : source_columns) + source_column = source_column->cut(key_begin, rows - key_begin); + + current_chunk = Chunk(source_columns, rows - key_begin); + block_end_reached = true; + need_generate = true; + cur_block_size = 0; + return; + } + + /// We create a new state for the new key and update res_key_columns + params->aggregator.createStatesAndFillKeyColumnsWithSingleKey(variants, key_columns, key_begin, res_key_columns); + ++cur_block_size; + } + } + block_end_reached = false; +} + + +void AggregatingInOrderTransform::work() +{ + if (is_consume_finished || need_generate) + { + generate(); + } + else + { + consume(std::move(current_chunk)); + } +} + + +IProcessor::Status AggregatingInOrderTransform::prepare() +{ + auto & output = outputs.front(); + auto & input = inputs.back(); + + /// Check can output. + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + + if (block_end_reached) + { + if (need_generate) + { + return Status::Ready; + } + else + { + output.push(std::move(to_push_chunk)); + return Status::Ready; + } + } + else + { + if (is_consume_finished) + { + output.push(std::move(to_push_chunk)); + output.finish(); + LOG_TRACE(log, "Aggregated. {} to {} rows (from {})", src_rows, res_rows, + formatReadableSizeWithBinarySuffix(src_bytes)); + return Status::Finished; + } + if (input.isFinished()) + { + is_consume_finished = true; + return Status::Ready; + } + } + if (!input.hasData()) + { + input.setNeeded(); + return Status::NeedData; + } + current_chunk = input.pull(!is_consume_finished); + return Status::Ready; +} + +void AggregatingInOrderTransform::generate() +{ + if (cur_block_size && is_consume_finished) + params->aggregator.fillAggregateColumnsWithSingleKey(variants, res_aggregate_columns); + + Block res = res_header.cloneEmpty(); + + for (size_t i = 0; i < res_key_columns.size(); ++i) + { + res.getByPosition(i).column = std::move(res_key_columns[i]); + } + for (size_t i = 0; i < res_aggregate_columns.size(); ++i) + { + res.getByPosition(i + res_key_columns.size()).column = std::move(res_aggregate_columns[i]); + } + to_push_chunk = convertToChunk(res); + res_rows += to_push_chunk.getNumRows(); + need_generate = false; +} + + +} diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.h b/src/Processors/Transforms/AggregatingInOrderTransform.h new file mode 100644 index 00000000000..10793e885ce --- /dev/null +++ b/src/Processors/Transforms/AggregatingInOrderTransform.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class AggregatingInOrderTransform : public IProcessor +{ + +public: + AggregatingInOrderTransform(Block header, AggregatingTransformParamsPtr params, + const SortDescription & group_by_description, size_t res_block_size, + ManyAggregatedDataPtr many_data, size_t current_variant); + + AggregatingInOrderTransform(Block header, AggregatingTransformParamsPtr params, + const SortDescription & group_by_description, size_t res_block_size); + + ~AggregatingInOrderTransform() override; + + String getName() const override { return "AggregatingInOrderTransform"; } + + Status prepare() override; + + void work() override; + + void consume(Chunk chunk); + +private: + void generate(); + + size_t res_block_size; + size_t cur_block_size = 0; + + MutableColumns res_key_columns; + MutableColumns res_aggregate_columns; + + AggregatingTransformParamsPtr params; + SortDescription group_by_description; + + Aggregator::AggregateColumns aggregate_columns; + + ManyAggregatedDataPtr many_data; + AggregatedDataVariants & variants; + + UInt64 src_rows = 0; + UInt64 src_bytes = 0; + UInt64 res_rows = 0; + + bool need_generate = false; + bool block_end_reached = false; + bool is_consume_started = false; + bool is_consume_finished = false; + + Block res_header; + Chunk current_chunk; + Chunk to_push_chunk; + + Poco::Logger * log = &Poco::Logger::get("AggregatingInOrderTransform"); +}; + + +class FinalizingSimpleTransform : public ISimpleTransform +{ +public: + FinalizingSimpleTransform(Block header, AggregatingTransformParamsPtr params_) + : ISimpleTransform({std::move(header)}, {params_->getHeader()}, true) + , params(params_) {} + + void transform(Chunk & chunk) override + { + if (params->final) + finalizeChunk(chunk); + else if (!chunk.getChunkInfo()) + { + auto info = std::make_shared(); + chunk.setChunkInfo(std::move(info)); + } + } + + String getName() const override { return "FinalizingSimpleTransform"; } + +private: + AggregatingTransformParamsPtr params; +}; + + +} diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 8be353b4d7a..c5be62e276a 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -19,23 +19,23 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +/// Convert block to chunk. +/// Adds additional info about aggregation. +Chunk convertToChunk(const Block & block) +{ + auto info = std::make_shared(); + info->bucket_num = block.info.bucket_num; + info->is_overflows = block.info.is_overflows; + + UInt64 num_rows = block.rows(); + Chunk chunk(block.getColumns(), num_rows); + chunk.setChunkInfo(std::move(info)); + + return chunk; +} + namespace { - /// Convert block to chunk. - /// Adds additional info about aggregation. - Chunk convertToChunk(const Block & block) - { - auto info = std::make_shared(); - info->bucket_num = block.info.bucket_num; - info->is_overflows = block.info.is_overflows; - - UInt64 num_rows = block.rows(); - Chunk chunk(block.getColumns(), num_rows); - chunk.setChunkInfo(std::move(info)); - - return chunk; - } - const AggregatedChunkInfo * getInfoFromChunk(const Chunk & chunk) { const auto & info = chunk.getChunkInfo(); diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index c2693579c67..235d01ebc77 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -28,6 +28,8 @@ struct AggregatingTransformParams : params(params_), aggregator(params), final(final_) {} Block getHeader() const { return aggregator.getHeader(final); } + + Block getCustomHeader(bool final_) const { return aggregator.getHeader(final_); } }; struct ManyAggregatedData @@ -117,4 +119,6 @@ private: void initGenerate(); }; +Chunk convertToChunk(const Block & block); + } diff --git a/src/Processors/Transforms/FinishSortingTransform.cpp b/src/Processors/Transforms/FinishSortingTransform.cpp index 4c904eb95a1..b58b008339d 100644 --- a/src/Processors/Transforms/FinishSortingTransform.cpp +++ b/src/Processors/Transforms/FinishSortingTransform.cpp @@ -112,7 +112,7 @@ void FinishSortingTransform::consume(Chunk chunk) } } - /// If we reach here, that means that current cunk is first in portion + /// If we reach here, that means that current chunk is first in portion /// or it all consists of rows with the same key as tail of a previous chunk. chunks.push_back(std::move(chunk)); } diff --git a/src/Processors/Transforms/TotalsHavingTransform.h b/src/Processors/Transforms/TotalsHavingTransform.h index b6069da66f3..f16b333ffd4 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.h +++ b/src/Processors/Transforms/TotalsHavingTransform.h @@ -1,5 +1,6 @@ -#include +#pragma once +#include #include namespace DB diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 62320f1c147..064505673d1 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -134,6 +134,7 @@ SRCS( Transforms/RollupTransform.cpp Transforms/SortingTransform.cpp Transforms/TotalsHavingTransform.cpp + Transforms/AggregatingInOrderTransform.cpp ) END() diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 6e9275540e5..7e17604c4c7 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -304,17 +304,17 @@ void TCPHandler::runImpl() * We will try to send exception to the client in any case - see below. */ state.io.onException(); - exception.emplace(Exception::CreateFromPoco, e); + exception.emplace(Exception::CreateFromPocoTag{}, e); } catch (const Poco::Exception & e) { state.io.onException(); - exception.emplace(Exception::CreateFromPoco, e); + exception.emplace(Exception::CreateFromPocoTag{}, e); } catch (const std::exception & e) { state.io.onException(); - exception.emplace(Exception::CreateFromSTD, e); + exception.emplace(Exception::CreateFromSTDTag{}, e); } catch (...) { diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index a5db9636a5d..a491cc411b1 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -8,8 +8,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -108,11 +110,19 @@ StorageDistributedDirectoryMonitor::~StorageDistributedDirectoryMonitor() void StorageDistributedDirectoryMonitor::flushAllData() { - if (!quit) + if (quit) + return; + + CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0}; + std::unique_lock lock{mutex}; + + const auto & files = getFiles(metric_pending_files); + if (!files.empty()) { - CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0}; - std::unique_lock lock{mutex}; - processFiles(metric_pending_files); + processFiles(files, metric_pending_files); + + /// Update counters + getFiles(metric_pending_files); } } @@ -139,20 +149,31 @@ void StorageDistributedDirectoryMonitor::run() while (!quit) { do_sleep = true; + + const auto & files = getFiles(metric_pending_files); + if (files.empty()) + break; + if (!monitor_blocker.isCancelled()) { try { - do_sleep = !processFiles(metric_pending_files); + do_sleep = !processFiles(files, metric_pending_files); + + std::unique_lock metrics_lock(metrics_mutex); + last_exception = std::exception_ptr{}; } catch (...) { + std::unique_lock metrics_lock(metrics_mutex); + do_sleep = true; ++error_count; sleep_time = std::min( std::chrono::milliseconds{Int64(default_sleep_time.count() * std::exp2(error_count))}, max_sleep_time); tryLogCurrentException(getLoggerName().data()); + last_exception = std::current_exception(); } } else @@ -163,6 +184,8 @@ void StorageDistributedDirectoryMonitor::run() const auto now = std::chrono::system_clock::now(); if (now - last_decrease_time > decrease_error_count_period) { + std::unique_lock metrics_lock(metrics_mutex); + error_count /= 2; last_decrease_time = now; } @@ -171,6 +194,9 @@ void StorageDistributedDirectoryMonitor::run() break; } + /// Update counters + getFiles(metric_pending_files); + if (!quit && do_sleep) task_handle->scheduleAfter(sleep_time.count()); } @@ -226,9 +252,10 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri } -bool StorageDistributedDirectoryMonitor::processFiles(CurrentMetrics::Increment & metric_pending_files) +std::map StorageDistributedDirectoryMonitor::getFiles(CurrentMetrics::Increment & metric_pending_files) { std::map files; + size_t new_bytes_count = 0; Poco::DirectoryIterator end; for (Poco::DirectoryIterator it{path}; it != end; ++it) @@ -237,16 +264,26 @@ bool StorageDistributedDirectoryMonitor::processFiles(CurrentMetrics::Increment Poco::Path file_path{file_path_str}; if (!it->isDirectory() && startsWith(file_path.getExtension(), "bin")) + { files[parse(file_path.getBaseName())] = file_path_str; + new_bytes_count += Poco::File(file_path).getSize(); + } } /// Note: the value of this metric will be kept if this function will throw an exception. /// This is needed, because in case of exception, files still pending. metric_pending_files.changeTo(files.size()); - if (files.empty()) - return false; + { + std::unique_lock metrics_lock(metrics_mutex); + files_count = files.size(); + bytes_count = new_bytes_count; + } + return files; +} +bool StorageDistributedDirectoryMonitor::processFiles(const std::map & files, CurrentMetrics::Increment & metric_pending_files) +{ if (should_batch_inserts) { processFilesWithBatching(files, metric_pending_files); @@ -593,6 +630,20 @@ bool StorageDistributedDirectoryMonitor::scheduleAfter(size_t ms) return task_handle->scheduleAfter(ms, false); } +StorageDistributedDirectoryMonitor::Status StorageDistributedDirectoryMonitor::getStatus() const +{ + std::unique_lock metrics_lock(metrics_mutex); + + return Status{ + path, + last_exception, + error_count, + files_count, + bytes_count, + monitor_blocker.isCancelled(), + }; +} + void StorageDistributedDirectoryMonitor::processFilesWithBatching( const std::map & files, CurrentMetrics::Increment & metric_pending_files) @@ -734,7 +785,10 @@ void StorageDistributedDirectoryMonitor::updatePath(const std::string & new_path task_handle->deactivate(); - path = new_path; + { + std::unique_lock metrics_lock(metrics_mutex); + path = new_path; + } current_batch_file_path = path + "current_batch.txt"; task_handle->activateAndSchedule(); diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 418cd430243..960d82f0716 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -1,7 +1,7 @@ #pragma once -#include #include +#include #include #include @@ -14,6 +14,10 @@ namespace CurrentMetrics { class Increment; } namespace DB { +class StorageDistributed; +class ActionBlocker; +class BackgroundSchedulePool; + /** Details of StorageDistributed. * This type is not designed for standalone use. */ @@ -37,9 +41,24 @@ public: /// For scheduling via DistributedBlockOutputStream bool scheduleAfter(size_t ms); + + /// system.distribution_queue interface + struct Status + { + std::string path; + std::exception_ptr last_exception; + size_t error_count; + size_t files_count; + size_t bytes_count; + bool is_blocked; + }; + Status getStatus() const; + private: void run(); - bool processFiles(CurrentMetrics::Increment & metric_pending_files); + + std::map getFiles(CurrentMetrics::Increment & metric_pending_files); + bool processFiles(const std::map & files, CurrentMetrics::Increment & metric_pending_files); void processFile(const std::string & file_path, CurrentMetrics::Increment & metric_pending_files); void processFilesWithBatching(const std::map & files, CurrentMetrics::Increment & metric_pending_files); @@ -61,7 +80,12 @@ private: struct BatchHeader; struct Batch; - size_t error_count{}; + mutable std::mutex metrics_mutex; + size_t error_count = 0; + size_t files_count = 0; + size_t bytes_count = 0; + std::exception_ptr last_exception; + const std::chrono::milliseconds default_sleep_time; std::chrono::milliseconds sleep_time; const std::chrono::milliseconds max_sleep_time; diff --git a/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/src/Storages/MergeTree/BackgroundProcessingPool.cpp index 8f6d7c19549..ec062d3d138 100644 --- a/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -16,30 +16,24 @@ namespace DB { -void BackgroundProcessingPoolTaskInfo::wake() +void BackgroundProcessingPoolTaskInfo::signalReadyToRun() { Poco::Timestamp current_time; - { std::unique_lock lock(pool.tasks_mutex); - /// This will ensure that iterator is valid. Must be done under the same mutex when the iterator is invalidated. + /// This check ensures that the iterator is valid. Must be performed under the same mutex as invalidation. if (removed) return; - auto next_time_to_execute = iterator->first; - auto this_task_handle = iterator->second; + /// If this task did nothing the previous time and still should sleep, then reschedule to cancel the sleep. + const auto & scheduled_time = iterator->first; + if (scheduled_time > current_time) + pool.rescheduleTask(iterator, current_time); - /// If this task was done nothing at previous time and it has to sleep, then cancel sleep time. - if (next_time_to_execute > current_time) - next_time_to_execute = current_time; - - pool.tasks.erase(iterator); - iterator = pool.tasks.emplace(next_time_to_execute, this_task_handle); + /// Note that if all threads are currently busy doing their work, this call will not wakeup any thread. + pool.wake_event.notify_one(); } - - /// Note that if all threads are currently do some work, this call will not wakeup any thread. - pool.wake_event.notify_one(); } @@ -56,7 +50,7 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_, threads.resize(size); for (auto & thread : threads) - thread = ThreadFromGlobalPool([this] { threadFunction(); }); + thread = ThreadFromGlobalPool([this] { workLoopFunc(); }); } @@ -65,16 +59,19 @@ BackgroundProcessingPool::TaskHandle BackgroundProcessingPool::createTask(const return std::make_shared(*this, task); } -void BackgroundProcessingPool::startTask(const TaskHandle & task) +void BackgroundProcessingPool::startTask(const TaskHandle & task, bool allow_execute_in_parallel) { Poco::Timestamp current_time; + task->allow_execute_in_parallel = allow_execute_in_parallel; + { std::unique_lock lock(tasks_mutex); task->iterator = tasks.emplace(current_time, task); + + wake_event.notify_all(); } - wake_event.notify_all(); } BackgroundProcessingPool::TaskHandle BackgroundProcessingPool::addTask(const Task & task) @@ -105,8 +102,12 @@ BackgroundProcessingPool::~BackgroundProcessingPool() { try { - shutdown = true; - wake_event.notify_all(); + { + std::lock_guard lock(tasks_mutex); + shutdown = true; + wake_event.notify_all(); + } + for (auto & thread : threads) thread.join(); } @@ -117,7 +118,7 @@ BackgroundProcessingPool::~BackgroundProcessingPool() } -void BackgroundProcessingPool::threadFunction() +void BackgroundProcessingPool::workLoopFunc() { setThreadName(thread_name); @@ -137,80 +138,82 @@ void BackgroundProcessingPool::threadFunction() } SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); - if (auto * memory_tracker = CurrentThread::getMemoryTracker()) + if (auto * const memory_tracker = CurrentThread::getMemoryTracker()) memory_tracker->setMetric(settings.memory_metric); pcg64 rng(randomSeed()); std::this_thread::sleep_for(std::chrono::duration(std::uniform_real_distribution(0, settings.thread_sleep_seconds_random_part)(rng))); - while (!shutdown) + Poco::Timestamp scheduled_task_start_time; + + while (true) { TaskResult task_result = TaskResult::ERROR; TaskHandle task; - try { - Poco::Timestamp min_time; + std::unique_lock lock(tasks_mutex); + while (!task && !shutdown) { - std::unique_lock lock(tasks_mutex); - - if (!tasks.empty()) + for (const auto & [time, handle] : tasks) { - for (const auto & time_handle : tasks) + if (!handle->removed + && (handle->allow_execute_in_parallel || handle->concurrent_executors == 0)) { - if (!time_handle.second->removed) - { - min_time = time_handle.first; - task = time_handle.second; - break; - } + task = handle; + scheduled_task_start_time = time; + ++task->concurrent_executors; + break; } } + + if (task) + { + Poco::Timestamp current_time; + + if (scheduled_task_start_time <= current_time) + continue; + + wake_event.wait_for(lock, + std::chrono::microseconds(scheduled_task_start_time - current_time + + std::uniform_int_distribution(0, settings.thread_sleep_seconds_random_part * 1000000)(rng))); + } + else + { + wake_event.wait_for(lock, + std::chrono::duration(settings.thread_sleep_seconds + + std::uniform_real_distribution(0, settings.thread_sleep_seconds_random_part)(rng))); + } } if (shutdown) break; + } - if (!task) - { - std::unique_lock lock(tasks_mutex); - wake_event.wait_for(lock, - std::chrono::duration(settings.thread_sleep_seconds - + std::uniform_real_distribution(0, settings.thread_sleep_seconds_random_part)(rng))); - continue; - } + std::shared_lock rlock(task->rwlock); - /// No tasks ready for execution. - Poco::Timestamp current_time; - if (min_time > current_time) - { - std::unique_lock lock(tasks_mutex); - wake_event.wait_for(lock, std::chrono::microseconds( - min_time - current_time + std::uniform_int_distribution(0, settings.thread_sleep_seconds_random_part * 1000000)(rng))); - } + if (task->removed) + continue; - std::shared_lock rlock(task->rwlock); - - if (task->removed) - continue; - - { - CurrentMetrics::Increment metric_increment{settings.tasks_metric}; - task_result = task->function(); - } + try + { + CurrentMetrics::Increment metric_increment{settings.tasks_metric}; + task_result = task->task_function(); } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } - if (shutdown) - break; - { std::unique_lock lock(tasks_mutex); + if (shutdown) + break; + + --task->concurrent_executors; + if (task->removed) continue; @@ -231,8 +234,7 @@ void BackgroundProcessingPool::threadFunction() else if (task_result == TaskResult::NOTHING_TO_DO) next_time_to_execute += 1000000 * settings.thread_sleep_seconds_if_nothing_to_do; - tasks.erase(task->iterator); - task->iterator = tasks.emplace(next_time_to_execute, task); + rescheduleTask(task->iterator, next_time_to_execute); } } } diff --git a/src/Storages/MergeTree/BackgroundProcessingPool.h b/src/Storages/MergeTree/BackgroundProcessingPool.h index 526cab0800e..8bed696ab2c 100644 --- a/src/Storages/MergeTree/BackgroundProcessingPool.h +++ b/src/Storages/MergeTree/BackgroundProcessingPool.h @@ -85,11 +85,13 @@ public: /// Create task and start it. TaskHandle addTask(const Task & task); + /// The following two methods are invoked by Storage*MergeTree at startup /// Create task but not start it. TaskHandle createTask(const Task & task); /// Start the task that was created but not started. Precondition: task was not started. - void startTask(const TaskHandle & task); + void startTask(const TaskHandle & task, bool allow_execute_in_parallel = true); + /// Invoked by Storage*MergeTree at shutdown void removeTask(const TaskHandle & task); ~BackgroundProcessingPool(); @@ -109,13 +111,20 @@ protected: Threads threads; - std::atomic shutdown {false}; + bool shutdown{false}; std::condition_variable wake_event; /// Thread group used for profiling purposes ThreadGroupStatusPtr thread_group; - void threadFunction(); + void workLoopFunc(); + + void rescheduleTask(Tasks::iterator & task_it, const Poco::Timestamp & new_scheduled_ts) + { + auto node_handle = tasks.extract(task_it); + node_handle.key() = new_scheduled_ts; + task_it = tasks.insert(std::move(node_handle)); + } private: PoolSettings settings; @@ -125,23 +134,29 @@ private: class BackgroundProcessingPoolTaskInfo { public: - /// Wake up any thread. - void wake(); + /// Signals random idle thread from the pool that this task is ready to be executed. + void signalReadyToRun(); BackgroundProcessingPoolTaskInfo(BackgroundProcessingPool & pool_, const BackgroundProcessingPool::Task & function_) - : pool(pool_), function(function_) {} + : pool(pool_), task_function(function_) {} protected: friend class BackgroundProcessingPool; BackgroundProcessingPool & pool; - BackgroundProcessingPool::Task function; + BackgroundProcessingPool::Task task_function; - /// Read lock is hold when task is executed. + /// Read lock is held while task is being executed. + /// Write lock is used for stopping BGProcPool std::shared_mutex rwlock; + + bool allow_execute_in_parallel = false; + size_t concurrent_executors = 0; + + /// Signals that this task must no longer be planned for execution and is about to be removed std::atomic removed {false}; - std::multimap>::iterator iterator; + BackgroundProcessingPool::Tasks::iterator iterator; /// For exponential backoff. size_t count_no_work_done = 0; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index d98457f7f4b..f5ca0fee070 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -107,6 +107,7 @@ public: virtual ~IMergeTreeDataPart(); using ColumnToSize = std::map; + /// Populates columns_to_size map (compressed size). void accumulateColumnSizes(ColumnToSize & /* column_to_size */) const; Type getType() const { return part_type; } @@ -117,6 +118,7 @@ public: const NamesAndTypesList & getColumns() const { return columns; } + /// Throws an exception if part is not stored in on-disk format. void assertOnDisk() const; void remove() const; @@ -161,6 +163,8 @@ public: VolumePtr volume; + /// A directory path (relative to storage's path) where part data is actually stored + /// Examples: 'detached/tmp_fetch_', 'tmp_', '' mutable String relative_path; MergeTreeIndexGranularityInfo index_granularity_info; @@ -290,10 +294,21 @@ public: void setBytesOnDisk(UInt64 bytes_on_disk_) { bytes_on_disk = bytes_on_disk_; } size_t getFileSizeOrZero(const String & file_name) const; + + /// Returns path to part dir relatively to disk mount point String getFullRelativePath() const; + + /// Returns full path to part dir String getFullPath() const; - void renameTo(const String & new_relative_path, bool remove_new_dir_if_exists = false) const; + + /// Makes checks and move part to new directory + /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly + void renameTo(const String & new_relative_path, bool remove_new_dir_if_exists = true) const; + + /// Moves a part to detached/ directory and adds prefix to its name void renameToDetached(const String & prefix) const; + + /// Makes clone of a part in detached/ directory via hard links void makeCloneInDetached(const String & prefix) const; /// Makes full clone of part in detached/ on another disk @@ -306,6 +321,7 @@ public: /// storage and pass it to this method. virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */) const{ return false; } + /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); void calculateColumnsSizesOnDisk(); @@ -358,6 +374,7 @@ private: void loadPartitionAndMinMaxIndex(); + /// Generate unique path to detach part String getRelativePathForDetachedPart(const String & prefix) const; }; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index ee381709dd4..dad73b6a003 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -849,7 +849,7 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, const Context & cont || const_value.getType() == Field::Types::Float64) { /// Zero in all types is represented in memory the same way as in UInt64. - out.function = const_value.get() + out.function = const_value.safeGet() ? RPNElement::ALWAYS_TRUE : RPNElement::ALWAYS_FALSE; diff --git a/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp index be3caf98ad4..b6376dd3779 100644 --- a/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockOutputStream.cpp @@ -28,7 +28,7 @@ void MergeTreeBlockOutputStream::write(const Block & block) /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. if (storage.merging_mutating_task_handle) - storage.merging_mutating_task_handle->wake(); + storage.merging_mutating_task_handle->signalReadyToRun(); } } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 1e81bb6d0a8..c1fc8184206 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -637,9 +637,9 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( reader_settings, result_projection); } - else if (settings.optimize_read_in_order && query_info.input_sorting_info) + else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && query_info.input_order_info) { - size_t prefix_size = query_info.input_sorting_info->order_key_prefix_descr.size(); + size_t prefix_size = query_info.input_order_info->order_key_prefix_descr.size(); auto order_key_prefix_ast = data.getSortingKey().expression_list_ast->clone(); order_key_prefix_ast->children.resize(prefix_size); @@ -855,7 +855,8 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( ExpressionActionsPtr & out_projection) const { size_t sum_marks = 0; - const InputSortingInfoPtr & input_sorting_info = query_info.input_sorting_info; + const InputOrderInfoPtr & input_order_info = query_info.input_order_info; + size_t adaptive_parts = 0; std::vector sum_marks_in_parts(parts.size()); const auto data_settings = data.getSettings(); @@ -998,10 +999,9 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( } parts.emplace_back(part); } + ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_order_info->direction); - ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_sorting_info->direction); - - if (input_sorting_info->direction == 1) + if (input_order_info->direction == 1) { pipes.emplace_back(std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, @@ -1024,9 +1024,9 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( if (pipes.size() > 1) { SortDescription sort_description; - for (size_t j = 0; j < input_sorting_info->order_key_prefix_descr.size(); ++j) + for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) sort_description.emplace_back(data.getSortingKey().column_names[j], - input_sorting_info->direction, 1); + input_order_info->direction, 1); /// Drop temporary columns, added by 'sorting_key_prefix_expr' out_projection = createProjection(pipes.back(), data); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 4ea7ddda738..a6dec4816bf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -559,7 +559,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } if (storage.queue_task_handle) - storage.queue_task_handle->wake(); + storage.queue_task_handle->signalReadyToRun(); } } @@ -641,7 +641,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C } if (some_active_mutations_were_killed) - storage.queue_task_handle->wake(); + storage.queue_task_handle->signalReadyToRun(); if (!entries_to_load.empty()) { @@ -754,7 +754,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( } if (mutation_was_active) - storage.queue_task_handle->wake(); + storage.queue_task_handle->signalReadyToRun(); return entry; } diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 5bbe5be9928..bfdbd7ef557 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -30,7 +30,7 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( forbidden_columns.insert(elem.first); } -InputSortingInfoPtr ReadInOrderOptimizer::getInputOrder(const StoragePtr & storage) const +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StoragePtr & storage) const { Names sorting_key_columns; if (const auto * merge_tree = dynamic_cast(storage.get())) @@ -122,7 +122,7 @@ InputSortingInfoPtr ReadInOrderOptimizer::getInputOrder(const StoragePtr & stora if (order_key_prefix_descr.empty()) return {}; - return std::make_shared(std::move(order_key_prefix_descr), read_direction); + return std::make_shared(std::move(order_key_prefix_descr), read_direction); } } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 8416d23a912..de858e8fd92 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -20,10 +20,10 @@ public: const SortDescription & required_sort_description, const SyntaxAnalyzerResultPtr & syntax_result); - InputSortingInfoPtr getInputOrder(const StoragePtr & storage) const; + InputOrderInfoPtr getInputOrder(const StoragePtr & storage) const; private: - /// Actions for every element of order expression to analyze functions for monotonicicy + /// Actions for every element of order expression to analyze functions for monotonicity ManyExpressionActions elements_actions; NameSet forbidden_columns; SortDescription required_sort_description; diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 84cf3a32aa1..c4cd1035ea7 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -2,6 +2,7 @@ #include #include +#include #include namespace DB @@ -35,25 +36,25 @@ struct FilterInfo bool do_remove_column = false; }; -struct InputSortingInfo +struct InputOrderInfo { SortDescription order_key_prefix_descr; int direction; - InputSortingInfo(const SortDescription & order_key_prefix_descr_, int direction_) + InputOrderInfo(const SortDescription & order_key_prefix_descr_, int direction_) : order_key_prefix_descr(order_key_prefix_descr_), direction(direction_) {} - bool operator ==(const InputSortingInfo & other) const + bool operator ==(const InputOrderInfo & other) const { return order_key_prefix_descr == other.order_key_prefix_descr && direction == other.direction; } - bool operator !=(const InputSortingInfo & other) const { return !(*this == other); } + bool operator !=(const InputOrderInfo & other) const { return !(*this == other); } }; using PrewhereInfoPtr = std::shared_ptr; using FilterInfoPtr = std::shared_ptr; -using InputSortingInfoPtr = std::shared_ptr; +using InputOrderInfoPtr = std::shared_ptr; struct SyntaxAnalyzerResult; using SyntaxAnalyzerResultPtr = std::shared_ptr; @@ -61,6 +62,7 @@ using SyntaxAnalyzerResultPtr = std::shared_ptr; class ReadInOrderOptimizer; using ReadInOrderOptimizerPtr = std::shared_ptr; + /** Query along with some additional data, * that can be used during query processing * inside storage engines. @@ -73,9 +75,9 @@ struct SelectQueryInfo PrewhereInfoPtr prewhere_info; - ReadInOrderOptimizerPtr order_by_optimizer; + ReadInOrderOptimizerPtr order_optimizer; /// We can modify it while reading from storage - mutable InputSortingInfoPtr input_sorting_info; + mutable InputOrderInfoPtr input_order_info; /// Prepared sets are used for indices by storage engine. /// Example: x IN (1, 2, 3) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 9b4fdcae23f..2d8c3fd9a2f 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -171,8 +171,8 @@ Pipes StorageBuffer::read( if (dst_has_same_structure) { - if (query_info.order_by_optimizer) - query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(destination); + if (query_info.order_optimizer) + query_info.input_order_info = query_info.order_optimizer->getInputOrder(destination); /// The destination table has the same structure of the requested columns and we can simply read blocks from there. pipes_from_dst = destination->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 2b10311d8cf..d80fee1e4dc 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -649,13 +648,23 @@ StorageDistributedDirectoryMonitor& StorageDistributed::requireDirectoryMonitor( auto & node_data = cluster_nodes_data[key]; if (!node_data.directory_monitor) { - node_data.conneciton_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); + node_data.connection_pool = StorageDistributedDirectoryMonitor::createPool(name, *this); node_data.directory_monitor = std::make_unique( - *this, path, node_data.conneciton_pool, monitors_blocker, global_context->getDistributedSchedulePool()); + *this, path, node_data.connection_pool, monitors_blocker, global_context->getDistributedSchedulePool()); } return *node_data.directory_monitor; } +std::vector StorageDistributed::getDirectoryMonitorsStatuses() const +{ + std::vector statuses; + std::lock_guard lock(cluster_nodes_mutex); + statuses.reserve(cluster_nodes_data.size()); + for (const auto & node : cluster_nodes_data) + statuses.push_back(node.second.directory_monitor->getStatus()); + return statuses; +} + size_t StorageDistributed::getShardCount() const { return getCluster()->getShardCount(); diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index a7e3a073af4..4067012c449 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -17,7 +18,6 @@ namespace DB { class Context; -class StorageDistributedDirectoryMonitor; class VolumeJBOD; using VolumeJBODPtr = std::shared_ptr; @@ -107,6 +107,9 @@ public: void createDirectoryMonitors(const std::string & disk); /// ensure directory monitor thread and connectoin pool creation by disk and subdirectory name StorageDistributedDirectoryMonitor & requireDirectoryMonitor(const std::string & disk, const std::string & name); + /// Return list of metrics for all created monitors + /// (note that monitors are created lazily, i.e. until at least one INSERT executed) + std::vector getDirectoryMonitorsStatuses() const; void flushClusterNodesAllData(); @@ -181,13 +184,13 @@ protected: struct ClusterNodeData { std::unique_ptr directory_monitor; - ConnectionPoolPtr conneciton_pool; + ConnectionPoolPtr connection_pool; void flushAllData() const; void shutdownAndDropAllData() const; }; std::unordered_map cluster_nodes_data; - std::mutex cluster_nodes_mutex; + mutable std::mutex cluster_nodes_mutex; }; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 1b8c6acb49f..f7233d67eca 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -180,8 +180,8 @@ Pipes StorageMaterializedView::read( auto lock = storage->lockStructureForShare( false, context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout); - if (query_info.order_by_optimizer) - query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(storage); + if (query_info.order_optimizer) + query_info.input_order_info = query_info.order_optimizer->getInputOrder(storage); Pipes pipes = storage->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index baf54a1d50a..64d0f11f853 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -172,12 +172,12 @@ Pipes StorageMerge::read( num_streams *= num_streams_multiplier; size_t remaining_streams = num_streams; - InputSortingInfoPtr input_sorting_info; - if (query_info.order_by_optimizer) + InputOrderInfoPtr input_sorting_info; + if (query_info.order_optimizer) { for (auto it = selected_tables.begin(); it != selected_tables.end(); ++it) { - auto current_info = query_info.order_by_optimizer->getInputOrder(std::get<0>(*it)); + auto current_info = query_info.order_optimizer->getInputOrder(std::get<0>(*it)); if (it == selected_tables.begin()) input_sorting_info = current_info; else if (!current_info || (input_sorting_info && *current_info != *input_sorting_info)) @@ -187,7 +187,7 @@ Pipes StorageMerge::read( break; } - query_info.input_sorting_info = input_sorting_info; + query_info.input_order_info = input_sorting_info; } for (const auto & table : selected_tables) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 7cd38e9081f..4650485847c 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -410,7 +410,7 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, String current_mutations_by_version.emplace(version, insertion.first->second); LOG_INFO(log, "Added mutation: {}", mutation_file_name); - merging_mutating_task_handle->wake(); + merging_mutating_task_handle->signalReadyToRun(); return version; } @@ -543,7 +543,7 @@ CancellationCode StorageMergeTree::killMutation(const String & mutation_id) } /// Maybe there is another mutation that was blocked by the killed one. Try to execute it immediately. - merging_mutating_task_handle->wake(); + merging_mutating_task_handle->signalReadyToRun(); return CancellationCode::CancelSent; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e65df7f7160..faa44ff7db1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5323,7 +5323,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI queue.pullLogsToQueue(getZooKeeper()); /// This is significant, because the execution of this task could be delayed at BackgroundPool. /// And we force it to be executed. - queue_task_handle->wake(); + queue_task_handle->signalReadyToRun(); Poco::Event target_size_event; auto callback = [&target_size_event, queue_size] (size_t new_queue_size) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index e8fd89c4505..918662750e4 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -23,6 +24,7 @@ #include +#include #include #include @@ -200,18 +202,24 @@ StorageS3::StorageS3( , format_name(format_name_) , min_upload_part_size(min_upload_part_size_) , compression_method(compression_method_) - , client(S3::ClientFactory::instance().create(uri_.endpoint, uri_.is_virtual_hosted_style, access_key_id_, secret_access_key_)) { context_global.getRemoteHostFilter().checkURL(uri_.uri); setColumns(columns_); setConstraints(constraints_); + + auto settings = context_.getStorageS3Settings().getSettings(uri.endpoint); + Aws::Auth::AWSCredentials credentials(access_key_id_, secret_access_key_); + if (access_key_id_.empty()) + credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key)); + + client = S3::ClientFactory::instance().create( + uri_.endpoint, uri_.is_virtual_hosted_style, access_key_id_, secret_access_key_, std::move(settings.headers)); } namespace { - -/* "Recursive" directory listing with matched paths as a result. + /* "Recursive" directory listing with matched paths as a result. * Have the same method in StorageFile. */ Strings listFilesWithRegexpMatching(Aws::S3::S3Client & client, const S3::URI & globbed_uri) diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp new file mode 100644 index 00000000000..5b443de6b9a --- /dev/null +++ b/src/Storages/StorageS3Settings.cpp @@ -0,0 +1,57 @@ +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INVALID_CONFIG_PARAMETER; +} + +void StorageS3Settings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config) +{ + std::lock_guard lock(mutex); + settings.clear(); + if (!config.has(config_elem)) + return; + + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(config_elem, config_keys); + + for (const String & key : config_keys) + { + auto endpoint = config.getString(config_elem + "." + key + ".endpoint"); + auto access_key_id = config.getString(config_elem + "." + key + ".access_key_id", ""); + auto secret_access_key = config.getString(config_elem + "." + key + ".secret_access_key", ""); + + HeaderCollection headers; + Poco::Util::AbstractConfiguration::Keys subconfig_keys; + config.keys(config_elem + "." + key, subconfig_keys); + for (const String & subkey : subconfig_keys) + { + if (subkey.starts_with("header")) + { + auto header_str = config.getString(config_elem + "." + key + "." + subkey); + auto delimiter = header_str.find(':'); + if (delimiter == String::npos) + throw Exception("Malformed s3 header value", ErrorCodes::INVALID_CONFIG_PARAMETER); + headers.emplace_back(HttpHeader{header_str.substr(0, delimiter), header_str.substr(delimiter + 1, String::npos)}); + } + } + + settings.emplace(endpoint, S3AuthSettings{std::move(access_key_id), std::move(secret_access_key), std::move(headers)}); + } +} + +S3AuthSettings StorageS3Settings::getSettings(const String & endpoint) const +{ + std::lock_guard lock(mutex); + if (auto setting = settings.find(endpoint); setting != settings.end()) + return setting->second; + return {}; +} + +} diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h new file mode 100644 index 00000000000..ac31928a240 --- /dev/null +++ b/src/Storages/StorageS3Settings.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include + +namespace Poco::Util +{ +class AbstractConfiguration; +} + +namespace DB +{ + +struct HttpHeader +{ + const String name; + const String value; +}; + +using HeaderCollection = std::vector; + +struct S3AuthSettings +{ + const String access_key_id; + const String secret_access_key; + + const HeaderCollection headers; +}; + +/// Settings for the StorageS3. +class StorageS3Settings +{ +public: + StorageS3Settings() = default; + void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); + + S3AuthSettings getSettings(const String & endpoint) const; + +private: + mutable std::mutex mutex; + std::map settings; +}; + +} diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index c6b37a50aa9..97403a359c3 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -124,7 +124,7 @@ ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Cont /// TODO: remove getTableExpressions and getTablesWithColumns { const auto & table_expressions = getTableExpressions(*outer_query); - const auto & tables_with_columns = getDatabaseAndTablesWithColumnNames(table_expressions, context); + const auto & tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context); replaceTableNameWithSubquery(outer_query, runtime_view_query); if (context.getSettingsRef().joined_subquery_requires_alias && tables_with_columns.size() > 1) diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp new file mode 100644 index 00000000000..2459be0ba71 --- /dev/null +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + + +NamesAndTypesList StorageSystemDistributionQueue::getNamesAndTypes() +{ + return { + { "database", std::make_shared() }, + { "table", std::make_shared() }, + { "data_path", std::make_shared() }, + { "is_blocked", std::make_shared() }, + { "error_count", std::make_shared() }, + { "data_files", std::make_shared() }, + { "data_compressed_bytes", std::make_shared() }, + { "last_exception", std::make_shared() }, + }; +} + + +void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const +{ + const auto access = context.getAccess(); + const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); + + std::map> tables; + for (const auto & db : DatabaseCatalog::instance().getDatabases()) + { + /// Lazy database can not contain distributed tables + if (db.second->getEngineName() == "Lazy") + continue; + + const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); + + for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) + { + StoragePtr table = iterator->table(); + if (!table) + continue; + + if (!dynamic_cast(table.get())) + continue; + if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) + continue; + tables[db.first][iterator->name()] = table; + } + } + + + MutableColumnPtr col_database_mut = ColumnString::create(); + MutableColumnPtr col_table_mut = ColumnString::create(); + + for (auto & db : tables) + { + for (auto & table : db.second) + { + col_database_mut->insert(db.first); + col_table_mut->insert(table.first); + } + } + + ColumnPtr col_database_to_filter = std::move(col_database_mut); + ColumnPtr col_table_to_filter = std::move(col_table_mut); + + /// Determine what tables are needed by the conditions in the query. + { + Block filtered_block + { + { col_database_to_filter, std::make_shared(), "database" }, + { col_table_to_filter, std::make_shared(), "table" }, + }; + + VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context); + + if (!filtered_block.rows()) + return; + + col_database_to_filter = filtered_block.getByName("database").column; + col_table_to_filter = filtered_block.getByName("table").column; + } + + for (size_t i = 0, tables_size = col_database_to_filter->size(); i < tables_size; ++i) + { + String database = (*col_database_to_filter)[i].safeGet(); + String table = (*col_table_to_filter)[i].safeGet(); + + auto & distributed_table = dynamic_cast(*tables[database][table]); + + for (const auto & status : distributed_table.getDirectoryMonitorsStatuses()) + { + size_t col_num = 0; + res_columns[col_num++]->insert(database); + res_columns[col_num++]->insert(table); + res_columns[col_num++]->insert(status.path); + res_columns[col_num++]->insert(status.is_blocked); + res_columns[col_num++]->insert(status.error_count); + res_columns[col_num++]->insert(status.files_count); + res_columns[col_num++]->insert(status.bytes_count); + + if (status.last_exception) + res_columns[col_num++]->insert(getExceptionMessage(status.last_exception, false)); + else + res_columns[col_num++]->insertDefault(); + } + } +} + +} diff --git a/src/Storages/System/StorageSystemDistributionQueue.h b/src/Storages/System/StorageSystemDistributionQueue.h new file mode 100644 index 00000000000..88e7fa45cf5 --- /dev/null +++ b/src/Storages/System/StorageSystemDistributionQueue.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class Context; + + +/** Implements the `distribution_queue` system table, which allows you to view the INSERT queues for the Distributed tables. + */ +class StorageSystemDistributionQueue final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ + friend struct ext::shared_ptr_helper; +public: + std::string getName() const override { return "SystemDistributionQueue"; } + + static NamesAndTypesList getNamesAndTypes(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 585eab2b4d8..2b52f0fe5cc 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -124,6 +125,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) system_database.attachTable("mutations", StorageSystemMutations::create("mutations")); system_database.attachTable("replicas", StorageSystemReplicas::create("replicas")); system_database.attachTable("replication_queue", StorageSystemReplicationQueue::create("replication_queue")); + system_database.attachTable("distribution_queue", StorageSystemDistributionQueue::create("distribution_queue")); system_database.attachTable("dictionaries", StorageSystemDictionaries::create("dictionaries")); system_database.attachTable("models", StorageSystemModels::create("models")); system_database.attachTable("clusters", StorageSystemClusters::create("clusters")); diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index bf86322a676..318d667d9b0 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -18,6 +18,8 @@ using namespace DB; /// NOTE How to do better? struct State { + State(const State&) = delete; + Context context; NamesAndTypesList columns{ {"column", std::make_shared()}, @@ -27,23 +29,24 @@ struct State {"create_time", std::make_shared()}, }; + static const State & instance() + { + static State state; + return state; + } + +private: explicit State() : context(getContext().context) { registerFunctions(); DatabasePtr database = std::make_shared("test", context); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); - context.makeGlobalContext(); DatabaseCatalog::instance().attachDatabase("test", database); context.setCurrentDatabase("test"); } }; -State getState() -{ - static State state; - return state; -} static void check(const std::string & query, const std::string & expected, const Context & context, const NamesAndTypesList & columns) { @@ -60,7 +63,7 @@ static void check(const std::string & query, const std::string & expected, const TEST(TransformQueryForExternalDatabase, InWithSingleElement) { - const State & state = getState(); + const State & state = State::instance(); check("SELECT column FROM test.table WHERE 1 IN (1)", R"(SELECT "column" FROM "test"."table" WHERE 1)", @@ -75,7 +78,7 @@ TEST(TransformQueryForExternalDatabase, InWithSingleElement) TEST(TransformQueryForExternalDatabase, Like) { - const State & state = getState(); + const State & state = State::instance(); check("SELECT column FROM test.table WHERE column LIKE '%hello%'", R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')", @@ -87,7 +90,7 @@ TEST(TransformQueryForExternalDatabase, Like) TEST(TransformQueryForExternalDatabase, Substring) { - const State & state = getState(); + const State & state = State::instance(); check("SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", R"(SELECT "column" FROM "test"."table")", @@ -96,7 +99,7 @@ TEST(TransformQueryForExternalDatabase, Substring) TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) { - const State & state = getState(); + const State & state = State::instance(); check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))", @@ -108,7 +111,7 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) TEST(TransformQueryForExternalDatabase, Issue7245) { - const State & state = getState(); + const State & state = State::instance(); check("select apply_id from test.table where apply_type = 2 and create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) and apply_status in (3,4)", R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))", diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 7e36e4145eb..18f62504e1f 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -121,6 +121,7 @@ SRCS( System/StorageSystemQuotasUsage.cpp System/StorageSystemReplicas.cpp System/StorageSystemReplicationQueue.cpp + System/StorageSystemDistributionQueue.cpp System/StorageSystemRoleGrants.cpp System/StorageSystemRoles.cpp System/StorageSystemRowPolicies.cpp @@ -164,6 +165,7 @@ SRCS( StorageMySQL.cpp StorageNull.cpp StorageReplicatedMergeTree.cpp + StorageS3Settings.cpp StorageSet.cpp StorageStripeLog.cpp StorageTinyLog.cpp diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index 3b3db1c2510..548db38515c 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -21,6 +21,7 @@ namespace DB namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int LOGICAL_ERROR; } @@ -44,6 +45,18 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, " structure, [random_seed, max_string_length, max_array_length].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + // All the arguments must be literals. + for (const auto & arg : args) + { + if (!arg->as()) + { + throw Exception(fmt::format( + "All arguments of table function '{}' must be literals. " + "Got '{}' instead", getName(), arg->formatForErrorMessage()), + ErrorCodes::BAD_ARGUMENTS); + } + } + /// Parsing first argument as table structure and creating a sample block std::string structure = args[0]->as().value.safeGet(); diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 4e166b10d8f..5ecd978146c 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -25,6 +25,7 @@ namespace DB namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -75,6 +76,13 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const C ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// Parsing first argument as table structure and creating a sample block + if (!args[0]->as()) + { + throw Exception(fmt::format( + "The first argument of table function '{}' must be a literal. " + "Got '{}' instead", getName(), args[0]->formatForErrorMessage()), + ErrorCodes::BAD_ARGUMENTS); + } std::string structure = args[0]->as().value.safeGet(); ColumnsDescription columns = parseColumnsListFromString(structure, context); diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/__init__.py b/tests/integration/test_s3_with_proxy/proxy-resolver/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh b/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh deleted file mode 100644 index e456be666a9..00000000000 --- a/tests/integration/test_s3_with_proxy/proxy-resolver/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -pip install bottle -python resolver.py diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 11176b19f0c..dc326b719bf 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -14,9 +14,7 @@ def run_resolver(cluster): current_dir = os.path.dirname(__file__) cluster.copy_file_to_container(container_id, os.path.join(current_dir, "proxy-resolver", "resolver.py"), "resolver.py") - cluster.copy_file_to_container(container_id, os.path.join(current_dir, "proxy-resolver", "entrypoint.sh"), - "entrypoint.sh") - cluster.exec_in_container(container_id, ["/bin/bash", "entrypoint.sh"], detach=True) + cluster.exec_in_container(container_id, ["python", "resolver.py"], detach=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml new file mode 100644 index 00000000000..26dc52f9e8f --- /dev/null +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -0,0 +1,8 @@ + + + + http://resolver:8080 +
Authorization: Bearer TOKEN
+ + + diff --git a/tests/integration/test_storage_s3/s3_mock/mock_s3.py b/tests/integration/test_storage_s3/s3_mock/mock_s3.py new file mode 100644 index 00000000000..35b477d6b10 --- /dev/null +++ b/tests/integration/test_storage_s3/s3_mock/mock_s3.py @@ -0,0 +1,17 @@ +from bottle import abort, route, run, request + + +@route('/<_bucket>/<_path>') +def server(_bucket, _path): + for name in request.headers: + if name == 'Authorization' and request.headers[name] == u'Bearer TOKEN': + return '1, 2, 3' + abort(403) + + +@route('/') +def ping(): + return 'OK' + + +run(host='0.0.0.0', port=8080) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 9f124507e14..b25e5907e62 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -2,6 +2,7 @@ import json import logging import random import threading +import os import pytest @@ -9,7 +10,6 @@ from helpers.cluster import ClickHouseCluster, ClickHouseInstance import helpers.client - logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) @@ -82,14 +82,16 @@ def get_nginx_access_logs(): def cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"], with_minio=True) - cluster.add_instance("dummy", with_minio=True) + cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"], + with_minio=True) + cluster.add_instance("dummy", with_minio=True, main_configs=["configs/defaultS3.xml"]) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") prepare_s3_bucket(cluster) logging.info("S3 bucket created") + run_s3_mock(cluster) yield cluster finally: @@ -199,14 +201,15 @@ def test_put_get_with_globs(cluster): for j in range(10): path = "{}_{}/{}.csv".format(i, random.choice(['a', 'b', 'c', 'd']), j) max_path = max(path, max_path) - values = "({},{},{})".format(i, j, i+j) + values = "({},{},{})".format(i, j, i + j) query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( cluster.minio_host, cluster.minio_port, bucket, path, table_format, values) run_query(instance, query) query = "select sum(column1), sum(column2), sum(column3), min(_file), max(_path) from s3('http://{}:{}/{}/*_{{a,b,c,d}}/%3f.csv', 'CSV', '{}')".format( cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, table_format) - assert run_query(instance, query).splitlines() == ["450\t450\t900\t0.csv\t{bucket}/{max_path}".format(bucket=bucket, max_path=max_path)] + assert run_query(instance, query).splitlines() == [ + "450\t450\t900\t0.csv\t{bucket}/{max_path}".format(bucket=bucket, max_path=max_path)] # Test multipart put. @@ -307,3 +310,29 @@ def test_s3_glob_scheherazade(cluster): query = "select count(), sum(column1), sum(column2), sum(column3) from s3('http://{}:{}/{}/night_*/tale.csv', 'CSV', '{}')".format( cluster.minio_redirect_host, cluster.minio_redirect_port, bucket, table_format) assert run_query(instance, query).splitlines() == ["1001\t1001\t1001\t1001"] + + +def run_s3_mock(cluster): + logging.info("Starting s3 mock") + container_id = cluster.get_container_id('resolver') + current_dir = os.path.dirname(__file__) + cluster.copy_file_to_container(container_id, os.path.join(current_dir, "s3_mock", "mock_s3.py"), "mock_s3.py") + cluster.exec_in_container(container_id, ["python", "mock_s3.py"], detach=True) + logging.info("S3 mock started") + + +# Test get values in CSV format with default settings. +def test_get_csv_default(cluster): + ping_response = cluster.exec_in_container(cluster.get_container_id('resolver'), ["curl", "-s", "http://resolver:8080"]) + assert ping_response == 'OK', 'Expected "OK", but got "{}"'.format(ping_response) + + table_format = "column1 UInt32, column2 UInt32, column3 UInt32" + filename = "test.csv" + get_query = "select * from s3('http://resolver:8080/{bucket}/{file}', 'CSV', '{table_format}')".format( + bucket=cluster.minio_restricted_bucket, + file=filename, + table_format=table_format) + + instance = cluster.instances["dummy"] # type: ClickHouseInstance + result = run_query(instance, get_query) + assert result == '1\t2\t3\n' diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 29169ad3c0e..a458db07a23 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -78,13 +78,13 @@ def test_ttl_many_columns(started_cluster): time.sleep(1) # sleep to allow use ttl merge selector for second time node1.query("OPTIMIZE TABLE test_ttl_2 FINAL", timeout=5) - + node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) expected = "1\t0\t0\t0\t0\n6\t7\t8\t9\t10\n" assert TSV(node1.query("SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id")) == TSV(expected) assert TSV(node2.query("SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id")) == TSV(expected) - + @pytest.mark.parametrize("delete_suffix", [ "", @@ -167,3 +167,67 @@ def test_ttl_double_delete_rule_returns_error(started_cluster): pass except: assert False + +@pytest.mark.parametrize("name,engine", [ + ("test_ttl_alter_delete", "MergeTree()"), + ("test_replicated_ttl_alter_delete", "ReplicatedMergeTree('/clickhouse/test_replicated_ttl_alter_delete', '1')"), +]) +def test_ttl_alter_delete(started_cluster, name, engine): + """Copyright 2019, Altinity LTD +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.""" + """Check compatibility with old TTL delete expressions to make sure + that: + * alter modify of column's TTL delete expression works + * alter to add new columns works + * alter modify to add TTL delete expression to a a new column works + for a table that has TTL delete expression defined but + no explicit storage policy assigned. + """ + drop_table([node1], name) + + def optimize_with_retry(retry=20): + for i in range(retry): + try: + node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name), settings={"optimize_throw_if_noop": "1"}) + break + except: + time.sleep(0.5) + node1.query( + """ + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 + INTERVAL 1 DAY DELETE + """.format(name=name, engine=engine)) + + node1.query("""ALTER TABLE {name} MODIFY COLUMN s1 String TTL d1 + INTERVAL 1 SECOND""".format(name=name)) + node1.query("""ALTER TABLE {name} ADD COLUMN b1 Int32""".format(name=name)) + + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello1', 1, toDateTime({time}))""".format(name=name, time=time.time())) + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello2', 2, toDateTime({time}))""".format(name=name, time=time.time() + 360)) + + time.sleep(1) + + optimize_with_retry() + r = node1.query("SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name)).splitlines() + assert r == ["\t1", "hello2\t2"] + + node1.query("""ALTER TABLE {name} MODIFY COLUMN b1 Int32 TTL d1""".format(name=name)) + node1.query("""INSERT INTO {name} (s1, b1, d1) VALUES ('hello3', 3, toDateTime({time}))""".format(name=name, time=time.time())) + + time.sleep(1) + + optimize_with_retry() + + r = node1.query("SELECT s1, b1 FROM {name} ORDER BY b1, s1".format(name=name)).splitlines() + assert r == ["\t0", "\t0", "hello2\t2"] diff --git a/tests/performance/aggregation_in_order.xml b/tests/performance/aggregation_in_order.xml new file mode 100644 index 00000000000..6e58865dab4 --- /dev/null +++ b/tests/performance/aggregation_in_order.xml @@ -0,0 +1,23 @@ + + + hits_10m_single + hits_100m_single + + + 1 + + + + table + + hits_10m_single + hits_100m_single + + + + + SELECT avg(length(URL)) as x from hits_100m_single GROUP BY CounterID FORMAT Null + SELECT avg(length(URL)) as x from {table} GROUP BY CounterID, EventDate FORMAT Null + SELECT avg(length(URL)) as x from hits_10m_single GROUP BY CounterID, EventDate, intHash32(UserID) FORMAT Null + + diff --git a/tests/queries/0_stateless/00398_url_functions.reference b/tests/queries/0_stateless/00398_url_functions.reference index acb605597d3..c926240b4f7 100644 --- a/tests/queries/0_stateless/00398_url_functions.reference +++ b/tests/queries/0_stateless/00398_url_functions.reference @@ -16,6 +16,17 @@ www.example.com example.com example.com example.com +====NETLOC==== +paul@www.example.com:80 +127.0.0.1:443 +127.0.0.1:443 +example.ru +example.ru +paul:zozo@example.ru +paul:zozo@example.ru +www.example.com +www.example.com +example.com ====DOMAIN==== com diff --git a/tests/queries/0_stateless/00398_url_functions.sql b/tests/queries/0_stateless/00398_url_functions.sql index d301cac5b15..c689844d08d 100644 --- a/tests/queries/0_stateless/00398_url_functions.sql +++ b/tests/queries/0_stateless/00398_url_functions.sql @@ -18,6 +18,17 @@ SELECT domain('example.com') as Host; SELECT domainWithoutWWW('//paul@www.example.com') AS Host; SELECT domainWithoutWWW('http://paul@www.example.com:80/') AS Host; +SELECT '====NETLOC===='; +SELECT netloc('http://paul@www.example.com:80/') AS Netloc; +SELECT netloc('http://127.0.0.1:443/') AS Netloc; +SELECT netloc('http://127.0.0.1:443') AS Netloc; +SELECT netloc('svn+ssh://example.ru/?q=hello%20world') AS Netloc; +SELECT netloc('svn+ssh://example.ru/?q=hello%20world') AS Netloc; +SELECT netloc('svn+ssh://paul:zozo@example.ru/?q=hello%20world') AS Netloc; +SELECT netloc('svn+ssh://paul:zozo@example.ru/?q=hello%20world') AS Netloc; +SELECT netloc('//www.example.com') AS Netloc; +SELECT netloc('www.example.com') as Netloc; +SELECT netloc('example.com') as Netloc; SELECT '====DOMAIN===='; SELECT topLevelDomain('http://paul@www.example.com:80/') AS Domain; diff --git a/tests/queries/0_stateless/00500_point_in_polygon_nan.reference b/tests/queries/0_stateless/00500_point_in_polygon_nan.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_nan.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/00500_point_in_polygon_nan.sql b/tests/queries/0_stateless/00500_point_in_polygon_nan.sql new file mode 100644 index 00000000000..37ed8dbeded --- /dev/null +++ b/tests/queries/0_stateless/00500_point_in_polygon_nan.sql @@ -0,0 +1 @@ +SELECT pointInPolygon((nan, 10.000100135803223), [(39.83154, 21.41527), (2., 1000.0001220703125), (39.90033, 21.37195), (1.000100016593933, 10.000100135803223), (39.83051, 21.42553), (39.82898, 21.41382), (39.83043, 21.41432), (39.83154, 21.41527)]); diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql index 96740d63778..c94c0f3c55b 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper.sql @@ -43,7 +43,7 @@ DETACH TABLE test.summing_r2; ALTER TABLE test.summing_r1 ADD COLUMN t UInt32 AFTER z, MODIFY ORDER BY (x, y, t * t) SETTINGS replication_alter_partitions_sync = 2; -- { serverError 341 } ATTACH TABLE test.summing_r2; -SELECT sleep(1) Format Null; +SYSTEM SYNC REPLICA test.summing_r2; SELECT '*** Check SHOW CREATE TABLE after offline ALTER ***'; SHOW CREATE TABLE test.summing_r2; diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference new file mode 100644 index 00000000000..d1b29b46df6 --- /dev/null +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference @@ -0,0 +1,29 @@ +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +0 \N Nullable(String) +0 \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +2 \N Nullable(String) +1 l \N Nullable(String) +2 \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) +- +1 l \N Nullable(String) +\N \N Nullable(String) +1 l \N Nullable(String) +\N \N Nullable(String) diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql new file mode 100644 index 00000000000..edaf2870e89 --- /dev/null +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql @@ -0,0 +1,50 @@ +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS nr; + +CREATE TABLE t (`x` UInt32, `lc` LowCardinality(String)) ENGINE = Memory; +CREATE TABLE nr (`x` Nullable(UInt32), `lc` Nullable(String)) ENGINE = Memory; + +INSERT INTO t VALUES (1, 'l'); +INSERT INTO nr VALUES (2, NULL); + +SET join_use_nulls = 0; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SET join_use_nulls = 1; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; + + +DROP TABLE t; +DROP TABLE nr; diff --git a/tests/queries/0_stateless/01143_trivial_count_with_join.reference b/tests/queries/0_stateless/01143_trivial_count_with_join.reference new file mode 100644 index 00000000000..9c3f6a570ce --- /dev/null +++ b/tests/queries/0_stateless/01143_trivial_count_with_join.reference @@ -0,0 +1,5 @@ +4 +4 +4 +4 +4 diff --git a/tests/queries/0_stateless/01143_trivial_count_with_join.sql b/tests/queries/0_stateless/01143_trivial_count_with_join.sql new file mode 100644 index 00000000000..d31750e37dc --- /dev/null +++ b/tests/queries/0_stateless/01143_trivial_count_with_join.sql @@ -0,0 +1,10 @@ +drop table if exists t; +create table t engine Memory as select * from numbers(2); + +select count(*) from t, numbers(2) r; +select count(*) from t cross join numbers(2) r; +select count() from t cross join numbers(2) r; +select count(t.number) from t cross join numbers(2) r; +select count(r.number) from t cross join numbers(2) r; + +drop table t; diff --git a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql index 3550ed64e8c..d0e8fa426cf 100644 --- a/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql +++ b/tests/queries/0_stateless/01271_optimize_arithmetic_operations_in_aggr_func.sql @@ -1,9 +1,11 @@ -set optimize_arithmetic_operations_in_agr_func = 1; +set optimize_arithmetic_operations_in_aggregate_functions = 1; + SELECT sum(number * -3) + min(2 * number * -3) - max(-1 * -2 * number * -3) FROM numbers(10000000); SELECT max(log(2) * number) FROM numbers(10000000); SELECT round(max(log(2) * 3 * sin(0.3) * number * 4)) FROM numbers(10000000); -set optimize_arithmetic_operations_in_agr_func = 0; +set optimize_arithmetic_operations_in_aggregate_functions = 0; + SELECT sum(number * -3) + min(2 * number * -3) - max(-1 * -2 * number * -3) FROM numbers(10000000); SELECT max(log(2) * number) FROM numbers(10000000); SELECT round(max(log(2) * 3 * sin(0.3) * number * 4)) FROM numbers(10000000); diff --git a/tests/queries/0_stateless/01291_aggregation_in_order.reference b/tests/queries/0_stateless/01291_aggregation_in_order.reference new file mode 100644 index 00000000000..c072a8aed3e --- /dev/null +++ b/tests/queries/0_stateless/01291_aggregation_in_order.reference @@ -0,0 +1,41 @@ +1 1 +1 2 +1 3 +1 4 +1 5 +1 6 +2 1 +2 2 +2 3 +2 4 +1 +2 +1 1 101 1 +1 2 102 1 +1 3 103 1 +1 4 104 1 +1 5 104 1 +1 6 105 1 +2 1 213 2 +2 2 107 2 +2 3 108 2 +2 4 109 2 +1 619 1 +2 537 2 +1 619 1 +2 537 2 +2019-05-05 00:00:00 -45363190 +2019-05-05 00:00:00 -1249512288 +2019-05-05 00:00:00 345522721 +2019-05-05 00:00:00 486601715 +2019-05-05 00:00:00 1449669396 +2019-05-05 00:00:00 45 +2019-05-06 00:00:00 46 +2019-05-07 00:00:00 47 +2019-05-08 00:00:00 48 +2019-05-09 00:00:00 49 +2019-05-05 00:00:00 0 1900940608 +2019-05-06 00:00:00 1 1857737272 +2019-05-07 00:00:00 2 1996614413 +2019-05-08 00:00:00 3 1873725230 +2019-05-09 00:00:00 4 1831412253 diff --git a/tests/queries/0_stateless/01291_aggregation_in_order.sql b/tests/queries/0_stateless/01291_aggregation_in_order.sql new file mode 100644 index 00000000000..753075f2757 --- /dev/null +++ b/tests/queries/0_stateless/01291_aggregation_in_order.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS pk_order; + +SET optimize_aggregation_in_order = 1; + +CREATE TABLE pk_order(a UInt64, b UInt64, c UInt64, d UInt64) ENGINE=MergeTree() ORDER BY (a, b); +INSERT INTO pk_order(a, b, c, d) VALUES (1, 1, 101, 1), (1, 2, 102, 1), (1, 3, 103, 1), (1, 4, 104, 1); +INSERT INTO pk_order(a, b, c, d) VALUES (1, 5, 104, 1), (1, 6, 105, 1), (2, 1, 106, 2), (2, 1, 107, 2); +INSERT INTO pk_order(a, b, c, d) VALUES (2, 2, 107, 2), (2, 3, 108, 2), (2, 4, 109, 2); + +-- Order after group by in order is determined + +SELECT a, b FROM pk_order GROUP BY a, b; +SELECT a FROM pk_order GROUP BY a; + +SELECT a, b, sum(c), avg(d) FROM pk_order GROUP BY a, b; +SELECT a, sum(c), avg(d) FROM pk_order GROUP BY a; +SELECT a, sum(c), avg(d) FROM pk_order GROUP BY -a; + +DROP TABLE IF EXISTS pk_order; + +CREATE TABLE pk_order (d DateTime, a Int32, b Int32) ENGINE = MergeTree ORDER BY (d, a) + PARTITION BY toDate(d) SETTINGS index_granularity=1; + +INSERT INTO pk_order + SELECT toDateTime('2019-05-05 00:00:00') + INTERVAL number % 10 DAY, number, intHash32(number) from numbers(100); + +set max_block_size = 1; + +SELECT d, max(b) FROM pk_order GROUP BY d, a LIMIT 5; +SELECT d, avg(a) FROM pk_order GROUP BY toString(d) LIMIT 5; +SELECT toStartOfHour(d) as d1, min(a), max(b) FROM pk_order GROUP BY d1 LIMIT 5; + +DROP TABLE pk_order; diff --git a/tests/queries/0_stateless/01293_system_distribution_queue.reference b/tests/queries/0_stateless/01293_system_distribution_queue.reference new file mode 100644 index 00000000000..a2c1e5f2a7b --- /dev/null +++ b/tests/queries/0_stateless/01293_system_distribution_queue.reference @@ -0,0 +1,6 @@ +INSERT +1 0 1 1 +FLUSH +1 0 0 0 +UNBLOCK +0 0 0 0 diff --git a/tests/queries/0_stateless/01293_system_distribution_queue.sql b/tests/queries/0_stateless/01293_system_distribution_queue.sql new file mode 100644 index 00000000000..c0ff6a21e8e --- /dev/null +++ b/tests/queries/0_stateless/01293_system_distribution_queue.sql @@ -0,0 +1,21 @@ +drop table if exists null_01293; +drop table if exists dist_01293; + +create table null_01293 (key Int) engine=Null(); +create table dist_01293 as null_01293 engine=Distributed(test_cluster_two_shards, currentDatabase(), null_01293, key); + +-- no rows, since no active monitor +select * from system.distribution_queue; + +select 'INSERT'; +system stop distributed sends dist_01293; +insert into dist_01293 select * from numbers(10); +select is_blocked, error_count, data_files, data_compressed_bytes>100 from system.distribution_queue; +system flush distributed dist_01293; + +select 'FLUSH'; +select is_blocked, error_count, data_files, data_compressed_bytes from system.distribution_queue; + +select 'UNBLOCK'; +system start distributed sends dist_01293; +select is_blocked, error_count, data_files, data_compressed_bytes from system.distribution_queue; diff --git a/tests/queries/0_stateless/01294_system_distributed_on_cluster.reference b/tests/queries/0_stateless/01294_system_distributed_on_cluster.reference new file mode 100644 index 00000000000..a8b5d159c9c --- /dev/null +++ b/tests/queries/0_stateless/01294_system_distributed_on_cluster.reference @@ -0,0 +1,3 @@ +localhost 9000 0 0 0 +localhost 9000 0 0 0 +localhost 9000 0 0 0 diff --git a/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql new file mode 100644 index 00000000000..d56bddba3c6 --- /dev/null +++ b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql @@ -0,0 +1,21 @@ +-- just a smoke test + +-- quirk for ON CLUSTER does not uses currentDatabase() +drop database if exists db_01294; +create database db_01294; + +drop table if exists db_01294.dist_01294; +create table db_01294.dist_01294 as system.one engine=Distributed(test_shard_localhost, system, one); +-- flush +system flush distributed db_01294.dist_01294; +system flush distributed on cluster test_shard_localhost db_01294.dist_01294; +-- stop +system stop distributed sends; +system stop distributed sends db_01294.dist_01294; +system stop distributed sends on cluster test_shard_localhost db_01294.dist_01294; +-- start +system start distributed sends; +system start distributed sends db_01294.dist_01294; +system start distributed sends on cluster test_shard_localhost db_01294.dist_01294; + +drop database db_01294; diff --git a/tests/queries/0_stateless/01296_codecs_bad_arguments.reference b/tests/queries/0_stateless/01296_codecs_bad_arguments.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01296_codecs_bad_arguments.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01296_codecs_bad_arguments.sql b/tests/queries/0_stateless/01296_codecs_bad_arguments.sql new file mode 100644 index 00000000000..d7eb53300ec --- /dev/null +++ b/tests/queries/0_stateless/01296_codecs_bad_arguments.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS delta_table; +DROP TABLE IF EXISTS zstd_table; +DROP TABLE IF EXISTS lz4_table; + +CREATE TABLE delta_table (`id` UInt64 CODEC(Delta(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} +CREATE TABLE zstd_table (`id` UInt64 CODEC(ZSTD(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} +CREATE TABLE lz4_table (`id` UInt64 CODEC(LZ4HC(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 433} + +CREATE TABLE lz4_table (`id` UInt64 CODEC(LZ4(tuple()))) ENGINE = MergeTree() ORDER BY tuple(); --{serverError 378} + +SELECT 1; + +DROP TABLE IF EXISTS delta_table; +DROP TABLE IF EXISTS zstd_table; +DROP TABLE IF EXISTS lz4_table; diff --git a/tests/queries/0_stateless/01296_pipeline_stuck.reference b/tests/queries/0_stateless/01296_pipeline_stuck.reference new file mode 100644 index 00000000000..ed8de641763 --- /dev/null +++ b/tests/queries/0_stateless/01296_pipeline_stuck.reference @@ -0,0 +1,13 @@ +1 +INSERT SELECT +1 +1 +INSERT SELECT max_threads +1 +1 +1 +INSERT SELECT max_insert_threads max_threads +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01296_pipeline_stuck.sql b/tests/queries/0_stateless/01296_pipeline_stuck.sql new file mode 100644 index 00000000000..eeb67362634 --- /dev/null +++ b/tests/queries/0_stateless/01296_pipeline_stuck.sql @@ -0,0 +1,18 @@ +drop table if exists data_01295; +create table data_01295 (key Int) Engine=AggregatingMergeTree() order by key; + +insert into data_01295 values (1); +select * from data_01295; + +select 'INSERT SELECT'; +insert into data_01295 select * from data_01295; -- no stuck for now +select * from data_01295; + +select 'INSERT SELECT max_threads'; +insert into data_01295 select * from data_01295 final settings max_threads=2; -- stuck with multiple threads +select * from data_01295; + +select 'INSERT SELECT max_insert_threads max_threads'; +set max_insert_threads=2; +insert into data_01295 select * from data_01295 final settings max_threads=2; -- no stuck for now +select * from data_01295; diff --git a/tests/queries/1_stateful/00004_top_counters.reference b/tests/queries/1_stateful/00004_top_counters.reference index cf2824e45b0..e2d584170c0 100644 --- a/tests/queries/1_stateful/00004_top_counters.reference +++ b/tests/queries/1_stateful/00004_top_counters.reference @@ -8,3 +8,13 @@ 59183 85379 33010362 77807 800784 77492 +1704509 523264 +732797 475698 +598875 337212 +792887 252197 +3807842 196036 +25703952 147211 +716829 90109 +59183 85379 +33010362 77807 +800784 77492 diff --git a/tests/queries/1_stateful/00004_top_counters.sql b/tests/queries/1_stateful/00004_top_counters.sql index 045f940da42..abdd5ac794a 100644 --- a/tests/queries/1_stateful/00004_top_counters.sql +++ b/tests/queries/1_stateful/00004_top_counters.sql @@ -1 +1,2 @@ -SELECT CounterID, count() AS c FROM test.hits GROUP BY CounterID ORDER BY c DESC LIMIT 10 +SELECT CounterID, count() AS c FROM test.hits GROUP BY CounterID ORDER BY c DESC LIMIT 10; +SELECT CounterID, count() AS c FROM test.hits GROUP BY CounterID ORDER BY c DESC LIMIT 10 SETTINGS optimize_aggregation_in_order = 1 diff --git a/tests/queries/1_stateful/00047_bar.reference b/tests/queries/1_stateful/00047_bar.reference index 61999ae73c9..c038f59946e 100644 --- a/tests/queries/1_stateful/00047_bar.reference +++ b/tests/queries/1_stateful/00047_bar.reference @@ -98,3 +98,103 @@ 7901143 10022 █▌ 194599 9997 █▌ 21052498 9780 █▍ +1704509 523264 ████████████████████████████████████████████████████████████████████████████████ +732797 475698 ████████████████████████████████████████████████████████████████████████▋ +598875 337212 ███████████████████████████████████████████████████▌ +792887 252197 ██████████████████████████████████████▌ +3807842 196036 █████████████████████████████▊ +25703952 147211 ██████████████████████▌ +716829 90109 █████████████▋ +59183 85379 █████████████ +33010362 77807 ███████████▊ +800784 77492 ███████████▋ +20810645 73213 ███████████▏ +25843850 68945 ██████████▌ +23447120 67570 ██████████▎ +14739804 64174 █████████▋ +32077710 60456 █████████▏ +22446879 58389 ████████▊ +170282 57017 ████████▋ +11482817 52345 ████████ +63469 52142 ███████▊ +29103473 47758 ███████▎ +10136747 44080 ██████▋ +27528801 43395 ██████▋ +10581377 43279 ██████▌ +9841201 40581 ██████▏ +20310963 37562 █████▋ +17337667 34301 █████▏ +28600281 32776 █████ +32046685 28788 ████▍ +10130880 26603 ████ +8676831 25733 ███▊ +53230 25595 ███▊ +20271226 25585 ███▊ +17420663 25496 ███▊ +631207 25270 ███▋ +633130 24744 ███▋ +14324015 23349 ███▌ +8537965 21270 ███▎ +11285298 20825 ███▏ +14937615 20788 ███▏ +185050 20785 ███▏ +16368233 19897 ███ +81602 19724 ███ +62896 19717 ███ +12967664 19402 ██▊ +15996597 18557 ██▋ +4379238 18370 ██▋ +90982 17443 ██▋ +18211045 17390 ██▋ +14625884 17302 ██▋ +12864910 17279 ██▋ +126096 16959 ██▌ +30296134 16849 ██▌ +26360482 16175 ██▍ +17788950 16017 ██▍ +5928716 15340 ██▎ +15469035 15171 ██▎ +29732125 15146 ██▎ +32946244 15104 ██▎ +20957241 14719 ██▎ +9495695 14584 ██▏ +29241146 14540 ██▏ +109805 14199 ██▏ +26905788 13972 ██▏ +212019 13930 ██▏ +171509 13792 ██ +23913162 13615 ██ +1861993 13509 ██ +125776 13308 ██ +11312316 13181 ██ +32667326 13181 ██ +28628973 12922 █▊ +122804 12520 █▊ +12322758 12352 █▊ +1301819 12283 █▊ +10769545 12183 █▋ +21566939 12170 █▋ +28905364 12158 █▋ +4250765 12049 █▋ +15009727 11818 █▋ +12761932 11733 █▋ +26995888 11658 █▋ +12759346 11514 █▋ +1507911 11452 █▋ +968488 11444 █▋ +15736172 11358 █▋ +54310 11193 █▋ +17027391 11047 █▋ +17439919 10936 █▋ +4480860 10747 █▋ +26738469 10738 █▋ +9986231 10656 █▋ +1539995 10655 █▋ +214556 10625 █▌ +219339 10522 █▌ +3266 10503 █▌ +30563429 10128 █▌ +1960469 10098 █▌ +7901143 10022 █▌ +194599 9997 █▌ +21052498 9780 █▍ diff --git a/tests/queries/1_stateful/00047_bar.sql b/tests/queries/1_stateful/00047_bar.sql index c7310763525..37c420b91ff 100644 --- a/tests/queries/1_stateful/00047_bar.sql +++ b/tests/queries/1_stateful/00047_bar.sql @@ -1 +1,2 @@ -SELECT CounterID, count() AS c, bar(c, 0, 523264) FROM test.hits GROUP BY CounterID ORDER BY c DESC, CounterID ASC LIMIT 100 +SELECT CounterID, count() AS c, bar(c, 0, 523264) FROM test.hits GROUP BY CounterID ORDER BY c DESC, CounterID ASC LIMIT 100; +SELECT CounterID, count() AS c, bar(c, 0, 523264) FROM test.hits GROUP BY CounterID ORDER BY c DESC, CounterID ASC LIMIT 100 SETTINGS optimize_aggregation_in_order = 1 diff --git a/tests/queries/1_stateful/00049_max_string_if.reference b/tests/queries/1_stateful/00049_max_string_if.reference index f87bc6d1fd2..6897a773c87 100644 --- a/tests/queries/1_stateful/00049_max_string_if.reference +++ b/tests/queries/1_stateful/00049_max_string_if.reference @@ -18,3 +18,23 @@ 11482817 52345 я скачать игры 63469 52142 яндекс марте рокус надписями я любимому у полосы фото минск 29103473 47758 +1704509 523264 نيك امريكي نيك افلام سكس جامد +732797 475698 نيك سكس سيحاق +598875 337212 سکس باصات +792887 252197 №2267 отзыв +3807842 196036 ярмаркетовара 200кг купить по неделю тебелье +25703952 147211 +716829 90109 яндекс повыш +59183 85379 франция машину угловы крузер из кофе +33010362 77807 ярмаркетовара 200кг купить по неделю тебелье +800784 77492 ярмаркур смерти теплицы из чего +20810645 73213 ярмаркетовара 200кг купить по неделю тебе перево метиков детский +25843850 68945 электросчет-фактура +23447120 67570 южная степанов +14739804 64174 штангал волк +32077710 60456 +22446879 58389 فیلم سكس امريكي نيك +170282 57017 ل افلام السكس +11482817 52345 я скачать игры +63469 52142 яндекс марте рокус надписями я любимому у полосы фото минск +29103473 47758 diff --git a/tests/queries/1_stateful/00049_max_string_if.sql b/tests/queries/1_stateful/00049_max_string_if.sql index af87123ef02..5c6d4274bab 100644 --- a/tests/queries/1_stateful/00049_max_string_if.sql +++ b/tests/queries/1_stateful/00049_max_string_if.sql @@ -1 +1,2 @@ -SELECT CounterID, count(), maxIf(SearchPhrase, notEmpty(SearchPhrase)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 +SELECT CounterID, count(), maxIf(SearchPhrase, notEmpty(SearchPhrase)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; +SELECT CounterID, count(), maxIf(SearchPhrase, notEmpty(SearchPhrase)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 SETTINGS optimize_aggregation_in_order = 1 diff --git a/tests/queries/1_stateful/00050_min_max.reference b/tests/queries/1_stateful/00050_min_max.reference index ab47fd7a69a..91473c4ea17 100644 --- a/tests/queries/1_stateful/00050_min_max.reference +++ b/tests/queries/1_stateful/00050_min_max.reference @@ -18,3 +18,23 @@ 11482817 4611708000353743073 9223337838355779113 63469 4611695097019173921 9223353530156141191 29103473 4611744585914335132 9223333530281362537 +1704509 4611700827100483880 9223360787015464643 +732797 4611701940806302259 9223355550934604746 +598875 4611701407242345792 9223362250391155632 +792887 4611699550286611812 9223290551912005343 +3807842 4611710821592843606 9223326163906184987 +25703952 4611709443519524003 9223353913449113943 +716829 4611852156092872082 9223361623076951140 +59183 4611730685242027332 9223354909338698162 +33010362 4611704682869732882 9223268545373999677 +800784 4611752907938305166 9223340418389788041 +20810645 4611712185532639162 9223218900001937412 +25843850 4611690025407720929 9223346023778617822 +23447120 4611796031755620254 9223329309291309758 +14739804 4611692230555590277 9223313509005166531 +32077710 4611884228437061959 9223352444952988904 +22446879 4611846229717089436 9223124373140579096 +170282 4611833225706935900 9223371583739401906 +11482817 4611708000353743073 9223337838355779113 +63469 4611695097019173921 9223353530156141191 +29103473 4611744585914335132 9223333530281362537 diff --git a/tests/queries/1_stateful/00050_min_max.sql b/tests/queries/1_stateful/00050_min_max.sql index 4c45f6fffa6..1ca93a5d620 100644 --- a/tests/queries/1_stateful/00050_min_max.sql +++ b/tests/queries/1_stateful/00050_min_max.sql @@ -1 +1,2 @@ -SELECT CounterID, min(WatchID), max(WatchID) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 +SELECT CounterID, min(WatchID), max(WatchID) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; +SELECT CounterID, min(WatchID), max(WatchID) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 SETTINGS optimize_aggregation_in_order = 1 diff --git a/tests/queries/1_stateful/00051_min_max_array.reference b/tests/queries/1_stateful/00051_min_max_array.reference index a5f1b6cdfef..b5555954099 100644 --- a/tests/queries/1_stateful/00051_min_max_array.reference +++ b/tests/queries/1_stateful/00051_min_max_array.reference @@ -18,3 +18,23 @@ 11482817 52345 [] [] [] 63469 52142 [] [] [] 29103473 47758 [6185451] [] [6185451] +1704509 523264 [271264] [] [271264] +732797 475698 [] [] [] +598875 337212 [] [] [] +792887 252197 [2094893,2028343] [] [1272031] +3807842 196036 [1710269] [] [1134660] +25703952 147211 [] [] [] +716829 90109 [4186138] [] [1800405] +59183 85379 [] [] [] +33010362 77807 [] [] [] +800784 77492 [4002316] [] [1270480] +20810645 73213 [] [] [] +25843850 68945 [4028285] [] [4028285] +23447120 67570 [6503091,2762273] [] [2098132] +14739804 64174 [4180720] [] [664490] +32077710 60456 [] [] [] +22446879 58389 [] [] [] +170282 57017 [4166114] [] [34386,1240412,1248634,1616213,2928740,1458582] +11482817 52345 [] [] [] +63469 52142 [] [] [] +29103473 47758 [6185451] [] [6185451] diff --git a/tests/queries/1_stateful/00051_min_max_array.sql b/tests/queries/1_stateful/00051_min_max_array.sql index 1027586372d..adf44fb9c22 100644 --- a/tests/queries/1_stateful/00051_min_max_array.sql +++ b/tests/queries/1_stateful/00051_min_max_array.sql @@ -1 +1,2 @@ -SELECT CounterID, count(), max(GoalsReached), min(GoalsReached), minIf(GoalsReached, notEmpty(GoalsReached)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 +SELECT CounterID, count(), max(GoalsReached), min(GoalsReached), minIf(GoalsReached, notEmpty(GoalsReached)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; +SELECT CounterID, count(), max(GoalsReached), min(GoalsReached), minIf(GoalsReached, notEmpty(GoalsReached)) FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 SETTINGS optimize_aggregation_in_order = 1 diff --git a/tests/queries/1_stateful/00087_where_0.sql b/tests/queries/1_stateful/00087_where_0.sql index c55617d2245..33c325e53b8 100644 --- a/tests/queries/1_stateful/00087_where_0.sql +++ b/tests/queries/1_stateful/00087_where_0.sql @@ -1,3 +1,5 @@ SET max_rows_to_read = 1000; SELECT CounterID, uniq(UserID) FROM test.hits WHERE 0 != 0 GROUP BY CounterID; +SELECT CounterID, uniq(UserID) FROM test.hits WHERE 0 != 0 GROUP BY CounterID SETTINGS optimize_aggregation_in_order = 1; SELECT CounterID, uniq(UserID) FROM test.hits WHERE 0 AND CounterID = 1704509 GROUP BY CounterID; +SELECT CounterID, uniq(UserID) FROM test.hits WHERE 0 AND CounterID = 1704509 GROUP BY CounterID SETTINGS optimize_aggregation_in_order = 1; diff --git a/tests/queries/1_stateful/00149_quantiles_timing_distributed.reference b/tests/queries/1_stateful/00149_quantiles_timing_distributed.reference index 8ac5f01c7cc..e31a1e90d87 100644 --- a/tests/queries/1_stateful/00149_quantiles_timing_distributed.reference +++ b/tests/queries/1_stateful/00149_quantiles_timing_distributed.reference @@ -1 +1,2 @@ 10726001768429413598 +10726001768429413598 diff --git a/tests/queries/1_stateful/00149_quantiles_timing_distributed.sql b/tests/queries/1_stateful/00149_quantiles_timing_distributed.sql index b195518e1e7..dc63cb5867f 100644 --- a/tests/queries/1_stateful/00149_quantiles_timing_distributed.sql +++ b/tests/queries/1_stateful/00149_quantiles_timing_distributed.sql @@ -1 +1,2 @@ SELECT sum(cityHash64(*)) FROM (SELECT CounterID, quantileTiming(0.5)(SendTiming), count() FROM remote('127.0.0.{1,2,3,4,5,6,7,8,9,10}', test.hits) WHERE SendTiming != -1 GROUP BY CounterID); +SELECT sum(cityHash64(*)) FROM (SELECT CounterID, quantileTiming(0.5)(SendTiming), count() FROM remote('127.0.0.{1,2,3,4,5,6,7,8,9,10}', test.hits) WHERE SendTiming != -1 GROUP BY CounterID) SETTINGS optimize_aggregation_in_order = 1; diff --git a/tests/queries/1_stateful/00150_quantiles_timing_precision.reference b/tests/queries/1_stateful/00150_quantiles_timing_precision.reference index 09aaf8449dc..79ef24af591 100644 --- a/tests/queries/1_stateful/00150_quantiles_timing_precision.reference +++ b/tests/queries/1_stateful/00150_quantiles_timing_precision.reference @@ -1 +1,2 @@ 4379238 1868 1879 5755 0.006 +4379238 1868 1879 5755 0.006 diff --git a/tests/queries/1_stateful/00150_quantiles_timing_precision.sql b/tests/queries/1_stateful/00150_quantiles_timing_precision.sql index 7d5b27fafd3..e858bcf34ff 100644 --- a/tests/queries/1_stateful/00150_quantiles_timing_precision.sql +++ b/tests/queries/1_stateful/00150_quantiles_timing_precision.sql @@ -1 +1,2 @@ SELECT CounterID, quantileTiming(0.5)(SendTiming) AS qt, least(30000, quantileExact(0.5)(SendTiming)) AS qe, count() AS c, round(abs(qt - qe) / greatest(qt, qe) AS diff, 3) AS rounded_diff FROM test.hits WHERE SendTiming != -1 GROUP BY CounterID HAVING diff != 0 ORDER BY diff DESC; +SELECT CounterID, quantileTiming(0.5)(SendTiming) AS qt, least(30000, quantileExact(0.5)(SendTiming)) AS qe, count() AS c, round(abs(qt - qe) / greatest(qt, qe) AS diff, 3) AS rounded_diff FROM test.hits WHERE SendTiming != -1 GROUP BY CounterID HAVING diff != 0 ORDER BY diff DESC SETTINGS optimize_aggregation_in_order = 1;