Merge branch 'master' of github.com:ClickHouse/ClickHouse

contrib/aws

@@ -1 +1 @@
-Subproject commit fb5c604525f5151d75a856462653e7e38b559b79
+Subproject commit f7d9ce39f41323300044567be007c233338bb94a

docker/packager/binary/build.sh

@@ -18,8 +18,7 @@ ccache --zero-stats ||:
 ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
 rm -f CMakeCache.txt
 cmake .. -LA -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DSANITIZE=$SANITIZER $CMAKE_FLAGS
-ninja
+ninja clickhouse-bundle
-ccache --show-stats ||:
 mv ./programs/clickhouse* /output
 mv ./src/unit_tests_dbms /output
 find . -name '*.so' -print -exec mv '{}' /output \;
@@ -47,3 +46,4 @@ then
     rm -r /output/*
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
+ccache --show-stats ||:

docker/packager/packager

@@ -120,6 +120,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
         result.append("CCACHE_BASEDIR=/build")
         result.append("CCACHE_NOHASHDIR=true")
         result.append("CCACHE_COMPILERCHECK=content")
+        result.append("CCACHE_MAXSIZE=15G")
         # result.append("CCACHE_UMASK=777")
 
     if distcc_hosts:

docker/test/integration/compose/docker_compose_minio.yml

@@ -43,7 +43,10 @@ services:
 
 # Empty container to run proxy resolver.
   resolver:
-    image: python:3
+    build:
+      context: ../../../docker/test/integration/
+      dockerfile: resolver/Dockerfile
+      network: host
     ports:
       - "4083:8080"
     tty: true

docker/test/integration/resolver/Dockerfile

@@ -0,0 +1,4 @@
+# Helper docker container to run python bottle apps
+
+FROM python:3
+RUN python -m pip install bottle

docker/test/performance-comparison/compare.sh

@@ -104,13 +104,12 @@ function run_tests
         # allows the tests to pass even when we add new functions and tests for
         # them, that are not supported in the old revision.
         test_prefix=left/performance
-    elif [ "$PR_TO_TEST" != "" ] && [ "$PR_TO_TEST" != "0" ]
+    else
-    then
         # For PRs, use newer test files so we can test these changes.
         test_prefix=right/performance
 
-        # If some tests were changed in the PR, we may want to run only these
+        # If only the perf tests were changed in the PR, we will run only these
-        # ones. The list of changed tests in changed-test.txt is prepared in
+        # tests. The list of changed tests in changed-test.txt is prepared in
         # entrypoint.sh from git diffs, because it has the cloned repo.  Used
         # to use rsync for that but it was really ugly and not always correct
         # (e.g. when the reference SHA is really old and has some other

docker/test/performance-comparison/config/config.d/perf-comparison-tweaks-config.xml

@@ -19,6 +19,5 @@
         <collect_interval_milliseconds>1000</collect_interval_milliseconds>
     </metric_log>
 
-    <use_uncompressed_cache>0</use_uncompressed_cache>
     <uncompressed_cache_size>1000000000</uncompressed_cache_size>
 </yandex>

docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml

@@ -5,6 +5,7 @@
             <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
             <allow_introspection_functions>1</allow_introspection_functions>
             <log_queries>1</log_queries>
+            <metrics_perf_events_enabled>1</metrics_perf_events_enabled>
         </default>
     </profiles>
 </yandex>

docker/test/performance-comparison/entrypoint.sh

@@ -83,10 +83,17 @@ if [ "$REF_PR" == "" ]; then echo Reference PR is not specified ; exit 1 ; fi
 
 if [ "$PR_TO_TEST" != "0" ]
 then
-    # Prepare the list of tests changed in the PR for use by compare.sh. Compare to
+    # If the PR only changes the tests and nothing else, prepare a list of these
-    # merge base, because master might be far in the future and have unrelated test
+    # tests for use by compare.sh. Compare to merge base, because master might be
-    # changes.
+    # far in the future and have unrelated test changes.
-    git -C ch diff --name-only "$SHA_TO_TEST" "$(git -C ch merge-base "$SHA_TO_TEST" master)" -- tests/performance | tee changed-tests.txt
+    base=$(git -C ch merge-base "$SHA_TO_TEST" master)
+    git -C ch diff --name-only "$SHA_TO_TEST" "$base" | tee changed-tests.txt
+    if grep -vq '^tests/performance' changed-tests.txt
+    then
+        # Have some other changes besides the tests, so truncate the test list,
+        # meaning, run all tests.
+        : > changed-tests.txt
+    fi
 fi
 
 # Set python output encoding so that we can print queries with Russian letters.
@@ -124,5 +131,5 @@ done
 
 dmesg -T > dmesg.log
 
-7z a /output/output.7z ./*.{log,tsv,html,txt,rep,svg} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
+7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} {right,left}/{performance,db/preprocessed_configs,scripts} report analyze
 cp compare.log /output

docker/test/performance-comparison/perf.py

@@ -100,11 +100,20 @@ for c in connections:
 
 report_stage_end('drop1')
 
-# Apply settings
+# Apply settings.
+# If there are errors, report them and continue -- maybe a new test uses a setting
+# that is not in master, but the queries can still run. If we have multiple
+# settings and one of them throws an exception, all previous settings for this
+# connection will be reset, because the driver reconnects on error (not
+# configurable). So the end result is uncertain, but hopefully we'll be able to
+# run at least some queries.
 settings = root.findall('settings/*')
 for c in connections:
     for s in settings:
+        try:
             c.execute("set {} = '{}'".format(s.tag, s.text))
+        except:
+            print(traceback.format_exc(), file=sys.stderr)
 
 report_stage_end('settings')
 

docs/en/development/developer-instruction.md

@@ -137,7 +137,7 @@ Official Yandex builds currently use GCC because it generates machine code of sl
 
 To install GCC on Ubuntu run: `sudo apt install gcc g++`
 
-Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9.
+Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-9.
 
 Mac OS X build is supported only for Clang. Just run `brew install llvm`
 
@@ -245,7 +245,7 @@ The Code Style Guide: https://clickhouse.tech/docs/en/development/style/
 
 Writing tests: https://clickhouse.tech/docs/en/development/tests/
 
-List of tasks: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+List of tasks: https://github.com/ClickHouse/ClickHouse/contribute
 
 ## Test Data {#test-data}
 

docs/en/operations/backup.md

@@ -31,6 +31,7 @@ For smaller volumes of data, a simple `INSERT INTO ... SELECT ...` to remote tab
 ## Manipulations with Parts {#manipulations-with-parts}
 
 ClickHouse allows using the `ALTER TABLE ... FREEZE PARTITION ...` query to create a local copy of table partitions. This is implemented using hardlinks to the `/var/lib/clickhouse/shadow/` folder, so it usually does not consume extra disk space for old data. The created copies of files are not handled by ClickHouse server, so you can just leave them there: you will have a simple backup that doesn’t require any additional external system, but it will still be prone to hardware issues. For this reason, it’s better to remotely copy them to another location and then remove the local copies. Distributed filesystems and object stores are still a good options for this, but normal attached file servers with a large enough capacity might work as well (in this case the transfer will occur via the network filesystem or maybe [rsync](https://en.wikipedia.org/wiki/Rsync)).
+Data can be restored from backup using the `ALTER TABLE ... ATTACH PARTITION ...`
 
 For more information about queries related to partition manipulations, see the [ALTER documentation](../sql-reference/statements/alter.md#alter_manipulations-with-partitions).
 

docs/en/operations/system-tables.md

@@ -18,9 +18,11 @@ System tables:
 - Available only for reading data.
 - Can't be dropped or altered, but can be detached.
 
-Most of system tables store their data in RAM. ClickHouse server creates such system tables at the start.
+Most of system tables store their data in RAM. A ClickHouse server creates such system tables at the start.
 
-The [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) system tables store data in a storage filesystem. You can alter them or remove from a disk manually. If you remove one of that tables from a disk, the ClickHouse server creates the table again at the time of the next recording. A storage period for these tables is not limited, and ClickHouse server doesn't delete their data automatically. You need to organize removing of outdated logs by yourself. For example, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. 
+Unlike other system tables, the system tables [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) are served by [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) table engine and store their data in a storage filesystem. If you remove a table from a filesystem, the ClickHouse server creates the empty one again at the time of the next data writing. If system table schema changed in a new release, then ClickHouse renames the current table and creates a new one.
+
+By default, table growth is unlimited. To control a size of a table, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. Also you can use the partitioning feature of `MergeTree`-engine tables.
 
 
 ### Sources of System Metrics {#system-tables-sources-of-system-metrics}
@@ -636,9 +638,9 @@ You can change settings of queries logging in the [query_log](server-configurati
 
 You can disable queries logging by setting [log_queries = 0](settings/settings.md#settings-log-queries). We don't recommend to turn off logging because information in this table is important for solving issues.
 
-The flushing period of logs is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing logs, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
+The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
 
-ClickHouse doesn't delete logs from the table automatically. See [Introduction](#system-tables-introduction) for more details.
+ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details.
 
 The `system.query_log` table registers two kinds of queries:
 
@@ -766,68 +768,117 @@ Settings.Values:      ['0','random','1','10000000000']
 
 ## system.query_thread_log {#system_tables-query_thread_log}
 
-The table contains information about each query execution thread.
+Contains information about threads which execute queries, for example, thread name, thread start time, duration of query processing.
 
-ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in.
+To start logging:
 
-To enable query logging, set the [log\_query\_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section.
+1. Configure parameters in the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) section.
+2. Set [log_query_threads](settings/settings.md#settings-log-query-threads) to 1.
+
+The flushing period of data is set in `flush_interval_milliseconds` parameter of the [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server settings section. To force flushing, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query.
+
+ClickHouse doesn't delete data from the table automatically. See [Introduction](#system-tables-introduction) for more details.
 
 Columns:
 
--   `event_date` (Date) — the date when the thread has finished execution of the query.
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query.
--   `event_time` (DateTime) — the date and time when the thread has finished execution of the query.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query.
--   `query_start_time` (DateTime) — Start time of query execution.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time of query execution.
--   `query_duration_ms` (UInt64) — Duration of query execution.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution.
--   `read_rows` (UInt64) — Number of read rows.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read rows.
--   `read_bytes` (UInt64) — Number of read bytes.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read bytes.
--   `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0.
--   `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0.
--   `memory_usage` (Int64) — The difference between the amount of allocated and freed memory in context of this thread.
+-   `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread.
--   `peak_memory_usage` (Int64) — The maximum difference between the amount of allocated and freed memory in context of this thread.
+-   `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread.
--   `thread_name` (String) — Name of the thread.
+-   `thread_name` ([String](../sql-reference/data-types/string.md)) — Name of the thread.
--   `thread_number` (UInt32) — Internal thread ID.
+-   `thread_number` ([UInt32](../sql-reference/data-types/int-uint.md)) — Internal thread ID.
--   `os_thread_id` (Int32) — OS thread ID.
+-   `thread_id` ([Int32](../sql-reference/data-types/int-uint.md)) — thread ID.
--   `master_thread_id` (UInt64) — OS initial ID of initial thread.
+-   `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread.
--   `query` (String) — Query string.
+-   `query` ([String](../sql-reference/data-types/string.md)) — Query string.
--   `is_initial_query` (UInt8) — Query type. Possible values:
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values:
     -   1 — Query was initiated by the client.
     -   0 — Query was initiated by another query for distributed query execution.
--   `user` (String) — Name of the user who initiated the current query.
+-   `user` ([String](../sql-reference/data-types/string.md)) — Name of the user who initiated the current query.
--   `query_id` (String) — ID of the query.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID of the query.
--   `address` (IPv6) — IP address that was used to make the query.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query.
--   `port` (UInt16) — The client port that was used to make the query.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query.
--   `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution).
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution).
--   `initial_query_id` (String) — ID of the initial query (for distributed query execution).
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution).
--   `initial_address` (IPv6) — IP address that the parent query was launched from.
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from.
--   `initial_port` (UInt16) — The client port that was used to make the parent query.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query.
--   `interface` (UInt8) — Interface that the query was initiated from. Possible values:
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — OS’s username who runs [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run.
--   `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name.
--   `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client.
--   `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version.
--   `http_method` (UInt8) — HTTP method that initiated the query. Possible values:
+-   `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP method that initiated the query. Possible values:
     -   0 — The query was launched from the TCP interface.
     -   1 — `GET` method was used.
     -   2 — `POST` method was used.
--   `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request.
--   `quota_key` (String) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`).
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`).
--   `revision` (UInt32) — ClickHouse revision.
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse revision.
--   `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events)
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events).
--   `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column.
 
-By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query.
+**Example**
 
-When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted.
+``` sql
+ SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical
+```
 
-!!! note "Note"
+``` text
-    The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself.
+Row 1:
+──────
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            1
+read_bytes:           1
+written_rows:         0
+written_bytes:        0
+memory_usage:         0
+peak_memory_usage:    0
+thread_name:          QueryPipelineEx
+thread_id:            28952
+master_thread_id:     28924
+query:                SELECT 1
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+ProfileEvents.Names:  ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds']
+ProfileEvents.Values: [1,97,81,5,81]
+...
+```
 
-You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `partition_by` parameter).
+**See Also**
+
+- [system.query_log](#system_tables-query_log) — Description of the `query_log` system table which contains common information about queries execution.
 
 ## system.trace\_log {#system_tables-trace_log}
 

docs/es/development/developer-instruction.md

@@ -141,7 +141,7 @@ Las compilaciones oficiales de Yandex actualmente usan GCC porque genera código
 
 Para instalar GCC en Ubuntu, ejecute: `sudo apt install gcc g++`
 
-Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse .tech/docs/en/development/build/\#install-gcc-9.
+Compruebe la versión de gcc: `gcc --version`. Si está por debajo de 9, siga las instrucciones aquí: https://clickhouse.tech/docs/es/development/build/#install-gcc-9.
 
 La compilación de Mac OS X solo es compatible con Clang. Sólo tiene que ejecutar `brew install llvm`
 
@@ -249,7 +249,7 @@ La Guía de estilo de código: https://clickhouse.tech/docs/en/development/style
 
 Pruebas de escritura: https://clickhouse.tech/docs/en/development/tests/
 
-Lista de tareas: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Lista de tareas: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Datos de prueba {#test-data}
 

docs/fa/development/developer-instruction.md

@@ -143,7 +143,7 @@ toc_title: "\u062F\u0633\u062A\u0648\u0631\u0627\u0644\u0639\u0645\u0644 \u062A\
 
 برای نصب شورای همکاری خلیج فارس در اوبونتو اجرای: `sudo apt install gcc g++`
 
-بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.فناوری / اسناد / ارتباطات / توسعه/ساختن / \#نصب شورای همکاری خلیج فارس-9.
+بررسی نسخه شورای همکاری خلیج فارس: `gcc --version`. اگر زیر است 9, سپس دستورالعمل اینجا را دنبال کنید: https://clickhouse.tech/docs/fa/development/build/#install-gcc-9.
 
 سیستم عامل مک ایکس ساخت فقط برای صدای جرنگ جرنگ پشتیبانی می شود. فقط فرار کن `brew install llvm`
 
@@ -251,7 +251,7 @@ KDevelop و QTCreator دیگر از جایگزین های بسیار خوبی ا
 
 تست نوشتن: https://clickhouse.فناوری / اسناد/توسعه/تست/
 
-فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+فهرست تکلیفها: https://github.com/ClickHouse/ClickHouse/contribute
 
 # داده های تست {#test-data}
 

docs/fr/development/developer-instruction.md

@@ -141,7 +141,7 @@ Les builds officiels de Yandex utilisent actuellement GCC car ils génèrent du
 
 Pour installer GCC sur Ubuntu Exécutez: `sudo apt install gcc g++`
 
-Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/développement/construction/\#install-gcc-9.
+Vérifiez la version de gcc: `gcc --version`. Si elle est inférieure à 9, suivez les instructions ici: https://clickhouse.tech/docs/fr/development/build/#install-gcc-9.
 
 Mac OS X build est pris en charge uniquement pour Clang. Il suffit d'exécuter `brew install llvm`
 
@@ -249,7 +249,7 @@ Le code Style Guide: https://clickhouse.tech/docs/fr/développement/style/
 
 Rédaction de tests: https://clickhouse.tech/docs/fr/développement/tests/
 
-Liste des tâches: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Liste des tâches: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Des Données De Test {#test-data}
 

docs/ja/development/developer-instruction.md

@@ -141,7 +141,7 @@ ClickHouseのビルドには、バージョン9以降のGCCとClangバージョ
 
 UBUNTUにGCCをインストールするには: `sudo apt install gcc g++`
 
-Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/en/development/build/\#install-gcc-9.
+Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/ja/development/build/#install-gcc-9.
 
 Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm`
 
@@ -249,7 +249,7 @@ KDevelopとQTCreatorは、ClickHouseを開発するためのIDEの他の優れ
 
 筆記試験：https://clickhouse.tech/docs/en/development/tests/
 
-タスクのリスト：https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+タスクのリスト：https://github.com/ClickHouse/ClickHouse/contribute
 
 # テストデータ {#test-data}
 

docs/ru/development/developer-instruction.md

@@ -135,7 +135,7 @@ ClickHouse использует для сборки некоторое коли
 
 Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`.
 
-Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/en/development/build/\#install-gcc-9
+Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/ru/development/build/#install-gcc-9.
 
 Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm`
 
@@ -244,7 +244,7 @@ Mac OS X:
 
 Разработка тестов: https://clickhouse.tech/docs/ru/development/tests/
 
-Список задач: https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_ru.md
+Список задач: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Тестовые данные {#testovye-dannye}
 

docs/ru/operations/system-tables.md

@@ -593,15 +593,9 @@ CurrentMetric_ReplicatedChecks:                             0
 
 Можно отключить логгирование настройкой [log_queries = 0](settings/settings.md#settings-log-queries). По-возможности, не отключайте логгирование, поскольку информация из таблицы важна при решении проблем.
 
-Период сброса логов в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
+Период сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
 
-ClickHouse не удаляет логи из таблица автоматически. Смотрите [Введение](#system-tables-introduction).
+ClickHouse не удаляет данные из таблица автоматически. Смотрите [Введение](#system-tables-introduction).
-
-Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`).
-
-
-
-Если таблицу удалить вручную, она создается заново автоматически «на лету». При этом все логи на момент удаления таблицы будут убраны.
 
 Таблица `system.query_log` содержит информацию о двух видах запросов:
 
@@ -729,71 +723,116 @@ Settings.Values:      ['0','random','1','10000000000']
 
 ## system.query_thread_log {#system_tables-query_thread_log}
 
-Содержит информацию о каждом потоке выполняемых запросов.
+Содержит информацию о потоках, которые выполняют запросы, например, имя потока, время его запуска, продолжительность обработки запроса.
 
-ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы.
+Чтобы начать логирование:
 
-Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md#settings).
+1. Настройте параметры [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) в конфигурации сервера.
+2. Установите значение [log_query_threads](settings/settings.md#settings-log-query-threads) равным 1.
+
+Интервал сброса данных в таблицу задаётся параметром `flush_interval_milliseconds` в разделе настроек сервера [query_thread_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Чтобы принудительно записать логи из буфера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs).
+
+ClickHouse не удаляет данные из таблицы автоматически. Подробности в разделе [Введение](#system-tables-introduction).
 
 Столбцы:
 
--   `event_date` (Date) — дата завершения выполнения запроса потоком.
+-   `event_date` ([Date](../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком.
--   `event_time` (DateTime) — дата и время завершения выполнения запроса потоком.
+-   `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком.
--   `query_start_time` (DateTime) — время начала обработки запроса.
+-   `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — время начала обработки запроса.
--   `query_duration_ms` (UInt64) — длительность обработки запроса в миллисекундах.
+-   `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах.
--   `read_rows` (UInt64) — количество прочитанных строк.
+-   `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных строк.
--   `read_bytes` (UInt64) — количество прочитанных байтов.
+-   `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество прочитанных байтов.
--   `written_rows` (UInt64) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0.
+-   `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0.
--   `written_bytes` (UInt64) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0.
+-   `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0.
--   `memory_usage` (Int64) — разница между выделенной и освобождённой памятью в контексте потока.
+-   `memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — разница между выделенной и освобождённой памятью в контексте потока.
--   `peak_memory_usage` (Int64) — максимальная разница между выделенной и освобождённой памятью в контексте потока.
+-   `peak_memory_usage` ([Int64](../sql-reference/data-types/int-uint.md)) — максимальная разница между выделенной и освобождённой памятью в контексте потока.
--   `thread_name` (String) — Имя потока.
+-   `thread_name` ([String](../sql-reference/data-types/string.md)) — Имя потока.
--   `thread_id` (UInt64) — tid (ID потока операционной системы).
+-   `thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы).
--   `master_thread_id` (UInt64) — tid (ID потока операционной системы) главного потока.
+-   `master_thread_id` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы) главного потока.
--   `query` (String) — текст запроса.
+-   `query` ([String](../sql-reference/data-types/string.md)) — текст запроса.
--   `is_initial_query` (UInt8) — вид запроса. Возможные значения:
+-   `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — вид запроса. Возможные значения:
     -   1 — запрос был инициирован клиентом.
     -   0 — запрос был инициирован другим запросом при распределенном запросе.
--   `user` (String) — пользователь, запустивший текущий запрос.
+-   `user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос.
--   `query_id` (String) — ID запроса.
+-   `query_id` ([String](../sql-reference/data-types/string.md)) — ID запроса.
--   `address` (IPv6) — IP адрес, с которого пришел запрос.
+-   `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос.
--   `port` (UInt16) — порт, с которого пришел запрос.
+-   `port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, с которого пришел запрос.
--   `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
+-   `initial_user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов).
--   `initial_query_id` (String) — ID родительского запроса.
+-   `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID родительского запроса.
--   `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос.
+-   `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос.
--   `initial_port` (UInt16) — порт, пришел родительский запрос.
+-   `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md#uint-ranges)) — порт, пришел родительский запрос.
--   `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения:
+-   `interface` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — интерфейс, с которого ушёл запрос. Возможные значения:
     -   1 — TCP.
     -   2 — HTTP.
--   `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md).
+-   `os_user` ([String](../sql-reference/data-types/string.md)) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md).
--   `client_hostname` (String) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_hostname` ([String](../sql-reference/data-types/string.md)) — hostname клиентской машины, с которой присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
+-   `client_name` ([String](../sql-reference/data-types/string.md)) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент.
--   `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
+-   `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента.
--   `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения:
+-   `http_method` ([UInt8](../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP метод, инициировавший запрос. Возможные значения:
     -   0 — запрос запущен с интерфейса TCP.
     -   1 — `GET`.
     -   2 — `POST`.
--   `http_user_agent` (String) — HTTP заголовок `UserAgent`.
+-   `http_user_agent` ([String](../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`.
--   `quota_key` (String) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`).
+-   `quota_key` ([String](../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`).
--   `revision` (UInt32) — ревизия ClickHouse.
+-   `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse.
--   `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events
+-   `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events).
--   `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`.
+-   `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`.
 
-По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`.
+**Пример**
 
-Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены.
+``` sql
+ SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical
+```
 
-!!! note "Примечание"
+``` text
-    Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов.
+Row 1:
+──────
+event_date:           2020-05-13
+event_time:           2020-05-13 14:02:28
+query_start_time:     2020-05-13 14:02:28
+query_duration_ms:    0
+read_rows:            1
+read_bytes:           1
+written_rows:         0
+written_bytes:        0
+memory_usage:         0
+peak_memory_usage:    0
+thread_name:          QueryPipelineEx
+thread_id:            28952
+master_thread_id:     28924
+query:                SELECT 1
+is_initial_query:     1
+user:                 default
+query_id:             5e834082-6f6d-4e34-b47b-cd1934f4002a
+address:              ::ffff:127.0.0.1
+port:                 57720
+initial_user:         default
+initial_query_id:     5e834082-6f6d-4e34-b47b-cd1934f4002a
+initial_address:      ::ffff:127.0.0.1
+initial_port:         57720
+interface:            1
+os_user:              bayonet
+client_hostname:      clickhouse.ru-central1.internal
+client_name:          ClickHouse client
+client_revision:      54434
+client_version_major: 20
+client_version_minor: 4
+client_version_patch: 1
+http_method:          0
+http_user_agent:
+quota_key:
+revision:             54434
+ProfileEvents.Names:  ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds']
+ProfileEvents.Values: [1,97,81,5,81]
+...
+```
 
-Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (параметр `partition_by`).
+**Смотрите также**
 
-## system.query_thread_log {#system_tables-query_thread_log}
+- [system.query_log](#system_tables-query_log) — описание системной таблицы `query_log`, которая содержит общую информацию о выполненных запросах.
-
-Содержит информацию о каждом потоке исполнения запроса.
 
 ## system.trace\_log {#system_tables-trace_log}
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md

@@ -32,7 +32,7 @@ ClickHouse поддерживает иерархические словари с
 
 ClickHouse поддерживает свойство [hierarchical](external-dicts-dict-structure.md#hierarchical-dict-attr) для атрибутов [внешнего словаря](index.md). Это свойство позволяет конфигурировать словари, подобные описанному выше.
 
-С помощью функции [dictGetHierarchy](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md#dictgethierarchy) можно получить цепочку предков элемента.
+С помощью функции [dictGetHierarchy](../../../sql-reference/functions/ext-dict-functions.md#dictgethierarchy) можно получить цепочку предков элемента.
 
 Структура словаря для нашего примера может выглядеть следующим образом:
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md

@@ -2,7 +2,7 @@
 
 Словари можно размещать в памяти множеством способов.
 
-Рекомендуем [flat](#flat), [hashed](#hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна.
+Рекомендуем [flat](#flat), [hashed](#dicts-external_dicts_dict_layout-hashed) и [complex\_key\_hashed](#complex-key-hashed). Скорость обработки словарей при этом максимальна.
 
 Размещение с кэшированием не рекомендуется использовать из-за потенциально низкой производительности и сложностей в подборе оптимальных параметров. Читайте об этом подробнее в разделе «[cache](#cache)».
 
@@ -34,7 +34,7 @@
 </yandex>
 ```
 
-Соответствущий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query):
+Соответствущий [DDL-запрос](../../statements/create.md#create-dictionary-query):
 
 ``` sql
 CREATE DICTIONARY (...)
@@ -46,7 +46,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
 ## Способы размещения словарей в памяти {#sposoby-razmeshcheniia-slovarei-v-pamiati}
 
 -   [flat](#flat)
--   [hashed](#hashed)
+-   [hashed](#dicts-external_dicts_dict_layout-hashed)
 -   [sparse\_hashed](#dicts-external_dicts_dict_layout-sparse_hashed)
 -   [cache](#cache)
 -   [direct](#direct)
@@ -80,7 +80,7 @@ LAYOUT(LAYOUT_TYPE(param value)) -- layout settings
 LAYOUT(FLAT())
 ```
 
-### hashed {#hashed}
+### hashed {#dicts-external_dicts_dict_layout-hashed}
 
 Словарь полностью хранится в оперативной памяти в виде хэш-таблиц. Словарь может содержать произвольное количество элементов с произвольными идентификаторами. На практике, количество ключей может достигать десятков миллионов элементов.
 

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md

@@ -19,7 +19,7 @@
 </yandex>
 ```
 
-Аналогичный [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query):
+Аналогичный [DDL-запрос](../../statements/create.md#create-dictionary-query):
 
 ``` sql
 CREATE DICTIONARY dict_name (...)
@@ -150,7 +150,7 @@ SOURCE(HTTP(
 ))
 ```
 
-Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) в конфигурации сервера.
+Чтобы ClickHouse смог обратиться к HTTPS-ресурсу, необходимо [настроить openSSL](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl) в конфигурации сервера.
 
 Поля настройки:
 
@@ -531,7 +531,7 @@ SOURCE(CLICKHOUSE(
 
 Поля настройки:
 
--   `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) и прописать её в дальнейших настройках.
+-   `host` — хост ClickHouse. Если host локальный, то запрос выполняется без сетевого взаимодействия. Чтобы повысить отказоустойчивость решения, можно создать таблицу типа [Distributed](../../../engines/table-engines/special/distributed.md) и прописать её в дальнейших настройках.
 -   `port` — порт сервера ClickHouse.
 -   `user` — имя пользователя ClickHouse.
 -   `password` — пароль пользователя ClickHouse.

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md

@@ -154,7 +154,7 @@ CREATE DICTIONARY somename (
 | Тег                                                  | Описание                                                                                                                                                                                                                                                                                                                                                      | Обязательный |
 |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
 | `name`                                               | Имя столбца.                                                                                                                                                                                                                                                                                                                                                  | Да           |
-| `type`                                               | Тип данных ClickHouse.<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md) не поддерживается. | Да           |
+| `type`                                               | Тип данных ClickHouse.<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/data-types/nullable.md) не поддерживается. | Да           |
 | `null_value`                                         | Значение по умолчанию для несуществующего элемента.<br/>В примере это пустая строка. Нельзя указать значение `NULL`.                                                                                                                                                                                                                                          | Да           |
 | `expression`                                         | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.<br/>Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.<br/><br/>Значение по умолчанию: нет выражения.                                                                  | Нет          |
 | <a name="hierarchical-dict-attr"></a> `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).<br/><br/>Default value: `false`.                                                                                                                                                                                   | No           |
@@ -162,6 +162,6 @@ CREATE DICTIONARY somename (
 
 ## Смотрите также {#smotrite-takzhe}
 
--   [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md).
+-   [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md).
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) <!--hide-->

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md

@@ -24,7 +24,7 @@ XML-конфигурация словаря имеет следующую стр
 </dictionary>
 ```
 
-Соответствующий [DDL-запрос](../../../sql-reference/statements/create.md#create-dictionary-query) имеет следующий вид:
+Соответствующий [DDL-запрос](../../statements/create.md#create-dictionary-query) имеет следующий вид:
 
 ``` sql
 CREATE DICTIONARY dict_name

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md

@@ -5,11 +5,11 @@
 ClickHouse:
 - Полностью или частично хранит словари в оперативной памяти.
 - Периодически обновляет их и динамически подгружает отсутствующие значения.
-- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../../sql-reference/statements/create.md#create-dictionary-query).
+- Позволяет создавать внешние словари с помощью xml-файлов или [DDL-запросов](../../statements/create.md#create-dictionary-query).
 
-Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
+Конфигурация внешних словарей может находится в одном или нескольких xml-файлах. Путь к конфигурации указывается в параметре [dictionaries\_config](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_config).
 
-Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
+Словари могут загружаться при старте сервера или при первом использовании, в зависимости от настройки [dictionaries\_lazy\_load](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load).
 
 Системная таблица [system.dictionaries](../../../operations/system-tables.md#system_tables-dictionaries) содержит информацию о словарях, сконфигурированных на сервере. Для каждого словаря там можно найти:
 
@@ -41,10 +41,10 @@ ClickHouse:
 
 В одном файле можно [сконфигурировать](external-dicts-dict.md) произвольное количество словарей.
 
-Если вы создаёте внешние словари [DDL-запросами](../../../sql-reference/statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера.
+Если вы создаёте внешние словари [DDL-запросами](../../statements/create.md#create-dictionary-query), то не задавайте конфигурацию словаря в конфигурации сервера.
 
 !!! attention "Внимание"
-    Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md)). Эта функциональность не связана с внешними словарями.
+    Можно преобразовывать значения по небольшому словарю, описав его в запросе `SELECT` (см. функцию [transform](../../../sql-reference/functions/other-functions.md)). Эта функциональность не связана с внешними словарями.
 
 ## Смотрите также {#ext-dicts-see-also}
 
@@ -53,6 +53,6 @@ ClickHouse:
 -   [Обновление словарей](external-dicts-dict-lifetime.md)
 -   [Источники внешних словарей](external-dicts-dict-sources.md)
 -   [Ключ и поля словаря](external-dicts-dict-structure.md)
--   [Функции для работы с внешними словарями](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md#ext_dict_functions)
+-   [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md)
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) <!--hide-->

docs/ru/sql-reference/index.md

@@ -10,7 +10,7 @@ toc_title: hidden
 -   [SELECT](statements/select/index.md)
 -   [INSERT INTO](statements/insert-into.md)
 -   [CREATE](statements/create.md)
--   [ALTER](statements/alter.md)
+-   [ALTER](statements/alter.md#query_language_queries_alter)
 -   [Прочие виды запросов](statements/misc.md)
 
 [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) <!--hide-->

docs/tr/development/developer-instruction.md

@@ -141,7 +141,7 @@ Resmi Yandex şu anda GCC'Yİ kullanıyor çünkü biraz daha iyi performansa sa
 
 Ubuntu run GCC yüklemek için: `sudo apt install gcc g++`
 
-Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech / docs/TR/development / build / \#ınstall-gcc-9.
+Gcc sürümünü kontrol edin: `gcc --version`. 9'un altındaysa, buradaki talimatları izleyin: https://clickhouse.tech/docs/tr/development/build/#install-gcc-9.
 
 Mac OS X build sadece Clang için desteklenir. Sadece koş `brew install llvm`
 
@@ -249,7 +249,7 @@ Kod stili Kılavuzu: https://clickhouse.tech / doscs / TR / development / style/
 
 Yazma testleri: https://clickhouse.teknoloji / doscs / TR / geliştirme / testler/
 
-Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
+Görevlerin listesi: https://github.com/ClickHouse/ClickHouse/contribute
 
 # Test Verileri {#test-data}
 

docs/zh/development/developer-instruction.md

@@ -129,7 +129,7 @@ Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性
 
 在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++`
 
-请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/en/development/build/\#install-gcc-9 。
+请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/zh/development/build/#an-zhuang-gcc-9 。
 
 在Mac OS X上安装GCC，请执行：`brew install gcc`
 
@@ -234,7 +234,7 @@ ClickHouse的架构描述可以在此处查看：https://clickhouse.tech/docs/en
 
 编写测试用例：https://clickhouse.tech/docs/en/development/tests/
 
-任务列表：https://github.com/ClickHouse/ClickHouse/blob/master/tests/instructions/easy\_tasks\_sorted\_en.md
+任务列表：https://github.com/ClickHouse/ClickHouse/contribute
 
 # 测试数据 {#ce-shi-shu-ju}
 

docs/zh/sql-reference/aggregate-functions/parametric-functions.md

@@ -313,7 +313,7 @@ ORDER BY level ASC
 └───────┴───┘
 ```
 
-## 保留 {#retention}
+## Retention {#retention}
 
 该函数将一组条件作为参数，类型为1到32个参数 `UInt8` 表示事件是否满足特定条件。
 任何条件都可以指定为参数（如 [WHERE](../../sql-reference/statements/select/where.md#select-where)).

programs/CMakeLists.txt

@@ -207,7 +207,7 @@ if (TARGET clickhouse-server AND TARGET copy-headers)
 endif ()
 
 if (ENABLE_TESTS AND USE_GTEST)
-    set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor)
+    set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms)
-    add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS})
+    add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS})
     add_dependencies(clickhouse-bundle clickhouse-tests)
 endif()

programs/client/Suggest.cpp

@@ -114,6 +114,8 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo
             << " UNION ALL "
             "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
             << " UNION ALL "
+            "SELECT DISTINCT name FROM system.dictionaries LIMIT " << limit_str
+            << " UNION ALL "
             "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
     }
 

programs/server/Server.cpp

@@ -125,6 +125,7 @@ namespace ErrorCodes
     extern const int FAILED_TO_GETPWUID;
     extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
     extern const int NETWORK_ERROR;
+    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
 }
 
 
@@ -210,6 +211,52 @@ void Server::defineOptions(Poco::Util::OptionSet & options)
     BaseDaemon::defineOptions(options);
 }
 
+
+/// Check that there is no user-level settings at the top level in config.
+/// This is a common source of mistake (user don't know where to write user-level setting).
+void checkForUserSettingsAtTopLevel(const Poco::Util::AbstractConfiguration & config, const std::string & path)
+{
+    if (config.getBool("skip_check_for_incorrect_settings", false))
+        return;
+
+    Settings settings;
+    for (const auto & setting : settings)
+    {
+        std::string name = setting.getName().toString();
+        if (config.has(name))
+        {
+            throw Exception(fmt::format("A setting '{}' appeared at top level in config {}."
+                " But it is user-level setting that should be located in users.xml inside <profiles> section for specific profile."
+                " You can add it to <profiles><default> if you want to change default value of this setting."
+                " You can also disable the check - specify <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings>"
+                " in the main configuration file.",
+                name, path),
+                ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
+        }
+    }
+}
+
+void checkForUsersNotInMainConfig(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_path,
+    const std::string & users_config_path,
+    Poco::Logger * log)
+{
+    if (config.getBool("skip_check_for_incorrect_settings", false))
+        return;
+
+    if (config.has("users") || config.has("profiles") || config.has("quotas"))
+    {
+        /// We cannot throw exception here, because we have support for obsolete 'conf.d' directory
+        /// (that does not correspond to config.d or users.d) but substitute configuration to both of them.
+
+        LOG_ERROR(log, "The <users>, <profiles> and <quotas> elements should be located in users config file: {} not in main config {}."
+            " Also note that you should place configuration changes to the appropriate *.d directory like 'users.d'.",
+            users_config_path, config_path);
+    }
+}
+
+
 int Server::main(const std::vector<std::string> & /*args*/)
 {
     Poco::Logger * log = &logger();
@@ -269,6 +316,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
         config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
     }
 
+    checkForUserSettingsAtTopLevel(config(), config_path);
+
     const auto memory_amount = getMemoryAmount();
 
 #if defined(OS_LINUX)
@@ -473,13 +522,16 @@ int Server::main(const std::vector<std::string> & /*args*/)
         SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
     }
 
-    auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
+    auto main_config_reloader = std::make_unique<ConfigReloader>(
+        config_path,
         include_from_path,
         config().getString("path", ""),
         std::move(main_config_zk_node_cache),
         main_config_zk_changed_event,
         [&](ConfigurationPtr config)
         {
+            checkForUserSettingsAtTopLevel(*config, config_path);
+
             // FIXME logging-related things need synchronization -- see the 'Logger * log' saved
             // in a lot of places. For now, disable updating log configuration without server restart.
             //setTextLog(global_context->getTextLog());
@@ -508,12 +560,21 @@ int Server::main(const std::vector<std::string> & /*args*/)
         if (Poco::File(config_dir + users_config_path).exists())
             users_config_path = config_dir + users_config_path;
     }
-    auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
+
+    if (users_config_path != config_path)
+        checkForUsersNotInMainConfig(config(), config_path, users_config_path, log);
+
+    auto users_config_reloader = std::make_unique<ConfigReloader>(
+        users_config_path,
         include_from_path,
         config().getString("path", ""),
         zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
         std::make_shared<Poco::Event>(),
-        [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
+        [&](ConfigurationPtr config)
+        {
+            global_context->setUsersConfig(config);
+            checkForUserSettingsAtTopLevel(*config, users_config_path);
+        },
         /* already_loaded = */ false);
 
     /// Reload config in SYSTEM RELOAD CONFIG query.

programs/server/config.xml

@@ -1,6 +1,9 @@
 <?xml version="1.0"?>
 <!--
   NOTE: User and query level settings are set up in "users.xml" file.
+  If you have accidentially specified user-level settings here, server won't start.
+  You can either move the settings to the right place inside "users.xml" file
+   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
 -->
 <yandex>
     <logger>

src/Access/IAccessStorage.cpp

@@ -12,10 +12,10 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int BAD_CAST;
     extern const int ACCESS_ENTITY_ALREADY_EXISTS;
     extern const int ACCESS_ENTITY_NOT_FOUND;
     extern const int ACCESS_STORAGE_READONLY;
+    extern const int LOGICAL_ERROR;
 }
 
 
@@ -403,7 +403,7 @@ void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String
 {
     throw Exception(
         "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type),
-        ErrorCodes::BAD_CAST);
+        ErrorCodes::LOGICAL_ERROR);
 }
 
 

src/AggregateFunctions/AggregateFunctionMLMethod.h

@@ -15,7 +15,6 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int BAD_CAST;
 }
 
 /**
@@ -381,7 +380,7 @@ public:
         auto * column = typeid_cast<ColumnFloat64 *>(&to);
         if (!column)
             throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to",
-                            ErrorCodes::BAD_CAST);
+                            ErrorCodes::LOGICAL_ERROR);
 
         this->data(place).predict(column->getData(), block, offset, limit, arguments, context);
     }

src/AggregateFunctions/IAggregateFunction.h

@@ -150,6 +150,8 @@ public:
     virtual void addBatchSinglePlaceNotNull(
         size_t batch_size, AggregateDataPtr place, const IColumn ** columns, const UInt8 * null_map, Arena * arena) const = 0;
 
+    virtual void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const = 0;
+
     /** In addition to addBatch, this method collects multiple rows of arguments into array "places"
       *  as long as they are between offsets[i-1] and offsets[i]. This is used for arrayReduce and
       *  -Array combinator. It might also be used generally to break data dependency when array
@@ -214,6 +216,12 @@ public:
                 static_cast<const Derived *>(this)->add(place, columns, i, arena);
     }
 
+    void addBatchSinglePlaceFromInterval(size_t batch_begin, size_t batch_end, AggregateDataPtr place, const IColumn ** columns, Arena * arena) const override
+    {
+        for (size_t i = batch_begin; i < batch_end; ++i)
+            static_cast<const Derived *>(this)->add(place, columns, i, arena);
+    }
+
     void addBatchArray(
         size_t batch_size, AggregateDataPtr * places, size_t place_offset, const IColumn ** columns, const UInt64 * offsets, Arena * arena)
         const override

src/AggregateFunctions/parseAggregateFunctionParameters.cpp

@@ -27,8 +27,12 @@ Array getAggregateFunctionParametersArray(const ASTPtr & expression_list, const
         const auto * literal = parameters[i]->as<ASTLiteral>();
         if (!literal)
         {
-            throw Exception("Parameters to aggregate functions must be literals" + (error_context.empty() ? "" : " (in " + error_context +")"),
+            throw Exception(
-                        ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS);
+                ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
+                "Parameters to aggregate functions must be literals. "
+                "Got parameter '{}'{}",
+                parameters[i]->formatForErrorMessage(),
+                (error_context.empty() ? "" : " (in " + error_context +")"));
         }
 
         params_row[i] = literal->value;

src/Columns/ColumnAggregateFunction.h

@@ -121,6 +121,7 @@ public:
 
     std::string getName() const override { return "AggregateFunction(" + func->getName() + ")"; }
     const char * getFamilyName() const override { return "AggregateFunction"; }
+    TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; }
 
     MutableColumnPtr predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const;
 

src/Columns/ColumnArray.h

@@ -52,6 +52,7 @@ public:
 
     std::string getName() const override;
     const char * getFamilyName() const override { return "Array"; }
+    TypeIndex getDataType() const override { return TypeIndex::Array; }
     MutableColumnPtr cloneResized(size_t size) const override;
     size_t size() const override;
     Field operator[](size_t n) const override;

src/Columns/ColumnConst.h

@@ -50,6 +50,11 @@ public:
         return "Const";
     }
 
+    TypeIndex getDataType() const override
+    {
+        return data->getDataType();
+    }
+
     MutableColumnPtr cloneResized(size_t new_size) const override
     {
         return ColumnConst::create(data, new_size);

src/Columns/ColumnDecimal.cpp

@@ -333,17 +333,6 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
     max = NearestFieldType<T>(cur_max, scale);
 }
 
-TypeIndex columnDecimalDataType(const IColumn * column)
-{
-    if (checkColumn<ColumnDecimal<Decimal32>>(column))
-        return TypeIndex::Decimal32;
-    else if (checkColumn<ColumnDecimal<Decimal64>>(column))
-        return TypeIndex::Decimal64;
-    else if (checkColumn<ColumnDecimal<Decimal128>>(column))
-        return TypeIndex::Decimal128;
-    return TypeIndex::Nothing;
-}
-
 template class ColumnDecimal<Decimal32>;
 template class ColumnDecimal<Decimal64>;
 template class ColumnDecimal<Decimal128>;

src/Columns/ColumnDecimal.h

@@ -81,6 +81,7 @@ private:
 
 public:
     const char * getFamilyName() const override { return TypeName<T>::get(); }
+    TypeIndex getDataType() const override { return TypeId<T>::value; }
 
     bool isNumeric() const override { return false; }
     bool canBeInsideNullable() const override { return true; }
@@ -197,6 +198,4 @@ ColumnPtr ColumnDecimal<T>::indexImpl(const PaddedPODArray<Type> & indexes, size
     return res;
 }
 
-TypeIndex columnDecimalDataType(const IColumn * column);
-
 }

src/Columns/ColumnFixedString.h

@@ -43,6 +43,7 @@ private:
 public:
     std::string getName() const override { return "FixedString(" + std::to_string(n) + ")"; }
     const char * getFamilyName() const override { return "FixedString"; }
+    TypeIndex getDataType() const override { return TypeIndex::FixedString; }
 
     MutableColumnPtr cloneResized(size_t size) const override;
 

src/Columns/ColumnFunction.h

@@ -29,6 +29,7 @@ private:
 
 public:
     const char * getFamilyName() const override { return "Function"; }
+    TypeIndex getDataType() const override { return TypeIndex::Function; }
 
     MutableColumnPtr cloneResized(size_t size) const override;
 

src/Columns/ColumnLowCardinality.h

@@ -39,6 +39,7 @@ public:
 
     std::string getName() const override { return "ColumnLowCardinality"; }
     const char * getFamilyName() const override { return "ColumnLowCardinality"; }
+    TypeIndex getDataType() const override { return TypeIndex::LowCardinality; }
 
     ColumnPtr convertToFullColumn() const { return getDictionary().getNestedColumn()->index(getIndexes(), 0); }
     ColumnPtr convertToFullColumnIfLowCardinality() const override { return convertToFullColumn(); }

src/Columns/ColumnNothing.h

@@ -21,6 +21,7 @@ private:
 public:
     const char * getFamilyName() const override { return "Nothing"; }
     MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); }
+    TypeIndex getDataType() const override { return TypeIndex::Nothing; }
 
     bool canBeInsideNullable() const override { return true; }
 

src/Columns/ColumnNullable.h

@@ -45,6 +45,7 @@ public:
 
     const char * getFamilyName() const override { return "Nullable"; }
     std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; }
+    TypeIndex getDataType() const override { return TypeIndex::Nullable; }
     MutableColumnPtr cloneResized(size_t size) const override;
     size_t size() const override { return nested_column->size(); }
     bool isNullAt(size_t n) const override { return assert_cast<const ColumnUInt8 &>(*null_map).getData()[n] != 0;}

src/Columns/ColumnSet.h

@@ -25,6 +25,7 @@ private:
 
 public:
     const char * getFamilyName() const override { return "Set"; }
+    TypeIndex getDataType() const override { return TypeIndex::Set; }
     MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnSet::create(s_, data); }
 
     ConstSetPtr getData() const { return data; }

src/Columns/ColumnString.h

@@ -56,6 +56,7 @@ private:
 
 public:
     const char * getFamilyName() const override { return "String"; }
+    TypeIndex getDataType() const override { return TypeIndex::String; }
 
     size_t size() const override
     {

src/Columns/ColumnTuple.h

@@ -40,6 +40,7 @@ public:
 
     std::string getName() const override;
     const char * getFamilyName() const override { return "Tuple"; }
+    TypeIndex getDataType() const override { return TypeIndex::Tuple; }
 
     MutableColumnPtr cloneEmpty() const override;
     MutableColumnPtr cloneResized(size_t size) const override;

src/Columns/ColumnVector.cpp

@@ -289,13 +289,6 @@ void ColumnVector<T>::updatePermutation(bool reverse, size_t limit, int nan_dire
     equal_range = std::move(new_ranges);
 }
 
-
-template <typename T>
-const char * ColumnVector<T>::getFamilyName() const
-{
-    return TypeName<T>::get();
-}
-
 template <typename T>
 MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
 {
@@ -517,33 +510,6 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
     max = NearestFieldType<T>(cur_max);
 }
 
-TypeIndex columnVectorDataType(const IColumn * column)
-{
-    if (checkColumn<ColumnVector<UInt8>>(column))
-        return TypeIndex::UInt8;
-    else if (checkColumn<ColumnVector<UInt16>>(column))
-        return TypeIndex::UInt16;
-    else if (checkColumn<ColumnVector<UInt32>>(column))
-        return TypeIndex::UInt32;
-    else if (checkColumn<ColumnVector<UInt64>>(column))
-        return TypeIndex::UInt64;
-    else if (checkColumn<ColumnVector<Int8>>(column))
-        return TypeIndex::Int8;
-    else if (checkColumn<ColumnVector<Int16>>(column))
-        return TypeIndex::Int16;
-    else if (checkColumn<ColumnVector<Int32>>(column))
-        return TypeIndex::Int32;
-    else if (checkColumn<ColumnVector<Int64>>(column))
-        return TypeIndex::Int64;
-    else if (checkColumn<ColumnVector<Int128>>(column))
-        return TypeIndex::Int128;
-    else if (checkColumn<ColumnVector<Float32>>(column))
-        return TypeIndex::Float32;
-    else if (checkColumn<ColumnVector<Float64>>(column))
-        return TypeIndex::Float64;
-    return TypeIndex::Nothing;
-}
-
 /// Explicit template instantiations - to avoid code bloat in headers.
 template class ColumnVector<UInt8>;
 template class ColumnVector<UInt16>;

src/Columns/ColumnVector.h

@@ -199,7 +199,8 @@ public:
         data.reserve(n);
     }
 
-    const char * getFamilyName() const override;
+    const char * getFamilyName() const override { return TypeName<T>::get(); }
+    TypeIndex getDataType() const override { return TypeId<T>::value; }
 
     MutableColumnPtr cloneResized(size_t size) const override;
 
@@ -320,6 +321,4 @@ ColumnPtr ColumnVector<T>::indexImpl(const PaddedPODArray<Type> & indexes, size_
     return res;
 }
 
-TypeIndex columnVectorDataType(const IColumn * column);
-
 }

src/Columns/IColumn.h

@@ -51,6 +51,9 @@ public:
     /// Name of a Column kind, without parameters (example: FixedString, Array).
     virtual const char * getFamilyName() const = 0;
 
+    /// Type of data that column contains. It's an underlying type: UInt16 for Date, UInt32 for DateTime, so on.
+    virtual TypeIndex getDataType() const = 0;
+
     /** If column isn't constant, returns itself.
       * If column is constant, transforms constant to full column (if column type allows such transform) and return it.
       */

src/Columns/IColumnUnique.h

@@ -66,6 +66,7 @@ public:
     virtual UInt128 getHash() const = 0;
 
     const char * getFamilyName() const override { return "ColumnUnique"; }
+    TypeIndex getDataType() const override { return getNestedColumn()->getDataType(); }
 
     void insert(const Field &) override
     {

src/Common/Arena.h

@@ -150,7 +150,7 @@ public:
         return res;
     }
 
-    /// Get peice of memory with alignment
+    /// Get piece of memory with alignment
     char * alignedAlloc(size_t size, size_t alignment)
     {
         do

src/Common/ErrorCodes.cpp

@@ -341,7 +341,6 @@ namespace ErrorCodes
     extern const int OUTPUT_IS_NOT_SORTED = 365;
     extern const int SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT = 366;
     extern const int TOO_MANY_FETCHES = 367;
-    extern const int BAD_CAST = 368;
     extern const int ALL_REPLICAS_ARE_STALE = 369;
     extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES = 370;
     extern const int INCONSISTENT_CLUSTER_DEFINITION = 371;
@@ -398,7 +397,6 @@ namespace ErrorCodes
     extern const int CANNOT_GETTIMEOFDAY = 423;
     extern const int CANNOT_LINK = 424;
     extern const int SYSTEM_ERROR = 425;
-    extern const int NULL_POINTER_DEREFERENCE = 426;
     extern const int CANNOT_COMPILE_REGEXP = 427;
     extern const int UNKNOWN_LOG_LEVEL = 428;
     extern const int FAILED_TO_GETPWUID = 429;
@@ -458,7 +456,6 @@ namespace ErrorCodes
     extern const int TOO_MANY_REDIRECTS = 483;
     extern const int INTERNAL_REDIS_ERROR = 484;
     extern const int SCALAR_ALREADY_EXISTS = 485;
-    extern const int UNKNOWN_SCALAR = 486;
     extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY = 487;
     extern const int UNKNOWN_DICTIONARY = 488;
     extern const int INCORRECT_DICTIONARY_DEFINITION = 489;

src/Common/Exception.h

@@ -8,6 +8,8 @@
 
 #include <Common/StackTrace.h>
 
+#include <fmt/format.h>
+
 namespace Poco { class Logger; }
 
 
@@ -20,8 +22,14 @@ public:
     Exception() = default;
     Exception(const std::string & msg, int code);
 
-    enum CreateFromPocoTag { CreateFromPoco };
+    // Format message with fmt::format, like the logging functions.
-    enum CreateFromSTDTag { CreateFromSTD };
+    template <typename ...Fmt>
+    Exception(int code, Fmt&&... fmt)
+        : Exception(fmt::format(std::forward<Fmt>(fmt)...), code)
+    {}
+
+    struct CreateFromPocoTag {};
+    struct CreateFromSTDTag {};
 
     Exception(CreateFromPocoTag, const Poco::Exception & exc);
     Exception(CreateFromSTDTag, const std::exception & exc);

src/Common/ProfileEvents.cpp

@@ -180,6 +180,25 @@
     M(OSWriteBytes, "Number of bytes written to disks or block devices. Doesn't include bytes that are in page cache dirty pages. May not include data that was written by OS asynchronously.") \
     M(OSReadChars, "Number of bytes read from filesystem, including page cache.") \
     M(OSWriteChars, "Number of bytes written to filesystem, including page cache.") \
+    \
+    M(PerfCpuCycles, "Total cycles. Be wary of what happens during CPU frequency scaling.")  \
+    M(PerfInstructions, "Retired instructions. Be careful, these can be affected by various issues, most notably hardware interrupt counts.") \
+    M(PerfCacheReferences, "Cache accesses. Usually this indicates Last Level Cache accesses but this may vary depending on your CPU. This may include prefetches and coherency messages; again this depends on the design of your CPU.") \
+    M(PerfCacheMisses, "Cache misses. Usually this indicates Last Level Cache misses; this is intended to be used in con‐junction with the PERFCOUNTHWCACHEREFERENCES event to calculate cache miss rates.") \
+    M(PerfBranchInstructions, "Retired branch instructions. Prior to Linux 2.6.35, this used the wrong event on AMD processors.") \
+    M(PerfBranchMisses, "Mispredicted branch instructions.") \
+    M(PerfBusCycles, "Bus cycles, which can be different from total cycles.") \
+    M(PerfStalledCyclesFrontend, "Stalled cycles during issue.") \
+    M(PerfStalledCyclesBackend, "Stalled cycles during retirement.") \
+    M(PerfRefCpuCycles, "Total cycles; not affected by CPU frequency scaling.") \
+    \
+    M(PerfCpuClock, "The CPU clock, a high-resolution per-CPU timer") \
+    M(PerfTaskClock, "A clock count specific to the task that is running") \
+    M(PerfContextSwitches, "Number of context switches") \
+    M(PerfCpuMigrations, "Number of times the process has migrated to a new CPU") \
+    M(PerfAlignmentFaults, "Number of alignment faults. These happen when unaligned memory accesses happen; the kernel can handle these but it reduces performance. This happens only on some architectures (never on x86).") \
+    M(PerfEmulationFaults, "Number of emulation faults. The kernel sometimes traps on unimplemented instructions and emulates them for user space. This can negatively impact performance.") \
+    \
     M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \
     \
     M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \

src/Common/ThreadProfileEvents.cpp

@@ -4,9 +4,22 @@
 
 #include "TaskStatsInfoGetter.h"
 #include "ProcfsMetricsProvider.h"
+#include "hasLinuxCapability.h"
 
+#include <filesystem>
+#include <fstream>
 #include <optional>
+#include <sstream>
+#include <unordered_set>
 
+#include <fcntl.h>
+#include <unistd.h>
+#include <linux/perf_event.h>
+#include <syscall.h>
+#include <sys/ioctl.h>
+#include <cerrno>
+#include <sys/types.h>
+#include <dirent.h>
 
 namespace DB
 {
@@ -104,6 +117,404 @@ void TasksStatsCounters::incrementProfileEvents(const ::taskstats & prev, const
     profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.read_bytes, curr.read_bytes));
     profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.write_bytes, curr.write_bytes));
 }
+
+}
+
+#endif
+
+#if defined(__linux__) && !defined(ARCADIA_BUILD)
+
+namespace DB
+{
+
+thread_local PerfEventsCounters current_thread_counters;
+
+#define SOFTWARE_EVENT(PERF_NAME, LOCAL_NAME) \
+    PerfEventInfo \
+    { \
+        .event_type = perf_type_id::PERF_TYPE_SOFTWARE, \
+        .event_config = (PERF_NAME), \
+        .profile_event = ProfileEvents::LOCAL_NAME, \
+        .settings_name = #LOCAL_NAME \
+    }
+
+#define HARDWARE_EVENT(PERF_NAME, LOCAL_NAME) \
+    PerfEventInfo \
+    { \
+        .event_type = perf_type_id::PERF_TYPE_HARDWARE, \
+        .event_config = (PERF_NAME), \
+        .profile_event = ProfileEvents::LOCAL_NAME, \
+        .settings_name = #LOCAL_NAME \
+    }
+
+// descriptions' source: http://man7.org/linux/man-pages/man2/perf_event_open.2.html
+static const PerfEventInfo raw_events_info[] = {
+    HARDWARE_EVENT(PERF_COUNT_HW_CPU_CYCLES, PerfCpuCycles),
+    HARDWARE_EVENT(PERF_COUNT_HW_INSTRUCTIONS, PerfInstructions),
+    HARDWARE_EVENT(PERF_COUNT_HW_CACHE_REFERENCES, PerfCacheReferences),
+    HARDWARE_EVENT(PERF_COUNT_HW_CACHE_MISSES, PerfCacheMisses),
+    HARDWARE_EVENT(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, PerfBranchInstructions),
+    HARDWARE_EVENT(PERF_COUNT_HW_BRANCH_MISSES, PerfBranchMisses),
+    HARDWARE_EVENT(PERF_COUNT_HW_BUS_CYCLES, PerfBusCycles),
+    HARDWARE_EVENT(PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, PerfStalledCyclesFrontend),
+    HARDWARE_EVENT(PERF_COUNT_HW_STALLED_CYCLES_BACKEND, PerfStalledCyclesBackend),
+    HARDWARE_EVENT(PERF_COUNT_HW_REF_CPU_CYCLES, PerfRefCpuCycles),
+    // `cpu-clock` is a bit broken according to this: https://stackoverflow.com/a/56967896
+    SOFTWARE_EVENT(PERF_COUNT_SW_CPU_CLOCK, PerfCpuClock),
+    SOFTWARE_EVENT(PERF_COUNT_SW_TASK_CLOCK, PerfTaskClock),
+    SOFTWARE_EVENT(PERF_COUNT_SW_CONTEXT_SWITCHES, PerfContextSwitches),
+    SOFTWARE_EVENT(PERF_COUNT_SW_CPU_MIGRATIONS, PerfCpuMigrations),
+    SOFTWARE_EVENT(PERF_COUNT_SW_ALIGNMENT_FAULTS, PerfAlignmentFaults),
+    SOFTWARE_EVENT(PERF_COUNT_SW_EMULATION_FAULTS, PerfEmulationFaults)
+};
+
+#undef HARDWARE_EVENT
+#undef SOFTWARE_EVENT
+
+// A map of event name -> event index, to parse event list in settings.
+static std::unordered_map<std::string, size_t> populateEventMap()
+{
+    std::unordered_map<std::string, size_t> name_to_index;
+    name_to_index.reserve(NUMBER_OF_RAW_EVENTS);
+
+    for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+    {
+        name_to_index.emplace(raw_events_info[i].settings_name, i);
+    }
+
+    return name_to_index;
+}
+
+static const auto event_name_to_index = populateEventMap();
+
+static int openPerfEvent(perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, UInt64 flags)
+{
+    return static_cast<int>(syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags));
+}
+
+static int openPerfEventDisabled(Int32 perf_event_paranoid, bool has_cap_sys_admin, UInt32 perf_event_type, UInt64 perf_event_config)
+{
+    perf_event_attr pe{};
+    pe.type = perf_event_type;
+    pe.size = sizeof(struct perf_event_attr);
+    pe.config = perf_event_config;
+    // disable by default to add as little extra time as possible
+    pe.disabled = 1;
+    // can record kernel only when `perf_event_paranoid` <= 1 or have CAP_SYS_ADMIN
+    pe.exclude_kernel = perf_event_paranoid >= 2 && !has_cap_sys_admin;
+    pe.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING;
+
+    return openPerfEvent(&pe, /* measure the calling thread */ 0, /* on any cpu */ -1, -1, 0);
+}
+
+static void enablePerfEvent(int event_fd)
+{
+    if (ioctl(event_fd, PERF_EVENT_IOC_ENABLE, 0))
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Can't enable perf event with file descriptor {}: '{}' ({})",
+            event_fd, strerror(errno), errno);
+    }
+}
+
+static void disablePerfEvent(int event_fd)
+{
+    if (ioctl(event_fd, PERF_EVENT_IOC_DISABLE, 0))
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Can't disable perf event with file descriptor {}: '{}' ({})",
+            event_fd, strerror(errno), errno);
+    }
+}
+
+static void releasePerfEvent(int event_fd)
+{
+    if (close(event_fd))
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Can't close perf event file descriptor {}: {} ({})",
+            event_fd, strerror(errno), errno);
+    }
+}
+
+static bool validatePerfEventDescriptor(int & fd)
+{
+    if (fcntl(fd, F_GETFL) != -1)
+        return true;
+
+    if (errno == EBADF)
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Event descriptor {} was closed from the outside; reopening", fd);
+    }
+    else
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Error while checking availability of event descriptor {}: {} ({})",
+            fd, strerror(errno), errno);
+
+        disablePerfEvent(fd);
+        releasePerfEvent(fd);
+    }
+
+    fd = -1;
+    return false;
+}
+
+bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_events_list)
+{
+    const auto valid_event_indices = eventIndicesFromString(needed_events_list);
+
+    // find state changes (if there are any)
+    bool old_state[NUMBER_OF_RAW_EVENTS];
+    for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+        old_state[i] = thread_events_descriptors_holder.descriptors[i] != -1;
+
+    bool new_state[NUMBER_OF_RAW_EVENTS];
+    std::fill_n(new_state, NUMBER_OF_RAW_EVENTS, false);
+    for (size_t opened_index : valid_event_indices)
+        new_state[opened_index] = true;
+
+    std::vector<size_t> events_to_open;
+    std::vector<size_t> events_to_release;
+    for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+    {
+        bool old_one = old_state[i];
+        bool new_one = new_state[i];
+
+        if (old_one == new_one)
+        {
+            if (old_one
+                && !validatePerfEventDescriptor(
+                    thread_events_descriptors_holder.descriptors[i]))
+            {
+                events_to_open.push_back(i);
+            }
+            continue;
+        }
+
+        if (new_one)
+            events_to_open.push_back(i);
+        else
+            events_to_release.push_back(i);
+    }
+
+    // release unused descriptors
+    for (size_t i : events_to_release)
+    {
+        int & fd = thread_events_descriptors_holder.descriptors[i];
+        disablePerfEvent(fd);
+        releasePerfEvent(fd);
+        fd = -1;
+    }
+
+    if (events_to_open.empty())
+    {
+        return true;
+    }
+
+    // check permissions
+    // cat /proc/sys/kernel/perf_event_paranoid
+    // -1: Allow use of (almost) all events by all users
+    // >=0: Disallow raw tracepoint access by users without CAP_IOC_LOCK
+    // >=1: Disallow CPU event access by users without CAP_SYS_ADMIN
+    // >=2: Disallow kernel profiling by users without CAP_SYS_ADMIN
+    // >=3: Disallow all event access by users without CAP_SYS_ADMIN
+    Int32 perf_event_paranoid = 0;
+    std::ifstream paranoid_file("/proc/sys/kernel/perf_event_paranoid");
+    paranoid_file >> perf_event_paranoid;
+
+    bool has_cap_sys_admin = hasLinuxCapability(CAP_SYS_ADMIN);
+    if (perf_event_paranoid >= 3 && !has_cap_sys_admin)
+    {
+        LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+            "Not enough permissions to record perf events: "
+            "perf_event_paranoid = {} and CAP_SYS_ADMIN = 0",
+            perf_event_paranoid);
+        return false;
+    }
+
+    // Open descriptors for new events.
+    // Theoretically, we can run out of file descriptors. Threads go up to 10k,
+    // and there might be a dozen perf events per thread, so we're looking at
+    // 100k open files. In practice, this is not likely -- perf events are
+    // mostly used in performance tests or other kinds of testing, and the
+    // number of threads stays below hundred.
+    // We used to check the number of open files by enumerating /proc/self/fd,
+    // but listing all open files before opening more files is obviously
+    // quadratic, and quadraticity never ends well.
+    for (size_t i : events_to_open)
+    {
+        const PerfEventInfo & event_info = raw_events_info[i];
+        int & fd = thread_events_descriptors_holder.descriptors[i];
+        // disable by default to add as little extra time as possible
+        fd = openPerfEventDisabled(perf_event_paranoid, has_cap_sys_admin, event_info.event_type, event_info.event_config);
+
+        if (fd == -1 && errno != ENOENT)
+        {
+            // ENOENT means that the event is not supported. Don't log it, because
+            // this is called for each thread and would be too verbose. Log other
+            // error codes because they might signify an error.
+            LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+                "Failed to open perf event {} (event_type={}, event_config={}): "
+                "'{}' ({})", event_info.settings_name, event_info.event_type,
+                event_info.event_config, strerror(errno), errno);
+        }
+    }
+
+    return true;
+}
+
+// Parse comma-separated list of event names. Empty means all available
+// events.
+std::vector<size_t> PerfEventsCounters::eventIndicesFromString(const std::string & events_list)
+{
+    std::vector<size_t> result;
+    result.reserve(NUMBER_OF_RAW_EVENTS);
+
+    if (events_list.empty())
+    {
+        for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+        {
+            result.push_back(i);
+        }
+        return result;
+    }
+
+    std::istringstream iss(events_list);
+    std::string event_name;
+    while (std::getline(iss, event_name, ','))
+    {
+        // Allow spaces at the beginning of the token, so that you can write
+        // 'a, b'.
+        event_name.erase(0, event_name.find_first_not_of(' '));
+
+        auto entry = event_name_to_index.find(event_name);
+        if (entry != event_name_to_index.end())
+        {
+            result.push_back(entry->second);
+        }
+        else
+        {
+            LOG_ERROR(&Poco::Logger::get("PerfEvents"),
+                "Unknown perf event name '{}' specified in settings", event_name);
+        }
+    }
+
+    return result;
+}
+
+void PerfEventsCounters::initializeProfileEvents(const std::string & events_list)
+{
+    if (!processThreadLocalChanges(events_list))
+        return;
+
+    for (int fd : thread_events_descriptors_holder.descriptors)
+    {
+        if (fd == -1)
+            continue;
+
+        // We don't reset the event, because the time_running and time_enabled
+        // can't be reset anyway and we have to calculate deltas.
+        enablePerfEvent(fd);
+    }
+}
+
+void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile_events)
+{
+    // Disable all perf events.
+    for (auto fd : thread_events_descriptors_holder.descriptors)
+    {
+        if (fd == -1)
+            continue;
+        disablePerfEvent(fd);
+    }
+
+    // Read the counter values.
+    PerfEventValue current_values[NUMBER_OF_RAW_EVENTS];
+    for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+    {
+        int fd = thread_events_descriptors_holder.descriptors[i];
+        if (fd == -1)
+            continue;
+
+        constexpr ssize_t bytes_to_read = sizeof(current_values[0]);
+        const int bytes_read = read(fd, &current_values[i], bytes_to_read);
+
+        if (bytes_read != bytes_to_read)
+        {
+            LOG_WARNING(&Poco::Logger::get("PerfEvents"),
+                "Can't read event value from file descriptor {}: '{}' ({})",
+                fd, strerror(errno), errno);
+            current_values[i] = {};
+        }
+    }
+
+    // actually process counters' values
+    for (size_t i = 0; i < NUMBER_OF_RAW_EVENTS; ++i)
+    {
+        int fd = thread_events_descriptors_holder.descriptors[i];
+        if (fd == -1)
+            continue;
+
+        const PerfEventInfo & info = raw_events_info[i];
+        const PerfEventValue & previous_value = previous_values[i];
+        const PerfEventValue & current_value = current_values[i];
+
+        // Account for counter multiplexing. time_running and time_enabled are
+        // not reset by PERF_EVENT_IOC_RESET, so we don't use it and calculate
+        // deltas from old values.
+        const UInt64 delta = (current_value.value - previous_value.value)
+            * (current_value.time_enabled - previous_value.time_enabled)
+            / std::max(1.f,
+                float(current_value.time_running - previous_value.time_running));
+
+        profile_events.increment(info.profile_event, delta);
+    }
+
+    // Store current counter values for the next profiling period.
+    memcpy(previous_values, current_values, sizeof(current_values));
+}
+
+void PerfEventsCounters::closeEventDescriptors()
+{
+    thread_events_descriptors_holder.releaseResources();
+}
+
+PerfDescriptorsHolder::PerfDescriptorsHolder()
+{
+    for (int & descriptor : descriptors)
+        descriptor = -1;
+}
+
+PerfDescriptorsHolder::~PerfDescriptorsHolder()
+{
+    releaseResources();
+}
+
+void PerfDescriptorsHolder::releaseResources()
+{
+    for (int & descriptor : descriptors)
+    {
+        if (descriptor == -1)
+            continue;
+
+        disablePerfEvent(descriptor);
+        releasePerfEvent(descriptor);
+        descriptor = -1;
+    }
+}
+
+}
+
+#else
+
+namespace DB
+{
+
+// Not on Linux or in Arcadia: the functionality is disabled.
+PerfEventsCounters current_thread_counters;
+
 }
 
 #endif

src/Common/ThreadProfileEvents.h

@@ -5,6 +5,7 @@
 #include <sys/time.h>
 #include <sys/resource.h>
 #include <pthread.h>
+#include <common/logger_useful.h>
 
 
 #if defined(__linux__)
@@ -34,6 +35,24 @@ namespace ProfileEvents
     extern const Event OSWriteChars;
     extern const Event OSReadBytes;
     extern const Event OSWriteBytes;
+
+    extern const Event PerfCpuCycles;
+    extern const Event PerfInstructions;
+    extern const Event PerfCacheReferences;
+    extern const Event PerfCacheMisses;
+    extern const Event PerfBranchInstructions;
+    extern const Event PerfBranchMisses;
+    extern const Event PerfBusCycles;
+    extern const Event PerfStalledCyclesFrontend;
+    extern const Event PerfStalledCyclesBackend;
+    extern const Event PerfRefCpuCycles;
+
+    extern const Event PerfCpuClock;
+    extern const Event PerfTaskClock;
+    extern const Event PerfContextSwitches;
+    extern const Event PerfCpuMigrations;
+    extern const Event PerfAlignmentFaults;
+    extern const Event PerfEmulationFaults;
 #endif
 }
 
@@ -116,6 +135,78 @@ struct RUsageCounters
     }
 };
 
+// thread_local is disabled in Arcadia, so we have to use a dummy implementation
+// there.
+#if defined(__linux__) && !defined(ARCADIA_BUILD)
+
+struct PerfEventInfo
+{
+    // see perf_event.h/perf_type_id enum
+    int event_type;
+    // see configs in perf_event.h
+    int event_config;
+    ProfileEvents::Event profile_event;
+    std::string settings_name;
+};
+
+struct PerfEventValue
+{
+    UInt64 value = 0;
+    UInt64 time_enabled = 0;
+    UInt64 time_running = 0;
+};
+
+static constexpr size_t NUMBER_OF_RAW_EVENTS = 16;
+
+struct PerfDescriptorsHolder : boost::noncopyable
+{
+    int descriptors[NUMBER_OF_RAW_EVENTS]{};
+
+    PerfDescriptorsHolder();
+
+    ~PerfDescriptorsHolder();
+
+    void releaseResources();
+};
+
+struct PerfEventsCounters
+{
+    PerfDescriptorsHolder thread_events_descriptors_holder;
+
+    // time_enabled and time_running can't be reset, so we have to store the
+    // data from the previous profiling period and calculate deltas to them,
+    // to be able to properly account for counter multiplexing.
+    PerfEventValue previous_values[NUMBER_OF_RAW_EVENTS]{};
+
+
+    void initializeProfileEvents(const std::string & events_list);
+    void finalizeProfileEvents(ProfileEvents::Counters & profile_events);
+    void closeEventDescriptors();
+    bool processThreadLocalChanges(const std::string & needed_events_list);
+
+
+    static std::vector<size_t> eventIndicesFromString(const std::string & events_list);
+};
+
+// Perf event creation is moderately heavy, so we create them once per thread and
+// then reuse.
+extern thread_local PerfEventsCounters current_thread_counters;
+
+#else
+
+// Not on Linux, or in Arcadia: the functionality is disabled.
+struct PerfEventsCounters
+{
+    void initializeProfileEvents(const std::string & /* events_list */) {}
+    void finalizeProfileEvents(ProfileEvents::Counters & /* profile_events */) {}
+    void closeEventDescriptors() {}
+};
+
+// thread_local is disabled in Arcadia, so we are going to use a static dummy.
+extern PerfEventsCounters current_thread_counters;
+
+#endif
+
 #if defined(__linux__)
 
 class TasksStatsCounters

src/Common/ThreadStatus.cpp

@@ -1,6 +1,5 @@
 #include <sstream>
 
-#include <Common/CurrentThread.h>
 #include <Common/Exception.h>
 #include <Common/ThreadProfileEvents.h>
 #include <Common/QueryProfiler.h>
@@ -57,36 +56,6 @@ ThreadStatus::~ThreadStatus()
     current_thread = nullptr;
 }
 
-void ThreadStatus::initPerformanceCounters()
-{
-    performance_counters_finalized = false;
-
-    /// Clear stats from previous query if a new query is started
-    /// TODO: make separate query_thread_performance_counters and thread_performance_counters
-    performance_counters.resetCounters();
-    memory_tracker.resetCounters();
-    memory_tracker.setDescription("(for thread)");
-
-    query_start_time_nanoseconds = getCurrentTimeNanoseconds();
-    query_start_time = time(nullptr);
-    ++queries_started;
-
-    *last_rusage = RUsageCounters::current(query_start_time_nanoseconds);
-    if (!taskstats)
-    {
-        try
-        {
-            taskstats = TasksStatsCounters::create(thread_id);
-        }
-        catch (...)
-        {
-            tryLogCurrentException(log);
-        }
-    }
-    if (taskstats)
-        taskstats->reset();
-}
-
 void ThreadStatus::updatePerformanceCounters()
 {
     try

src/Common/ThreadStatus.h

@@ -33,6 +33,7 @@ class QueryProfilerCpu;
 class QueryThreadLog;
 class TasksStatsCounters;
 struct RUsageCounters;
+struct PerfEventsCounters;
 class TaskStatsInfoGetter;
 class InternalTextLogsQueue;
 using InternalTextLogsQueuePtr = std::shared_ptr<InternalTextLogsQueue>;

src/Common/ZooKeeper/ZooKeeperHolder.cpp

@@ -5,7 +5,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int NULL_POINTER_DEREFERENCE;
+        extern const int LOGICAL_ERROR;
     }
 }
 
@@ -57,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe
 ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR);
 
     return zk_ptr.get();
 }
@@ -65,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
 const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR);
     return zk_ptr.get();
 }
 
 ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*()
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR);
     return *zk_ptr;
 }
 
 const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const
 {
     if (zk_ptr == nullptr)
-        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
+        throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::LOGICAL_ERROR);
     return *zk_ptr;
 }

src/Common/assert_cast.h

@@ -13,7 +13,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int BAD_CAST;
+        extern const int LOGICAL_ERROR;
     }
 }
 
@@ -41,11 +41,11 @@ To assert_cast(From && from)
     }
     catch (const std::exception & e)
     {
-        throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST);
+        throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
     }
 
     throw DB::Exception("Bad cast from type " + demangle(typeid(from).name()) + " to " + demangle(typeid(To).name()),
-                        DB::ErrorCodes::BAD_CAST);
+                        DB::ErrorCodes::LOGICAL_ERROR);
 #else
     return static_cast<To>(from);
 #endif

src/Common/typeid_cast.h

@@ -15,7 +15,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int BAD_CAST;
+        extern const int LOGICAL_ERROR;
     }
 }
 
@@ -34,11 +34,11 @@ std::enable_if_t<std::is_reference_v<To>, To> typeid_cast(From & from)
     }
     catch (const std::exception & e)
     {
-        throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST);
+        throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
     }
 
     throw DB::Exception("Bad cast from type " + demangle(typeid(from).name()) + " to " + demangle(typeid(To).name()),
-                        DB::ErrorCodes::BAD_CAST);
+                        DB::ErrorCodes::LOGICAL_ERROR);
 }
 
 
@@ -54,7 +54,7 @@ std::enable_if_t<std::is_pointer_v<To>, To> typeid_cast(From * from)
     }
     catch (const std::exception & e)
     {
-        throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST);
+        throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
     }
 }
 
@@ -71,6 +71,6 @@ std::enable_if_t<ext::is_shared_ptr_v<To>, To> typeid_cast(const std::shared_ptr
     }
     catch (const std::exception & e)
     {
-        throw DB::Exception(e.what(), DB::ErrorCodes::BAD_CAST);
+        throw DB::Exception(e.what(), DB::ErrorCodes::LOGICAL_ERROR);
     }
 }

src/Compression/CompressionCodecDelta.cpp

@@ -166,6 +166,9 @@ void registerCodecDelta(CompressionCodecFactory & factory)
 
             const auto children = arguments->children;
             const auto * literal = children[0]->as<ASTLiteral>();
+            if (!literal)
+                throw Exception("Delta codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
+
             size_t user_bytes_size = literal->value.safeGet<UInt64>();
             if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8)
                 throw Exception("Delta value for delta codec can be 1, 2, 4 or 8, given " + toString(user_bytes_size), ErrorCodes::ILLEGAL_CODEC_PARAMETER);

src/Compression/CompressionCodecDoubleDelta.cpp

@@ -166,6 +166,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
         throw Exception("Cannot compress, data size " + toString(source_size)
                         + " is not aligned to " + toString(sizeof(ValueType)), ErrorCodes::CANNOT_COMPRESS);
     const char * source_end = source + source_size;
+    const char * dest_start = dest;
 
     const UInt32 items_count = source_size / sizeof(ValueType);
     unalignedStore<UInt32>(dest, items_count);
@@ -229,7 +230,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
 
     writer.flush();
 
-    return sizeof(items_count) + sizeof(prev_value) + sizeof(prev_delta) + writer.count() / 8;
+    return (dest - dest_start) + (writer.count() + 7) / 8;
 }
 
 template <typename ValueType>
@@ -237,7 +238,6 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
 {
     static_assert(is_unsigned_v<ValueType>, "ValueType must be unsigned.");
     using UnsignedDeltaType = ValueType;
-    using SignedDeltaType = typename std::make_signed<UnsignedDeltaType>::type;
 
     const char * source_end = source + source_size;
 
@@ -286,12 +286,13 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
         if (write_spec.data_bits != 0)
         {
             const UInt8 sign = reader.readBit();
-            SignedDeltaType signed_dd = static_cast<SignedDeltaType>(reader.readBits(write_spec.data_bits - 1) + 1);
+            double_delta = reader.readBits(write_spec.data_bits - 1) + 1;
             if (sign)
             {
-                signed_dd *= -1;
+                /// It's well defined for unsigned data types.
+                /// In constrast, it's undefined to do negation of the most negative signed number due to overflow.
+                double_delta = -double_delta;
             }
-            double_delta = static_cast<UnsignedDeltaType>(signed_dd);
         }
 
         const UnsignedDeltaType delta = double_delta + prev_delta;

src/Compression/CompressionCodecGorilla.cpp

@@ -90,6 +90,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest,
     if (source_size % sizeof(T) != 0)
         throw Exception("Cannot compress, data size " + toString(source_size) + " is not aligned to " + toString(sizeof(T)), ErrorCodes::CANNOT_COMPRESS);
     const char * source_end = source + source_size;
+    const char * dest_start = dest;
     const char * dest_end = dest + dest_size;
 
     const UInt32 items_count = source_size / sizeof(T);
@@ -145,7 +146,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest,
 
     writer.flush();
 
-    return sizeof(items_count) + sizeof(prev_value) + writer.count() / 8;
+    return (dest - dest_start) + (writer.count() + 7) / 8;
 }
 
 template <typename T>

src/Compression/CompressionCodecLZ4.cpp

@@ -19,6 +19,7 @@ namespace ErrorCodes
 {
 extern const int CANNOT_COMPRESS;
 extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
+extern const int ILLEGAL_CODEC_PARAMETER;
 }
 
 
@@ -84,6 +85,9 @@ void registerCodecLZ4HC(CompressionCodecFactory & factory)
 
             const auto children = arguments->children;
             const auto * literal = children[0]->as<ASTLiteral>();
+            if (!literal)
+                throw Exception("LZ4HC codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
+
             level = literal->value.safeGet<UInt64>();
         }
 

src/Compression/CompressionCodecZSTD.cpp

@@ -74,6 +74,9 @@ void registerCodecZSTD(CompressionCodecFactory & factory)
 
             const auto children = arguments->children;
             const auto * literal = children[0]->as<ASTLiteral>();
+            if (!literal)
+                throw Exception("ZSTD codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
+
             level = literal->value.safeGet<UInt64>();
             if (level > ZSTD_maxCLevel())
                 throw Exception("ZSTD codec can't have level more that " + toString(ZSTD_maxCLevel()) + ", given " + toString(level), ErrorCodes::ILLEGAL_CODEC_PARAMETER);

src/Compression/ICompressionCodec.cpp

@@ -21,6 +21,8 @@ namespace ErrorCodes
 
 UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char * dest) const
 {
+    assert(source != nullptr && dest != nullptr);
+
     dest[0] = getMethodByte();
     UInt8 header_size = getHeaderSize();
     /// Write data from header_size
@@ -33,8 +35,9 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char
 
 UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, char * dest) const
 {
-    UInt8 header_size = getHeaderSize();
+    assert(source != nullptr && dest != nullptr);
 
+    UInt8 header_size = getHeaderSize();
     if (source_size < header_size)
         throw Exception("Can't decompress data: the compressed data size (" + toString(source_size)
             + ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);

src/Compression/tests/gtest_compressionCodec.cpp

@@ -220,7 +220,7 @@ template <typename T, typename ContainerLeft, typename ContainerRight>
 
     if (l_size != r_size)
     {
-        result = ::testing::AssertionFailure() << "size mismatch expected: " << l_size << " got:" << r_size;
+        result = ::testing::AssertionFailure() << "size mismatch" << " expected: " << l_size << " got:" << r_size;
     }
     if (l_size == 0 || r_size == 0)
     {
@@ -403,11 +403,6 @@ CodecTestSequence generateSeq(Generator gen, const char* gen_name, B Begin = 0,
     {
         const T v = gen(static_cast<T>(i));
 
-//        if constexpr (debug_log_items)
-//        {
-//            std::cerr << "#" << i << " " << type_name<T>() << "(" << sizeof(T) << " bytes) : " << v << std::endl;
-//        }
-
         unalignedStore<T>(write_pos, v);
         write_pos += sizeof(v);
     }
@@ -483,6 +478,7 @@ void testTranscoding(Timer & timer, ICompressionCodec & codec, const CodecTestSe
 
     timer.start();
 
+    assert(source_data.data() != nullptr); // Codec assumes that source buffer is not null.
     const UInt32 encoded_size = codec.compress(source_data.data(), source_data.size(), encoded.data());
     timer.report("encoding");
 
@@ -800,7 +796,8 @@ std::vector<CodecTestSequence> generatePyramidOfSequences(const size_t sequences
     std::vector<CodecTestSequence> sequences;
     sequences.reserve(sequences_count);
 
-    sequences.push_back(makeSeq<T>()); // sequence of size 0
+    // Don't test against sequence of size 0, since it causes a nullptr source buffer as codec input and produces an error.
+    // sequences.push_back(makeSeq<T>()); // sequence of size 0
     for (size_t i = 1; i < sequences_count; ++i)
     {
         std::string name = generator_name + std::string(" from 0 to ") + std::to_string(i);

src/Core/Settings.h

@@ -28,6 +28,7 @@ class IColumn;
 
 
 /** Settings of query execution.
+  * These settings go to users.xml.
   */
 struct Settings : public SettingsCollection<Settings>
 {
@@ -276,6 +277,8 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
     M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
+    M(SettingBool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
+    M(SettingString, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \
     \
     \
     /** Limits during query execution are part of the settings. \
@@ -385,6 +388,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, enable_debug_queries, false, "Enables debug queries such as AST.", 0) \
     M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \
     M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
+    M(SettingBool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \
     M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
     M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
     M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \
@@ -411,7 +415,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \
     M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \
     M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
-    M(SettingBool, optimize_arithmetic_operations_in_agr_func, true, "Removing arithmetic operations from aggregation functions", 0) \
+    M(SettingBool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \
     M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
     M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \
     M(SettingBool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \
@@ -441,6 +445,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingUInt64, mark_cache_min_lifetime, 0, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
     M(SettingBool, partial_merge_join, false, "Obsolete. Use join_algorithm='prefer_partial_merge' instead.", 0) \
     M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \
+    \
     M(SettingBool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \
 
     DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)

src/Core/SortCursor.h

@@ -63,7 +63,7 @@ struct SortCursorImpl
         for (auto & column_desc : desc)
         {
             if (!column_desc.column_name.empty())
-                throw Exception("SortDesctiption should contain column position if SortCursor was used without header.",
+                throw Exception("SortDescription should contain column position if SortCursor was used without header.",
                         ErrorCodes::LOGICAL_ERROR);
         }
         reset(columns, {});

src/Core/SortDescription.h

@@ -59,6 +59,13 @@ struct SortColumnDescription
     {
         return !(*this == other);
     }
+
+    std::string dump() const
+    {
+        std::stringstream ss;
+        ss << column_name << ":" << column_number << ":dir " << direction << "nulls " << nulls_direction;
+        return ss.str();
+    }
 };
 
 /// Description of the sorting rule for several columns.

src/DataTypes/DataTypeAggregateFunction.cpp

@@ -362,8 +362,11 @@ static DataTypePtr create(const ASTPtr & arguments)
         {
             const auto * literal = parameters[i]->as<ASTLiteral>();
             if (!literal)
-                throw Exception("Parameters to aggregate functions must be literals",
+                throw Exception(
-                    ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS);
+                    ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
+                    "Parameters to aggregate functions must be literals. "
+                    "Got parameter '{}' for function '{}'",
+                    parameters[i]->formatForErrorMessage(), function_name);
 
             params_row[i] = literal->value;
         }

src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp

@@ -82,8 +82,11 @@ static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const ASTPtr & argum
         {
             const ASTLiteral * lit = parameters[i]->as<ASTLiteral>();
             if (!lit)
-                throw Exception("Parameters to aggregate functions must be literals",
+                throw Exception(
-                                ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS);
+                    ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
+                    "Parameters to aggregate functions must be literals. "
+                    "Got parameter '{}' for function '{}'",
+                    parameters[i]->formatForErrorMessage(), function_name);
 
             params_row[i] = lit->value;
         }

src/Databases/DatabaseOnDisk.cpp

@@ -294,7 +294,7 @@ void DatabaseOnDisk::renameTable(
     {
         attachTable(table_name, table, table_data_relative_path);
         /// Better diagnostics.
-        throw Exception{Exception::CreateFromPoco, e};
+        throw Exception{Exception::CreateFromPocoTag{}, e};
     }
 
     /// Now table data are moved to new database, so we must add metadata and attach table to new database

src/Functions/PolygonUtils.h

@@ -358,6 +358,9 @@ bool PointInPolygonWithGrid<CoordinateType>::contains(CoordinateType x, Coordina
     if (has_empty_bound)
         return false;
 
+    if (std::isnan(x) || std::isnan(y))
+        return false;
+
     CoordinateType float_row = (y + y_shift) * y_scale;
     CoordinateType float_col = (x + x_shift) * x_scale;
 

src/Functions/URL/FunctionsURL.h

@@ -21,6 +21,7 @@ namespace DB
   *  queryString
   *  fragment
   *  queryStringAndFragment
+  *  netloc
   *
   * Functions, removing parts from URL.
   * If URL has nothing like, then it is returned unchanged.

src/Functions/URL/netloc.cpp

@@ -0,0 +1,17 @@
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionStringToString.h>
+#include "netloc.h"
+
+namespace DB
+{
+
+struct NameNetloc { static constexpr auto name = "netloc"; };
+using FunctionNetloc = FunctionStringToString<ExtractSubstringImpl<ExtractNetloc>, NameNetloc>;
+
+void registerFunctionNetloc(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionNetloc>();
+}
+
+}
+

src/Functions/URL/netloc.h

@@ -0,0 +1,129 @@
+#pragma once
+
+#include "FunctionsURL.h"
+#include <common/find_symbols.h>
+#include "protocol.h"
+#include <cstring>
+#include <Common/StringUtils/StringUtils.h>
+
+
+namespace DB
+{
+
+struct ExtractNetloc
+{
+    /// We use the same as domain function
+    static size_t getReserveLengthForElement() { return 15; }
+
+    static inline StringRef getNetworkLocation(const char * data, size_t size)
+    {
+        Pos pos = data;
+        Pos end = data + size;
+
+        if (*pos == '/' && *(pos + 1) == '/')
+        {
+            pos += 2;
+        }
+        else
+        {
+            Pos scheme_end = data + std::min(size, 16UL);
+            for (++pos; pos < scheme_end; ++pos)
+            {
+                if (!isAlphaNumericASCII(*pos))
+                {
+                    switch (*pos)
+                    {
+                        case '.':
+                        case '-':
+                        case '+':
+                            break;
+                        case ' ': /// restricted symbols
+                        case '\t':
+                        case '<':
+                        case '>':
+                        case '%':
+                        case '{':
+                        case '}':
+                        case '|':
+                        case '\\':
+                        case '^':
+                        case '~':
+                        case '[':
+                        case ']':
+                        case ';':
+                        case '=':
+                        case '&':
+                            return StringRef{};
+                        default:
+                            goto exloop;
+                    }
+                }
+            }
+exloop: if ((scheme_end - pos) > 2 && *pos == ':' && *(pos + 1) == '/' && *(pos + 2) == '/')
+            pos += 3;
+        else
+            pos = data;
+        }
+
+        bool has_identification = false;
+        Pos question_mark_pos = end;
+        Pos slash_pos = end;
+        auto start_of_host = pos;
+        for (; pos < end; ++pos)
+        {
+            switch (*pos)
+            {
+                case '/':
+                    if (has_identification)
+                        return StringRef(start_of_host, pos - start_of_host);
+                    else
+                        slash_pos = pos;
+                    break;
+                case '?':
+                    if (has_identification)
+                        return StringRef(start_of_host, pos - start_of_host);
+                    else
+                        question_mark_pos = pos;
+                    break;
+                case '#':
+                    return StringRef(start_of_host, pos - start_of_host);
+                case '@': /// foo:bar@example.ru
+                    has_identification = true;
+                    break;
+                case ' ': /// restricted symbols in whole URL
+                case '\t':
+                case '<':
+                case '>':
+                case '%':
+                case '{':
+                case '}':
+                case '|':
+                case '\\':
+                case '^':
+                case '~':
+                case '[':
+                case ']':
+                case ';':
+                case '=':
+                case '&':
+                    return StringRef(start_of_host, std::min(std::min(pos - 1, question_mark_pos), slash_pos) - start_of_host);
+            }
+        }
+
+        if (has_identification)
+            return StringRef(start_of_host, pos - start_of_host);
+        else
+            return StringRef(start_of_host, std::min(std::min(pos, question_mark_pos), slash_pos) - start_of_host);
+    }
+
+    static void execute(Pos data, size_t size, Pos & res_data, size_t & res_size)
+    {
+        StringRef host = getNetworkLocation(data, size);
+
+        res_data = host.data;
+        res_size = host.size;
+    }
+};
+
+}
+

src/Functions/URL/registerFunctionsURL.cpp

@@ -26,6 +26,7 @@ void registerFunctionCutFragment(FunctionFactory & factory);
 void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory);
 void registerFunctionCutURLParameter(FunctionFactory & factory);
 void registerFunctionDecodeURLComponent(FunctionFactory & factory);
+void registerFunctionNetloc(FunctionFactory & factory);
 
 void registerFunctionsURL(FunctionFactory & factory)
 {
@@ -52,6 +53,7 @@ void registerFunctionsURL(FunctionFactory & factory)
     registerFunctionCutQueryStringAndFragment(factory);
     registerFunctionCutURLParameter(factory);
     registerFunctionDecodeURLComponent(factory);
+    registerFunctionNetloc(factory);
 }
 
 }

src/Functions/bitBoolMaskAnd.cpp

@@ -7,7 +7,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int BAD_CAST;
+        extern const int BAD_ARGUMENTS;
     }
 
     /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h).
@@ -23,8 +23,10 @@ namespace DB
         template <typename Result = ResultType>
         static inline Result apply(A left, B right)
         {
+            // Should be a logical error, but this function is callable from SQL.
+            // Need to investigate this.
             if constexpr (!std::is_same_v<A, ResultType> || !std::is_same_v<B, ResultType>)
-                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskAnd.", ErrorCodes::BAD_CAST);
+                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskAnd.", ErrorCodes::BAD_ARGUMENTS);
             return static_cast<ResultType>(
                     ((static_cast<ResultType>(left) & static_cast<ResultType>(right)) & 1)
                     | ((((static_cast<ResultType>(left) >> 1) | (static_cast<ResultType>(right) >> 1)) & 1) << 1));

src/Functions/bitBoolMaskOr.cpp

@@ -7,7 +7,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int BAD_CAST;
+        extern const int BAD_ARGUMENTS;
     }
 
     /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h).
@@ -24,7 +24,9 @@ namespace DB
         static inline Result apply(A left, B right)
         {
             if constexpr (!std::is_same_v<A, ResultType> || !std::is_same_v<B, ResultType>)
-                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskOr.", ErrorCodes::BAD_CAST);
+                // Should be a logical error, but this function is callable from SQL.
+                // Need to investigate this.
+                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitBoolMaskOr.", ErrorCodes::BAD_ARGUMENTS);
             return static_cast<ResultType>(
                     ((static_cast<ResultType>(left) | static_cast<ResultType>(right)) & 1)
                     | ((((static_cast<ResultType>(left) >> 1) & (static_cast<ResultType>(right) >> 1)) & 1) << 1));

src/Functions/bitSwapLastTwo.cpp

@@ -7,7 +7,7 @@ namespace DB
     namespace ErrorCodes
     {
         extern const int LOGICAL_ERROR;
-        extern const int BAD_CAST;
+        extern const int BAD_ARGUMENTS;
     }
 
     /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h).
@@ -21,7 +21,9 @@ namespace DB
         static inline ResultType NO_SANITIZE_UNDEFINED apply(A a)
         {
             if constexpr (!std::is_same_v<A, ResultType>)
-                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitSwapLastTwo.", ErrorCodes::BAD_CAST);
+                // Should be a logical error, but this function is callable from SQL.
+                // Need to investigate this.
+                throw DB::Exception("It's a bug! Only UInt8 type is supported by __bitSwapLastTwo.", ErrorCodes::BAD_ARGUMENTS);
             return static_cast<ResultType>(
                     ((static_cast<ResultType>(a) & 1) << 1) | ((static_cast<ResultType>(a) >> 1) & 1));
         }

src/Functions/bitWrapperFunc.cpp

@@ -6,7 +6,7 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int BAD_CAST;
+        extern const int BAD_ARGUMENTS;
     }
 
     /// Working with UInt8: last bit = can be true, previous = can be false (Like src/Storages/MergeTree/BoolMask.h).
@@ -20,8 +20,10 @@ namespace DB
 
         static inline ResultType NO_SANITIZE_UNDEFINED apply(A a)
         {
+            // Should be a logical error, but this function is callable from SQL.
+            // Need to investigate this.
             if constexpr (!is_integral_v<A>)
-                throw DB::Exception("It's a bug! Only integer types are supported by __bitWrapperFunc.", ErrorCodes::BAD_CAST);
+                throw DB::Exception("It's a bug! Only integer types are supported by __bitWrapperFunc.", ErrorCodes::BAD_ARGUMENTS);
             return a == 0 ? static_cast<ResultType>(0b10) : static_cast<ResultType >(0b1);
         }
 

src/Functions/ya.make

@@ -423,6 +423,7 @@ SRCS(
     URL/extractURLParameters.cpp
     URL/firstSignificantSubdomain.cpp
     URL/fragment.cpp
+    URL/netloc.cpp
     URL/path.cpp
     URL/pathFull.cpp
     URL/port.cpp

src/IO/S3Common.cpp

@@ -4,6 +4,7 @@
 
 #    include <IO/S3Common.h>
 #    include <IO/WriteBufferFromString.h>
+#    include <Storages/StorageS3Settings.h>
 
 #    include <aws/core/auth/AWSCredentialsProvider.h>
 #    include <aws/core/utils/logging/LogMacros.h>
@@ -60,6 +61,47 @@ public:
 private:
     Poco::Logger * log = &Poco::Logger::get("AWSClient");
 };
+
+class S3AuthSigner : public Aws::Client::AWSAuthV4Signer
+{
+public:
+    S3AuthSigner(
+        const Aws::Client::ClientConfiguration & client_configuration,
+        const Aws::Auth::AWSCredentials & credentials,
+        const DB::HeaderCollection & headers_)
+        : Aws::Client::AWSAuthV4Signer(
+            std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(credentials),
+            "s3",
+            client_configuration.region,
+            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+            false)
+        , headers(headers_)
+    {
+    }
+
+    bool SignRequest(Aws::Http::HttpRequest & request, const char * region, bool sign_body) const override
+    {
+        auto result = Aws::Client::AWSAuthV4Signer::SignRequest(request, region, sign_body);
+        for (const auto & header : headers)
+            request.SetHeaderValue(header.name, header.value);
+        return result;
+    }
+
+    bool PresignRequest(
+        Aws::Http::HttpRequest & request,
+        const char * region,
+        const char * serviceName,
+        long long expiration_time_sec) const override // NOLINT
+    {
+        auto result = Aws::Client::AWSAuthV4Signer::PresignRequest(request, region, serviceName, expiration_time_sec);
+        for (const auto & header : headers)
+            request.SetHeaderValue(header.name, header.value);
+        return result;
+    }
+
+private:
+    const DB::HeaderCollection headers;
+};
 }
 
 namespace DB
@@ -139,6 +181,25 @@ namespace S3
         );
     }
 
+    std::shared_ptr<Aws::S3::S3Client> ClientFactory::create( // NOLINT
+        const String & endpoint,
+        bool is_virtual_hosted_style,
+        const String & access_key_id,
+        const String & secret_access_key,
+        HeaderCollection headers)
+    {
+        Aws::Client::ClientConfiguration cfg;
+        if (!endpoint.empty())
+            cfg.endpointOverride = endpoint;
+
+        Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key);
+        return std::make_shared<Aws::S3::S3Client>(
+            std::make_shared<S3AuthSigner>(cfg, std::move(credentials), std::move(headers)),
+            std::move(cfg), // Client configuration.
+            is_virtual_hosted_style || cfg.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified.
+        );
+    }
+
     URI::URI(const Poco::URI & uri_)
     {
         /// Case when bucket name represented in domain name of S3 URL.

src/IO/S3Common.h

@@ -5,7 +5,7 @@
 #if USE_AWS_S3
 
 #include <Core/Types.h>
-#include <Poco/URI.h>
+#include <Interpreters/Context.h>
 #include <aws/core/Aws.h>
 
 namespace Aws::S3
@@ -13,6 +13,12 @@ namespace Aws::S3
     class S3Client;
 }
 
+namespace DB
+{
+    struct HttpHeader;
+    using HeaderCollection = std::vector<HttpHeader>;
+}
+
 namespace DB::S3
 {
 
@@ -34,6 +40,14 @@ public:
         bool is_virtual_hosted_style,
         const String & access_key_id,
         const String & secret_access_key);
+
+    std::shared_ptr<Aws::S3::S3Client> create(
+        const String & endpoint,
+        bool is_virtual_hosted_style,
+        const String & access_key_id,
+        const String & secret_access_key,
+        HeaderCollection headers);
+
 private:
     ClientFactory();
 

src/Interpreters/ActionsVisitor.cpp

@@ -512,7 +512,8 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
                 if (data.only_consts)
                     arguments_present = false;
                 else
-                    throw Exception("Unknown identifier: " + child_column_name, ErrorCodes::UNKNOWN_IDENTIFIER);
+                    throw Exception("Unknown identifier: " + child_column_name + " there are columns: " + data.getSampleBlock().dumpNames(),
+                                    ErrorCodes::UNKNOWN_IDENTIFIER);
             }
         }
     }

src/Interpreters/Aggregator.cpp

@@ -530,63 +530,33 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl(
 }
 
 
-bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & result,
+void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl(
-    ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys)
+        AggregatedDataWithoutKey & res,
+        size_t row_begin,
+        size_t row_end,
+        AggregateFunctionInstruction * aggregate_instructions,
+        Arena * arena)
 {
-    UInt64 num_rows = block.rows();
+    /// Adding values
-    return executeOnBlock(block.getColumns(), num_rows, result, key_columns, aggregate_columns, no_more_keys);
+    for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst)
+    {
+        if (inst->offsets)
+            inst->batch_that->addBatchSinglePlaceFromInterval(inst->offsets[row_begin], inst->offsets[row_end - 1], res + inst->state_offset, inst->batch_arguments, arena);
+        else
+            inst->batch_that->addBatchSinglePlaceFromInterval(row_begin, row_end, res + inst->state_offset, inst->batch_arguments, arena);
+    }
 }
 
-bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result,
+
-    ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys)
+void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns,
+                                              AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder)
 {
-    if (isCancelled())
-        return true;
-
-    /// `result` will destroy the states of aggregate functions in the destructor
-    result.aggregator = this;
-
-    /// How to perform the aggregation?
-    if (result.empty())
-    {
-        result.init(method_chosen);
-        result.keys_size = params.keys_size;
-        result.key_sizes = key_sizes;
-        LOG_TRACE(log, "Aggregation method: {}", result.getMethodName());
-    }
-
-    if (isCancelled())
-        return true;
-
     for (size_t i = 0; i < params.aggregates_size; ++i)
         aggregate_columns[i].resize(params.aggregates[i].arguments.size());
 
-    /** Constant columns are not supported directly during aggregation.
+    aggregate_functions_instructions.resize(params.aggregates_size + 1);
-      * To make them work anyway, we materialize them.
-      */
-    Columns materialized_columns;
-
-    /// Remember the columns we will work with
-    for (size_t i = 0; i < params.keys_size; ++i)
-    {
-        materialized_columns.push_back(columns.at(params.keys[i])->convertToFullColumnIfConst());
-        key_columns[i] = materialized_columns.back().get();
-
-        if (!result.isLowCardinality())
-        {
-            auto column_no_lc = recursiveRemoveLowCardinality(key_columns[i]->getPtr());
-            if (column_no_lc.get() != key_columns[i])
-            {
-                materialized_columns.emplace_back(std::move(column_no_lc));
-                key_columns[i] = materialized_columns.back().get();
-            }
-        }
-    }
-
-    AggregateFunctionInstructions aggregate_functions_instructions(params.aggregates_size + 1);
     aggregate_functions_instructions[params.aggregates_size].that = nullptr;
 
-    std::vector<std::vector<const IColumn *>> nested_columns_holder;
     for (size_t i = 0; i < params.aggregates_size; ++i)
     {
         for (size_t j = 0; j < aggregate_columns[i].size(); ++j)
@@ -627,6 +597,62 @@ bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedData
 
         aggregate_functions_instructions[i].batch_that = that;
     }
+}
+
+
+bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & result,
+                                ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys)
+{
+    UInt64 num_rows = block.rows();
+    return executeOnBlock(block.getColumns(), num_rows, result, key_columns, aggregate_columns, no_more_keys);
+}
+
+
+bool Aggregator::executeOnBlock(Columns columns, UInt64 num_rows, AggregatedDataVariants & result,
+    ColumnRawPtrs & key_columns, AggregateColumns & aggregate_columns, bool & no_more_keys)
+{
+    if (isCancelled())
+        return true;
+
+    /// `result` will destroy the states of aggregate functions in the destructor
+    result.aggregator = this;
+
+    /// How to perform the aggregation?
+    if (result.empty())
+    {
+        result.init(method_chosen);
+        result.keys_size = params.keys_size;
+        result.key_sizes = key_sizes;
+        LOG_TRACE(log, "Aggregation method: {}", result.getMethodName());
+    }
+
+    if (isCancelled())
+        return true;
+
+    /** Constant columns are not supported directly during aggregation.
+      * To make them work anyway, we materialize them.
+      */
+    Columns materialized_columns;
+
+    /// Remember the columns we will work with
+    for (size_t i = 0; i < params.keys_size; ++i)
+    {
+        materialized_columns.push_back(columns.at(params.keys[i])->convertToFullColumnIfConst());
+        key_columns[i] = materialized_columns.back().get();
+
+        if (!result.isLowCardinality())
+        {
+            auto column_no_lc = recursiveRemoveLowCardinality(key_columns[i]->getPtr());
+            if (column_no_lc.get() != key_columns[i])
+            {
+                materialized_columns.emplace_back(std::move(column_no_lc));
+                key_columns[i] = materialized_columns.back().get();
+            }
+        }
+    }
+    NestedColumnsHolder nested_columns_holder;
+    AggregateFunctionInstructions aggregate_functions_instructions;
+    prepareAggregateInstructions(columns, aggregate_columns, materialized_columns, aggregate_functions_instructions, nested_columns_holder);
 
     if (isCancelled())
         return true;
@@ -1112,7 +1138,39 @@ Block Aggregator::prepareBlockAndFill(
 
     return res;
 }
+void Aggregator::fillAggregateColumnsWithSingleKey(
+    AggregatedDataVariants & data_variants,
+    MutableColumns & final_aggregate_columns)
+{
+    AggregatedDataWithoutKey & data = data_variants.without_key;
 
+    for (size_t i = 0; i < params.aggregates_size; ++i)
+    {
+        ColumnAggregateFunction & column_aggregate_func = assert_cast<ColumnAggregateFunction &>(*final_aggregate_columns[i]);
+        for (auto & pool : data_variants.aggregates_pools)
+        {
+            column_aggregate_func.addArena(pool);
+        }
+        column_aggregate_func.getData().push_back(data + offsets_of_aggregate_states[i]);
+    }
+    data = nullptr;
+}
+
+void Aggregator::createStatesAndFillKeyColumnsWithSingleKey(
+    AggregatedDataVariants & data_variants,
+    Columns & key_columns,
+    size_t key_row,
+    MutableColumns & final_key_columns)
+{
+    AggregateDataPtr place = data_variants.aggregates_pool->alignedAlloc(total_size_of_aggregate_states, align_aggregate_states);
+    createAggregateStates(place);
+    data_variants.without_key = place;
+
+    for (size_t i = 0; i < params.keys_size; ++i)
+    {
+        final_key_columns[i]->insertFrom(*key_columns[i].get(), key_row);
+    }
+}
 
 Block Aggregator::prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const
 {

src/Interpreters/Aggregator.h

@@ -1002,6 +1002,7 @@ protected:
     friend class MergingAndConvertingBlockInputStream;
     friend class ConvertingAggregatedToChunksTransform;
     friend class ConvertingAggregatedToChunksSource;
+    friend class AggregatingInOrderTransform;
 
     Params params;
 
@@ -1033,12 +1034,13 @@ protected:
     };
 
     using AggregateFunctionInstructions = std::vector<AggregateFunctionInstruction>;
+    using NestedColumnsHolder = std::vector<std::vector<const IColumn *>>;
 
     Sizes offsets_of_aggregate_states;    /// The offset to the n-th aggregate function in a row of aggregate functions.
     size_t total_size_of_aggregate_states = 0;    /// The total size of the row from the aggregate functions.
 
     // add info to track alignment requirement
-    // If there are states whose alignmentment are v1, ..vn, align_aggregate_states will be max(v1, ... vn)
+    // If there are states whose alignment are v1, ..vn, align_aggregate_states will be max(v1, ... vn)
     size_t align_aggregate_states = 1;
 
     bool all_aggregates_has_trivial_destructor = false;
@@ -1105,6 +1107,13 @@ protected:
         AggregateFunctionInstruction * aggregate_instructions,
         Arena * arena);
 
+    static void executeOnIntervalWithoutKeyImpl(
+        AggregatedDataWithoutKey & res,
+        size_t row_begin,
+        size_t row_end,
+        AggregateFunctionInstruction * aggregate_instructions,
+        Arena * arena);
+
     template <typename Method>
     void writeToTemporaryFileImpl(
         AggregatedDataVariants & data_variants,
@@ -1250,6 +1259,22 @@ protected:
       * - sets the variable no_more_keys to true.
       */
     bool checkLimits(size_t result_size, bool & no_more_keys) const;
+
+    void prepareAggregateInstructions(
+        Columns columns,
+        AggregateColumns & aggregate_columns,
+        Columns & materialized_columns,
+        AggregateFunctionInstructions & instructions,
+        NestedColumnsHolder & nested_columns_holder);
+
+    void fillAggregateColumnsWithSingleKey(
+        AggregatedDataVariants & data_variants,
+        MutableColumns & final_aggregate_columns);
+
+    void createStatesAndFillKeyColumnsWithSingleKey(
+        AggregatedDataVariants & data_variants,
+        Columns & key_columns, size_t key_row,
+        MutableColumns & final_key_columns);
 };
 
 

src/Interpreters/CollectJoinOnKeysVisitor.h

@@ -26,8 +26,8 @@ public:
     struct Data
     {
         TableJoin & analyzed_join;
-        const TableWithColumnNames & left_table;
+        const TableWithColumnNamesAndTypes & left_table;
-        const TableWithColumnNames & right_table;
+        const TableWithColumnNamesAndTypes & right_table;
         const Aliases & aliases;
         const bool is_asof{false};
         ASTPtr asof_left_key{};

src/Interpreters/Context.cpp

@@ -22,6 +22,7 @@
 #include <Storages/MergeTree/MergeList.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/CompressionCodecSelector.h>
+#include <Storages/StorageS3Settings.h>
 #include <Disks/DiskLocal.h>
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Interpreters/ActionLocksManager.h>
@@ -101,7 +102,6 @@ namespace ErrorCodes
     extern const int SESSION_NOT_FOUND;
     extern const int SESSION_IS_LOCKED;
     extern const int LOGICAL_ERROR;
-    extern const int UNKNOWN_SCALAR;
     extern const int AUTHENTICATION_FAILED;
     extern const int NOT_IMPLEMENTED;
 }
@@ -351,6 +351,7 @@ struct ContextShared
     String format_schema_path;                              /// Path to a directory that contains schema files used by input formats.
     ActionLocksManagerPtr action_locks_manager;             /// Set of storages' action lockers
     std::optional<SystemLogs> system_logs;                  /// Used to log queries and operations on parts
+    std::optional<StorageS3Settings> storage_s3_settings;   /// Settings of S3 storage
 
     RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml
 
@@ -821,7 +822,11 @@ const Block & Context::getScalar(const String & name) const
 {
     auto it = scalars.find(name);
     if (scalars.end() == it)
-        throw Exception("Scalar " + backQuoteIfNeed(name) + " doesn't exist (internal bug)", ErrorCodes::UNKNOWN_SCALAR);
+    {
+        // This should be a logical error, but it fails the sql_fuzz test too
+        // often, so 'bad arguments' for now.
+        throw Exception("Scalar " + backQuoteIfNeed(name) + " doesn't exist (internal bug)", ErrorCodes::BAD_ARGUMENTS);
+    }
     return it->second;
 }
 
@@ -1764,6 +1769,11 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration
             LOG_ERROR(shared->log, "An error has occured while reloading storage policies, storage policies were not applied: {}", e.message());
         }
     }
+
+    if (shared->storage_s3_settings)
+    {
+        shared->storage_s3_settings->loadFromConfig("s3", config);
+    }
 }
 
 
@@ -1782,6 +1792,18 @@ const MergeTreeSettings & Context::getMergeTreeSettings() const
     return *shared->merge_tree_settings;
 }
 
+const StorageS3Settings & Context::getStorageS3Settings() const
+{
+    auto lock = getLock();
+
+    if (!shared->storage_s3_settings)
+    {
+        const auto & config = getConfigRef();
+        shared->storage_s3_settings.emplace().loadFromConfig("s3", config);
+    }
+
+    return *shared->storage_s3_settings;
+}
 
 void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const
 {

src/Interpreters/Context.h

@@ -81,6 +81,7 @@ class TextLog;
 class TraceLog;
 class MetricLog;
 struct MergeTreeSettings;
+class StorageS3Settings;
 class IDatabase;
 class DDLWorker;
 class ITableFunction;
@@ -531,6 +532,7 @@ public:
     std::shared_ptr<PartLog> getPartLog(const String & part_database);
 
     const MergeTreeSettings & getMergeTreeSettings() const;
+    const StorageS3Settings & getStorageS3Settings() const;
 
     /// Prevents DROP TABLE if its size is greater than max_size (50GB by default, max_size=0 turn off this check)
     void setMaxTableSizeToDrop(size_t max_size);

src/Interpreters/DatabaseAndTableWithAlias.h

@@ -45,34 +45,6 @@ struct DatabaseAndTableWithAlias
     }
 };
 
-struct TableWithColumnNames
-{
-    DatabaseAndTableWithAlias table;
-    Names columns;
-    Names hidden_columns; /// Not general columns like MATERIALIZED and ALIAS. They are omitted in * and t.* results.
-
-    TableWithColumnNames(const DatabaseAndTableWithAlias & table_, const Names & columns_)
-        : table(table_)
-        , columns(columns_)
-    {
-        columns_set.insert(columns.begin(), columns.end());
-    }
-
-    TableWithColumnNames(const DatabaseAndTableWithAlias table_, Names && columns_, Names && hidden_columns_)
-        : table(table_)
-        , columns(columns_)
-        , hidden_columns(hidden_columns_)
-    {
-        columns_set.insert(columns.begin(), columns.end());
-        columns_set.insert(hidden_columns.begin(), hidden_columns.end());
-    }
-
-    bool hasColumn(const String & name) const { return columns_set.count(name); }
-
-private:
-    NameSet columns_set;
-};
-
 struct TableWithColumnNamesAndTypes
 {
     DatabaseAndTableWithAlias table;
@@ -96,21 +68,6 @@ struct TableWithColumnNamesAndTypes
             names.insert(col.name);
     }
 
-    TableWithColumnNames removeTypes() const
-    {
-        Names out_columns;
-        out_columns.reserve(columns.size());
-        for (auto & col : columns)
-            out_columns.push_back(col.name);
-
-        Names out_hidden_columns;
-        out_hidden_columns.reserve(hidden_columns.size());
-        for (auto & col : hidden_columns)
-            out_hidden_columns.push_back(col.name);
-
-        return TableWithColumnNames(table, std::move(out_columns), std::move(out_hidden_columns));
-    }
-
 private:
     NameSet names;
 };
@@ -118,7 +75,6 @@ private:
 std::vector<DatabaseAndTableWithAlias> getDatabaseAndTables(const ASTSelectQuery & select_query, const String & current_database);
 std::optional<DatabaseAndTableWithAlias> getDatabaseAndTable(const ASTSelectQuery & select, size_t table_number);
 
-using TablesWithColumnNames = std::vector<TableWithColumnNames>;
+using TablesWithColumns = std::vector<TableWithColumnNamesAndTypes>;
-using TablesWithColumnNamesAndTypes = std::vector<TableWithColumnNames>;
 
 }

src/Interpreters/DatabaseCatalog.cpp

@@ -25,7 +25,6 @@ namespace ErrorCodes
     extern const int DATABASE_NOT_EMPTY;
     extern const int DATABASE_ACCESS_DENIED;
     extern const int LOGICAL_ERROR;
-    extern const int NULL_POINTER_DEREFERENCE;
 }
 
 TemporaryTableHolder::TemporaryTableHolder(const Context & context_,
@@ -385,38 +384,46 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database,
     it->second = std::make_pair(std::move(database), std::move(table));
 }
 
+std::unique_ptr<DatabaseCatalog> DatabaseCatalog::database_catalog;
+
 DatabaseCatalog::DatabaseCatalog(Context * global_context_)
     : global_context(global_context_), log(&Poco::Logger::get("DatabaseCatalog"))
 {
     if (!global_context)
-        throw Exception("DatabaseCatalog is not initialized. It's a bug.", ErrorCodes::NULL_POINTER_DEREFERENCE);
+        throw Exception("DatabaseCatalog is not initialized. It's a bug.", ErrorCodes::LOGICAL_ERROR);
 }
 
 DatabaseCatalog & DatabaseCatalog::init(Context * global_context_)
 {
-    static DatabaseCatalog database_catalog(global_context_);
+    if (database_catalog)
-    return database_catalog;
+    {
+        throw Exception("Database catalog is initialized twice. This is a bug.",
+            ErrorCodes::LOGICAL_ERROR);
+    }
+
+    database_catalog.reset(new DatabaseCatalog(global_context_));
+
+    return *database_catalog;
 }
 
 DatabaseCatalog & DatabaseCatalog::instance()
 {
-    return init(nullptr);
+    if (!database_catalog)
+    {
+        throw Exception("Database catalog is not initialized. This is a bug.",
+            ErrorCodes::LOGICAL_ERROR);
+    }
+
+    return *database_catalog;
 }
 
 void DatabaseCatalog::shutdown()
 {
-    try
+    // The catalog might not be initialized yet by init(global_context). It can
+    // happen if some exception was thrown on first steps of startup.
+    if (database_catalog)
     {
-        instance().shutdownImpl();
+        database_catalog->shutdownImpl();
-    }
-    catch (const Exception & e)
-    {
-        /// If catalog was not initialized yet by init(global_context), instance() throws NULL_POINTER_DEREFERENCE.
-        /// It can happen if some exception was thrown on first steps of startup (e.g. command line arguments parsing).
-        /// Ignore it.
-        if (e.code() == ErrorCodes::NULL_POINTER_DEREFERENCE)
-            return;
-        throw;
     }
 }
 
@@ -724,5 +731,3 @@ DDLGuard::~DDLGuard()
 }
 
 }
-
-

src/Interpreters/DatabaseCatalog.h

@@ -169,6 +169,11 @@ public:
     void enqueueDroppedTableCleanup(StorageID table_id, StoragePtr table, String dropped_metadata_path, bool ignore_delay = false);
 
 private:
+    // The global instance of database catalog. unique_ptr is to allow
+    // deferred initialization. Thought I'd use std::optional, but I can't
+    // make emplace(global_context_) compile with private constructor ¯\_(ツ)_/¯.
+    static std::unique_ptr<DatabaseCatalog> database_catalog;
+
     DatabaseCatalog(Context * global_context_);
     void assertDatabaseExistsUnlocked(const String & database_name) const;
     void assertDatabaseDoesntExistUnlocked(const String & database_name) const;

src/Interpreters/ExpressionAnalyzer.cpp

@@ -726,7 +726,8 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain,
     return true;
 }
 
-bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain, bool only_types)
+bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain, bool only_types, bool optimize_aggregation_in_order,
+                                                  ManyExpressionActions & group_by_elements_actions)
 {
     const auto * select_query = getAggregatingQuery();
 
@@ -743,6 +744,20 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
         getRootActions(ast, only_types, step.actions);
     }
 
+    if (optimize_aggregation_in_order)
+    {
+        auto all_columns = sourceWithJoinedColumns();
+        for (auto & child : asts)
+        {
+            group_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(all_columns, context));
+            getRootActions(child, only_types, group_by_elements_actions.back());
+        }
+//        std::cerr << "group_by_elements_actions\n";
+//        for (const auto & elem : group_by_elements_actions) {
+//            std::cerr << elem->dumpActions() << "\n";
+//        }
+    }
+
     return true;
 }
 
@@ -834,8 +849,11 @@ bool SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain
             order_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(all_columns, context));
             getRootActions(child, only_types, order_by_elements_actions.back());
         }
+//        std::cerr << "order_by_elements_actions\n";
+//        for (const auto & elem : order_by_elements_actions) {
+//            std::cerr << elem->dumpActions() << "\n";
+//        }
     }
-
     return true;
 }
 
@@ -1115,7 +1133,12 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
 
         if (need_aggregate)
         {
-            query_analyzer.appendGroupBy(chain, only_types || !first_stage);
+            /// TODO correct conditions
+            optimize_aggregation_in_order =
+                    context.getSettingsRef().optimize_aggregation_in_order
+                    && storage && query.groupBy();
+
+            query_analyzer.appendGroupBy(chain, only_types || !first_stage, optimize_aggregation_in_order, group_by_elements_actions);
             query_analyzer.appendAggregateFunctionsArguments(chain, only_types || !first_stage);
             before_aggregation = chain.getLastActions();
 
@@ -1128,13 +1151,13 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
             }
         }
 
-        bool has_stream_with_non_joned_rows = (before_join && before_join->getTableJoinAlgo()->hasStreamWithNonJoinedRows());
+        bool has_stream_with_non_joined_rows = (before_join && before_join->getTableJoinAlgo()->hasStreamWithNonJoinedRows());
         optimize_read_in_order =
             settings.optimize_read_in_order
             && storage && query.orderBy()
             && !query_analyzer.hasAggregation()
             && !query.final()
-            && !has_stream_with_non_joned_rows;
+            && !has_stream_with_non_joined_rows;
 
         /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers.
         query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage));