From dcb5d48f1b598d8135ba460ff36506c1399b21d3 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 18 Jun 2019 17:00:18 +0300 Subject: [PATCH 001/312] Create SECURITY.md --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..034e8480320 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 5.1.x | :white_check_mark: | +| 5.0.x | :x: | +| 4.0.x | :white_check_mark: | +| < 4.0 | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. From 0d4f3d8cc725d6e8f5297c7cfc1df851146aa15b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 19 Aug 2019 14:09:21 +0300 Subject: [PATCH 002/312] [experimental] auto-mark documentation PRs with labels --- .github/label-pr.yml | 2 ++ .github/main.workflow | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 .github/label-pr.yml create mode 100644 .github/main.workflow diff --git a/.github/label-pr.yml b/.github/label-pr.yml new file mode 100644 index 00000000000..4ae73a2e720 --- /dev/null +++ b/.github/label-pr.yml @@ -0,0 +1,2 @@ +- regExp: ".*\\.md$" + labels: ["documentation", "pr-documentation"] diff --git a/.github/main.workflow b/.github/main.workflow new file mode 100644 index 00000000000..a450195b955 --- /dev/null +++ b/.github/main.workflow @@ -0,0 +1,9 @@ +workflow "Main workflow" { + resolves = ["Label PR"] + on = "pull_request" +} + +action "Label PR" { + uses = "decathlon/pull-request-labeler-action@v1.0.0" + secrets = ["GITHUB_TOKEN"] +} From 453020593f1c8372100b70e47e9c4b26c9be09d5 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 19 Aug 2019 16:14:39 +0300 Subject: [PATCH 003/312] revert #6544 --- .github/label-pr.yml | 2 -- .github/main.workflow | 9 --------- 2 files changed, 11 deletions(-) delete mode 100644 .github/label-pr.yml delete mode 100644 .github/main.workflow diff --git a/.github/label-pr.yml b/.github/label-pr.yml deleted file mode 100644 index 4ae73a2e720..00000000000 --- a/.github/label-pr.yml +++ /dev/null @@ -1,2 +0,0 @@ -- regExp: ".*\\.md$" - labels: ["documentation", "pr-documentation"] diff --git a/.github/main.workflow b/.github/main.workflow deleted file mode 100644 index a450195b955..00000000000 --- a/.github/main.workflow +++ /dev/null @@ -1,9 +0,0 @@ -workflow "Main workflow" { - resolves = ["Label PR"] - on = "pull_request" -} - -action "Label PR" { - uses = "decathlon/pull-request-labeler-action@v1.0.0" - secrets = ["GITHUB_TOKEN"] -} From a187fa736035f247d8ae30fe6a25d827951afa25 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 20 Aug 2019 15:50:42 +0300 Subject: [PATCH 004/312] Sync RPM packages instructions to other docs languages --- docs/en/getting_started/index.md | 4 ++-- docs/fa/getting_started/index.md | 27 +++++++++++++++++++++++++ docs/ru/getting_started/index.md | 22 ++++++++++++++++----- docs/zh/getting_started/index.md | 34 ++++++++++++++++++++++++-------- 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md index 8f6308cd0ab..8cdbae86e5e 100644 --- a/docs/en/getting_started/index.md +++ b/docs/en/getting_started/index.md @@ -37,9 +37,9 @@ You can also download and install packages manually from here: :ﺪﯿﻨﮐ ﺐﺼﻧ ﻭ ﯼﺮﯿﮔﺭﺎﺑ ﺎﺠﻨ + + Docker Image ﺯﺍ ### + +.ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht + + ### نصب از طریق Source برای Compile، دستورالعمل های فایل build.md را دنبال کنید: diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index 8091f297019..e3fb2ab0985 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -37,13 +37,25 @@ sudo apt-get install clickhouse-client clickhouse-server ### Из RPM пакетов -Яндекс не использует ClickHouse на поддерживающих `rpm` дистрибутивах Linux, а также `rpm` пакеты менее тщательно тестируются. Таким образом, использовать их стоит на свой страх и риск, но, тем не менее, многие другие компании успешно работают на них в production без каких-либо серьезных проблем. +Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. -Для CentOS, RHEL и Fedora возможны следующие варианты: +Сначала нужно подключить официальный репозиторий: +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` -* Пакеты из генерируются на основе официальных `deb` пакетов от Яндекса и содержат в точности тот же исполняемый файл. -* Пакеты из собираются независимой компанией Altinity, но широко используются без каких-либо нареканий. -* Либо можно использовать Docker (см. ниже). +Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). + +Then run these commands to actually install packages: +Для, собственно, установки пакетов необходимо выполнить следующие команды: + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +Также есть возможность установить пакеты вручную, скачав отсюда: . ### Из Docker образа diff --git a/docs/zh/getting_started/index.md b/docs/zh/getting_started/index.md index 20d3c8ff9b1..f51323ce7e8 100644 --- a/docs/zh/getting_started/index.md +++ b/docs/zh/getting_started/index.md @@ -43,6 +43,32 @@ ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config 默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 有关更多信息，请参考"Configuration files"部分。 +###来自RPM包 + +Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 + +首先，您需要添加官方存储库： + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 + +然后运行这些命令以实际安装包： + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +您也可以从此处手动下载和安装软件包：。 + +###来自Docker + +要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 + ### 使用源码安装 具体编译方式可以参考build.md。 @@ -67,14 +93,6 @@ Server: dbms/programs/clickhouse-server 日志的路径可以在server config (src/dbms/programs/server/config.xml)中配置。 -### 其他的安装方法 - -Docker image： - -CentOS或RHEL安装包： - -Gentoo：`emerge clickhouse` - ## 启动 可以运行如下命令在后台启动服务： From 551de6ec31aaa3f0b54ea679616147b6c90ecd08 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 23 Aug 2019 15:41:23 +0300 Subject: [PATCH 005/312] Move tutorial to documentation with old content (for now) --- docs/en/getting_started/tutorial.md | 341 +++++++++++++++ docs/toc_en.yml | 1 + website/nginx/default.conf | 3 +- website/tutorial.html | 649 ---------------------------- 4 files changed, 344 insertions(+), 650 deletions(-) create mode 100644 docs/en/getting_started/tutorial.md delete mode 100644 website/tutorial.html diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md new file mode 100644 index 00000000000..48faf5bd327 --- /dev/null +++ b/docs/en/getting_started/tutorial.md @@ -0,0 +1,341 @@ +# ClickHouse Tutorial + +## Setup + +Let's get started with sample dataset from open sources. We will use USA civil flights data from 1987 to 2015. It's hard to call this sample a Big Data (contains 166 millions rows, 63 Gb of uncompressed data) but this allows us to quickly get to work. Dataset is available for download [here](https://yadi.sk/d/pOZxpa42sDdgm). Also you may download it from the original datasource [as described here](example_datasets/ontime.md). + +At first we will deploy ClickHouse to a single server. Later we will also review the process of deployment to a cluster with support for sharding and replication. + +ClickHouse is usually installed from [deb](index.md#from-deb-packages) or [rpm](index.md#from-rpm-packages) packages, but there are [alternatives](index.md#from-docker-image) for the operating systems that do no support them. What do we have in those packages: + +* `clickhouse-client` package contains [clickhouse-client](../interfaces/cli.md) application, interactive ClickHouse console client. +* `clickhouse-common` package contains a ClickHouse executable file. +* `clickhouse-server` package contains configuration files to run ClickHouse as a server. + +Server config files are located in /etc/clickhouse-server/. Before getting to work please notice the `path` element in config. Path dtermines the location for data storage. It's not really handy to directly edit `config.xml` file considering package updates. Recommended way to override the config elements is to create [files in config.d directory](../operations/configuration_files.md). Also you may want to [set up access rights](../operations/access_rights.md) early on.

+ +`clickhouse-server` won't be launched automatically after package installation. It won't be automatically restarted after updates either. Start the server with: +``` bash +sudo service clickhouse-server start +``` + +The default location for server logs is `/var/log/clickhouse-server/`. + +Server is ready to handle client connections once `Ready for connections` message was logged. + +Use `clickhouse-client` to connect to the server.

+ +
Tips for clickhouse-client +Interactive mode: +``` bash +clickhouse-client +clickhouse-client --host=... --port=... --user=... --password=... +``` + +Enable multiline queries: +``` bash +clickhouse-client -m +clickhouse-client --multiline +``` + +Run queries in batch-mode: +``` bash +clickhouse-client --query='SELECT 1' +echo 'SELECT 1' | clickhouse-client +``` + +Insert data from file of a specified format: +``` bash +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +``` +
+ +## Create Table for Sample Dataset +
Create table query +``` bash +$ clickhouse-client --multiline +ClickHouse client version 0.0.53720. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.53720. + +:) CREATE TABLE ontime +( + Year UInt16, + Quarter UInt8, + Month UInt8, + DayofMonth UInt8, + DayOfWeek UInt8, + FlightDate Date, + UniqueCarrier FixedString(7), + AirlineID Int32, + Carrier FixedString(2), + TailNum String, + FlightNum String, + OriginAirportID Int32, + OriginAirportSeqID Int32, + OriginCityMarketID Int32, + Origin FixedString(5), + OriginCityName String, + OriginState FixedString(2), + OriginStateFips String, + OriginStateName String, + OriginWac Int32, + DestAirportID Int32, + DestAirportSeqID Int32, + DestCityMarketID Int32, + Dest FixedString(5), + DestCityName String, + DestState FixedString(2), + DestStateFips String, + DestStateName String, + DestWac Int32, + CRSDepTime Int32, + DepTime Int32, + DepDelay Int32, + DepDelayMinutes Int32, + DepDel15 Int32, + DepartureDelayGroups String, + DepTimeBlk String, + TaxiOut Int32, + WheelsOff Int32, + WheelsOn Int32, + TaxiIn Int32, + CRSArrTime Int32, + ArrTime Int32, + ArrDelay Int32, + ArrDelayMinutes Int32, + ArrDel15 Int32, + ArrivalDelayGroups Int32, + ArrTimeBlk String, + Cancelled UInt8, + CancellationCode FixedString(1), + Diverted UInt8, + CRSElapsedTime Int32, + ActualElapsedTime Int32, + AirTime Int32, + Flights Int32, + Distance Int32, + DistanceGroup UInt8, + CarrierDelay Int32, + WeatherDelay Int32, + NASDelay Int32, + SecurityDelay Int32, + LateAircraftDelay Int32, + FirstDepTime String, + TotalAddGTime String, + LongestAddGTime String, + DivAirportLandings String, + DivReachedDest String, + DivActualElapsedTime String, + DivArrDelay String, + DivDistance String, + Div1Airport String, + Div1AirportID Int32, + Div1AirportSeqID Int32, + Div1WheelsOn String, + Div1TotalGTime String, + Div1LongestGTime String, + Div1WheelsOff String, + Div1TailNum String, + Div2Airport String, + Div2AirportID Int32, + Div2AirportSeqID Int32, + Div2WheelsOn String, + Div2TotalGTime String, + Div2LongestGTime String, + Div2WheelsOff String, + Div2TailNum String, + Div3Airport String, + Div3AirportID Int32, + Div3AirportSeqID Int32, + Div3WheelsOn String, + Div3TotalGTime String, + Div3LongestGTime String, + Div3WheelsOff String, + Div3TailNum String, + Div4Airport String, + Div4AirportID Int32, + Div4AirportSeqID Int32, + Div4WheelsOn String, + Div4TotalGTime String, + Div4LongestGTime String, + Div4WheelsOff String, + Div4TailNum String, + Div5Airport String, + Div5AirportID Int32, + Div5AirportSeqID Int32, + Div5WheelsOn String, + Div5TotalGTime String, + Div5LongestGTime String, + Div5WheelsOff String, + Div5TailNum String +) +ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192); +``` +
+ +Now we have a table of [MergeTree type](../operations/table_engines/mergetree.md). MergeTree table engine family is recommended for usage in production. Tables of this kind has a primary key used for incremental sort of table data. This allows fast execution of queries in ranges of a primary key. + +## Load Data +``` bash +xz -v -c -d < ontime.csv.xz | clickhouse-client --query="INSERT INTO ontime FORMAT CSV" +``` + +ClickHouse INSERT query allows to load data in any [supported format](../interfaces/formats.md). Data load requires just O(1) RAM consumption. INSERT query can receive any data volume as input. It is strongly recommended to insert data with [not so small blocks](../introduction/performance/#performance-when-inserting-data). Notice that insert of blocks with size up to max_insert_block_size (= 1 048 576 + rows by default) is an atomic operation: data block will be inserted completely or not inserted at all. In case of disconnect during insert operation you may not know if the block was inserted successfully. To achieve exactly-once semantics ClickHouse supports idempotency for [replicated tables](../operations/table_engines/replication.md). This means that you may retry insert of the same data block (possibly on a different replicas) but this block will be inserted just once. Anyway in this guide we will load data from our localhost so we may not take care about data blocks generation and exactly-once semantics. + +INSERT query into tables of MergeTree type is non-blocking (so does a SELECT query). You can execute SELECT queries right after of during insert operation. + +Our sample dataset is a bit not optimal. There are two reasons: + +* The first is that String data type is used in cases when [Enum](../data_types/enum.md) or numeric type would fit better. +* The second is that dataset contains redundant fields like Year, Quarter, Month, DayOfMonth, DayOfWeek. In fact a single FlightDate would be enough. Most likely they have been added to improve performance for other DBMS'eswhere DateTime handling functions may be not efficient. + +!!! note "Tip" + ClickHouse [functions for manipulating DateTime fields](../query_language/functions/date_time_functions/) are well-optimized so such redundancy is not required. Anyway many columns is not a reason to worry, because ClickHouse is a [column-oriented DBMS](https://en.wikipedia.org/wiki/Column-oriented_DBMS). This allows you to have as many fields as you need with minimal impact on performance. Hundreds of columns in a table is totally fine for ClickHouse. + +## Querying the Sample Dataset + +TODO + +## Cluster Deployment + +ClickHouse cluster is a homogenous cluster. Steps to set up: + +1. Install ClickHouse server on all machines of the cluster +2. Set up cluster configs in configuration file +3. Create local tables on each instance +4. Create a [Distributed table](../operations/table_engines/distributed.md) + +[Distributed table](../operations/table_engines/distributed.md) is actually a kind of "view" to local tables of ClickHouse cluster. SELECT query from a distributed table will be executed using resources of all cluster's shards. You may specify configs for multiple clusters and create multiple distributed tables providing views to different clusters. + +
Config for cluster with three shards, one replica each +``` xml + + + + + example-perftest01j.yandex.ru + 9000 + + + + + example-perftest02j.yandex.ru + 9000 + + + + + example-perftest03j.yandex.ru + 9000 + + + + +``` +
+ +Creating a local table: +``` sql +CREATE TABLE ontime_local (...) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192); +``` + +Creating a distributed table providing a view into local tables of the cluster: +``` sql +CREATE TABLE ontime_all AS ontime_local +ENGINE = Distributed(perftest_3shards_1replicas, default, ontime_local, rand()); +``` + +You can create a Distributed table on all machines in the cluster. This would allow to run distributed queries on any machine of the cluster. Besides distributed table you can also use [remote](../query_language/table_functions/remote.md) table function. + +Let's run [INSERT SELECT](../query_language/insert_into.md) into Distributed table to spread the table to multiple servers. + +``` sql +INSERT INTO ontime_all SELECT * FROM ontime; +``` + +!!! warning "Notice" + The approach given above is not suitable for sharding of large tables. + +As you could expect heavy queries are executed N times faster being launched on 3 servers instead of one.< + +In this case we have used a cluster with 3 shards each contains a single replica. + +To provide for resilience in production environment we recommend that each shard should contain 2-3 replicas distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas. + +
Config for cluster of one shard containing three replicas +``` xml + + ... + + + + example-perftest01j.yandex.ru + 9000 + + + example-perftest02j.yandex.ru + 9000 + + + example-perftest03j.yandex.ru + 9000 + + + + +``` +
+ +To enable replication ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure + automatically. It's recommended to deploy ZooKeeper cluster to separate servers. + +ZooKeeper is not a requirement — in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is not recommended — in this case ClickHouse is not able to + guarantee data consistency on all replicas. This remains the responsibility of your application. + +
Specify ZooKeeper locations in configuration file +``` xml + + + zoo01.yandex.ru + 2181 + + + zoo02.yandex.ru + 2181 + + + zoo03.yandex.ru + 2181 + + +``` +
+ +Also we need to set macros for identifying shard and replica — it will be used on table creation: +``` xml + + 01 + 01 + +``` +If there are no replicas at the moment on replicated table creation — a new first replica will be instantiated. If there are already live replicas — new replica will clone the data from existing ones. You have an option to create all replicated tables first and that insert data to it. Another option is to create some replicas and add the others after or during data insertion. + +``` sql +CREATE TABLE ontime_replica (...) +ENGINE = ReplcatedMergeTree( + '/clickhouse_perftest/tables/{shard}/ontime', + '{replica}', + FlightDate, + (Year, FlightDate), + 8192); +``` + +Here we use [ReplicatedMergeTree](../operations/table_engines/replication.md) table engine. In parameters we specify ZooKeeper path containing shard and replica identifiers. + +``` sql +INSERT INTO ontime_replica SELECT * FROM ontime; +``` +Replication operates in multi-master mode. Data can be loaded into any replica — it will be synced with other instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and repair consistency once they will become active again. Please notice that such scheme allows for the possibility of just appended data loss. diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 5e6854002ca..d823e026164 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -8,6 +8,7 @@ nav: - 'The Yandex.Metrica Task': 'introduction/ya_metrika_task.md' - 'Getting Started': + - 'Tutorial': 'getting_started/tutorial.md' - 'Deploying and Running': 'getting_started/index.md' - 'Example Datasets': - 'OnTime': 'getting_started/example_datasets/ontime.md' diff --git a/website/nginx/default.conf b/website/nginx/default.conf index 1600225836d..cb8c15a9f8c 100644 --- a/website/nginx/default.conf +++ b/website/nginx/default.conf @@ -14,9 +14,10 @@ server { } rewrite ^/docs/$ https://clickhouse.yandex/docs/en/ permanent; + rewrite ^/tutorial.html$ https://clickhouse.yandex/docs/en/getting_started/tutorial/ permanent; + rewrite ^/presentations/(.*)$ https://yandex.github.io/clickhouse-presentations/$1 permanent; rewrite ^/reference_en.html$ https://clickhouse.yandex/docs/en/single/ permanent; rewrite ^/reference_ru.html$ https://clickhouse.yandex/docs/ru/single/ permanent; - rewrite ^/presentations/(.*)$ https://yandex.github.io/clickhouse-presentations/$1 permanent; include /usr/share/nginx/html/docs/redirects.conf; diff --git a/website/tutorial.html b/website/tutorial.html deleted file mode 100644 index 2d8a2584f06..00000000000 --- a/website/tutorial.html +++ /dev/null @@ -1,649 +0,0 @@ - - - - - ClickHouse Quick Start Guide - - - - - - - - - - -
- -
- - -

ClickHouse

-

Tutorial

-
- -

Let's get started with sample dataset from open sources. We will use USA civil flights data since 1987 till 2015. - It's hard to call this sample a Big Data (contains 166 millions rows, 63 Gb of uncompressed data) but this - allows us to quickly get to work. Dataset is available for download here. - Also you may download it from the original datasource as described here.

- -

Firstly we will deploy ClickHouse to a single server. Below that we will also review the process of deployment to - a cluster with support for sharding and replication.

- -

On Ubuntu and Debian Linux ClickHouse can be installed from packages. - For other Linux distributions you can compile - ClickHouse from sources and then install.

- -

clickhouse-client package contains clickhouse-client application — - interactive ClickHouse client. clickhouse-common contains a clickhouse-server binary file. clickhouse-server - — contains config files for the clickhouse-server.

- -

Server config files are located in /etc/clickhouse-server/. Before getting to work please notice the path - element in config. Path determines the location for data storage. It's not really handy to directly - edit config.xml file considering package updates. Recommended way is to override the config elements in - files of config.d directory. - Also you may want to set up access - rights at the start.

- -

clickhouse-server won't be launched automatically after package installation. It won't be automatically - restarted after updates either. Start the server with: - 

sudo service clickhouse-server start
- Default location for server logs is /var/log/clickhouse-server/ - Server is ready to handle client connections once "Ready for connections" message was logged.

- -

Use clickhouse-client to connect to the server.

- -
Tips for clickhouse-client -
- Interactive mode: -
-clickhouse-client
-clickhouse-client --host=... --port=... --user=... --password=...
-
- Enable multiline queries: -
-clickhouse-client -m
-clickhouse-client --multiline
-
- Run queries in batch-mode: -
-clickhouse-client --query='SELECT 1'
-echo 'SELECT 1' | clickhouse-client
-
- Insert data from file of a specified format: -
-clickhouse-client --query='INSERT INTO table VALUES' < data.txt
-clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv
-
-
-
- -

Create table for sample dataset

-
Create table query -
-
-$ clickhouse-client --multiline
-ClickHouse client version 0.0.53720.
-Connecting to localhost:9000.
-Connected to ClickHouse server version 0.0.53720.
-
-:) CREATE TABLE ontime
-(
-    Year UInt16,
-    Quarter UInt8,
-    Month UInt8,
-    DayofMonth UInt8,
-    DayOfWeek UInt8,
-    FlightDate Date,
-    UniqueCarrier FixedString(7),
-    AirlineID Int32,
-    Carrier FixedString(2),
-    TailNum String,
-    FlightNum String,
-    OriginAirportID Int32,
-    OriginAirportSeqID Int32,
-    OriginCityMarketID Int32,
-    Origin FixedString(5),
-    OriginCityName String,
-    OriginState FixedString(2),
-    OriginStateFips String,
-    OriginStateName String,
-    OriginWac Int32,
-    DestAirportID Int32,
-    DestAirportSeqID Int32,
-    DestCityMarketID Int32,
-    Dest FixedString(5),
-    DestCityName String,
-    DestState FixedString(2),
-    DestStateFips String,
-    DestStateName String,
-    DestWac Int32,
-    CRSDepTime Int32,
-    DepTime Int32,
-    DepDelay Int32,
-    DepDelayMinutes Int32,
-    DepDel15 Int32,
-    DepartureDelayGroups String,
-    DepTimeBlk String,
-    TaxiOut Int32,
-    WheelsOff Int32,
-    WheelsOn Int32,
-    TaxiIn Int32,
-    CRSArrTime Int32,
-    ArrTime Int32,
-    ArrDelay Int32,
-    ArrDelayMinutes Int32,
-    ArrDel15 Int32,
-    ArrivalDelayGroups Int32,
-    ArrTimeBlk String,
-    Cancelled UInt8,
-    CancellationCode FixedString(1),
-    Diverted UInt8,
-    CRSElapsedTime Int32,
-    ActualElapsedTime Int32,
-    AirTime Int32,
-    Flights Int32,
-    Distance Int32,
-    DistanceGroup UInt8,
-    CarrierDelay Int32,
-    WeatherDelay Int32,
-    NASDelay Int32,
-    SecurityDelay Int32,
-    LateAircraftDelay Int32,
-    FirstDepTime String,
-    TotalAddGTime String,
-    LongestAddGTime String,
-    DivAirportLandings String,
-    DivReachedDest String,
-    DivActualElapsedTime String,
-    DivArrDelay String,
-    DivDistance String,
-    Div1Airport String,
-    Div1AirportID Int32,
-    Div1AirportSeqID Int32,
-    Div1WheelsOn String,
-    Div1TotalGTime String,
-    Div1LongestGTime String,
-    Div1WheelsOff String,
-    Div1TailNum String,
-    Div2Airport String,
-    Div2AirportID Int32,
-    Div2AirportSeqID Int32,
-    Div2WheelsOn String,
-    Div2TotalGTime String,
-    Div2LongestGTime String,
-    Div2WheelsOff String,
-    Div2TailNum String,
-    Div3Airport String,
-    Div3AirportID Int32,
-    Div3AirportSeqID Int32,
-    Div3WheelsOn String,
-    Div3TotalGTime String,
-    Div3LongestGTime String,
-    Div3WheelsOff String,
-    Div3TailNum String,
-    Div4Airport String,
-    Div4AirportID Int32,
-    Div4AirportSeqID Int32,
-    Div4WheelsOn String,
-    Div4TotalGTime String,
-    Div4LongestGTime String,
-    Div4WheelsOff String,
-    Div4TailNum String,
-    Div5Airport String,
-    Div5AirportID Int32,
-    Div5AirportSeqID Int32,
-    Div5WheelsOn String,
-    Div5TotalGTime String,
-    Div5LongestGTime String,
-    Div5WheelsOff String,
-    Div5TailNum String
-)
-ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192);
-
-
-
- -

Now we have a table of MergeTree type. - MergeTree table type is recommended for usage in production. Table of this kind has a primary key used for - incremental sort of table data. This allows fast execution of queries in ranges of a primary key.

- - -

Note - We store ad network banners impressions logs in ClickHouse. Each table entry looks like: - [Advertiser ID, Impression ID, attribute1, attribute2, …]. - Let assume that our aim is to provide a set of reports for each advertiser. Common and frequently demanded query - would be to count impressions for a specific Advertiser ID. This means that table primary key should start with - Advertiser ID. In this case ClickHouse needs to read smaller amount of data to perform the query for a - given Advertiser ID. -

- -

Load data

- 
xz -v -c -d < ontime.csv.xz | clickhouse-client --query="INSERT INTO ontime FORMAT CSV"
-

ClickHouse INSERT query allows to load data in any supported - format. Data load requires just O(1) RAM consumption. INSERT query can receive any data volume as input. - It's strongly recommended to insert data with not too small - size blocks. Notice that insert of blocks with size up to max_insert_block_size (= 1 048 576 - rows by default) is an atomic operation: data block will be inserted completely or not inserted at all. In case - of disconnect during insert operation you may not know if the block was inserted successfully. To achieve - exactly-once semantics ClickHouse supports idempotency for replicated tables. This means - that you may retry insert of the same data block (possibly on a different replicas) but this block will be - inserted just once. Anyway in this guide we will load data from our localhost so we may not take care about data - blocks generation and exactly-once semantics.

- -

INSERT query into tables of MergeTree type is non-blocking (so does a SELECT query). You can execute SELECT - queries right after of during insert operation.

- -

Our sample dataset is a bit not optimal. There are two reasons.

- -

The first is that String data type is used in cases when Enum or numeric type would fit best.

- -

⚖ When set of possible values is determined and known to be small. (E.g. OS name, browser - vendors etc.) it's recommended to use Enums or numbers to improve performance. - When set of possible values is not limited (search query, URL, etc.) just go ahead with String.

- -

The second is that dataset contains redundant fields like Year, Quarter, Month, DayOfMonth, DayOfWeek. In fact a - single FlightDate would be enough. Most likely they have been added to improve performance for other DBMS'es - which DateTime handling functions may be not efficient.

- -

✯ ClickHouse functions - for operating with DateTime fields are well-optimized so such redundancy is not required. Anyway much - columns is not a reason to worry — ClickHouse is a column-oriented - DBMS. This allows you to have as much fields as you need. Hundreds of columns in a table is fine for - ClickHouse.

- -

Querying the sample dataset

- -

Here are some examples of the queries from our test data.

- -
    -
  • -
    the most popular destinations in 2015; -
    -
    -SELECT
-    OriginCityName,
-    DestCityName,
-    count(*) AS flights,
-    bar(flights, 0, 20000, 40)
-FROM ontime WHERE Year = 2015 GROUP BY OriginCityName, DestCityName ORDER BY flights DESC LIMIT 20
-
    - -
    -SELECT
-    OriginCityName < DestCityName ? OriginCityName : DestCityName AS a,
-    OriginCityName < DestCityName ? DestCityName : OriginCityName AS b,
-    count(*) AS flights,
-    bar(flights, 0, 40000, 40)
-FROM ontime WHERE Year = 2015 GROUP BY a, b ORDER BY flights DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    the most popular cities of departure; -
    -
    -SELECT OriginCityName, count(*) AS flights
-FROM ontime GROUP BY OriginCityName ORDER BY flights DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    cities of departure which offer maximum variety of - destinations; -
    -
    -SELECT OriginCityName, uniq(Dest) AS u
-FROM ontime GROUP BY OriginCityName ORDER BY u DESC LIMIT 20
-
    -
    -
    -
    • -
  • -
    flight delay dependence on the day of week; -
    -
    -SELECT DayOfWeek, count() AS c, avg(DepDelay >  60) AS delays
-FROM ontime GROUP BY DayOfWeek ORDER BY DayOfWeek
-
    -
    -
    -
    • -
  • -
    cities of departure with most frequent delays for 1 hour or - longer; -
    -
    -SELECT OriginCityName, count() AS c, avg(DepDelay >  60) AS delays
-FROM ontime
-GROUP BY OriginCityName
-HAVING c >  100000
-ORDER BY delays DESC
-LIMIT 20
-
    -
    -
    -
    • -
  • -
    flights of maximum duration; -
    -
    -SELECT OriginCityName, DestCityName, count(*) AS flights, avg(AirTime) AS duration
-FROM ontime
-GROUP BY OriginCityName, DestCityName
-ORDER BY duration DESC
-LIMIT 20
-
    -
    -
    -
    • -
  • -
    distribution of arrival time delays split by aircompanies; -
    -
    -SELECT Carrier, count() AS c, round(quantileTDigest(0.99)(DepDelay), 2) AS q
-FROM ontime GROUP BY Carrier ORDER BY q DESC
-
    -
    -
    -
    • -
  • -
    aircompanies who stopped flights operation; -
    -
    -SELECT Carrier, min(Year), max(Year), count()
-FROM ontime GROUP BY Carrier HAVING max(Year) < 2015 ORDER BY count() DESC
-
    -
    -
    -
    • -
  • -
    most trending destination cities in 2015; -
    -
    -SELECT
-    DestCityName,
-    sum(Year = 2014) AS c2014,
-    sum(Year = 2015) AS c2015,
-    c2015 / c2014 AS diff
-FROM ontime
-WHERE Year IN (2014, 2015)
-GROUP BY DestCityName
-HAVING c2014 >  10000 AND c2015 >  1000 AND diff >  1
-ORDER BY diff DESC
-
    -
    -
    -
    • -
  • -
    destination cities with maximum popularity-season - dependency. -
    -
    -SELECT
-    DestCityName,
-    any(total),
-    avg(abs(monthly * 12 - total) / total) AS avg_month_diff
-FROM
-(
-    SELECT DestCityName, count() AS total
-    FROM ontime GROUP BY DestCityName HAVING total > 100000
-)
-ALL INNER JOIN
-(
-    SELECT DestCityName, Month, count() AS monthly
-    FROM ontime GROUP BY DestCityName, Month HAVING monthly > 10000
-)
-USING DestCityName
-GROUP BY DestCityName
-ORDER BY avg_month_diff DESC
-LIMIT 20
-
    -
    -
    -
    • -
- -

ClickHouse deployment to cluster

-

ClickHouse cluster is a homogenous cluster. Steps to set up: -

    -
  1. Install ClickHouse server on all machines of the cluster
    2. -
  2. Set up cluster configs in configuration file
    3. -
  3. Create local tables on each instance
    4. -
  4. Create a Distributed table
    5. -
-

- -

Distributed-table is actually a kind of - "view" to local tables of ClickHouse cluster. SELECT query from a distributed table will be executed using - resources of all cluster's shards. You may specify configs for multiple clusters and create multiple - Distributed-tables providing views to different clusters.

- -
Config for cluster of three shards. Each shard stores data on a single - replica -
-
-<remote_servers>
-    <perftest_3shards_1replicas>
-        <shard>
-            <replica>
-                <host>example-perftest01j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-        <shard>
-            <replica>
-                <host>example-perftest02j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-        <shard>
-            <replica>
-                <host>example-perftest03j.yandex.ru</host>
-                <port>9000</port>
-            </replica>
-        </shard>
-    </perftest_3shards_1replicas>
-</remote_servers>
-
-
-
- Creating a local table: - 
CREATE TABLE ontime_local (...) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192);
- Creating a distributed table providing a view into local tables of the cluster: -
CREATE TABLE ontime_all AS ontime_local
-    ENGINE = Distributed(perftest_3shards_1replicas, default, ontime_local, rand());
- -

You can create a Distributed table on all machines in the cluster. This would allow to run distributed queries on - any machine of the cluster. Besides distributed table you can also use *remote* table function.

- -

Let's run INSERT SELECT into Distributed table - to spread the table to multiple servers.

- - 
INSERT INTO ontime_all SELECT * FROM ontime;
- -

⚠ Worth to notice that the approach given above wouldn't fit for sharding of large - tables.

- -

As you could expect heavy queries are executed N times faster being launched on 3 servers instead of one.

-
See here -
- - -

You may have noticed that quantiles calculation are slightly different. This happens due to t-digest - algorithm implementation which is non-deterministic — it depends on the order of data processing.

-
-
- -

In this case we have used a cluster with 3 shards each contains a single replica.

- -

To provide for resilience in production environment we recommend that each shard should contain 2-3 replicas - distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas.

- -
Config for cluster of one shard containing three replicas -
-
-<remote_servers>
-    ...
-    <perftest_1shards_3replicas>
-        <shard>
-            <replica>
-                <host>example-perftest01j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-             <replica>
-                <host>example-perftest02j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-             <replica>
-                <host>example-perftest03j.yandex.ru</host>
-                <port>9000</port>
-             </replica>
-        </shard>
-    </perftest_1shards_3replicas>
-</remote_servers>
-
-
-
- -

To enable replication ZooKeeper is required. - ClickHouse will take care of data consistency on all replicas and run restore procedure after failure - automatically. It's recommended to deploy ZooKeeper cluster to separate servers.

- -

ZooKeeper is not a requirement — in some simple cases you can duplicate the data by writing it into all the - replicas from your application code. This approach is not recommended — in this case ClickHouse is not able to - guarantee data consistency on all replicas. This remains the responsibility of your application.

- -
Set ZooKeeper locations in configuration file -
-
-<zookeeper-servers>
-    <node>
-        <host>zoo01.yandex.ru</host>
-        <port>2181</port>
-    </node>
-    <node>
-        <host>zoo02.yandex.ru</host>
-        <port>2181</port>
-    </node>
-    <node>
-        <host>zoo03.yandex.ru</host>
-        <port>2181</port>
-    </node>
-</zookeeper-servers>
-
-
-
- -

Also we need to set macros for identifying shard and replica — it will be used on table creation

-
-<macros>
-    <shard>01</shard>
-    <replica>01</replica>
-</macros>
-
-

If there are no replicas at the moment on replicated table creation — a new first replica will be instantiated. - If there are already live replicas — new replica will clone the data from existing ones. You have an option to - create all replicated tables first and that insert data to it. Another option is to create some replicas and add - the others after or during data insertion.

- -
-CREATE TABLE ontime_replica (...)
-ENGINE = ReplicatedMergeTree(
-    '/clickhouse_perftest/tables/{shard}/ontime',
-    '{replica}',
-    FlightDate,
-    (Year, FlightDate),
-    8192);
-
-

Here we use ReplicatedMergeTree - table type. In parameters we specify ZooKeeper path containing shard and replica identifiers.

- - 
INSERT INTO ontime_replica SELECT * FROM ontime;
-

Replication operates in multi-master mode. Data can be loaded into any replica — it will be synced with other - instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain - recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and - repair consistency once they will become active again. Please notice that such scheme allows for the possibility - of just appended data loss.

- -

- ClickHouse source code is published under Apache 2.0 License. Software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied.

- - - -
- - - - - - - - - - - From 9de561d8655c0e75614fc32f636582936c3338f8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 23 Aug 2019 17:36:51 +0300 Subject: [PATCH 006/312] refactor installation guide a bit --- .../getting_started/{index.md => install.md} | 6 +- docs/fa/getting_started/index.md | 198 +----------------- docs/fa/getting_started/install.md | 197 +++++++++++++++++ docs/ru/getting_started/index.md | 140 +------------ docs/ru/getting_started/install.md | 142 +++++++++++++ docs/toc_en.yml | 3 +- docs/toc_fa.yml | 12 +- docs/toc_ru.yml | 4 +- docs/toc_zh.yml | 4 +- docs/zh/getting_started/index.md | 160 +------------- docs/zh/getting_started/install.md | 156 ++++++++++++++ 11 files changed, 529 insertions(+), 493 deletions(-) rename docs/en/getting_started/{index.md => install.md} (98%) create mode 100644 docs/fa/getting_started/install.md create mode 100644 docs/ru/getting_started/install.md create mode 100644 docs/zh/getting_started/install.md diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/install.md similarity index 98% rename from docs/en/getting_started/index.md rename to docs/en/getting_started/install.md index 8cdbae86e5e..779abba905b 100644 --- a/docs/en/getting_started/index.md +++ b/docs/en/getting_started/install.md @@ -1,4 +1,4 @@ -# Getting Started +# Installation ## System Requirements @@ -10,7 +10,7 @@ Though pre-built binaries are typically compiled to leverage SSE 4.2 instruction $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" ``` -## Installation +## Available Installation Options ### From DEB Packages @@ -148,4 +148,4 @@ SELECT 1 To continue experimenting, you can download one of test data sets or go through [tutorial](https://clickhouse.yandex/tutorial.html). -[Original article](https://clickhouse.yandex/docs/en/getting_started/) +[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) diff --git a/docs/fa/getting_started/index.md b/docs/fa/getting_started/index.md index 778393aed91..57496c474e2 100644 --- a/docs/fa/getting_started/index.md +++ b/docs/fa/getting_started/index.md @@ -1,197 +1,11 @@
+# ﻥﺪﺷ ﻉﻭﺮﺷ -# شروع به کار +ﻖﯾﺮﻃ ﺯﺍ ﺪﯾﺎﺑ ﻪﻤﻫ ﺯﺍ ﻝﻭﺍ ، ﺪﯿﻨﮐ ﺱﺎﺴﺣﺍ ﺍﺭ ﻥﺁ ﺩﺮﮑﻠﻤﻋ ﺪﯿﻫﺍﻮﺧ ﯽﻣ ﻭ ﺪﯿﺘﺴﻫ ﺩﺭﺍﻭ ﻩﺯﺎﺗ[ﺐﺼﻧ ﻞﺣﺍﺮﻣ](install.md). +ﺪﯿﻨﮐ ﺏﺎﺨﺘﻧﺍ ﺍﺭ ﺮﯾﺯ ﯼﺎﻫ ﻪﻨﯾﺰﮔ ﺯﺍ ﯽﮑﯾ ﺪﯿﻧﺍﻮﺗ ﯽﻣ ﻥﺁ ﺯﺍ ﺲﭘ: -## نیازمندی های سیستم - -این یک سیستم چند سکویی (Cross-Platform) نمی باشد. این ابزار نیاز به Linux Ubuntu Precise (12.04) یا جدیدتر، با معماری x86\_64 و پشتیبانی از SSE 4.2 می باشد. برای چک کردن SSE 4.2 خروجی دستور زیر را بررسی کنید: +* [ﺪﯿﻨﮐ ﯽﻃ ﺍﺭ ﻞﺼﻔﻣ ﺵﺯﻮﻣﺁ](tutorial.md) +* [ﺪﯿﻨﮐ ﺶﯾﺎﻣﺯﺁ ﻪﻧﻮﻤﻧ ﯼﺎﻫ ﻩﺩﺍﺩ ﺎﺑ](example_datasets/ontime.md) +[ﯽﻠﺻﺍ ﻪﻟﺎﻘﻣ](https://clickhouse.yandex/docs/fa/getting_started/)
- -```bash -grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -
- -پیشنهاد می کنیم از Ubuntu TrustyT، Ubuntu Xenial یا Ubuntu Precise استفاده کنید. ترمینال باید از UTF-8 پشتیبانی کند. (به صورت پیش فرض در Ubuntu پشتیبانی می شود). - -## نصب - -### نصب از طریق پکیج های Debian/Ubuntu - -در فایل `/etc/apt/sources.list` (یا در یک فایل جدا `/etc/apt/sources.list.d/clickhouse.list`)، Repo زیر را اضافه کنید: - -
- -``` -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -
- -اگر شما میخوایید جدیدترین نسخه ی تست را استفاده کنید، 'stable' رو به 'testing' تغییر بدید. - -سپس دستورات زیر را اجرا کنید: - -
- -```bash -sudo apt-get install dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -
- -شما همچنین می توانید از طریق لینک زیر پکیج ClickHouse را به صورت دستی دانلود و نصب کنید: . - -ClickHouse دارای تنظیمات محدودیت دسترسی می باشد. این تنظیمات در فایل 'users.xml' (کنار 'config.xml') می باشد. به صورت پیش فرض دسترسی برای کاربر 'default' از همه جا بدون نیاز به پسورد وجود دارد. 'user/default/networks' را مشاهده کنید. برای اطلاعات بیشتر قسمت "تنظیمات فایل ها" را مشاهده کنید. - - RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ### - -.ﺪﻨﮐ ﯽﻣ ﻪﯿﺻﻮﺗ ﺲﮐﻮﻨﯿﻟ ﺮﺑ ﯽﻨﺘﺒﻣ rpm ﺮﺑ ﯽﻨﺘﺒﻣ ﯼﺎﻫ ﻊﯾﺯﻮﺗ ﺮﯾﺎﺳ ﻭ CentOS ، RedHat ﯼﺍ - - :ﺪﯿﻨﮐ ﻪﻓﺎﺿﺍ ﺍﺭ ﯽﻤﺳﺭ ﻥﺰﺨﻣ ﺪﯾﺎﺑ ﺍﺪﺘﺑﺍ - -```bash -sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -.(ﺩﻮﺷ ﯽﻣ ﻪﯿﺻﻮﺗ ﺎﻤﺷ ﺶﯾﺎﻣﺯﺁ ﯼﺎﻫ ﻂﯿﺤﻣ ﯼﺍﺮﺑ ﻦﯾﺍ) ﺪﯿﻨﮐ ﻦﯾﺰﮕﯾﺎﺟ "ﺖﺴﺗ" ﺎﺑ ﺍﺭ "ﺭﺍﺪﯾﺎﭘ" - - :ﺪﯿﻨﮐ ﺐﺼﻧ ﺍﺭ ﺎﻫ ﻪﺘﺴﺑ ﻊﻗﺍﻭ ﺭﺩ ﺎﺗ ﺪﯿﻨﮐ ﺍﺮﺟﺍ ﺍﺭ ﺕﺍﺭﻮﺘﺳﺩ ﻦﯾﺍ ﺲﭙﺳ - -```bash -sudo yum install clickhouse-server clickhouse-client -``` - -. :ﺪﯿﻨﮐ ﺐﺼﻧ ﻭ ﯼﺮﯿﮔﺭﺎﺑ ﺎﺠﻨ - - Docker Image ﺯﺍ ### - -.ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht - - -### نصب از طریق Source - -برای Compile، دستورالعمل های فایل build.md را دنبال کنید: - -شما میتوانید پکیج را compile و نصب کنید. شما همچنین می توانید بدون نصب پکیج از برنامه ها استفاده کنید. - -
- -``` -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -
- -برای سرور، یک کاتالوگ با دیتا بسازید، مانند - -
- -``` -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -
- -(قابل تنظیم در تنظیمات سرور). 'chown' را برای کاربر دلخواه اجرا کنید. - -به مسیر لاگ ها در تنظیمات سرور توجه کنید (src/dbms/programs/config.xml). - -### روش های دیگر نصب - -Docker image: - -پکیج RPM برای CentOS یا RHEL: - -Gentoo: `emerge clickhouse` - -## راه اندازی - -برای استارت سرور (به صورت daemon)، دستور زیر را اجرا کنید: - -
- -```bash -sudo service clickhouse-server start -``` - -
- -لاگ های دایرکتوری `/var/log/clickhouse-server/` directory. را مشاهده کنید. - -اگر سرور استارت نشد، فایل تنظیمات را بررسی کنید `/etc/clickhouse-server/config.xml.` - -شما همچنین می توانید سرور را از طریق کنسول راه اندازی کنید: - -
- -```bash -clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -
- -در این مورد که مناسب زمان توسعه می باشد، لاگ ها در کنسول پرینت می شوند. اگر فایل تنظیمات در دایرکتوری جاری باشد، نیازی به مشخص کردن '--config-file' نمی باشد. به صورت پیش فرض از './config.xml' استفاده می شود. - -شما می توانید از کلاینت command-line برای اتصال به سرور استفاده کنید: - -
- -```bash -clickhouse-client -``` - -
- -پارامترهای پیش فرض، نشان از اتصال به localhost:9000 از طرف کاربر 'default' بدون پسورد را می دهد. از کلاینت میتوان برای اتصال به یک سرور remote استفاده کرد. مثال: - -
- -```bash -clickhouse-client --host=example.com -``` - -
- -برای اطلاعات بیشتر، بخش "کلاینت Command-line" را مشاهده کنید. - -چک کردن سیستم: - -
- -```bash -milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -
- -**تبریک میگم، سیستم کار می کنه!** - -برای ادامه آزمایشات، شما میتوانید دیتاست های تستی را دریافت و امتحان کنید. - -
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/) diff --git a/docs/fa/getting_started/install.md b/docs/fa/getting_started/install.md new file mode 100644 index 00000000000..2313d29578b --- /dev/null +++ b/docs/fa/getting_started/install.md @@ -0,0 +1,197 @@ +
+ +# ﯼﺯﺍﺪﻧﺍ ﻩﺍﺭ ﻭ ﺐﺼﻧ + +## نیازمندی های سیستم + +این یک سیستم چند سکویی (Cross-Platform) نمی باشد. این ابزار نیاز به Linux Ubuntu Precise (12.04) یا جدیدتر، با معماری x86\_64 و پشتیبانی از SSE 4.2 می باشد. برای چک کردن SSE 4.2 خروجی دستور زیر را بررسی کنید: + +
+ +```bash +grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +
+ +پیشنهاد می کنیم از Ubuntu Trusty، Ubuntu Xenial یا Ubuntu Precise استفاده کنید. ترمینال باید از UTF-8 پشتیبانی کند. (به صورت پیش فرض در Ubuntu پشتیبانی می شود). + +##ﺩﻮﺟﻮﻣ ﺐﺼﻧ ﯼﺎﻫ ﻪﻨﯾﺰﮔ + +### نصب از طریق پکیج های Debian/Ubuntu + +در فایل `/etc/apt/sources.list` (یا در یک فایل جدا `/etc/apt/sources.list.d/clickhouse.list`)، Repo زیر را اضافه کنید: + +
+ +``` +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +
+ +اگر شما میخوایید جدیدترین نسخه ی تست را استفاده کنید، 'stable' رو به 'testing' تغییر بدید. + +سپس دستورات زیر را اجرا کنید: + +
+ +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +
+ +شما همچنین می توانید از طریق لینک زیر پکیج ClickHouse را به صورت دستی دانلود و نصب کنید: . + +ClickHouse دارای تنظیمات محدودیت دسترسی می باشد. این تنظیمات در فایل 'users.xml' (کنار 'config.xml') می باشد. به صورت پیش فرض دسترسی برای کاربر 'default' از همه جا بدون نیاز به پسورد وجود دارد. 'user/default/networks' را مشاهده کنید. برای اطلاعات بیشتر قسمت "تنظیمات فایل ها" را مشاهده کنید. + + RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ### + +.ﺪﻨﮐ ﯽﻣ ﻪﯿﺻﻮﺗ ﺲﮐﻮﻨﯿﻟ ﺮﺑ ﯽﻨﺘﺒﻣ rpm ﺮﺑ ﯽﻨﺘﺒﻣ ﯼﺎﻫ ﻊﯾﺯﻮﺗ ﺮﯾﺎﺳ ﻭ CentOS ، RedHat ﯼﺍ + + :ﺪﯿﻨﮐ ﻪﻓﺎﺿﺍ ﺍﺭ ﯽﻤﺳﺭ ﻥﺰﺨﻣ ﺪﯾﺎﺑ ﺍﺪﺘﺑﺍ + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +.(ﺩﻮﺷ ﯽﻣ ﻪﯿﺻﻮﺗ ﺎﻤﺷ ﺶﯾﺎﻣﺯﺁ ﯼﺎﻫ ﻂﯿﺤﻣ ﯼﺍﺮﺑ ﻦﯾﺍ) ﺪﯿﻨﮐ ﻦﯾﺰﮕﯾﺎﺟ "ﺖﺴﺗ" ﺎﺑ ﺍﺭ "ﺭﺍﺪﯾﺎﭘ" + + :ﺪﯿﻨﮐ ﺐﺼﻧ ﺍﺭ ﺎﻫ ﻪﺘﺴﺑ ﻊﻗﺍﻭ ﺭﺩ ﺎﺗ ﺪﯿﻨﮐ ﺍﺮﺟﺍ ﺍﺭ ﺕﺍﺭﻮﺘﺳﺩ ﻦﯾﺍ ﺲﭙﺳ + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +. :ﺪﯿﻨﮐ ﺐﺼﻧ ﻭ ﯼﺮﯿﮔﺭﺎﺑ ﺎﺠﻨ + + Docker Image ﺯﺍ ### + +.ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht + + +### نصب از طریق Source + +برای Compile، دستورالعمل های فایل build.md را دنبال کنید: + +شما میتوانید پکیج را compile و نصب کنید. شما همچنین می توانید بدون نصب پکیج از برنامه ها استفاده کنید. + +
+ +``` +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +
+ +برای سرور، یک کاتالوگ با دیتا بسازید، مانند + +
+ +``` +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +
+ +(قابل تنظیم در تنظیمات سرور). 'chown' را برای کاربر دلخواه اجرا کنید. + +به مسیر لاگ ها در تنظیمات سرور توجه کنید (src/dbms/programs/config.xml). + +### روش های دیگر نصب + +Docker image: + +پکیج RPM برای CentOS یا RHEL: + +Gentoo: `emerge clickhouse` + +## راه اندازی + +برای استارت سرور (به صورت daemon)، دستور زیر را اجرا کنید: + +
+ +```bash +sudo service clickhouse-server start +``` + +
+ +لاگ های دایرکتوری `/var/log/clickhouse-server/` directory. را مشاهده کنید. + +اگر سرور استارت نشد، فایل تنظیمات را بررسی کنید `/etc/clickhouse-server/config.xml.` + +شما همچنین می توانید سرور را از طریق کنسول راه اندازی کنید: + +
+ +```bash +clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +
+ +در این مورد که مناسب زمان توسعه می باشد، لاگ ها در کنسول پرینت می شوند. اگر فایل تنظیمات در دایرکتوری جاری باشد، نیازی به مشخص کردن '--config-file' نمی باشد. به صورت پیش فرض از './config.xml' استفاده می شود. + +شما می توانید از کلاینت command-line برای اتصال به سرور استفاده کنید: + +
+ +```bash +clickhouse-client +``` + +
+ +پارامترهای پیش فرض، نشان از اتصال به localhost:9000 از طرف کاربر 'default' بدون پسورد را می دهد. از کلاینت میتوان برای اتصال به یک سرور remote استفاده کرد. مثال: + +
+ +```bash +clickhouse-client --host=example.com +``` + +
+ +برای اطلاعات بیشتر، بخش "کلاینت Command-line" را مشاهده کنید. + +چک کردن سیستم: + +
+ +```bash +milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +
+ +**تبریک میگم، سیستم کار می کنه!** + +برای ادامه آزمایشات، شما میتوانید دیتاست های تستی را دریافت و امتحان کنید. + +
+[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/install/) diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md index e3fb2ab0985..a8d0fbaa5b1 100644 --- a/docs/ru/getting_started/index.md +++ b/docs/ru/getting_started/index.md @@ -1,142 +1,10 @@ # Начало работы -## Системные требования +Если вы новичок в ClickHouse и хотите получить вживую оценить его производительность, прежде всего нужно пройти через [процесс установки](install.md). -ClickHouse может работать на любом Linux, FreeBSD или Mac OS X с архитектурой процессора x86\_64. +После этого можно выбрать один из следующих вариантов: -Хотя предсобранные релизы обычно компилируются с использованием набора инструкций SSE 4.2, что добавляет использование поддерживающего его процессора в список системных требований. Команда для проверки наличия поддержки инструкций SSE 4.2 на текущем процессоре: - -```bash -$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -## Установка - -### Из DEB пакетов - -Яндекс рекомендует использовать официальные скомпилированные `deb` пакеты для Debian или Ubuntu. - -Чтобы установить официальные пакеты, пропишите репозиторий Яндекса в `/etc/apt/sources.list` или в отдельный файл `/etc/apt/sources.list.d/clickhouse.list`: - -``` -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -Если вы хотите использовать наиболее свежую тестовую, замените `stable` на `testing` (не рекомендуется для production окружений). - -Затем для самой установки пакетов выполните: - -```bash -sudo apt-get install dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -Также эти пакеты можно скачать и установить вручную отсюда: . - -### Из RPM пакетов - -Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. - -Сначала нужно подключить официальный репозиторий: -```bash -sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). - -Then run these commands to actually install packages: -Для, собственно, установки пакетов необходимо выполнить следующие команды: - -```bash -sudo yum install clickhouse-server clickhouse-client -``` - -Также есть возможность установить пакеты вручную, скачав отсюда: . - -### Из Docker образа - -Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты. - -### Из исходникого кода - -Для компиляции ClickHouse вручную, используйте инструкцию для [Linux](../development/build.md) или [Mac OS X](../development/build_osx.md). - -Можно скомпилировать пакеты и установить их, либо использовать программы без установки пакетов. Также при ручой сборке можно отключить необходимость поддержки набора инструкций SSE 4.2 или собрать под процессоры архитектуры AArch64. - -``` -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -Для работы собранного вручную сервера необходимо создать директории для данных и метаданных, а также сделать их `chown` для желаемого пользователя. Пути к этим директориям могут быть изменены в конфигурационном файле сервера (src/dbms/programs/server/config.xml), по умолчанию используются следующие: - -``` -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -На Gentoo для установки ClickHouse из исходного кода можно использовать просто `emerge clickhouse`. - -## Запуск - -Для запуска сервера в качестве демона, выполните: - -``` bash -$ sudo service clickhouse-server start -``` - -Смотрите логи в директории `/var/log/clickhouse-server/`. - -Если сервер не стартует, проверьте корректность конфигурации в файле `/etc/clickhouse-server/config.xml` - -Также можно запустить сервер вручную из консоли: - -``` bash -$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -При этом, лог будет выводиться в консоль, что удобно для разработки. -Если конфигурационный файл лежит в текущей директории, то указывать параметр `--config-file` не требуется, по умолчанию будет использован файл `./config.xml`. - -После запуска сервера, соединиться с ним можно с помощью клиента командной строки: - -``` bash -$ clickhouse-client -``` - -По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргумента `--host`. - -Терминал должен использовать кодировку UTF-8. - -Более подробная информация о клиенте располагается в разделе [«Клиент командной строки»](../interfaces/cli.md). - -Пример проверки работоспособности системы: - -``` bash -$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -**Поздравляем, система работает!** - -Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.yandex/tutorial.html). +* [Пройти подробное руководство для начинающих](tutorial.md) +* [Поэкспериментировать с тестовыми наборами данных](example_datasets/ontime.md) [Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/) diff --git a/docs/ru/getting_started/install.md b/docs/ru/getting_started/install.md new file mode 100644 index 00000000000..ae8075e0157 --- /dev/null +++ b/docs/ru/getting_started/install.md @@ -0,0 +1,142 @@ +# Установка + +## Системные требования + +ClickHouse может работать на любом Linux, FreeBSD или Mac OS X с архитектурой процессора x86\_64. + +Хотя предсобранные релизы обычно компилируются с использованием набора инструкций SSE 4.2, что добавляет использование поддерживающего его процессора в список системных требований. Команда для проверки наличия поддержки инструкций SSE 4.2 на текущем процессоре: + +```bash +$ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +## Доступные варианты установки + +### Из DEB пакетов + +Яндекс рекомендует использовать официальные скомпилированные `deb` пакеты для Debian или Ubuntu. + +Чтобы установить официальные пакеты, пропишите репозиторий Яндекса в `/etc/apt/sources.list` или в отдельный файл `/etc/apt/sources.list.d/clickhouse.list`: + +``` +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +Если вы хотите использовать наиболее свежую тестовую, замените `stable` на `testing` (не рекомендуется для production окружений). + +Затем для самой установки пакетов выполните: + +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +Также эти пакеты можно скачать и установить вручную отсюда: . + +### Из RPM пакетов + +Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. + +Сначала нужно подключить официальный репозиторий: +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +Для использования наиболее свежих версий нужно заменить `stable` на `testing` (рекомендуется для тестовых окружений). + +Then run these commands to actually install packages: +Для, собственно, установки пакетов необходимо выполнить следующие команды: + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +Также есть возможность установить пакеты вручную, скачав отсюда: . + +### Из Docker образа + +Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты. + +### Из исходникого кода + +Для компиляции ClickHouse вручную, используйте инструкцию для [Linux](../development/build.md) или [Mac OS X](../development/build_osx.md). + +Можно скомпилировать пакеты и установить их, либо использовать программы без установки пакетов. Также при ручой сборке можно отключить необходимость поддержки набора инструкций SSE 4.2 или собрать под процессоры архитектуры AArch64. + +``` +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +Для работы собранного вручную сервера необходимо создать директории для данных и метаданных, а также сделать их `chown` для желаемого пользователя. Пути к этим директориям могут быть изменены в конфигурационном файле сервера (src/dbms/programs/server/config.xml), по умолчанию используются следующие: + +``` +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +На Gentoo для установки ClickHouse из исходного кода можно использовать просто `emerge clickhouse`. + +## Запуск + +Для запуска сервера в качестве демона, выполните: + +``` bash +$ sudo service clickhouse-server start +``` + +Смотрите логи в директории `/var/log/clickhouse-server/`. + +Если сервер не стартует, проверьте корректность конфигурации в файле `/etc/clickhouse-server/config.xml` + +Также можно запустить сервер вручную из консоли: + +``` bash +$ clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +При этом, лог будет выводиться в консоль, что удобно для разработки. +Если конфигурационный файл лежит в текущей директории, то указывать параметр `--config-file` не требуется, по умолчанию будет использован файл `./config.xml`. + +После запуска сервера, соединиться с ним можно с помощью клиента командной строки: + +``` bash +$ clickhouse-client +``` + +По умолчанию он соединяется с localhost:9000, от имени пользователя `default` без пароля. Также клиент может быть использован для соединения с удалённым сервером с помощью аргумента `--host`. + +Терминал должен использовать кодировку UTF-8. + +Более подробная информация о клиенте располагается в разделе [«Клиент командной строки»](../interfaces/cli.md). + +Пример проверки работоспособности системы: + +``` bash +$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**Поздравляем, система работает!** + +Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.yandex/tutorial.html). + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/install/) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index d823e026164..bf05e68e04a 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -8,8 +8,9 @@ nav: - 'The Yandex.Metrica Task': 'introduction/ya_metrika_task.md' - 'Getting Started': + - 'hidden': 'getting_started/index.md' + - 'Installation': 'getting_started/install.md' - 'Tutorial': 'getting_started/tutorial.md' - - 'Deploying and Running': 'getting_started/index.md' - 'Example Datasets': - 'OnTime': 'getting_started/example_datasets/ontime.md' - 'New York Taxi Data': 'getting_started/example_datasets/nyc_taxi.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 1799093df24..682e9197bac 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -1,6 +1,6 @@ nav: -- 'Introduction': +- 'ﯽﻓﺮﻌﻣ': - 'ClickHouse چیست؟': 'index.md' - ' ویژگی های برجسته ClickHouse': 'introduction/distinctive_features.md' - ' ویژگی های از ClickHouse که می تواند معایبی باشد': 'introduction/features_considered_disadvantages.md' @@ -8,8 +8,10 @@ nav: - 'The Yandex.Metrica task': 'introduction/ya_metrika_task.md' - 'Getting started': - - ' شروع به کار': 'getting_started/index.md' - - 'Example datasets': + - 'hidden': 'getting_started/index.md' + - 'ﯼﺯﺍﺪﻧﺍ ﻩﺍﺭ ﻭ ﺐﺼﻧ': 'getting_started/install.md' + - 'ﺵﺯﻮﻣﺁ': 'getting_started/tutorial.md' + - 'ﻪﻧﻮﻤﻧ ﯼﺎﻫ ﻩﺩﺍﺩ ﻪﻋﻮﻤﺠﻣ': - 'OnTime': 'getting_started/example_datasets/ontime.md' - ' داده های تاکسی New York': 'getting_started/example_datasets/nyc_taxi.md' - ' بنچمارک AMPLab Big Data': 'getting_started/example_datasets/amplab_benchmark.md' @@ -18,7 +20,7 @@ nav: - ' بنچمارک Star Schema': 'getting_started/example_datasets/star_schema.md' - 'Yandex.Metrica Data': 'getting_started/example_datasets/metrica.md' -- 'Interfaces': +- 'ﻂﺑﺍﺭ': - 'Interface ها': 'interfaces/index.md' - ' کلاینت Command-line': 'interfaces/cli.md' - 'Native interface (TCP)': 'interfaces/tcp.md' @@ -32,7 +34,7 @@ nav: - 'رابط های بصری': 'interfaces/third-party/gui.md' - 'پروکسی': 'interfaces/third-party/proxy.md' -- 'Data types': +- 'ﻩﺩﺍﺩ ﻉﺍﻮﻧﺍ': - 'Introduction': 'data_types/index.md' - 'UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64': 'data_types/int_uint.md' - 'Float32, Float64': 'data_types/float.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 682f171a1f3..60b1a8afd23 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -9,7 +9,9 @@ nav: - 'Информационная поддержка': 'introduction/info.md' - 'Начало работы': - - 'Установка и запуск': 'getting_started/index.md' + - 'hidden': 'getting_started/index.html' + - 'Установка': 'getting_started/install.md' + - 'Руководство для начинающих': 'getting_started/tutorial.md' - 'Тестовые наборы данных': - 'OnTime': 'getting_started/example_datasets/ontime.md' - 'Данные о такси в Нью-Йорке': 'getting_started/example_datasets/nyc_taxi.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index d140ec88d64..ef4f05c6172 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -8,7 +8,9 @@ nav: - 'Yandex.Metrica使用案例': 'introduction/ya_metrika_task.md' - '入门指南': - - '部署运行': 'getting_started/index.md' + - 'hidden': 'getting_started/index.md' + - '安装': 'getting_started/install.md' + - '教程': 'getting_started/tutorial.md' - '示例数据集': - '航班飞行数据': 'getting_started/example_datasets/ontime.md' - '纽约市出租车数据': 'getting_started/example_datasets/nyc_taxi.md' diff --git a/docs/zh/getting_started/index.md b/docs/zh/getting_started/index.md index f51323ce7e8..c73181a6068 100644 --- a/docs/zh/getting_started/index.md +++ b/docs/zh/getting_started/index.md @@ -1,158 +1,10 @@ -# 入门指南 +# 入门 -## 系统要求 +如果您是ClickHouse的新手，并希望亲身体验它的性能，首先您需要通过 [安装过程](install.md). -如果从官方仓库安装，需要确保您使用的是x86\_64处理器构架的Linux并且支持SSE 4.2指令集 +之后，您可以选择以下选项之一： -检查是否支持SSE 4.2： +* [通过详细的教程](tutorial.md) +* [试验示例数据集](example_datasets/ontime.md) -```bash -grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" -``` - -我们推荐使用Ubuntu或者Debian。终端必须使用UTF-8编码。 - -基于rpm的系统,你可以使用第三方的安装包：https://packagecloud.io/altinity/clickhouse 或者直接安装debian安装包。 - -ClickHouse还可以在FreeBSD与Mac OS X上工作。同时它可以在不支持SSE 4.2的x86\_64构架和AArch64 CPUs上编译。 - -## 安装 - -### 为Debian/Ubuntu安装 - -在`/etc/apt/sources.list` (或创建`/etc/apt/sources.list.d/clickhouse.list`文件)中添加仓库： - -```text -deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ -``` - -如果你想使用最新的测试版本，请使用'testing'替换'stable'。 - -然后运行： - -```bash -sudo apt-get install dirmngr # optional -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional -sudo apt-get update -sudo apt-get install clickhouse-client clickhouse-server -``` - -你也可以从这里手动下载安装包：。 - -ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config.xml'同目录)。 -默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 -有关更多信息，请参考"Configuration files"部分。 - -###来自RPM包 - -Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 - -首先，您需要添加官方存储库： - -```bash -sudo yum install yum-utils -sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG -sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 -``` - -如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 - -然后运行这些命令以实际安装包： - -```bash -sudo yum install clickhouse-server clickhouse-client -``` - -您也可以从此处手动下载和安装软件包：。 - -###来自Docker - -要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 - -### 使用源码安装 - -具体编译方式可以参考build.md。 - -你可以编译并安装它们。 -你也可以直接使用而不进行安装。 - -```text -Client: dbms/programs/clickhouse-client -Server: dbms/programs/clickhouse-server -``` - -在服务器中为数据创建如下目录： - -```text -/opt/clickhouse/data/default/ -/opt/clickhouse/metadata/default/ -``` - -(它们可以在server config中配置。) -为需要的用户运行‘chown’ - -日志的路径可以在server config (src/dbms/programs/server/config.xml)中配置。 - -## 启动 - -可以运行如下命令在后台启动服务： - -```bash -sudo service clickhouse-server start -``` - -可以在`/var/log/clickhouse-server/`目录中查看日志。 - -如果服务没有启动，请检查配置文件 `/etc/clickhouse-server/config.xml`。 - -你也可以在控制台中直接启动服务： - -```bash -clickhouse-server --config-file=/etc/clickhouse-server/config.xml -``` - -在这种情况下，日志将被打印到控制台中，这在开发过程中很方便。 -如果配置文件在当前目录中，你可以不指定‘--config-file’参数。它默认使用‘./config.xml’。 - -你可以使用命令行客户端连接到服务： - -```bash -clickhouse-client -``` - -默认情况下它使用‘default’用户无密码的与localhost:9000服务建立连接。 -客户端也可以用于连接远程服务，例如： - -```bash -clickhouse-client --host=example.com -``` - -有关更多信息，请参考"Command-line client"部分。 - -检查系统是否工作： - -```bash -milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client -ClickHouse client version 0.0.18749. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.18749. - -:) SELECT 1 - -SELECT 1 - -┌─1─┐ -│ 1 │ -└───┘ - -1 rows in set. Elapsed: 0.003 sec. - -:) -``` - -**恭喜，系统已经工作了!** - -为了继续进行实验，你可以尝试下载测试数据集。 - - -[Original article](https://clickhouse.yandex/docs/en/getting_started/) +[来源文章](https://clickhouse.yandex/docs/zh/getting_started/) diff --git a/docs/zh/getting_started/install.md b/docs/zh/getting_started/install.md new file mode 100644 index 00000000000..29aee915bfa --- /dev/null +++ b/docs/zh/getting_started/install.md @@ -0,0 +1,156 @@ +## 系统要求 + +如果从官方仓库安装，需要确保您使用的是x86\_64处理器构架的Linux并且支持SSE 4.2指令集 + +检查是否支持SSE 4.2： + +```bash +grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" +``` + +我们推荐使用Ubuntu或者Debian。终端必须使用UTF-8编码。 + +基于rpm的系统,你可以使用第三方的安装包：https://packagecloud.io/altinity/clickhouse 或者直接安装debian安装包。 + +ClickHouse还可以在FreeBSD与Mac OS X上工作。同时它可以在不支持SSE 4.2的x86\_64构架和AArch64 CPUs上编译。 + +##可用的安装选项 + +### 为Debian/Ubuntu安装 + +在`/etc/apt/sources.list` (或创建`/etc/apt/sources.list.d/clickhouse.list`文件)中添加仓库： + +```text +deb http://repo.yandex.ru/clickhouse/deb/stable/ main/ +``` + +如果你想使用最新的测试版本，请使用'testing'替换'stable'。 + +然后运行： + +```bash +sudo apt-get install dirmngr # optional +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 # optional +sudo apt-get update +sudo apt-get install clickhouse-client clickhouse-server +``` + +你也可以从这里手动下载安装包：。 + +ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config.xml'同目录)。 +默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 +有关更多信息，请参考"Configuration files"部分。 + +###来自RPM包 + +Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 + +首先，您需要添加官方存储库： + +```bash +sudo yum install yum-utils +sudo rpm --import https://repo.yandex.ru/clickhouse/CLICKHOUSE-KEY.GPG +sudo yum-config-manager --add-repo https://repo.yandex.ru/clickhouse/rpm/stable/x86_64 +``` + +如果您想使用最新版本，请将`stable`替换为`testing`（建议您在测试环境中使用）。 + +然后运行这些命令以实际安装包： + +```bash +sudo yum install clickhouse-server clickhouse-client +``` + +您也可以从此处手动下载和安装软件包：。 + +###来自Docker + +要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 + +### 使用源码安装 + +具体编译方式可以参考build.md。 + +你可以编译并安装它们。 +你也可以直接使用而不进行安装。 + +```text +Client: dbms/programs/clickhouse-client +Server: dbms/programs/clickhouse-server +``` + +在服务器中为数据创建如下目录： + +```text +/opt/clickhouse/data/default/ +/opt/clickhouse/metadata/default/ +``` + +(它们可以在server config中配置。) +为需要的用户运行‘chown’ + +日志的路径可以在server config (src/dbms/programs/server/config.xml)中配置。 + +## 启动 + +可以运行如下命令在后台启动服务： + +```bash +sudo service clickhouse-server start +``` + +可以在`/var/log/clickhouse-server/`目录中查看日志。 + +如果服务没有启动，请检查配置文件 `/etc/clickhouse-server/config.xml`。 + +你也可以在控制台中直接启动服务： + +```bash +clickhouse-server --config-file=/etc/clickhouse-server/config.xml +``` + +在这种情况下，日志将被打印到控制台中，这在开发过程中很方便。 +如果配置文件在当前目录中，你可以不指定‘--config-file’参数。它默认使用‘./config.xml’。 + +你可以使用命令行客户端连接到服务： + +```bash +clickhouse-client +``` + +默认情况下它使用‘default’用户无密码的与localhost:9000服务建立连接。 +客户端也可以用于连接远程服务，例如： + +```bash +clickhouse-client --host=example.com +``` + +有关更多信息，请参考"Command-line client"部分。 + +检查系统是否工作： + +```bash +milovidov@hostname:~/work/metrica/src/dbms/src/Client$ ./clickhouse-client +ClickHouse client version 0.0.18749. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.18749. + +:) SELECT 1 + +SELECT 1 + +┌─1─┐ +│ 1 │ +└───┘ + +1 rows in set. Elapsed: 0.003 sec. + +:) +``` + +**恭喜，系统已经工作了!** + +为了继续进行实验，你可以尝试下载测试数据集。 + + +[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) From 9e8e7709f6d91eb752dec57f79cc2eabbcfbb534 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 23 Aug 2019 18:29:58 +0300 Subject: [PATCH 007/312] add ../en/getting_started/index.md --- docs/en/getting_started/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/en/getting_started/index.md diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md new file mode 100644 index 00000000000..390cb3183c4 --- /dev/null +++ b/docs/en/getting_started/index.md @@ -0,0 +1,10 @@ +# Getting Started + +If you are new to ClickHouse and want to get a hands-on feeling of it's performance, first of all you need to go through the [installation process](install.md). + +After that you can choose one of the following options: + +* [Go through detailed tutorial](tutorial.md) +* [Experiment with example datasets](example_datasets/ontime.md) + +[Original article](https://clickhouse.yandex/docs/en/getting_started/) From 6b936b0c04b375ed9c1042444acd3a1bcb10b465 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 28 Aug 2019 11:51:03 +0300 Subject: [PATCH 008/312] Rename ya_metrica_task.md --- docs/en/introduction/history.md | 50 +++++++++++++++++++++++++++++++++ docs/fa/introduction/history.md | 49 ++++++++++++++++++++++++++++++++ docs/redirects.txt | 1 + docs/ru/introduction/history.md | 49 ++++++++++++++++++++++++++++++++ docs/toc_en.yml | 2 +- docs/toc_fa.yml | 4 +-- docs/toc_ru.yml | 2 +- docs/toc_zh.yml | 2 +- docs/zh/introduction/history.md | 50 +++++++++++++++++++++++++++++++++ 9 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 docs/en/introduction/history.md create mode 100644 docs/fa/introduction/history.md create mode 100644 docs/ru/introduction/history.md create mode 100644 docs/zh/introduction/history.md diff --git a/docs/en/introduction/history.md b/docs/en/introduction/history.md new file mode 100644 index 00000000000..e8f373880f1 --- /dev/null +++ b/docs/en/introduction/history.md @@ -0,0 +1,50 @@ +# ClickHouse History + +ClickHouse was originally developed to power [Yandex.Metrica](https://metrica.yandex.com/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all), and continues to be the core component of this system. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows generating custom reports on the fly directly from non-aggregated data. This article briefly covers the goals of ClickHouse in the early stages of its development. + +Yandex.Metrica builds customized reports on the fly based on hits and sessions, with arbitrary segments defined by the user. This often requires building complex aggregates, such as the number of unique users. New data for building a report is received in real time. + +As of April 2014, Yandex.Metrica was tracking about 12 billion events (page views and clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning millions of rows within a few hundred milliseconds, or hundreds of millions of rows in just a few seconds. + +## Usage in Yandex.Metrica and Other Yandex Services + +ClickHouse is used for multiple purposes in Yandex.Metrica. +Its main task is to build reports in online mode using non-aggregated data. It uses a cluster of 374 servers, which store over 20.3 trillion rows in the database. The volume of compressed data, without counting duplication and replication, is about 2 PB. The volume of uncompressed data (in TSV format) would be approximately 17 PB. + +ClickHouse is also used for: + +- Storing data for Session Replay from Yandex.Metrica. +- Processing intermediate data. +- Building global reports with Analytics. +- Running queries for debugging the Yandex.Metrica engine. +- Analyzing logs from the API and the user interface. + +ClickHouse has at least a dozen installations in other Yandex services: in search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services, and others. + +## Aggregated and Non-aggregated Data + +There is a popular opinion that in order to effectively calculate statistics, you must aggregate data, since this reduces the volume of data. + +But data aggregation is a very limited solution, for the following reasons: + +- You must have a pre-defined list of reports the user will need. +- The user can't make custom reports. +- When aggregating a large quantity of keys, the volume of data is not reduced, and aggregation is useless. +- For a large number of reports, there are too many aggregation variations (combinatorial explosion). +- When aggregating keys with high cardinality (such as URLs), the volume of data is not reduced by much (less than twofold). +- For this reason, the volume of data with aggregation might grow instead of shrink. +- Users do not view all the reports we generate for them. A large portion of calculations are useless. +- The logical integrity of data may be violated for various aggregations. + +If we do not aggregate anything and work with non-aggregated data, this might actually reduce the volume of calculations. + +However, with aggregation, a significant part of the work is taken offline and completed relatively calmly. In contrast, online calculations require calculating as fast as possible, since the user is waiting for the result. + +Yandex.Metrica has a specialized system for aggregating data called Metrage, which is used for the majority of reports. +Starting in 2009, Yandex.Metrica also used a specialized OLAP database for non-aggregated data called OLAPServer, which was previously used for the report builder. +OLAPServer worked well for non-aggregated data, but it had many restrictions that did not allow it to be used for all reports as desired. These included the lack of support for data types (only numbers), and the inability to incrementally update data in real-time (it could only be done by rewriting data daily). OLAPServer is not a DBMS, but a specialized DB. + +To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS. + + +[Original article](https://clickhouse.yandex/docs/en/introduction/history/) diff --git a/docs/fa/introduction/history.md b/docs/fa/introduction/history.md new file mode 100644 index 00000000000..abde10aa6f3 --- /dev/null +++ b/docs/fa/introduction/history.md @@ -0,0 +1,49 @@ +
+ +# ClickHouse ﻪﭽﺨﯾﺭﺎﺗ + +ClickHouse در ابتدا برای قدرت به Yandex.Metrica دومین بستر آنالیز وب در دنیا توسعه داده شد، و همچنان جز اصلی آن است. ClickHouse اجازه می دهند که با بیش از 13 تریلیون رکورد در دیتابیس و بیش از 20 میلیارد event در روز، گزارش های مستقیم (On the fly) از داده های non-aggregate تهیه کنیم. این مقاله پیشنیه ی تاریخی در ارتباط با اهداف اصلی ClickHouse قبل از آنکه به یک محصول open source تبدیل شود، می دهد. + +Yandex.Metrica تولید گزارش های برپایه بازدید و session ها به صورت on the fly و با استفده از بخش های دلخواه و دوره ی زمانی که توسط کاربر انتخاب می شود را انجام می دهد. aggregate های پیچیده معمولا مورد نیاز هستند، مانند تعداد بازدیدکنندگان unique. داده های جدید برای تهیه گزارش گیری به صورت real-time می رسند. + +از آوریل 2014، Yandex.Metrica تقریبا 12 میلیارد event شامل page view و click در روز دریافت کرد. تمام این event ها باید به ترتیب برای ساخت گزارش های سفارشی ذخیره سازی می شدند. یک query ممکن است نیاز به اسکن کردن میلیون ها سطر با زمان کمتر از چند صد میلی ثانیه، یا چند صد میلیون سطر در عرض چند ثانیه داشته باشد. + +## استفاده در Yandex.Metrica و دیگر سرویس های Yandex + +ClickHouse با چندین اهداف در Yandex.Metrica استفاده می شود. وظیفه اصلی آن ساخت گزارش های آنلاین از داده های non-aggregate می باشد. ClickHouse در یک کلاستر با سایز 374 سرور، که بیش از 20.3 تریلیون سطر در دیتابیس را دارد مورد استفاده قرار می گیرد. اندازه فشرده داده ها، بدون شمارش داده های تکراری و replication، حدود 2 پتابایت می باشد. اندازه ی غیرفشرده داده ها (در فرمت TSV) حدودا 17 پتابایت می باشد. + +ClickHouse همچنین در موارد زیراستفاده می شود: + +- ذخیره سازی داده ها برای Session replay از Yandex.Metrica. +- پردازش داده های Intermediate. +- ساخت گزارش های سراسری از آنالیز ها. +- اجرای query ها برای debug کردن موتور Yandex.Metrica. +- آنالیز لاگ های به دست آمده از API ها و user interface. + +ClickHouse حداقل در دوازده جای دیگر سرویس Yandex نصب شده است: در search verticals، Market، Direct، Business Analytics، Mobile Development، AdFox، سرویس های شخصی و.. + +## داده های Aggregate , Non-Aggregate + +یک دیدگاه محبوب وجود دارد که شما باید، داده های خود را به منظور کاهش اندازه داده ها Aggregate کنید. + +اما به دلایل زیر، aggregate کردن داده ها راه حل بسیار محدودی است: + +- شما باید لیست گزارش های از قبل تعریف شده توسط کاربر که نیاز به تهیه گزارش آنها را دارید، داشته باشید. +- کاربر نمیتواند گزارش های سفارشی تهیه کند. +- در هنگام aggregate کردن تعداد بسیار زیاد key، اندازه ی داده ها کم نمی شود و aggregate بی فایده است. +- برای تعداد زیادی از گزارش ها، aggregate های متنوع و تغییرپذیر زیادی وجود دارد. (انفجار ترکیبی). +- هنگام aggregate کردن key ها با cardinality بالا (مثل URL ها)، اندازه داده ها به اندازه کافی کاهش پیدا نمی کند (کمتر از دو برابر). +- به این دلیل اندازه ی داده ها با aggregate کردن ممکن است به جای شکستن، رشد هم بکند. +- کاربر تمام گزارش هایی که ما تولید کردیم را نگاه نمی کند. بخش بزرگی از محاسبات بی فایده است. +- یکپارچگی منطقی داده ها ممکن است برای aggregate های مختلف نقض شود. + +اگر ما هیچ چیزی را aggregate نکنیم و با داده های non-aggregate کار کنیم، در واقع این ممکن است باعث کاهش اندازه ی محاسبات شود. + +با این حال، با aggregate کردن، بخش قابل توجهی از کار به صورت آفلاین انجام می شود و نسبتا آرام به پایان می رسد. در مقابل، محاسبات آنلاین به دلیل اینکه کاربر منتظر نمایش نتایج می باشد، نیازمند محاسبه سریع تا جایی که ممکن است می باشد. + +Yandex.Metrica دارای یک سیستم تخصصی برای aggregate کردن داده ها به اسم Metrage می باشد، که برای اکثریت گزارش های مورد استفاده قرار می گیرد. شروع سال 2009، Yandex.Metrica همچنین از یک دیتابیس تخصصی OLAP برای داده های non-aggregate به نام OLAPServer، که قبلا برای ساخت گزارش ها استفاده می شد، استفاده می کرد. OLAPServer به خوبی روی داده های Non-Aggregate کار می کرد، اما محدودیت های بسیار زیادی داشت که اجازه ی استفاده در تمام گزارش های دلخواه را نمی داد. مواردی از قبیل عدم پشتیبانی از data type ها (فقط عدد)، و عدم توانایی در بروزرسانی افزایشی داده ها به صورت real-time (این کار فقط به rewrite کردن داده ها به صورت روزانه امکام پذیر بود). OLAPServer یک مدیریت دیتابیس نبود اما یک دیتابیس تخصصی بود. + +برای حذف محدودیت های OLAPServer و حل مشکلات کار با داده های Non-Aggregate برای تمام گزارش ها، ما مدیریت دیتابیس ClicHouse را توسعه دادیم.. + +
+[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/ya_metrika_task/) diff --git a/docs/redirects.txt b/docs/redirects.txt index 0ff077b660c..b38f6d242f2 100644 --- a/docs/redirects.txt +++ b/docs/redirects.txt @@ -1,3 +1,4 @@ +introduction/ya_metrika_task.md introduction/history.md system_tables.md operations/system_tables.md system_tables/system.asynchronous_metrics.md operations/system_tables.md system_tables/system.clusters.md operations/system_tables.md diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md new file mode 100644 index 00000000000..c0035b51f82 --- /dev/null +++ b/docs/ru/introduction/history.md @@ -0,0 +1,49 @@ +# История ClickHouse + +ClickHouse изначально разрабатывался для обеспечения работы [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики, и продолжает быть её ключевым компонентом. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Данная статья вкратце демонстрирует какие цели исторически стояли перед ClickHouse на ранних этапах его развития. + +Яндекс.Метрика на лету строит индивидуальные отчёты на основе хитов и визитов, с периодом и произвольными сегментами, задаваемыми конечным пользователем. Часто требуется построение сложных агрегатов, например числа уникальных пользователей. Новые данные для построения отчета поступают в реальном времени. + +На апрель 2014, в Яндекс.Метрику поступало около 12 миллиардов событий (показов страниц и кликов мыши) ежедневно. Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать миллионы строк за время не более нескольких сотен миллисекунд, или сотни миллионов строк за время не более нескольких секунд. + +## Использование в Яндекс.Метрике и других отделах Яндекса + +В Яндекс.Метрике ClickHouse используется для нескольких задач. +Основная задача - построение отчётов в режиме онлайн по неагрегированным данным. Для решения этой задачи используется кластер из 374 серверов, хранящий более 20,3 триллионов строк в базе данных. Объём сжатых данных, без учёта дублирования и репликации, составляет около 2 ПБ. Объём несжатых данных (в формате tsv) составил бы, приблизительно, 17 ПБ. + +Также ClickHouse используется: + +- для хранения данных Вебвизора; +- для обработки промежуточных данных; +- для построения глобальных отчётов Аналитиками; +- для выполнения запросов в целях отладки движка Метрики; +- для анализа логов работы API и пользовательского интерфейса. + +ClickHouse имеет более десятка инсталляций в других отделах Яндекса: в Вертикальных сервисах, Маркете, Директе, БК, Бизнес аналитике, Мобильной разработке, AdFox, Персональных сервисах и т п. + +## Агрегированные и неагрегированные данные + +Существует мнение, что для того, чтобы эффективно считать статистику, данные нужно агрегировать, так как это позволяет уменьшить объём данных. + +Но агрегированные данные являются очень ограниченным решением, по следующим причинам: + +- вы должны заранее знать перечень отчётов, необходимых пользователю; +- то есть, пользователь не может построить произвольный отчёт; +- при агрегации по большому количеству ключей, объём данных не уменьшается и агрегация бесполезна; +- при большом количестве отчётов, получается слишком много вариантов агрегации (комбинаторный взрыв); +- при агрегации по ключам высокой кардинальности (например, URL) объём данных уменьшается не сильно (менее чем в 2 раза); +- из-за этого, объём данных при агрегации может не уменьшиться, а вырасти; +- пользователи будут смотреть не все отчёты, которые мы для них посчитаем - то есть, большая часть вычислений бесполезна; +- возможно нарушение логической целостности данных для разных агрегаций; + +Как видно, если ничего не агрегировать, и работать с неагрегированными данными, то это даже может уменьшить объём вычислений. + +Впрочем, при агрегации, существенная часть работы выносится в оффлайне, и её можно делать сравнительно спокойно. Для сравнения, при онлайн вычислениях, вычисления надо делать так быстро, как это возможно, так как именно в момент вычислений пользователь ждёт результата. + +В Яндекс.Метрике есть специализированная система для агрегированных данных - Metrage, на основе которой работает большинство отчётов. +Также в Яндекс.Метрике с 2009 года использовалась специализированная OLAP БД для неагрегированных данных - OLAPServer, на основе которой раньше работал конструктор отчётов. +OLAPServer хорошо подходил для неагрегированных данных, но содержал много ограничений, не позволяющих использовать его для всех отчётов так, как хочется: отсутствие поддержки типов данных (только числа), невозможность инкрементального обновления данных в реальном времени (только перезаписью данных за сутки). OLAPServer не является СУБД, а является специализированной БД. + +Чтобы снять ограничения OLAPServer-а и решить задачу работы с неагрегированными данными для всех отчётов, разработана СУБД ClickHouse. + +[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/ya_metrika_task/) diff --git a/docs/toc_en.yml b/docs/toc_en.yml index bf05e68e04a..756dc15414d 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -5,7 +5,7 @@ nav: - 'Distinctive Features of ClickHouse': 'introduction/distinctive_features.md' - 'ClickHouse Features that Can Be Considered Disadvantages': 'introduction/features_considered_disadvantages.md' - 'Performance': 'introduction/performance.md' - - 'The Yandex.Metrica Task': 'introduction/ya_metrika_task.md' + - 'History': 'introduction/history.md' - 'Getting Started': - 'hidden': 'getting_started/index.md' diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index 682e9197bac..31484cd596c 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -4,8 +4,8 @@ nav: - 'ClickHouse چیست؟': 'index.md' - ' ویژگی های برجسته ClickHouse': 'introduction/distinctive_features.md' - ' ویژگی های از ClickHouse که می تواند معایبی باشد': 'introduction/features_considered_disadvantages.md' - - 'Performance': 'introduction/performance.md' - - 'The Yandex.Metrica task': 'introduction/ya_metrika_task.md' + - 'ﯽﯾﺍﺭﺎﮐ': 'introduction/performance.md' + - 'ﺦﯾﺭﺎﺗ': 'introduction/history.md' - 'Getting started': - 'hidden': 'getting_started/index.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 60b1a8afd23..f14dce709ac 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -5,7 +5,7 @@ nav: - 'Отличительные возможности ClickHouse': 'introduction/distinctive_features.md' - 'Особенности ClickHouse, которые могут считаться недостатками': 'introduction/features_considered_disadvantages.md' - 'Производительность': 'introduction/performance.md' - - 'Постановка задачи в Яндекс.Метрике': 'introduction/ya_metrika_task.md' + - 'История': 'introduction/history.md' - 'Информационная поддержка': 'introduction/info.md' - 'Начало работы': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index ef4f05c6172..3aef09a24ed 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -5,7 +5,7 @@ nav: - 'ClickHouse的独特功能': 'introduction/distinctive_features.md' - 'ClickHouse功能可被视为缺点': 'introduction/features_considered_disadvantages.md' - '性能': 'introduction/performance.md' - - 'Yandex.Metrica使用案例': 'introduction/ya_metrika_task.md' + - '历史': 'introduction/history.md' - '入门指南': - 'hidden': 'getting_started/index.md' diff --git a/docs/zh/introduction/history.md b/docs/zh/introduction/history.md new file mode 100644 index 00000000000..86fe02f84d5 --- /dev/null +++ b/docs/zh/introduction/history.md @@ -0,0 +1,50 @@ +# ClickHouse历史 + +ClickHouse最初是为 [Yandex.Metrica](https://metrica.yandex.com/) [世界第二大Web分析平台](http://w3techs.com/technologies/overview/traffic_analysis/all) 而开发的。多年来一直作为该系统的核心组件被该系统持续使用着。目前为止，该系统在ClickHouse中有超过13万亿条记录，并且每天超过200多亿个事件被处理。它允许直接从原始数据中动态查询并生成报告。本文简要介绍了ClickHouse在其早期发展阶段的目标。 + +Yandex.Metrica基于用户定义的字段，对实时访问、连接会话，生成实时的统计报表。这种需求往往需要复杂聚合方式，比如对访问用户进行去重。构建报表的数据，是实时接收存储的新数据。 + +截至2014年4月，Yandex.Metrica每天跟踪大约120亿个事件（用户的点击和浏览）。为了可以创建自定义的报表，我们必须存储全部这些事件。同时，这些查询可能需要在几百毫秒内扫描数百万行的数据，或在几秒内扫描数亿行的数据。 + +## Yandex.Metrica以及其他Yandex服务的使用案例 + +在Yandex.Metrica中，ClickHouse被用于多个场景中。 +它的主要任务是使用原始数据在线的提供各种数据报告。它使用374台服务器的集群，存储了20.3万亿行的数据。在去除重复与副本数据的情况下，压缩后的数据达到了2PB。未压缩前（TSV格式）它大概有17PB。 + +ClickHouse还被使用在： + +- 存储来自Yandex.Metrica回话重放数据。 +- 处理中间数据 +- 与Analytics一起构建全球报表。 +- 为调试Yandex.Metrica引擎运行查询 +- 分析来自API和用户界面的日志数据 + +ClickHouse在其他Yandex服务中至少有12个安装：search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services等。 + +## 聚合与非聚合数据 + +有一种流行的观点认为，想要有效的计算统计数据，必须要聚合数据，因为聚合将降低数据量。 + +但是数据聚合是一个有诸多限制的解决方案，例如： + +- 你必须提前知道用户定义的报表的字段列表 +- 用户无法自定义报表 +- 当聚合条件过多时，可能不会减少数据，聚合是无用的。 +- 存在大量报表时，有太多的聚合变化（组合爆炸） +- 当聚合条件有非常大的基数时（如：url），数据量没有太大减少（少于两倍） +- 聚合的数据量可能会增长而不是收缩 +- 用户不会查看我们为他生成的所有报告，大部分计算将是无用的 +- 各种聚合可能违背了数据的逻辑完整性 + +如果我们直接使用非聚合数据而不进行任何聚合时，我们的计算量可能是减少的。 + +然而，相对于聚合中很大一部分工作被离线完成，在线计算需要尽快的完成计算，因为用户在等待结果。 + +Yandex.Metrica 有一个专门用于聚合数据的系统，称为Metrage，它可以用作大部分报表。 +从2009年开始，Yandex.Metrica还为非聚合数据使用专门的OLAP数据库，称为OLAPServer，它以前用于报表构建系统。 +OLAPServer可以很好的工作在非聚合数据上，但是它有诸多限制，导致无法根据需要将其用于所有报表中。如，缺少对数据类型的支持（只支持数据），无法实时增量的更新数据（只能通过每天重写数据完成）。OLAPServer不是一个数据库管理系统，它只是一个数据库。 + +为了消除OLAPServer的这些局限性，解决所有报表使用非聚合数据的问题，我们开发了ClickHouse数据库管理系统。 + + +[来源文章](https://clickhouse.yandex/docs/en/introduction/ya_metrika_task/) From 86de9486cf5fca46c6e31046e4eee8287327edba Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 28 Aug 2019 11:51:27 +0300 Subject: [PATCH 009/312] Rename ya_metrica_task.md --- docs/en/introduction/ya_metrika_task.md | 50 ------------------------- docs/fa/introduction/ya_metrika_task.md | 49 ------------------------ docs/ru/introduction/ya_metrika_task.md | 49 ------------------------ docs/zh/introduction/ya_metrika_task.md | 50 ------------------------- 4 files changed, 198 deletions(-) delete mode 100644 docs/en/introduction/ya_metrika_task.md delete mode 100644 docs/fa/introduction/ya_metrika_task.md delete mode 100644 docs/ru/introduction/ya_metrika_task.md delete mode 100644 docs/zh/introduction/ya_metrika_task.md diff --git a/docs/en/introduction/ya_metrika_task.md b/docs/en/introduction/ya_metrika_task.md deleted file mode 100644 index 41b33eff581..00000000000 --- a/docs/en/introduction/ya_metrika_task.md +++ /dev/null @@ -1,50 +0,0 @@ -# Yandex.Metrica Use Case - -ClickHouse was originally developed to power [Yandex.Metrica](https://metrica.yandex.com/), [the second largest web analytics platform in the world](http://w3techs.com/technologies/overview/traffic_analysis/all), and continues to be the core component of this system. With more than 13 trillion records in the database and more than 20 billion events daily, ClickHouse allows generating custom reports on the fly directly from non-aggregated data. This article briefly covers the goals of ClickHouse in the early stages of its development. - -Yandex.Metrica builds customized reports on the fly based on hits and sessions, with arbitrary segments defined by the user. This often requires building complex aggregates, such as the number of unique users. New data for building a report is received in real time. - -As of April 2014, Yandex.Metrica was tracking about 12 billion events (page views and clicks) daily. All these events must be stored in order to build custom reports. A single query may require scanning millions of rows within a few hundred milliseconds, or hundreds of millions of rows in just a few seconds. - -## Usage in Yandex.Metrica and Other Yandex Services - -ClickHouse is used for multiple purposes in Yandex.Metrica. -Its main task is to build reports in online mode using non-aggregated data. It uses a cluster of 374 servers, which store over 20.3 trillion rows in the database. The volume of compressed data, without counting duplication and replication, is about 2 PB. The volume of uncompressed data (in TSV format) would be approximately 17 PB. - -ClickHouse is also used for: - -- Storing data for Session Replay from Yandex.Metrica. -- Processing intermediate data. -- Building global reports with Analytics. -- Running queries for debugging the Yandex.Metrica engine. -- Analyzing logs from the API and the user interface. - -ClickHouse has at least a dozen installations in other Yandex services: in search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services, and others. - -## Aggregated and Non-aggregated Data - -There is a popular opinion that in order to effectively calculate statistics, you must aggregate data, since this reduces the volume of data. - -But data aggregation is a very limited solution, for the following reasons: - -- You must have a pre-defined list of reports the user will need. -- The user can't make custom reports. -- When aggregating a large quantity of keys, the volume of data is not reduced, and aggregation is useless. -- For a large number of reports, there are too many aggregation variations (combinatorial explosion). -- When aggregating keys with high cardinality (such as URLs), the volume of data is not reduced by much (less than twofold). -- For this reason, the volume of data with aggregation might grow instead of shrink. -- Users do not view all the reports we generate for them. A large portion of calculations are useless. -- The logical integrity of data may be violated for various aggregations. - -If we do not aggregate anything and work with non-aggregated data, this might actually reduce the volume of calculations. - -However, with aggregation, a significant part of the work is taken offline and completed relatively calmly. In contrast, online calculations require calculating as fast as possible, since the user is waiting for the result. - -Yandex.Metrica has a specialized system for aggregating data called Metrage, which is used for the majority of reports. -Starting in 2009, Yandex.Metrica also used a specialized OLAP database for non-aggregated data called OLAPServer, which was previously used for the report builder. -OLAPServer worked well for non-aggregated data, but it had many restrictions that did not allow it to be used for all reports as desired. These included the lack of support for data types (only numbers), and the inability to incrementally update data in real-time (it could only be done by rewriting data daily). OLAPServer is not a DBMS, but a specialized DB. - -To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS. - - -[Original article](https://clickhouse.yandex/docs/en/introduction/ya_metrika_task/) diff --git a/docs/fa/introduction/ya_metrika_task.md b/docs/fa/introduction/ya_metrika_task.md deleted file mode 100644 index 1ea434f248c..00000000000 --- a/docs/fa/introduction/ya_metrika_task.md +++ /dev/null @@ -1,49 +0,0 @@ -
- -# Yandex.Metrica use case - -ClickHouse در ابتدا برای قدرت به Yandex.Metrica دومین بستر آنالیز وب در دنیا توسعه داده شد، و همچنان جز اصلی آن است. ClickHouse اجازه می دهند که با بیش از 13 تریلیون رکورد در دیتابیس و بیش از 20 میلیارد event در روز، گزارش های مستقیم (On the fly) از داده های non-aggregate تهیه کنیم. این مقاله پیشنیه ی تاریخی در ارتباط با اهداف اصلی ClickHouse قبل از آنکه به یک محصول open source تبدیل شود، می دهد. - -Yandex.Metrica تولید گزارش های برپایه بازدید و session ها به صورت on the fly و با استفده از بخش های دلخواه و دوره ی زمانی که توسط کاربر انتخاب می شود را انجام می دهد. aggregate های پیچیده معمولا مورد نیاز هستند، مانند تعداد بازدیدکنندگان unique. داده های جدید برای تهیه گزارش گیری به صورت real-time می رسند. - -از آوریل 2014، Yandex.Metrica تقریبا 12 میلیارد event شامل page view و click در روز دریافت کرد. تمام این event ها باید به ترتیب برای ساخت گزارش های سفارشی ذخیره سازی می شدند. یک query ممکن است نیاز به اسکن کردن میلیون ها سطر با زمان کمتر از چند صد میلی ثانیه، یا چند صد میلیون سطر در عرض چند ثانیه داشته باشد. - -## استفاده در Yandex.Metrica و دیگر سرویس های Yandex - -ClickHouse با چندین اهداف در Yandex.Metrica استفاده می شود. وظیفه اصلی آن ساخت گزارش های آنلاین از داده های non-aggregate می باشد. ClickHouse در یک کلاستر با سایز 374 سرور، که بیش از 20.3 تریلیون سطر در دیتابیس را دارد مورد استفاده قرار می گیرد. اندازه فشرده داده ها، بدون شمارش داده های تکراری و replication، حدود 2 پتابایت می باشد. اندازه ی غیرفشرده داده ها (در فرمت TSV) حدودا 17 پتابایت می باشد. - -ClickHouse همچنین در موارد زیراستفاده می شود: - -- ذخیره سازی داده ها برای Session replay از Yandex.Metrica. -- پردازش داده های Intermediate. -- ساخت گزارش های سراسری از آنالیز ها. -- اجرای query ها برای debug کردن موتور Yandex.Metrica. -- آنالیز لاگ های به دست آمده از API ها و user interface. - -ClickHouse حداقل در دوازده جای دیگر سرویس Yandex نصب شده است: در search verticals، Market، Direct، Business Analytics، Mobile Development، AdFox، سرویس های شخصی و.. - -## داده های Aggregate , Non-Aggregate - -یک دیدگاه محبوب وجود دارد که شما باید، داده های خود را به منظور کاهش اندازه داده ها Aggregate کنید. - -اما به دلایل زیر، aggregate کردن داده ها راه حل بسیار محدودی است: - -- شما باید لیست گزارش های از قبل تعریف شده توسط کاربر که نیاز به تهیه گزارش آنها را دارید، داشته باشید. -- کاربر نمیتواند گزارش های سفارشی تهیه کند. -- در هنگام aggregate کردن تعداد بسیار زیاد key، اندازه ی داده ها کم نمی شود و aggregate بی فایده است. -- برای تعداد زیادی از گزارش ها، aggregate های متنوع و تغییرپذیر زیادی وجود دارد. (انفجار ترکیبی). -- هنگام aggregate کردن key ها با cardinality بالا (مثل URL ها)، اندازه داده ها به اندازه کافی کاهش پیدا نمی کند (کمتر از دو برابر). -- به این دلیل اندازه ی داده ها با aggregate کردن ممکن است به جای شکستن، رشد هم بکند. -- کاربر تمام گزارش هایی که ما تولید کردیم را نگاه نمی کند. بخش بزرگی از محاسبات بی فایده است. -- یکپارچگی منطقی داده ها ممکن است برای aggregate های مختلف نقض شود. - -اگر ما هیچ چیزی را aggregate نکنیم و با داده های non-aggregate کار کنیم، در واقع این ممکن است باعث کاهش اندازه ی محاسبات شود. - -با این حال، با aggregate کردن، بخش قابل توجهی از کار به صورت آفلاین انجام می شود و نسبتا آرام به پایان می رسد. در مقابل، محاسبات آنلاین به دلیل اینکه کاربر منتظر نمایش نتایج می باشد، نیازمند محاسبه سریع تا جایی که ممکن است می باشد. - -Yandex.Metrica دارای یک سیستم تخصصی برای aggregate کردن داده ها به اسم Metrage می باشد، که برای اکثریت گزارش های مورد استفاده قرار می گیرد. شروع سال 2009، Yandex.Metrica همچنین از یک دیتابیس تخصصی OLAP برای داده های non-aggregate به نام OLAPServer، که قبلا برای ساخت گزارش ها استفاده می شد، استفاده می کرد. OLAPServer به خوبی روی داده های Non-Aggregate کار می کرد، اما محدودیت های بسیار زیادی داشت که اجازه ی استفاده در تمام گزارش های دلخواه را نمی داد. مواردی از قبیل عدم پشتیبانی از data type ها (فقط عدد)، و عدم توانایی در بروزرسانی افزایشی داده ها به صورت real-time (این کار فقط به rewrite کردن داده ها به صورت روزانه امکام پذیر بود). OLAPServer یک مدیریت دیتابیس نبود اما یک دیتابیس تخصصی بود. - -برای حذف محدودیت های OLAPServer و حل مشکلات کار با داده های Non-Aggregate برای تمام گزارش ها، ما مدیریت دیتابیس ClicHouse را توسعه دادیم.. - -
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/ya_metrika_task/) diff --git a/docs/ru/introduction/ya_metrika_task.md b/docs/ru/introduction/ya_metrika_task.md deleted file mode 100644 index c7e22346ae5..00000000000 --- a/docs/ru/introduction/ya_metrika_task.md +++ /dev/null @@ -1,49 +0,0 @@ -# Постановка задачи в Яндекс.Метрике - -ClickHouse изначально разрабатывался для обеспечения работы [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики, и продолжает быть её ключевым компонентом. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Данная статья вкратце демонстрирует какие цели исторически стояли перед ClickHouse на ранних этапах его развития. - -Яндекс.Метрика на лету строит индивидуальные отчёты на основе хитов и визитов, с периодом и произвольными сегментами, задаваемыми конечным пользователем. Часто требуется построение сложных агрегатов, например числа уникальных пользователей. Новые данные для построения отчета поступают в реальном времени. - -На апрель 2014, в Яндекс.Метрику поступало около 12 миллиардов событий (показов страниц и кликов мыши) ежедневно. Все эти события должны быть сохранены для возможности строить произвольные отчёты. Один запрос может потребовать просканировать миллионы строк за время не более нескольких сотен миллисекунд, или сотни миллионов строк за время не более нескольких секунд. - -## Использование в Яндекс.Метрике и других отделах Яндекса - -В Яндекс.Метрике ClickHouse используется для нескольких задач. -Основная задача - построение отчётов в режиме онлайн по неагрегированным данным. Для решения этой задачи используется кластер из 374 серверов, хранящий более 20,3 триллионов строк в базе данных. Объём сжатых данных, без учёта дублирования и репликации, составляет около 2 ПБ. Объём несжатых данных (в формате tsv) составил бы, приблизительно, 17 ПБ. - -Также ClickHouse используется: - -- для хранения данных Вебвизора; -- для обработки промежуточных данных; -- для построения глобальных отчётов Аналитиками; -- для выполнения запросов в целях отладки движка Метрики; -- для анализа логов работы API и пользовательского интерфейса. - -ClickHouse имеет более десятка инсталляций в других отделах Яндекса: в Вертикальных сервисах, Маркете, Директе, БК, Бизнес аналитике, Мобильной разработке, AdFox, Персональных сервисах и т п. - -## Агрегированные и неагрегированные данные - -Существует мнение, что для того, чтобы эффективно считать статистику, данные нужно агрегировать, так как это позволяет уменьшить объём данных. - -Но агрегированные данные являются очень ограниченным решением, по следующим причинам: - -- вы должны заранее знать перечень отчётов, необходимых пользователю; -- то есть, пользователь не может построить произвольный отчёт; -- при агрегации по большому количеству ключей, объём данных не уменьшается и агрегация бесполезна; -- при большом количестве отчётов, получается слишком много вариантов агрегации (комбинаторный взрыв); -- при агрегации по ключам высокой кардинальности (например, URL) объём данных уменьшается не сильно (менее чем в 2 раза); -- из-за этого, объём данных при агрегации может не уменьшиться, а вырасти; -- пользователи будут смотреть не все отчёты, которые мы для них посчитаем - то есть, большая часть вычислений бесполезна; -- возможно нарушение логической целостности данных для разных агрегаций; - -Как видно, если ничего не агрегировать, и работать с неагрегированными данными, то это даже может уменьшить объём вычислений. - -Впрочем, при агрегации, существенная часть работы выносится в оффлайне, и её можно делать сравнительно спокойно. Для сравнения, при онлайн вычислениях, вычисления надо делать так быстро, как это возможно, так как именно в момент вычислений пользователь ждёт результата. - -В Яндекс.Метрике есть специализированная система для агрегированных данных - Metrage, на основе которой работает большинство отчётов. -Также в Яндекс.Метрике с 2009 года использовалась специализированная OLAP БД для неагрегированных данных - OLAPServer, на основе которой раньше работал конструктор отчётов. -OLAPServer хорошо подходил для неагрегированных данных, но содержал много ограничений, не позволяющих использовать его для всех отчётов так, как хочется: отсутствие поддержки типов данных (только числа), невозможность инкрементального обновления данных в реальном времени (только перезаписью данных за сутки). OLAPServer не является СУБД, а является специализированной БД. - -Чтобы снять ограничения OLAPServer-а и решить задачу работы с неагрегированными данными для всех отчётов, разработана СУБД ClickHouse. - -[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/ya_metrika_task/) diff --git a/docs/zh/introduction/ya_metrika_task.md b/docs/zh/introduction/ya_metrika_task.md deleted file mode 100644 index da4b18826e0..00000000000 --- a/docs/zh/introduction/ya_metrika_task.md +++ /dev/null @@ -1,50 +0,0 @@ -# Yandex.Metrica的使用案例 - -ClickHouse最初是为 [Yandex.Metrica](https://metrica.yandex.com/) [世界第二大Web分析平台](http://w3techs.com/technologies/overview/traffic_analysis/all) 而开发的。多年来一直作为该系统的核心组件被该系统持续使用着。目前为止，该系统在ClickHouse中有超过13万亿条记录，并且每天超过200多亿个事件被处理。它允许直接从原始数据中动态查询并生成报告。本文简要介绍了ClickHouse在其早期发展阶段的目标。 - -Yandex.Metrica基于用户定义的字段，对实时访问、连接会话，生成实时的统计报表。这种需求往往需要复杂聚合方式，比如对访问用户进行去重。构建报表的数据，是实时接收存储的新数据。 - -截至2014年4月，Yandex.Metrica每天跟踪大约120亿个事件（用户的点击和浏览）。为了可以创建自定义的报表，我们必须存储全部这些事件。同时，这些查询可能需要在几百毫秒内扫描数百万行的数据，或在几秒内扫描数亿行的数据。 - -## Yandex.Metrica以及其他Yandex服务的使用案例 - -在Yandex.Metrica中，ClickHouse被用于多个场景中。 -它的主要任务是使用原始数据在线的提供各种数据报告。它使用374台服务器的集群，存储了20.3万亿行的数据。在去除重复与副本数据的情况下，压缩后的数据达到了2PB。未压缩前（TSV格式）它大概有17PB。 - -ClickHouse还被使用在： - -- 存储来自Yandex.Metrica回话重放数据。 -- 处理中间数据 -- 与Analytics一起构建全球报表。 -- 为调试Yandex.Metrica引擎运行查询 -- 分析来自API和用户界面的日志数据 - -ClickHouse在其他Yandex服务中至少有12个安装：search verticals, Market, Direct, business analytics, mobile development, AdFox, personal services等。 - -## 聚合与非聚合数据 - -有一种流行的观点认为，想要有效的计算统计数据，必须要聚合数据，因为聚合将降低数据量。 - -但是数据聚合是一个有诸多限制的解决方案，例如： - -- 你必须提前知道用户定义的报表的字段列表 -- 用户无法自定义报表 -- 当聚合条件过多时，可能不会减少数据，聚合是无用的。 -- 存在大量报表时，有太多的聚合变化（组合爆炸） -- 当聚合条件有非常大的基数时（如：url），数据量没有太大减少（少于两倍） -- 聚合的数据量可能会增长而不是收缩 -- 用户不会查看我们为他生成的所有报告，大部分计算将是无用的 -- 各种聚合可能违背了数据的逻辑完整性 - -如果我们直接使用非聚合数据而不进行任何聚合时，我们的计算量可能是减少的。 - -然而，相对于聚合中很大一部分工作被离线完成，在线计算需要尽快的完成计算，因为用户在等待结果。 - -Yandex.Metrica 有一个专门用于聚合数据的系统，称为Metrage，它可以用作大部分报表。 -从2009年开始，Yandex.Metrica还为非聚合数据使用专门的OLAP数据库，称为OLAPServer，它以前用于报表构建系统。 -OLAPServer可以很好的工作在非聚合数据上，但是它有诸多限制，导致无法根据需要将其用于所有报表中。如，缺少对数据类型的支持（只支持数据），无法实时增量的更新数据（只能通过每天重写数据完成）。OLAPServer不是一个数据库管理系统，它只是一个数据库。 - -为了消除OLAPServer的这些局限性，解决所有报表使用非聚合数据的问题，我们开发了ClickHouse数据库管理系统。 - - -[来源文章](https://clickhouse.yandex/docs/en/introduction/ya_metrika_task/) From 4a7f22040991b4054a4e2bee4cf2504e688d6d8d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 28 Aug 2019 11:52:07 +0300 Subject: [PATCH 010/312] Refactor Yandex.Metrica dataset description --- .../example_datasets/metrica.md | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/en/getting_started/example_datasets/metrica.md b/docs/en/getting_started/example_datasets/metrica.md index 75741ba0b54..e3a9556adb4 100644 --- a/docs/en/getting_started/example_datasets/metrica.md +++ b/docs/en/getting_started/example_datasets/metrica.md @@ -1,9 +1,13 @@ # Anonymized Yandex.Metrica Data -Dataset consists of two tables containing anonymized data about hits (`hits_v1`) and visits (`visits_v1`) of Yandex.Metrica. Each of the tables can be downloaded as a compressed `tsv.xz` file or as prepared partitions. In addition to that, an extended version of the `hits` table containing 100 million rows is available as TSV at `https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_100m_obfuscated_v1.tsv.xz` and as prepared partitions at `https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz`. +Dataset consists of two tables containing anonymized data about hits (`hits_v1`) and visits (`visits_v1`) of Yandex.Metrica. You can read more about Yandex.Metrica in [ClickHouse history](../../introduction/history.md) section. + +The dataset consists of two tables, either of them can be downloaded as a compressed `tsv.xz` file or as prepared partitions. In addition to that, an extended version of the `hits` table containing 100 million rows is available as TSV at and as prepared partitions at . ## Obtaining Tables from Prepared Partitions -**Download and import hits:** -```bash + +Download and import hits table: + +``` bash curl -O https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar tar xvf hits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory # check permissions on unpacked data, fix if required @@ -11,8 +15,9 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -**Download and import visits:** -```bash +Download and import visits: + +``` bash curl -O https://clickhouse-datasets.s3.yandex.net/visits/partitions/visits_v1.tar tar xvf visits_v1.tar -C /var/lib/clickhouse # path to ClickHouse data directory # check permissions on unpacked data, fix if required @@ -20,9 +25,11 @@ sudo service clickhouse-server restart clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Obtaining Tables from Compressed tsv-file -**Download and import hits from compressed tsv-file** -```bash +## Obtaining Tables from Compressed TSV File + +Download and import hits from compressed TSV file: + +``` bash curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv # now create table clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" @@ -34,8 +41,9 @@ clickhouse-client --query "OPTIMIZE TABLE datasets.hits_v1 FINAL" clickhouse-client --query "SELECT COUNT(*) FROM datasets.hits_v1" ``` -**Download and import visits from compressed tsv-file** -```bash +Download and import visits from compressed tsv-file: + +``` bash curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv # now create table clickhouse-client --query "CREATE DATABASE IF NOT EXISTS datasets" @@ -48,4 +56,4 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` ## Queries -Examples of queries to these tables (they are named `test.hits` and `test.visits`) can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) and in some [performance tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/performance/test_hits) of ClickHouse. +Examples of queries to these tables (they are named `test.hits` and `test.visits`) can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) of ClickHouse. From a95105453f4a4b176edf198294a5678ccaee80da Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 28 Aug 2019 14:14:42 +0300 Subject: [PATCH 011/312] WIP on rewriting tutorial --- .../example_datasets/metrica.md | 7 +- docs/en/getting_started/install.md | 6 +- docs/en/getting_started/tutorial.md | 577 +++++++++++++----- 3 files changed, 442 insertions(+), 148 deletions(-) diff --git a/docs/en/getting_started/example_datasets/metrica.md b/docs/en/getting_started/example_datasets/metrica.md index e3a9556adb4..b26f26f5acb 100644 --- a/docs/en/getting_started/example_datasets/metrica.md +++ b/docs/en/getting_started/example_datasets/metrica.md @@ -55,5 +55,8 @@ clickhouse-client --query "OPTIMIZE TABLE datasets.visits_v1 FINAL" clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ``` -## Queries -Examples of queries to these tables (they are named `test.hits` and `test.visits`) can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) of ClickHouse. +## Example Queries + +[ClickHouse tutorial](../tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. + +Additional examples of queries to these tables can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there). diff --git a/docs/en/getting_started/install.md b/docs/en/getting_started/install.md index 779abba905b..6bb7a17d340 100644 --- a/docs/en/getting_started/install.md +++ b/docs/en/getting_started/install.md @@ -2,14 +2,16 @@ ## System Requirements -ClickHouse can run on any Linux, FreeBSD or Mac OS X with x86\_64 CPU architecture. +ClickHouse can run on any Linux, FreeBSD or Mac OS X with x86\_64, AArch64 or PowerPC64LE CPU architecture. -Though pre-built binaries are typically compiled to leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: +Though pre-built binaries are typically compiled for x86\_64 and leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: ``` bash $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" ``` +To run ClickHouse on processors that do not support SSE 4.2 or have AArch64 or PowerPC64LE architecture, you should [build ClickHouse from sources](#from-sources) with proper configuration adjustments. + ## Available Installation Options ### From DEB Packages diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md index 48faf5bd327..222d6d7517b 100644 --- a/docs/en/getting_started/tutorial.md +++ b/docs/en/getting_started/tutorial.md @@ -1,31 +1,48 @@ # ClickHouse Tutorial -## Setup +## What to Expect from This Tutorial? -Let's get started with sample dataset from open sources. We will use USA civil flights data from 1987 to 2015. It's hard to call this sample a Big Data (contains 166 millions rows, 63 Gb of uncompressed data) but this allows us to quickly get to work. Dataset is available for download [here](https://yadi.sk/d/pOZxpa42sDdgm). Also you may download it from the original datasource [as described here](example_datasets/ontime.md). +By going through this tutorial you'll learn how to set up basic ClickHouse cluster, it'll be small, but fault tolerant and scalable. We will use one of example datasets to fill it with data and execute some demo queries. -At first we will deploy ClickHouse to a single server. Later we will also review the process of deployment to a cluster with support for sharding and replication. +## Single Node Setup -ClickHouse is usually installed from [deb](index.md#from-deb-packages) or [rpm](index.md#from-rpm-packages) packages, but there are [alternatives](index.md#from-docker-image) for the operating systems that do no support them. What do we have in those packages: +To postpone complexities of distributed environment, we'll start with deploying ClickHouse on a single server or virtual machine. ClickHouse is usually installed from [deb](index.md#from-deb-packages) or [rpm](index.md#from-rpm-packages) packages, but there are [alternatives](index.md#from-docker-image) for the operating systems that do no support them. + +For example, you have chosen `deb` packages and executed: +``` bash +sudo apt-get install dirmngr +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E0C56BD4 + +echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" | sudo tee /etc/apt/sources.list.d/clickhouse.list +sudo apt-get update + +sudo apt-get install -y clickhouse-server clickhouse-client +``` + +What do we have in the packages that got installed: * `clickhouse-client` package contains [clickhouse-client](../interfaces/cli.md) application, interactive ClickHouse console client. * `clickhouse-common` package contains a ClickHouse executable file. * `clickhouse-server` package contains configuration files to run ClickHouse as a server. -Server config files are located in /etc/clickhouse-server/. Before getting to work please notice the `path` element in config. Path dtermines the location for data storage. It's not really handy to directly edit `config.xml` file considering package updates. Recommended way to override the config elements is to create [files in config.d directory](../operations/configuration_files.md). Also you may want to [set up access rights](../operations/access_rights.md) early on.

+Server config files are located in `/etc/clickhouse-server/`. Before going further please notice the `` element in `config.xml`. Path determines the location for data storage, so it should be located on volume with large disk capacity, the default value is `/var/lib/clickhouse/`. If you want to adjust the configuration it's not really handy to directly edit `config.xml` file, considering it might get rewritten on future package updates. Recommended way to override the config elements is to create [files in config.d directory](../operations/configuration_files.md) which serve as "patches" to config.xml. + +As you might have noticed, `clickhouse-server` is not launched automatically after package installation. It won't be automatically restarted after updates either. The way you start the server depends on your init system, usually it's: -`clickhouse-server` won't be launched automatically after package installation. It won't be automatically restarted after updates either. Start the server with: ``` bash sudo service clickhouse-server start ``` +or -The default location for server logs is `/var/log/clickhouse-server/`. +``` bash +sudo /etc/init.d/clickhouse-server start +``` -Server is ready to handle client connections once `Ready for connections` message was logged. +The default location for server logs is `/var/log/clickhouse-server/`. Server will be ready to handle client connections once `Ready for connections` message was logged. -Use `clickhouse-client` to connect to the server.

+Once the `clickhouse-server` is up and running, we can use `clickhouse-client` to connect to the server and run some test queries like `SELECT "Hello, world!";`. -
Tips for clickhouse-client +
Quick tips for clickhouse-client Interactive mode: ``` bash clickhouse-client @@ -42,160 +59,432 @@ Run queries in batch-mode: ``` bash clickhouse-client --query='SELECT 1' echo 'SELECT 1' | clickhouse-client +clickhouse-client <<< 'SELECT 1' ``` -Insert data from file of a specified format: +Insert data from a file in specified format: ``` bash -clickhouse-client --query='INSERT INTO table VALUES' < data.txt -clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv ```
-## Create Table for Sample Dataset -
Create table query -``` bash -$ clickhouse-client --multiline -ClickHouse client version 0.0.53720. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.53720. +## Import Sample Dataset -:) CREATE TABLE ontime +Now it's time to fill our ClickHouse server with some sample data. In this tutorial we'll use anonymized data of Yandex.Metrica, the first service that run ClickHouse in production way before it became open-source (more on that in [history section](../introduction/history.md)). There are [multiple ways to import Yandex.Metrica dataset](example_datasets/metrica.md) and for the sake of the tutorial we'll go with the most realistic one. + +### Download and Extract Table Data + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +``` + +The extracted files are about 10GB in size. + +### Create Tables + +Tables are logically grouped into "databases". There's a `default` database, but we'll create a new one named `tutorial`: + +``` bash +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" +``` + +Syntax for creating tables is way more complicated compared to databases (see [reference](../query_language/create.md). In general `CREATE TABLE` statement has to specify three key things: + +1. Name of table to create. +2. Table schema, i.e. list of columns and their [data types](../data_types/index.md). +3. [Table engine](../operations/table_engines/index.md) and it's settings, which determines all the details on how queries to this table will be physically executed. + +Yandex.Metrica is a web analytics service and sample dataset doesn't cover it's full functionality, so there are only two tables to create: + +* `hits` is a table with each action done by all users on all websites covered by the service. +* `visits` is a table that contains pre-built sessions instead of individual actions. + +Let's see and execute the real create table queries for these tables: + +``` sql +CREATE TABLE tutorial.hits_v1 ( - Year UInt16, - Quarter UInt8, - Month UInt8, - DayofMonth UInt8, - DayOfWeek UInt8, - FlightDate Date, - UniqueCarrier FixedString(7), - AirlineID Int32, - Carrier FixedString(2), - TailNum String, - FlightNum String, - OriginAirportID Int32, - OriginAirportSeqID Int32, - OriginCityMarketID Int32, - Origin FixedString(5), - OriginCityName String, - OriginState FixedString(2), - OriginStateFips String, - OriginStateName String, - OriginWac Int32, - DestAirportID Int32, - DestAirportSeqID Int32, - DestCityMarketID Int32, - Dest FixedString(5), - DestCityName String, - DestState FixedString(2), - DestStateFips String, - DestStateName String, - DestWac Int32, - CRSDepTime Int32, - DepTime Int32, - DepDelay Int32, - DepDelayMinutes Int32, - DepDel15 Int32, - DepartureDelayGroups String, - DepTimeBlk String, - TaxiOut Int32, - WheelsOff Int32, - WheelsOn Int32, - TaxiIn Int32, - CRSArrTime Int32, - ArrTime Int32, - ArrDelay Int32, - ArrDelayMinutes Int32, - ArrDel15 Int32, - ArrivalDelayGroups Int32, - ArrTimeBlk String, - Cancelled UInt8, - CancellationCode FixedString(1), - Diverted UInt8, - CRSElapsedTime Int32, - ActualElapsedTime Int32, - AirTime Int32, - Flights Int32, - Distance Int32, - DistanceGroup UInt8, - CarrierDelay Int32, - WeatherDelay Int32, - NASDelay Int32, - SecurityDelay Int32, - LateAircraftDelay Int32, - FirstDepTime String, - TotalAddGTime String, - LongestAddGTime String, - DivAirportLandings String, - DivReachedDest String, - DivActualElapsedTime String, - DivArrDelay String, - DivDistance String, - Div1Airport String, - Div1AirportID Int32, - Div1AirportSeqID Int32, - Div1WheelsOn String, - Div1TotalGTime String, - Div1LongestGTime String, - Div1WheelsOff String, - Div1TailNum String, - Div2Airport String, - Div2AirportID Int32, - Div2AirportSeqID Int32, - Div2WheelsOn String, - Div2TotalGTime String, - Div2LongestGTime String, - Div2WheelsOff String, - Div2TailNum String, - Div3Airport String, - Div3AirportID Int32, - Div3AirportSeqID Int32, - Div3WheelsOn String, - Div3TotalGTime String, - Div3LongestGTime String, - Div3WheelsOff String, - Div3TailNum String, - Div4Airport String, - Div4AirportID Int32, - Div4AirportSeqID Int32, - Div4WheelsOn String, - Div4TotalGTime String, - Div4LongestGTime String, - Div4WheelsOff String, - Div4TailNum String, - Div5Airport String, - Div5AirportID Int32, - Div5AirportSeqID Int32, - Div5WheelsOn String, - Div5TotalGTime String, - Div5LongestGTime String, - Div5WheelsOff String, - Div5TailNum String + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 ) -ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192); +ENGINE = MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 ``` -
-Now we have a table of [MergeTree type](../operations/table_engines/mergetree.md). MergeTree table engine family is recommended for usage in production. Tables of this kind has a primary key used for incremental sort of table data. This allows fast execution of queries in ranges of a primary key. +``` sql +CREATE TABLE tutorial.visits_v1 +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals` Nested( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource` Nested( + ID Int8, + SearchEngineID UInt16, + AdvEngineID UInt8, + PlaceID UInt16, + SocialSourceNetworkID UInt8, + Domain String, + SearchPhrase String, + SocialSourcePage String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `Market` Nested( + Type UInt8, + GoalID UInt32, + OrderID String, + OrderPrice Int64, + PP UInt32, + DirectPlaceID UInt32, + DirectOrderID UInt32, + DirectBannerID UInt32, + GoodID String, + GoodName String, + GoodQuantity Int32, + GoodPrice Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +You can execute those queries using interactive mode of `clickhouse-client` (just launch it in terminal without specifying a query in advance) or try some [alternative interface](../interfaces/index.md) if you ant. + +As we can see, `hits_v1` uses the [basic MergeTree engine](../operations/table_engines/mergetree.md), while the `visits_v1` uses the [Collapsing](../operations/table_engines/collapsingmergetree.md) variant. + +### Import Data + +Data import to ClickHouse is done via [INSERT INTO](../query_language/insert_into.md) query like in many other SQL databases. However data is usually provided in one of the [supported formats](../interfaces/formats.md) instead of `VALUES` clause (which is also supported). + +The files we downloaded earlier are in tab-separated format, so here's how to import them via console client: -## Load Data ``` bash -xz -v -c -d < ontime.csv.xz | clickhouse-client --query="INSERT INTO ontime FORMAT CSV" +clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv +clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv ``` -ClickHouse INSERT query allows to load data in any [supported format](../interfaces/formats.md). Data load requires just O(1) RAM consumption. INSERT query can receive any data volume as input. It is strongly recommended to insert data with [not so small blocks](../introduction/performance/#performance-when-inserting-data). Notice that insert of blocks with size up to max_insert_block_size (= 1 048 576 - rows by default) is an atomic operation: data block will be inserted completely or not inserted at all. In case of disconnect during insert operation you may not know if the block was inserted successfully. To achieve exactly-once semantics ClickHouse supports idempotency for [replicated tables](../operations/table_engines/replication.md). This means that you may retry insert of the same data block (possibly on a different replicas) but this block will be inserted just once. Anyway in this guide we will load data from our localhost so we may not take care about data blocks generation and exactly-once semantics. +ClickHouse has a lot of [settings to tune](../operations/settings.md) and one way to specify them in console client is via arguments, as we can see with `--max_insert_block_size`. The easiest way to figure out what settings are available, what do they mean and what the defaults are is to query the `system.settings` table: -INSERT query into tables of MergeTree type is non-blocking (so does a SELECT query). You can execute SELECT queries right after of during insert operation. +``` sql +SELECT name, value, changed, description +FROM system.settings +WHERE name LIKE '%max_insert_b%' +FORMAT TSV -Our sample dataset is a bit not optimal. There are two reasons: +max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." +``` -* The first is that String data type is used in cases when [Enum](../data_types/enum.md) or numeric type would fit better. -* The second is that dataset contains redundant fields like Year, Quarter, Month, DayOfMonth, DayOfWeek. In fact a single FlightDate would be enough. Most likely they have been added to improve performance for other DBMS'eswhere DateTime handling functions may be not efficient. +Optionally you can [OPTIMIZE](../query_language/misc/#misc_operations-optimize) the tables after import. Tables that are configured with MergeTree-family engine always do merges of data parts in background to optimize data storage (or at least check if it makes sense). These queries will just force table engine to do storage optimization right now instead of some time later: +``` bash +clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" +clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" +``` -!!! note "Tip" - ClickHouse [functions for manipulating DateTime fields](../query_language/functions/date_time_functions/) are well-optimized so such redundancy is not required. Anyway many columns is not a reason to worry, because ClickHouse is a [column-oriented DBMS](https://en.wikipedia.org/wiki/Column-oriented_DBMS). This allows you to have as many fields as you need with minimal impact on performance. Hundreds of columns in a table is totally fine for ClickHouse. +This is I/O and CPU intensive operation so if the table constantly receives new data it's better to leave it alone and let merges run in background. -## Querying the Sample Dataset +Now we can check that the tables are successfully imported: +``` bash +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" +``` + +## Queries TODO From 431968d69b4ed39f8502ed01713295b007cb710a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 15 Nov 2019 19:23:48 +0300 Subject: [PATCH 012/312] Processors and dydtem.numbers --- .../Storages/System/StorageSystemNumbers.cpp | 70 ++++++++++++------- .../Storages/System/StorageSystemNumbers.h | 4 +- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 2f155e22a11..aa457eee9ca 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -5,6 +5,9 @@ #include #include +#include +#include +#include namespace DB { @@ -12,21 +15,16 @@ namespace DB namespace { -class NumbersBlockInputStream : public IBlockInputStream +class NumbersSource : public SourceWithProgress { public: - NumbersBlockInputStream(UInt64 block_size_, UInt64 offset_, UInt64 step_) - : block_size(block_size_), next(offset_), step(step_) {} + NumbersSource(UInt64 block_size_, UInt64 offset_, UInt64 step_) + : SourceWithProgress(createHeader()), block_size(block_size_), next(offset_), step(step_) {} String getName() const override { return "Numbers"; } - Block getHeader() const override - { - return { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number") }; - } - protected: - Block readImpl() override + Chunk generate() override { auto column = ColumnUInt64::create(block_size); ColumnUInt64::Container & vec = column->getData(); @@ -38,12 +36,19 @@ protected: *pos++ = curr++; next += step; - return { ColumnWithTypeAndName(std::move(column), std::make_shared(), "number") }; + + return { Columns {std::move(column)}, block_size }; } + private: UInt64 block_size; UInt64 next; UInt64 step; + + static Block createHeader() + { + return { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number") }; + } }; @@ -55,21 +60,19 @@ struct NumbersMultiThreadedState using NumbersMultiThreadedStatePtr = std::shared_ptr; -class NumbersMultiThreadedBlockInputStream : public IBlockInputStream +class NumbersMultiThreadedSource : public SourceWithProgress { public: - NumbersMultiThreadedBlockInputStream(NumbersMultiThreadedStatePtr state_, UInt64 block_size_, UInt64 max_counter_) - : state(std::move(state_)), block_size(block_size_), max_counter(max_counter_) {} + NumbersMultiThreadedSource(NumbersMultiThreadedStatePtr state_, UInt64 block_size_, UInt64 max_counter_) + : SourceWithProgress(createHeader()) + , state(std::move(state_)) + , block_size(block_size_) + , max_counter(max_counter_) {} String getName() const override { return "NumbersMt"; } - Block getHeader() const override - { - return { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number") }; - } - protected: - Block readImpl() override + Chunk generate() override { if (block_size == 0) return {}; @@ -90,7 +93,7 @@ protected: while (pos < end) *pos++ = curr++; - return { ColumnWithTypeAndName(std::move(column), std::make_shared(), "number") }; + return { Columns {std::move(column)}, block_size }; } private: @@ -98,6 +101,11 @@ private: UInt64 block_size; UInt64 max_counter; + + Block createHeader() const + { + return { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared(), "number") }; + } }; } @@ -109,7 +117,7 @@ StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multi setColumns(ColumnsDescription({{"number", std::make_shared()}})); } -BlockInputStreams StorageSystemNumbers::read( +Pipes StorageSystemNumbers::readWithProcessors( const Names & column_names, const SelectQueryInfo &, const Context & /*context*/, @@ -128,7 +136,8 @@ BlockInputStreams StorageSystemNumbers::read( if (!multithreaded) num_streams = 1; - BlockInputStreams res(num_streams); + Pipes res; + res.reserve(num_streams); if (num_streams > 1 && !even_distribution && *limit) { @@ -136,17 +145,26 @@ BlockInputStreams StorageSystemNumbers::read( UInt64 max_counter = offset + *limit; for (size_t i = 0; i < num_streams; ++i) - res[i] = std::make_shared(state, max_block_size, max_counter); + res.emplace_back(std::make_shared(state, max_block_size, max_counter)); return res; } for (size_t i = 0; i < num_streams; ++i) { - res[i] = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); + auto source = std::make_shared(max_block_size, offset + i * max_block_size, num_streams * max_block_size); - if (limit) /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. - res[i] = std::make_shared(res[i], *limit * (i + 1) / num_streams - *limit * i / num_streams, 0, false, true); + if (limit && i == 0) + source->addTotalRowsApprox(*limit); + + res.emplace_back(std::move(source)); + + if (limit) + { + /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly. + res.back().addSimpleTransform(std::make_shared( + res.back().getHeader(), *limit * (i + 1) / num_streams - *limit * i / num_streams, 0, false)); + } } return res; diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 76070839012..2db97683279 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -31,7 +31,7 @@ public: std::string getTableName() const override { return name; } std::string getDatabaseName() const override { return "system"; } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -39,6 +39,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + private: const std::string name; bool multithreaded; From ac89b60d5af7ee07407bc4c239617804c517132b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 3 Dec 2019 16:13:14 +0300 Subject: [PATCH 013/312] tmp commit --- docs/en/getting_started/tutorial.md | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md index 222d6d7517b..9251967d567 100644 --- a/docs/en/getting_started/tutorial.md +++ b/docs/en/getting_started/tutorial.md @@ -76,8 +76,8 @@ Now it's time to fill our ClickHouse server with some sample data. In this tutor ### Download and Extract Table Data ``` bash -curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv ``` The extracted files are about 10GB in size. @@ -484,9 +484,28 @@ clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" ``` -## Queries +## Example Queries -TODO +``` sql +SELECT + StartURL AS URL, + AVG(Duration) AS AvgDuration +FROM tutorial.visits_v1 +WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' +GROUP BY URL +ORDER BY AvgDuration DESC +LIMIT 10 +``` + + +``` sql +SELECT + sum(Sign) AS visits, + sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, + (100. * goal_visits) / visits AS goal_percent +FROM tutorial.visits_v1 +WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') +``` ## Cluster Deployment @@ -525,7 +544,7 @@ ClickHouse cluster is a homogenous cluster. Steps to set up: ```
- + Creating a local table: ``` sql CREATE TABLE ontime_local (...) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192); @@ -577,7 +596,7 @@ To provide for resilience in production environment we recommend that each shard ``` - + To enable replication ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure automatically. It's recommended to deploy ZooKeeper cluster to separate servers. From c136bee4ce9cfbec249aa1d729b5f88d34b90d2f Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 3 Dec 2019 19:19:07 +0300 Subject: [PATCH 014/312] lots of docs fixes --- .../example_datasets/metrica.md | 2 +- docs/en/getting_started/tutorial.md | 2 +- docs/en/index.md | 108 +++++++------ docs/fa/getting_started/install.md | 8 +- docs/fa/getting_started/tutorial.md | 1 + docs/ja/changelog.md | 1 + docs/ja/data_types/array.md | 1 + docs/ja/data_types/boolean.md | 1 + docs/ja/data_types/date.md | 1 + docs/ja/data_types/datetime.md | 1 + docs/ja/data_types/decimal.md | 1 + docs/ja/data_types/domains/ipv4.md | 1 + docs/ja/data_types/domains/ipv6.md | 1 + docs/ja/data_types/domains/overview.md | 1 + docs/ja/data_types/enum.md | 1 + docs/ja/data_types/fixedstring.md | 1 + docs/ja/data_types/float.md | 1 + docs/ja/data_types/index.md | 1 + docs/ja/data_types/int_uint.md | 1 + .../aggregatefunction.md | 1 + .../nested_data_structures/index.md | 1 + .../nested_data_structures/nested.md | 1 + docs/ja/data_types/nullable.md | 1 + .../special_data_types/expression.md | 1 + .../ja/data_types/special_data_types/index.md | 1 + .../data_types/special_data_types/interval.md | 1 + .../data_types/special_data_types/nothing.md | 1 + docs/ja/data_types/special_data_types/set.md | 1 + docs/ja/data_types/string.md | 1 + docs/ja/data_types/tuple.md | 1 + docs/ja/data_types/uuid.md | 1 + docs/ja/database_engines/index.md | 1 + docs/ja/database_engines/lazy.md | 1 + docs/ja/database_engines/mysql.md | 1 + docs/ja/development/architecture.md | 1 + docs/ja/development/build.md | 1 + docs/ja/development/build_cross_arm.md | 1 + docs/ja/development/build_cross_osx.md | 1 + docs/ja/development/build_osx.md | 1 + docs/ja/development/contrib.md | 1 + docs/ja/development/developer_instruction.md | 1 + docs/ja/development/index.md | 1 + docs/ja/development/style.md | 1 + docs/ja/development/tests.md | 1 + .../tests/developer_instruction_ru.md | 1 + .../development/tests/easy_tasks_sorted_ru.md | 1 + docs/ja/development/tests/sanitizers.md | 1 + docs/ja/faq/general.md | 1 + .../example_datasets/amplab_benchmark.md | 1 + .../example_datasets/criteo.md | 1 + .../example_datasets/metrica.md | 1 + .../example_datasets/nyc_taxi.md | 1 + .../example_datasets/ontime.md | 1 + .../example_datasets/star_schema.md | 1 + .../example_datasets/wikistat.md | 1 + docs/ja/getting_started/index.md | 1 + docs/ja/getting_started/install.md | 1 + docs/ja/getting_started/tutorial.md | 1 + docs/ja/guides/apply_catboost_model.md | 1 + docs/ja/guides/index.md | 1 + docs/ja/images/column_oriented.gif | Bin 0 -> 45485 bytes docs/ja/images/logo.svg | 12 ++ docs/ja/images/row_oriented.gif | Bin 0 -> 41571 bytes docs/ja/index.md | 143 +----------------- docs/ja/interfaces/cli.md | 1 + docs/ja/interfaces/cpp.md | 1 + docs/ja/interfaces/formats.md | 1 + docs/ja/interfaces/http.md | 1 + docs/ja/interfaces/index.md | 1 + docs/ja/interfaces/jdbc.md | 1 + docs/ja/interfaces/odbc.md | 1 + docs/ja/interfaces/tcp.md | 1 + .../third-party/client_libraries.md | 1 + docs/ja/interfaces/third-party/gui.md | 1 + .../ja/interfaces/third-party/integrations.md | 1 + docs/ja/interfaces/third-party/proxy.md | 1 + docs/ja/introduction/distinctive_features.md | 1 + .../features_considered_disadvantages.md | 1 + docs/ja/introduction/history.md | 1 + docs/ja/introduction/performance.md | 1 + docs/ja/operations/access_rights.md | 1 + docs/ja/operations/backup.md | 1 + docs/ja/operations/configuration_files.md | 1 + docs/ja/operations/index.md | 1 + docs/ja/operations/monitoring.md | 1 + docs/ja/operations/quotas.md | 1 + docs/ja/operations/requirements.md | 1 + docs/ja/operations/server_settings/index.md | 1 + .../ja/operations/server_settings/settings.md | 1 + .../settings/constraints_on_settings.md | 1 + docs/ja/operations/settings/index.md | 1 + .../settings/permissions_for_queries.md | 1 + .../operations/settings/query_complexity.md | 1 + docs/ja/operations/settings/settings.md | 1 + .../operations/settings/settings_profiles.md | 1 + docs/ja/operations/settings/settings_users.md | 1 + docs/ja/operations/system_tables.md | 1 + .../table_engines/aggregatingmergetree.md | 1 + docs/ja/operations/table_engines/buffer.md | 1 + .../table_engines/collapsingmergetree.md | 1 + .../table_engines/custom_partitioning_key.md | 1 + .../ja/operations/table_engines/dictionary.md | 1 + .../operations/table_engines/distributed.md | 1 + .../operations/table_engines/external_data.md | 1 + docs/ja/operations/table_engines/file.md | 1 + .../table_engines/graphitemergetree.md | 1 + docs/ja/operations/table_engines/hdfs.md | 1 + docs/ja/operations/table_engines/index.md | 1 + docs/ja/operations/table_engines/jdbc.md | 1 + docs/ja/operations/table_engines/join.md | 1 + docs/ja/operations/table_engines/kafka.md | 1 + docs/ja/operations/table_engines/log.md | 1 + .../ja/operations/table_engines/log_family.md | 1 + .../table_engines/materializedview.md | 1 + docs/ja/operations/table_engines/memory.md | 1 + docs/ja/operations/table_engines/merge.md | 1 + docs/ja/operations/table_engines/mergetree.md | 1 + docs/ja/operations/table_engines/mysql.md | 1 + docs/ja/operations/table_engines/null.md | 1 + docs/ja/operations/table_engines/odbc.md | 1 + .../table_engines/replacingmergetree.md | 1 + .../operations/table_engines/replication.md | 1 + docs/ja/operations/table_engines/set.md | 1 + docs/ja/operations/table_engines/stripelog.md | 1 + .../table_engines/summingmergetree.md | 1 + docs/ja/operations/table_engines/tinylog.md | 1 + docs/ja/operations/table_engines/url.md | 1 + .../versionedcollapsingmergetree.md | 1 + docs/ja/operations/table_engines/view.md | 1 + docs/ja/operations/tips.md | 1 + docs/ja/operations/troubleshooting.md | 1 + docs/ja/operations/update.md | 1 + docs/ja/operations/utils/clickhouse-copier.md | 1 + docs/ja/operations/utils/clickhouse-local.md | 1 + docs/ja/operations/utils/index.md | 1 + .../agg_functions/combinators.md | 1 + docs/ja/query_language/agg_functions/index.md | 1 + .../agg_functions/parametric_functions.md | 1 + .../query_language/agg_functions/reference.md | 1 + docs/ja/query_language/alter.md | 1 + docs/ja/query_language/create.md | 1 + .../ja/query_language/dicts/external_dicts.md | 1 + .../dicts/external_dicts_dict.md | 1 + .../dicts/external_dicts_dict_layout.md | 1 + .../dicts/external_dicts_dict_lifetime.md | 1 + .../dicts/external_dicts_dict_sources.md | 1 + .../dicts/external_dicts_dict_structure.md | 1 + docs/ja/query_language/dicts/index.md | 1 + .../ja/query_language/dicts/internal_dicts.md | 1 + .../functions/arithmetic_functions.md | 1 + .../functions/array_functions.md | 1 + .../ja/query_language/functions/array_join.md | 1 + .../query_language/functions/bit_functions.md | 1 + .../functions/bitmap_functions.md | 1 + .../functions/comparison_functions.md | 1 + .../functions/conditional_functions.md | 1 + .../functions/date_time_functions.md | 1 + .../functions/encoding_functions.md | 1 + .../functions/ext_dict_functions.md | 1 + .../functions/functions_for_nulls.md | 1 + docs/ja/query_language/functions/geo.md | 1 + .../functions/hash_functions.md | 1 + .../functions/higher_order_functions.md | 1 + .../query_language/functions/in_functions.md | 1 + docs/ja/query_language/functions/index.md | 1 + .../functions/ip_address_functions.md | 1 + .../functions/json_functions.md | 1 + .../functions/logical_functions.md | 1 + .../functions/machine_learning_functions.md | 1 + .../functions/math_functions.md | 1 + .../functions/other_functions.md | 1 + .../functions/random_functions.md | 1 + .../functions/rounding_functions.md | 1 + .../functions/splitting_merging_functions.md | 1 + .../functions/string_functions.md | 1 + .../functions/string_replace_functions.md | 1 + .../functions/string_search_functions.md | 1 + .../functions/type_conversion_functions.md | 1 + .../query_language/functions/url_functions.md | 1 + .../functions/uuid_functions.md | 1 + .../functions/ym_dict_functions.md | 1 + docs/ja/query_language/index.md | 1 + docs/ja/query_language/insert_into.md | 1 + docs/ja/query_language/misc.md | 1 + docs/ja/query_language/operators.md | 1 + docs/ja/query_language/select.md | 1 + docs/ja/query_language/show.md | 1 + docs/ja/query_language/syntax.md | 1 + docs/ja/query_language/system.md | 1 + .../ja/query_language/table_functions/file.md | 1 + .../ja/query_language/table_functions/hdfs.md | 1 + .../query_language/table_functions/index.md | 1 + .../query_language/table_functions/input.md | 1 + .../ja/query_language/table_functions/jdbc.md | 1 + .../query_language/table_functions/merge.md | 1 + .../query_language/table_functions/mysql.md | 1 + .../query_language/table_functions/numbers.md | 1 + .../ja/query_language/table_functions/odbc.md | 1 + .../query_language/table_functions/remote.md | 1 + docs/ja/query_language/table_functions/url.md | 1 + docs/ja/roadmap.md | 1 + docs/ja/security_changelog.md | 1 + docs/ru/getting_started/install.md | 6 +- docs/ru/getting_started/tutorial.md | 1 + docs/toc_en.yml | 3 +- docs/toc_ja.yml | 11 +- docs/toc_ru.yml | 4 +- docs/toc_zh.yml | 3 +- docs/tools/make_links.sh | 2 +- .../mkdocs-material-theme/assets/flags/ja.svg | 11 +- .../partials/language/ja.html | 6 + .../example_datasets/metrica.md | 1 + docs/zh/getting_started/install.md | 6 +- docs/zh/getting_started/tutorial.md | 1 + 214 files changed, 308 insertions(+), 216 deletions(-) create mode 120000 docs/fa/getting_started/tutorial.md create mode 120000 docs/ja/changelog.md create mode 120000 docs/ja/data_types/array.md create mode 120000 docs/ja/data_types/boolean.md create mode 120000 docs/ja/data_types/date.md create mode 120000 docs/ja/data_types/datetime.md create mode 120000 docs/ja/data_types/decimal.md create mode 120000 docs/ja/data_types/domains/ipv4.md create mode 120000 docs/ja/data_types/domains/ipv6.md create mode 120000 docs/ja/data_types/domains/overview.md create mode 120000 docs/ja/data_types/enum.md create mode 120000 docs/ja/data_types/fixedstring.md create mode 120000 docs/ja/data_types/float.md create mode 120000 docs/ja/data_types/index.md create mode 120000 docs/ja/data_types/int_uint.md create mode 120000 docs/ja/data_types/nested_data_structures/aggregatefunction.md create mode 120000 docs/ja/data_types/nested_data_structures/index.md create mode 120000 docs/ja/data_types/nested_data_structures/nested.md create mode 120000 docs/ja/data_types/nullable.md create mode 120000 docs/ja/data_types/special_data_types/expression.md create mode 120000 docs/ja/data_types/special_data_types/index.md create mode 120000 docs/ja/data_types/special_data_types/interval.md create mode 120000 docs/ja/data_types/special_data_types/nothing.md create mode 120000 docs/ja/data_types/special_data_types/set.md create mode 120000 docs/ja/data_types/string.md create mode 120000 docs/ja/data_types/tuple.md create mode 120000 docs/ja/data_types/uuid.md create mode 120000 docs/ja/database_engines/index.md create mode 120000 docs/ja/database_engines/lazy.md create mode 120000 docs/ja/database_engines/mysql.md create mode 120000 docs/ja/development/architecture.md create mode 120000 docs/ja/development/build.md create mode 120000 docs/ja/development/build_cross_arm.md create mode 120000 docs/ja/development/build_cross_osx.md create mode 120000 docs/ja/development/build_osx.md create mode 120000 docs/ja/development/contrib.md create mode 120000 docs/ja/development/developer_instruction.md create mode 120000 docs/ja/development/index.md create mode 120000 docs/ja/development/style.md create mode 120000 docs/ja/development/tests.md create mode 120000 docs/ja/development/tests/developer_instruction_ru.md create mode 120000 docs/ja/development/tests/easy_tasks_sorted_ru.md create mode 120000 docs/ja/development/tests/sanitizers.md create mode 120000 docs/ja/faq/general.md create mode 120000 docs/ja/getting_started/example_datasets/amplab_benchmark.md create mode 120000 docs/ja/getting_started/example_datasets/criteo.md create mode 120000 docs/ja/getting_started/example_datasets/metrica.md create mode 120000 docs/ja/getting_started/example_datasets/nyc_taxi.md create mode 120000 docs/ja/getting_started/example_datasets/ontime.md create mode 120000 docs/ja/getting_started/example_datasets/star_schema.md create mode 120000 docs/ja/getting_started/example_datasets/wikistat.md create mode 120000 docs/ja/getting_started/index.md create mode 120000 docs/ja/getting_started/install.md create mode 120000 docs/ja/getting_started/tutorial.md create mode 120000 docs/ja/guides/apply_catboost_model.md create mode 120000 docs/ja/guides/index.md create mode 100644 docs/ja/images/column_oriented.gif create mode 100644 docs/ja/images/logo.svg create mode 100644 docs/ja/images/row_oriented.gif mode change 100644 => 120000 docs/ja/index.md create mode 120000 docs/ja/interfaces/cli.md create mode 120000 docs/ja/interfaces/cpp.md create mode 120000 docs/ja/interfaces/formats.md create mode 120000 docs/ja/interfaces/http.md create mode 120000 docs/ja/interfaces/index.md create mode 120000 docs/ja/interfaces/jdbc.md create mode 120000 docs/ja/interfaces/odbc.md create mode 120000 docs/ja/interfaces/tcp.md create mode 120000 docs/ja/interfaces/third-party/client_libraries.md create mode 120000 docs/ja/interfaces/third-party/gui.md create mode 120000 docs/ja/interfaces/third-party/integrations.md create mode 120000 docs/ja/interfaces/third-party/proxy.md create mode 120000 docs/ja/introduction/distinctive_features.md create mode 120000 docs/ja/introduction/features_considered_disadvantages.md create mode 120000 docs/ja/introduction/history.md create mode 120000 docs/ja/introduction/performance.md create mode 120000 docs/ja/operations/access_rights.md create mode 120000 docs/ja/operations/backup.md create mode 120000 docs/ja/operations/configuration_files.md create mode 120000 docs/ja/operations/index.md create mode 120000 docs/ja/operations/monitoring.md create mode 120000 docs/ja/operations/quotas.md create mode 120000 docs/ja/operations/requirements.md create mode 120000 docs/ja/operations/server_settings/index.md create mode 120000 docs/ja/operations/server_settings/settings.md create mode 120000 docs/ja/operations/settings/constraints_on_settings.md create mode 120000 docs/ja/operations/settings/index.md create mode 120000 docs/ja/operations/settings/permissions_for_queries.md create mode 120000 docs/ja/operations/settings/query_complexity.md create mode 120000 docs/ja/operations/settings/settings.md create mode 120000 docs/ja/operations/settings/settings_profiles.md create mode 120000 docs/ja/operations/settings/settings_users.md create mode 120000 docs/ja/operations/system_tables.md create mode 120000 docs/ja/operations/table_engines/aggregatingmergetree.md create mode 120000 docs/ja/operations/table_engines/buffer.md create mode 120000 docs/ja/operations/table_engines/collapsingmergetree.md create mode 120000 docs/ja/operations/table_engines/custom_partitioning_key.md create mode 120000 docs/ja/operations/table_engines/dictionary.md create mode 120000 docs/ja/operations/table_engines/distributed.md create mode 120000 docs/ja/operations/table_engines/external_data.md create mode 120000 docs/ja/operations/table_engines/file.md create mode 120000 docs/ja/operations/table_engines/graphitemergetree.md create mode 120000 docs/ja/operations/table_engines/hdfs.md create mode 120000 docs/ja/operations/table_engines/index.md create mode 120000 docs/ja/operations/table_engines/jdbc.md create mode 120000 docs/ja/operations/table_engines/join.md create mode 120000 docs/ja/operations/table_engines/kafka.md create mode 120000 docs/ja/operations/table_engines/log.md create mode 120000 docs/ja/operations/table_engines/log_family.md create mode 120000 docs/ja/operations/table_engines/materializedview.md create mode 120000 docs/ja/operations/table_engines/memory.md create mode 120000 docs/ja/operations/table_engines/merge.md create mode 120000 docs/ja/operations/table_engines/mergetree.md create mode 120000 docs/ja/operations/table_engines/mysql.md create mode 120000 docs/ja/operations/table_engines/null.md create mode 120000 docs/ja/operations/table_engines/odbc.md create mode 120000 docs/ja/operations/table_engines/replacingmergetree.md create mode 120000 docs/ja/operations/table_engines/replication.md create mode 120000 docs/ja/operations/table_engines/set.md create mode 120000 docs/ja/operations/table_engines/stripelog.md create mode 120000 docs/ja/operations/table_engines/summingmergetree.md create mode 120000 docs/ja/operations/table_engines/tinylog.md create mode 120000 docs/ja/operations/table_engines/url.md create mode 120000 docs/ja/operations/table_engines/versionedcollapsingmergetree.md create mode 120000 docs/ja/operations/table_engines/view.md create mode 120000 docs/ja/operations/tips.md create mode 120000 docs/ja/operations/troubleshooting.md create mode 120000 docs/ja/operations/update.md create mode 120000 docs/ja/operations/utils/clickhouse-copier.md create mode 120000 docs/ja/operations/utils/clickhouse-local.md create mode 120000 docs/ja/operations/utils/index.md create mode 120000 docs/ja/query_language/agg_functions/combinators.md create mode 120000 docs/ja/query_language/agg_functions/index.md create mode 120000 docs/ja/query_language/agg_functions/parametric_functions.md create mode 120000 docs/ja/query_language/agg_functions/reference.md create mode 120000 docs/ja/query_language/alter.md create mode 120000 docs/ja/query_language/create.md create mode 120000 docs/ja/query_language/dicts/external_dicts.md create mode 120000 docs/ja/query_language/dicts/external_dicts_dict.md create mode 120000 docs/ja/query_language/dicts/external_dicts_dict_layout.md create mode 120000 docs/ja/query_language/dicts/external_dicts_dict_lifetime.md create mode 120000 docs/ja/query_language/dicts/external_dicts_dict_sources.md create mode 120000 docs/ja/query_language/dicts/external_dicts_dict_structure.md create mode 120000 docs/ja/query_language/dicts/index.md create mode 120000 docs/ja/query_language/dicts/internal_dicts.md create mode 120000 docs/ja/query_language/functions/arithmetic_functions.md create mode 120000 docs/ja/query_language/functions/array_functions.md create mode 120000 docs/ja/query_language/functions/array_join.md create mode 120000 docs/ja/query_language/functions/bit_functions.md create mode 120000 docs/ja/query_language/functions/bitmap_functions.md create mode 120000 docs/ja/query_language/functions/comparison_functions.md create mode 120000 docs/ja/query_language/functions/conditional_functions.md create mode 120000 docs/ja/query_language/functions/date_time_functions.md create mode 120000 docs/ja/query_language/functions/encoding_functions.md create mode 120000 docs/ja/query_language/functions/ext_dict_functions.md create mode 120000 docs/ja/query_language/functions/functions_for_nulls.md create mode 120000 docs/ja/query_language/functions/geo.md create mode 120000 docs/ja/query_language/functions/hash_functions.md create mode 120000 docs/ja/query_language/functions/higher_order_functions.md create mode 120000 docs/ja/query_language/functions/in_functions.md create mode 120000 docs/ja/query_language/functions/index.md create mode 120000 docs/ja/query_language/functions/ip_address_functions.md create mode 120000 docs/ja/query_language/functions/json_functions.md create mode 120000 docs/ja/query_language/functions/logical_functions.md create mode 120000 docs/ja/query_language/functions/machine_learning_functions.md create mode 120000 docs/ja/query_language/functions/math_functions.md create mode 120000 docs/ja/query_language/functions/other_functions.md create mode 120000 docs/ja/query_language/functions/random_functions.md create mode 120000 docs/ja/query_language/functions/rounding_functions.md create mode 120000 docs/ja/query_language/functions/splitting_merging_functions.md create mode 120000 docs/ja/query_language/functions/string_functions.md create mode 120000 docs/ja/query_language/functions/string_replace_functions.md create mode 120000 docs/ja/query_language/functions/string_search_functions.md create mode 120000 docs/ja/query_language/functions/type_conversion_functions.md create mode 120000 docs/ja/query_language/functions/url_functions.md create mode 120000 docs/ja/query_language/functions/uuid_functions.md create mode 120000 docs/ja/query_language/functions/ym_dict_functions.md create mode 120000 docs/ja/query_language/index.md create mode 120000 docs/ja/query_language/insert_into.md create mode 120000 docs/ja/query_language/misc.md create mode 120000 docs/ja/query_language/operators.md create mode 120000 docs/ja/query_language/select.md create mode 120000 docs/ja/query_language/show.md create mode 120000 docs/ja/query_language/syntax.md create mode 120000 docs/ja/query_language/system.md create mode 120000 docs/ja/query_language/table_functions/file.md create mode 120000 docs/ja/query_language/table_functions/hdfs.md create mode 120000 docs/ja/query_language/table_functions/index.md create mode 120000 docs/ja/query_language/table_functions/input.md create mode 120000 docs/ja/query_language/table_functions/jdbc.md create mode 120000 docs/ja/query_language/table_functions/merge.md create mode 120000 docs/ja/query_language/table_functions/mysql.md create mode 120000 docs/ja/query_language/table_functions/numbers.md create mode 120000 docs/ja/query_language/table_functions/odbc.md create mode 120000 docs/ja/query_language/table_functions/remote.md create mode 120000 docs/ja/query_language/table_functions/url.md create mode 120000 docs/ja/roadmap.md create mode 120000 docs/ja/security_changelog.md create mode 120000 docs/ru/getting_started/tutorial.md create mode 120000 docs/zh/getting_started/example_datasets/metrica.md create mode 120000 docs/zh/getting_started/tutorial.md diff --git a/docs/en/getting_started/example_datasets/metrica.md b/docs/en/getting_started/example_datasets/metrica.md index b26f26f5acb..d89fe54f4eb 100644 --- a/docs/en/getting_started/example_datasets/metrica.md +++ b/docs/en/getting_started/example_datasets/metrica.md @@ -57,6 +57,6 @@ clickhouse-client --query "SELECT COUNT(*) FROM datasets.visits_v1" ## Example Queries -[ClickHouse tutorial](../tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. +[ClickHouse tutorial](../../getting_started/tutorial.md) is based on Yandex.Metrica dataset and the recommended way to get started with this dataset is to just go through tutorial. Additional examples of queries to these tables can be found among [stateful tests](https://github.com/yandex/ClickHouse/tree/master/dbms/tests/queries/1_stateful) of ClickHouse (they are named `test.hists` and `test.visits` there). diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md index 9251967d567..f1fd9ee2beb 100644 --- a/docs/en/getting_started/tutorial.md +++ b/docs/en/getting_started/tutorial.md @@ -459,7 +459,7 @@ clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv ``` -ClickHouse has a lot of [settings to tune](../operations/settings.md) and one way to specify them in console client is via arguments, as we can see with `--max_insert_block_size`. The easiest way to figure out what settings are available, what do they mean and what the defaults are is to query the `system.settings` table: +ClickHouse has a lot of [settings to tune](../operations/settings/index.md) and one way to specify them in console client is via arguments, as we can see with `--max_insert_block_size`. The easiest way to figure out what settings are available, what do they mean and what the defaults are is to query the `system.settings` table: ``` sql SELECT name, value, changed, description diff --git a/docs/en/index.md b/docs/en/index.md index 40158f524ec..6dea5f6570b 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -1,8 +1,8 @@ -# What is ClickHouse? +# ClickHouseとは? -ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP). +ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向のデータベース管理システム（DBMS）です。 -In a "normal" row-oriented DBMS, data is stored in this order: +「通常の」行指向のDBMSでは、データは次の順序で保存されます。 | Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | | ------ | ------------------- | ---------- | ------------------ | --------- | ------------------- | @@ -11,12 +11,12 @@ In a "normal" row-oriented DBMS, data is stored in this order: | #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | | #N | ... | ... | ... | ... | ... | -In other words, all the values related to a row are physically stored next to each other. +つまり、行に関連するすべての値は物理的に隣り合わせに格納されます。 -Examples of a row-oriented DBMS are MySQL, Postgres, and MS SQL Server. +行指向のDBMSの例：MySQL, Postgres および MS SQL Server {: .grey } -In a column-oriented DBMS, data is stored like this: +列指向のDBMSでは、データは次のように保存されます： | Row: | #0 | #1 | #2 | #N | | ----------- | ------------------- | ------------------- | ------------------- | ------------------- | @@ -26,68 +26,74 @@ In a column-oriented DBMS, data is stored like this: | GoodEvent: | 1 | 1 | 1 | ... | | EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | ... | -These examples only show the order that data is arranged in. -The values from different columns are stored separately, and data from the same column is stored together. +これらの例は、データが配置される順序のみを示しています。 +異なる列の値は別々に保存され、同じ列のデータは一緒に保存されます。 -Examples of a column-oriented DBMS: Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, and kdb+. +列指向DBMSの例：Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid および kdb+ {: .grey } -Different orders for storing data are better suited to different scenarios. -The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query – rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. +異なったデータ格納の順序は、異なったシナリオにより適します。 +データアクセスシナリオとは、クエリの実行内容、頻度、割合を指します。クエリで読み取られるの各種データの量（行、列、バイト）。データの読み取りと更新の関係。作業データのサイズとローカルでの使用方法。トランザクションが使用されるかどうか、およびそれらがどの程度分離されているか。データ複製と論理的整合性の要件。クエリの種類ごとの遅延とスループットの要件など。 -The higher the load on the system, the more important it is to customize the system set up to match the requirements of the usage scenario, and the more fine grained this customization becomes. There is no system that is equally well-suited to significantly different scenarios. If a system is adaptable to a wide set of scenarios, under a high load, the system will handle all the scenarios equally poorly, or will work well for just one or few of possible scenarios. +システムの負荷が高いほど、使用シナリオの要件に一致するようにセットアップされたシステムをカスタマイズすることがより重要になり、このカスタマイズはより細かくなります。大きく異なるシナリオに等しく適したシステムはありません。システムがさまざまなシナリオに適応可能である場合、高負荷下では、システムはすべてのシナリオを同等に不十分に処理するか、1つまたはいくつかの可能なシナリオでうまく機能します。 -## Key Properties of the OLAP scenario +## OLAPシナリオの主要なプロパティ -- The vast majority of requests are for read access. -- Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. -- Data is added to the DB but is not modified. -- For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. -- Tables are "wide," meaning they contain a large number of columns. -- Queries are relatively rare (usually hundreds of queries per server or less per second). -- For simple queries, latencies around 50 ms are allowed. -- Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). -- Requires high throughput when processing a single query (up to billions of rows per second per server). -- Transactions are not necessary. -- Low requirements for data consistency. -- There is one large table per query. All tables are small, except for one. -- A query result is significantly smaller than the source data. In other words, data is filtered or aggregated, so the result fits in a single server's RAM. +- リクエストの大部分は読み取りアクセス用である。 +- データは、単一行ではなく、かなり大きなバッチ（> 1000行）で更新されます。または、まったく更新されない。 +- データはDBに追加されるが、変更されない。 +- 読み取りの場合、非常に多くの行がDBから抽出されるが、一部の列のみ。 +- テーブルは「幅が広く」、多数の列が含まれる。 +- クエリは比較的まれ（通常、サーバーあたり毎秒数百あるいはそれ以下の数のクエリ）。 +- 単純なクエリでは、約50ミリ秒の遅延が容認される。 +- 列の値はかなり小さく、数値や短い文字列（たとえば、URLごとに60バイト）。 +- 単一のクエリを処理する場合、高いスループットが必要（サーバーあたり毎秒最大数十億行）。 +- トランザクションは必要ない。 +- データの一貫性の要件が低い。 +- クエリごとに1つの大きなテーブルがある。 1つを除くすべてのテーブルは小さい。 +- クエリ結果は、ソースデータよりも大幅に小さくなる。つまり、データはフィルター処理または集計されるため、結果は単一サーバーのRAMに収まる。 -It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn't make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Redis for analytics, you will get very poor performance compared to OLAP databases. +OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセスなど）とは非常に異なることが容易にわかります。 したがって、まともなパフォーマンスを得るには、OLTPまたはKey-Value DBを使用して分析クエリを処理しようとするのは無意味です。 たとえば、分析にMongoDBまたはRedisを使用しようとすると、OLAPデータベースに比べてパフォーマンスが非常に低下します。 -## Why Column-Oriented Databases Work Better in the OLAP Scenario +## OLAPシナリオで列指向データベースがよりよく機能する理由 -Column-oriented databases are better suited to OLAP scenarios: they are at least 100 times faster in processing most queries. The reasons are explained in detail below, but the fact is easier to demonstrate visually: +列指向データベースは、OLAPシナリオにより適しています。ほとんどのクエリの処理が少なくとも100倍高速です。 理由を以下に詳しく説明しますが、その根拠は視覚的に簡単に説明できます： -**Row-oriented DBMS** +**行指向DBMS** ![Row-oriented](images/row_oriented.gif#) -**Column-oriented DBMS** +**列指向DBMS** ![Column-oriented](images/column_oriented.gif#) -See the difference? +違いがわかりましたか？ ### Input/output -1. For an analytical query, only a small number of table columns need to be read. In a column-oriented database, you can read just the data you need. For example, if you need 5 columns out of 100, you can expect a 20-fold reduction in I/O. -2. Since data is read in packets, it is easier to compress. Data in columns is also easier to compress. This further reduces the I/O volume. -3. Due to the reduced I/O, more data fits in the system cache. +1. 分析クエリでは、少数のテーブル列のみを読み取る必要があります。列指向のデータベースでは、必要なデータのみを読み取ることができます。たとえば、100のうち5つの列が必要な場合、I/Oが20倍削減されることが期待できます。 +2. データはパケットで読み取られるため、圧縮が容易です。列のデータも圧縮が簡単です。これにより、I/Oボリュームがさらに削減されます。 +3. I/Oの削減により、より多くのデータがシステムキャッシュに収まります。 -For example, the query "count the number of records for each advertising platform" requires reading one "advertising platform ID" column, which takes up 1 byte uncompressed. If most of the traffic was not from advertising platforms, you can expect at least 10-fold compression of this column. When using a quick compression algorithm, data decompression is possible at a speed of at least several gigabytes of uncompressed data per second. In other words, this query can be processed at a speed of approximately several billion rows per second on a single server. This speed is actually achieved in practice. +たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。
Example -```bash +``` $ clickhouse-client ClickHouse client version 0.0.52053. Connecting to localhost:9000. Connected to ClickHouse server version 0.0.52053. -``` -```sql -SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 -``` -```text + +:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 + +SELECT +CounterID, +count() +FROM hits +GROUP BY CounterID +ORDER BY count() DESC +LIMIT 20 + ┌─CounterID─┬──count()─┐ │ 114208 │ 56057344 │ │ 115080 │ 51619590 │ @@ -110,23 +116,27 @@ SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIM │ 115079 │ 8837972 │ │ 337234 │ 8205961 │ └───────────┴──────────┘ + +20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) + +:) ```
### CPU -Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don't do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. -It makes sense to both store data in columns and process it, when possible, by columns. +クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 +データを列に格納し、可能な場合は列ごとに処理することは理にかなっています。 -There are two ways to do this: +これを行うには2つの方法があります: -1. A vector engine. All operations are written for vectors, instead of for separate values. This means you don't need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. +1. ベクトルエンジン。 すべての操作は、個別の値ではなく、ベクトルに対して記述されます。 これは、オペレーションを頻繁に呼び出す必要がなく、ディスパッチコストが無視できることを意味します。 操作コードには、最適化された内部サイクルが含まれています。 -2. Code generation. The code generated for the query has all the indirect calls in it. +2. コード生成。 クエリ用に生成されたコードには、すべての間接的な呼び出しが含まれています。 -This is not done in "normal" databases, because it doesn't make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) +これは、単純なクエリを実行する場合には意味がないため、「通常の」データベースでは実行されません。 ただし、例外があります。 たとえば、MemSQLはコード生成を使用して、SQLクエリを処理する際の遅延を減らします。 （比較のために、分析DBMSではレイテンシではなくスループットの最適化が必要です。） -Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization. +CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。 [Original article](https://clickhouse.yandex/docs/en/) diff --git a/docs/fa/getting_started/install.md b/docs/fa/getting_started/install.md index 4922724bf6d..790c9381007 100644 --- a/docs/fa/getting_started/install.md +++ b/docs/fa/getting_started/install.md @@ -20,7 +20,7 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su ##ﺩﻮﺟﻮﻣ ﺐﺼﻧ ﯼﺎﻫ ﻪﻨﯾﺰﮔ -### نصب از طریق پکیج های Debian/Ubuntu +### نصب از طریق پکیج های Debian/Ubuntu {#from-deb-packages} در فایل `/etc/apt/sources.list` (یا در یک فایل جدا `/etc/apt/sources.list.d/clickhouse.list`)، Repo زیر را اضافه کنید: @@ -51,7 +51,7 @@ sudo apt-get install clickhouse-client clickhouse-server ClickHouse دارای تنظیمات محدودیت دسترسی می باشد. این تنظیمات در فایل 'users.xml' (کنار 'config.xml') می باشد. به صورت پیش فرض دسترسی برای کاربر 'default' از همه جا بدون نیاز به پسورد وجود دارد. 'user/default/networks' را مشاهده کنید. برای اطلاعات بیشتر قسمت "تنظیمات فایل ها" را مشاهده کنید. -### RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ +### RPM ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ {#from-rpm-packages} .ﺪﻨﮐ ﯽﻣ ﻪﯿﺻﻮﺗ ﺲﮐﻮﻨﯿﻟ ﺮﺑ ﯽﻨﺘﺒﻣ rpm ﺮﺑ ﯽﻨﺘﺒﻣ ﯼﺎﻫ ﻊﯾﺯﻮﺗ ﺮﯾﺎﺳ ﻭ CentOS ، RedHat ﯼﺍ @@ -78,7 +78,7 @@ sudo yum install clickhouse-server clickhouse-client .ﺪﻨﻨﮐ ﯽﻣ ﻩﺩﺎﻔﺘﺳﺍ ﻞﺧﺍﺩ ﺭﺩ "deb" ﯽﻤﺳﺭ ﯼﺎﻫ ﻪﺘﺴﺑ ﺯﺍ ﺮﯾﻭﺎﺼﺗ ﻦﯾﺍ .ﺪﯿﻨﮐ ﻝﺎﺒﻧﺩ ﺍﺭ (/ht -### نصب از طریق Source +### نصب از طریق Source {#from-sources} برای Compile، دستورالعمل های فایل build.md را دنبال کنید: @@ -108,7 +108,7 @@ Server: dbms/programs/clickhouse-server به مسیر لاگ ها در تنظیمات سرور توجه کنید (src/dbms/programs/config.xml). -### روش های دیگر نصب +### روش های دیگر نصب {#from-docker-image} Docker image: diff --git a/docs/fa/getting_started/tutorial.md b/docs/fa/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/fa/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/ja/changelog.md b/docs/ja/changelog.md new file mode 120000 index 00000000000..699cc9e7b7c --- /dev/null +++ b/docs/ja/changelog.md @@ -0,0 +1 @@ +../../CHANGELOG.md \ No newline at end of file diff --git a/docs/ja/data_types/array.md b/docs/ja/data_types/array.md new file mode 120000 index 00000000000..808c98bf91a --- /dev/null +++ b/docs/ja/data_types/array.md @@ -0,0 +1 @@ +../../en/data_types/array.md \ No newline at end of file diff --git a/docs/ja/data_types/boolean.md b/docs/ja/data_types/boolean.md new file mode 120000 index 00000000000..42e84f1e52a --- /dev/null +++ b/docs/ja/data_types/boolean.md @@ -0,0 +1 @@ +../../en/data_types/boolean.md \ No newline at end of file diff --git a/docs/ja/data_types/date.md b/docs/ja/data_types/date.md new file mode 120000 index 00000000000..d1ebc137e8f --- /dev/null +++ b/docs/ja/data_types/date.md @@ -0,0 +1 @@ +../../en/data_types/date.md \ No newline at end of file diff --git a/docs/ja/data_types/datetime.md b/docs/ja/data_types/datetime.md new file mode 120000 index 00000000000..2eb9f44e6eb --- /dev/null +++ b/docs/ja/data_types/datetime.md @@ -0,0 +1 @@ +../../en/data_types/datetime.md \ No newline at end of file diff --git a/docs/ja/data_types/decimal.md b/docs/ja/data_types/decimal.md new file mode 120000 index 00000000000..ccea440adfa --- /dev/null +++ b/docs/ja/data_types/decimal.md @@ -0,0 +1 @@ +../../en/data_types/decimal.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/ipv4.md b/docs/ja/data_types/domains/ipv4.md new file mode 120000 index 00000000000..eb4cc7d57b5 --- /dev/null +++ b/docs/ja/data_types/domains/ipv4.md @@ -0,0 +1 @@ +../../../en/data_types/domains/ipv4.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/ipv6.md b/docs/ja/data_types/domains/ipv6.md new file mode 120000 index 00000000000..cca37a22458 --- /dev/null +++ b/docs/ja/data_types/domains/ipv6.md @@ -0,0 +1 @@ +../../../en/data_types/domains/ipv6.md \ No newline at end of file diff --git a/docs/ja/data_types/domains/overview.md b/docs/ja/data_types/domains/overview.md new file mode 120000 index 00000000000..13465d655ee --- /dev/null +++ b/docs/ja/data_types/domains/overview.md @@ -0,0 +1 @@ +../../../en/data_types/domains/overview.md \ No newline at end of file diff --git a/docs/ja/data_types/enum.md b/docs/ja/data_types/enum.md new file mode 120000 index 00000000000..23ebe64773e --- /dev/null +++ b/docs/ja/data_types/enum.md @@ -0,0 +1 @@ +../../en/data_types/enum.md \ No newline at end of file diff --git a/docs/ja/data_types/fixedstring.md b/docs/ja/data_types/fixedstring.md new file mode 120000 index 00000000000..53092fcb884 --- /dev/null +++ b/docs/ja/data_types/fixedstring.md @@ -0,0 +1 @@ +../../en/data_types/fixedstring.md \ No newline at end of file diff --git a/docs/ja/data_types/float.md b/docs/ja/data_types/float.md new file mode 120000 index 00000000000..d2ae6bd11de --- /dev/null +++ b/docs/ja/data_types/float.md @@ -0,0 +1 @@ +../../en/data_types/float.md \ No newline at end of file diff --git a/docs/ja/data_types/index.md b/docs/ja/data_types/index.md new file mode 120000 index 00000000000..c9f29d637f3 --- /dev/null +++ b/docs/ja/data_types/index.md @@ -0,0 +1 @@ +../../en/data_types/index.md \ No newline at end of file diff --git a/docs/ja/data_types/int_uint.md b/docs/ja/data_types/int_uint.md new file mode 120000 index 00000000000..3a913c9328e --- /dev/null +++ b/docs/ja/data_types/int_uint.md @@ -0,0 +1 @@ +../../en/data_types/int_uint.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/aggregatefunction.md b/docs/ja/data_types/nested_data_structures/aggregatefunction.md new file mode 120000 index 00000000000..36544324d2b --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/aggregatefunction.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/aggregatefunction.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/index.md b/docs/ja/data_types/nested_data_structures/index.md new file mode 120000 index 00000000000..a5659a9c5cd --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/index.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/index.md \ No newline at end of file diff --git a/docs/ja/data_types/nested_data_structures/nested.md b/docs/ja/data_types/nested_data_structures/nested.md new file mode 120000 index 00000000000..653a1ce31c3 --- /dev/null +++ b/docs/ja/data_types/nested_data_structures/nested.md @@ -0,0 +1 @@ +../../../en/data_types/nested_data_structures/nested.md \ No newline at end of file diff --git a/docs/ja/data_types/nullable.md b/docs/ja/data_types/nullable.md new file mode 120000 index 00000000000..0233f91d954 --- /dev/null +++ b/docs/ja/data_types/nullable.md @@ -0,0 +1 @@ +../../en/data_types/nullable.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/expression.md b/docs/ja/data_types/special_data_types/expression.md new file mode 120000 index 00000000000..4cec632b416 --- /dev/null +++ b/docs/ja/data_types/special_data_types/expression.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/expression.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/index.md b/docs/ja/data_types/special_data_types/index.md new file mode 120000 index 00000000000..f3ca4a47f98 --- /dev/null +++ b/docs/ja/data_types/special_data_types/index.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/index.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/interval.md b/docs/ja/data_types/special_data_types/interval.md new file mode 120000 index 00000000000..6829f5ced00 --- /dev/null +++ b/docs/ja/data_types/special_data_types/interval.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/interval.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/nothing.md b/docs/ja/data_types/special_data_types/nothing.md new file mode 120000 index 00000000000..197a752ce9c --- /dev/null +++ b/docs/ja/data_types/special_data_types/nothing.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/nothing.md \ No newline at end of file diff --git a/docs/ja/data_types/special_data_types/set.md b/docs/ja/data_types/special_data_types/set.md new file mode 120000 index 00000000000..5beb14114d3 --- /dev/null +++ b/docs/ja/data_types/special_data_types/set.md @@ -0,0 +1 @@ +../../../en/data_types/special_data_types/set.md \ No newline at end of file diff --git a/docs/ja/data_types/string.md b/docs/ja/data_types/string.md new file mode 120000 index 00000000000..7bdd739398f --- /dev/null +++ b/docs/ja/data_types/string.md @@ -0,0 +1 @@ +../../en/data_types/string.md \ No newline at end of file diff --git a/docs/ja/data_types/tuple.md b/docs/ja/data_types/tuple.md new file mode 120000 index 00000000000..d30a8463aeb --- /dev/null +++ b/docs/ja/data_types/tuple.md @@ -0,0 +1 @@ +../../en/data_types/tuple.md \ No newline at end of file diff --git a/docs/ja/data_types/uuid.md b/docs/ja/data_types/uuid.md new file mode 120000 index 00000000000..aba05e889ac --- /dev/null +++ b/docs/ja/data_types/uuid.md @@ -0,0 +1 @@ +../../en/data_types/uuid.md \ No newline at end of file diff --git a/docs/ja/database_engines/index.md b/docs/ja/database_engines/index.md new file mode 120000 index 00000000000..bbdb762a4ad --- /dev/null +++ b/docs/ja/database_engines/index.md @@ -0,0 +1 @@ +../../en/database_engines/index.md \ No newline at end of file diff --git a/docs/ja/database_engines/lazy.md b/docs/ja/database_engines/lazy.md new file mode 120000 index 00000000000..66830dcdb2f --- /dev/null +++ b/docs/ja/database_engines/lazy.md @@ -0,0 +1 @@ +../../en/database_engines/lazy.md \ No newline at end of file diff --git a/docs/ja/database_engines/mysql.md b/docs/ja/database_engines/mysql.md new file mode 120000 index 00000000000..51ac4126e2d --- /dev/null +++ b/docs/ja/database_engines/mysql.md @@ -0,0 +1 @@ +../../en/database_engines/mysql.md \ No newline at end of file diff --git a/docs/ja/development/architecture.md b/docs/ja/development/architecture.md new file mode 120000 index 00000000000..abda4dd48a8 --- /dev/null +++ b/docs/ja/development/architecture.md @@ -0,0 +1 @@ +../../en/development/architecture.md \ No newline at end of file diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md new file mode 120000 index 00000000000..480dbc2e9f5 --- /dev/null +++ b/docs/ja/development/build.md @@ -0,0 +1 @@ +../../en/development/build.md \ No newline at end of file diff --git a/docs/ja/development/build_cross_arm.md b/docs/ja/development/build_cross_arm.md new file mode 120000 index 00000000000..983a9872dc1 --- /dev/null +++ b/docs/ja/development/build_cross_arm.md @@ -0,0 +1 @@ +../../en/development/build_cross_arm.md \ No newline at end of file diff --git a/docs/ja/development/build_cross_osx.md b/docs/ja/development/build_cross_osx.md new file mode 120000 index 00000000000..72e64e8631f --- /dev/null +++ b/docs/ja/development/build_cross_osx.md @@ -0,0 +1 @@ +../../en/development/build_cross_osx.md \ No newline at end of file diff --git a/docs/ja/development/build_osx.md b/docs/ja/development/build_osx.md new file mode 120000 index 00000000000..f9adaf24584 --- /dev/null +++ b/docs/ja/development/build_osx.md @@ -0,0 +1 @@ +../../en/development/build_osx.md \ No newline at end of file diff --git a/docs/ja/development/contrib.md b/docs/ja/development/contrib.md new file mode 120000 index 00000000000..4749f95f9ef --- /dev/null +++ b/docs/ja/development/contrib.md @@ -0,0 +1 @@ +../../en/development/contrib.md \ No newline at end of file diff --git a/docs/ja/development/developer_instruction.md b/docs/ja/development/developer_instruction.md new file mode 120000 index 00000000000..bdfa9047aa2 --- /dev/null +++ b/docs/ja/development/developer_instruction.md @@ -0,0 +1 @@ +../../en/development/developer_instruction.md \ No newline at end of file diff --git a/docs/ja/development/index.md b/docs/ja/development/index.md new file mode 120000 index 00000000000..1e2ad97dcc5 --- /dev/null +++ b/docs/ja/development/index.md @@ -0,0 +1 @@ +../../en/development/index.md \ No newline at end of file diff --git a/docs/ja/development/style.md b/docs/ja/development/style.md new file mode 120000 index 00000000000..c1bbf11f421 --- /dev/null +++ b/docs/ja/development/style.md @@ -0,0 +1 @@ +../../en/development/style.md \ No newline at end of file diff --git a/docs/ja/development/tests.md b/docs/ja/development/tests.md new file mode 120000 index 00000000000..c03d36c3916 --- /dev/null +++ b/docs/ja/development/tests.md @@ -0,0 +1 @@ +../../en/development/tests.md \ No newline at end of file diff --git a/docs/ja/development/tests/developer_instruction_ru.md b/docs/ja/development/tests/developer_instruction_ru.md new file mode 120000 index 00000000000..c053faed45e --- /dev/null +++ b/docs/ja/development/tests/developer_instruction_ru.md @@ -0,0 +1 @@ +../../../en/development/tests/developer_instruction_ru.md \ No newline at end of file diff --git a/docs/ja/development/tests/easy_tasks_sorted_ru.md b/docs/ja/development/tests/easy_tasks_sorted_ru.md new file mode 120000 index 00000000000..fb8630a5e8b --- /dev/null +++ b/docs/ja/development/tests/easy_tasks_sorted_ru.md @@ -0,0 +1 @@ +../../../en/development/tests/easy_tasks_sorted_ru.md \ No newline at end of file diff --git a/docs/ja/development/tests/sanitizers.md b/docs/ja/development/tests/sanitizers.md new file mode 120000 index 00000000000..c71a3d7aa25 --- /dev/null +++ b/docs/ja/development/tests/sanitizers.md @@ -0,0 +1 @@ +../../../en/development/tests/sanitizers.md \ No newline at end of file diff --git a/docs/ja/faq/general.md b/docs/ja/faq/general.md new file mode 120000 index 00000000000..bc267395b1b --- /dev/null +++ b/docs/ja/faq/general.md @@ -0,0 +1 @@ +../../en/faq/general.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/amplab_benchmark.md b/docs/ja/getting_started/example_datasets/amplab_benchmark.md new file mode 120000 index 00000000000..78c93906bb0 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/amplab_benchmark.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/amplab_benchmark.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/criteo.md b/docs/ja/getting_started/example_datasets/criteo.md new file mode 120000 index 00000000000..507dc68cd62 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/criteo.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/criteo.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/metrica.md b/docs/ja/getting_started/example_datasets/metrica.md new file mode 120000 index 00000000000..984023973eb --- /dev/null +++ b/docs/ja/getting_started/example_datasets/metrica.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/metrica.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/nyc_taxi.md b/docs/ja/getting_started/example_datasets/nyc_taxi.md new file mode 120000 index 00000000000..c47fc83a293 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/nyc_taxi.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/nyc_taxi.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/ontime.md b/docs/ja/getting_started/example_datasets/ontime.md new file mode 120000 index 00000000000..87cfbb8be91 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/ontime.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/ontime.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/star_schema.md b/docs/ja/getting_started/example_datasets/star_schema.md new file mode 120000 index 00000000000..1c26392dd23 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/star_schema.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/star_schema.md \ No newline at end of file diff --git a/docs/ja/getting_started/example_datasets/wikistat.md b/docs/ja/getting_started/example_datasets/wikistat.md new file mode 120000 index 00000000000..bf6e780fb27 --- /dev/null +++ b/docs/ja/getting_started/example_datasets/wikistat.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/wikistat.md \ No newline at end of file diff --git a/docs/ja/getting_started/index.md b/docs/ja/getting_started/index.md new file mode 120000 index 00000000000..1acedb0f03e --- /dev/null +++ b/docs/ja/getting_started/index.md @@ -0,0 +1 @@ +../../en/getting_started/index.md \ No newline at end of file diff --git a/docs/ja/getting_started/install.md b/docs/ja/getting_started/install.md new file mode 120000 index 00000000000..60aa3fb93a4 --- /dev/null +++ b/docs/ja/getting_started/install.md @@ -0,0 +1 @@ +../../en/getting_started/install.md \ No newline at end of file diff --git a/docs/ja/getting_started/tutorial.md b/docs/ja/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/ja/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/ja/guides/apply_catboost_model.md b/docs/ja/guides/apply_catboost_model.md new file mode 120000 index 00000000000..dd36e885974 --- /dev/null +++ b/docs/ja/guides/apply_catboost_model.md @@ -0,0 +1 @@ +../../en/guides/apply_catboost_model.md \ No newline at end of file diff --git a/docs/ja/guides/index.md b/docs/ja/guides/index.md new file mode 120000 index 00000000000..162dcbc3b8f --- /dev/null +++ b/docs/ja/guides/index.md @@ -0,0 +1 @@ +../../en/guides/index.md \ No newline at end of file diff --git a/docs/ja/images/column_oriented.gif b/docs/ja/images/column_oriented.gif new file mode 100644 index 0000000000000000000000000000000000000000..15f4b12e697ac40c60bf77f964645316410da946 GIT binary patch literal 45485 zcmb@tWmFvB)+Y=h2^JuM;DHc=2Y1&5hY$!Z3Enutp>fy7p>c=C9U6CcZ``eMcY5-_ z^W0}<-ZghVz4f8$oL$>a)m~@S-u2s3GLpP}`c5xiAn$k~A(D^~RY(X?QSE9NV%r^Y zn2A{TMzkOy>O~PYqR-pg2o@y7EbGbg65?hS(c=BMW{p@?vv5$dAPiHQgjq^HTr$NeM(IXPl2^Lc`#Y5f#&ae)|pb9J(Y zP*+Dh9%yT-BlgtoT9*;Q-iXuooAbSl$sGg^4q`tNQIq!^85!LigLs2n-)((+*a&HJ zKpgubO3{~3uC6vC2YO_jp|uEk^r@-It*y-;-{trB_f3opT3VW8q9gnJ`~KKiRaaF6 z1^OY*5)k)Wh^w*lyGO+8Z^UIEq8)v)j}@U!JHE4tm={Itcq9HE!$^NmpAeXe-+rn{ zh$($zXJ$n~MnXdROVUtKqWG%sp#S$ODB@zPZ)jocKw)5PYGy4&dD7HINnvIrM5)fD zz^Y&?VGJ;ncC#~9aZ~(h=w@NaXGAF?Od;sP?_yDib?<*2OE2UjV*z1)YMvp_%nR zMEZY8DJbyESlc`3TN@h7d=;YntB2Xl%!r?pk5@uMT#}DRl2x3I?HiXkD>ti{I42hu z8xK1#r#RogWxv`OI$9Z9JN#SL=zqzI{YTk<0>aAnujQ}Cc4kh-Mv`_mRuunQIKSC{ z_J!v^>iaKQqyOv+?|+nK`RfeJKd<+Hd)@yg{f(i2hW{PBe<%MP{>IjSW8Us>a3dq0 zpB^9X?{06duP!gn&rVN{j}8yu`+K`P+gqC(>uakk%S($3^RT(undzy?iSe<~k>R1i zf&RYUp6;&Bj`p_Jmgc6$255aeX-RQWA*3KbFE=MUD>EZKEj1-M37nV^ z9~T=F9Tf?R2oDPl2@VPj@b~le@%Hlca0j}%x;Q&II@sI&v9+`cp;uhthXN1$jAH8EGlWZ(k+E#Y9Dfg#-oo`FMG_xi~r4*;rYa znHaw?(9_Y;ic-CERxwa!~z4so^cF zc$q#*L4<<)5@6?WDp1>xd*0X6m#!ZxMVJ+qdT7tFJRHHQTyZvU8Uf{$72>W!t=1d2 z`K$u|nYg1cz=eev&OOA~l^CkF5pKtFjW1HJFhId1cSVcfjpkf@414H}Ag*AavBke! z-yq@iALdm&^WTZ(u}a^1jLMGK5(eyWxv-c2Noh_Q4D5dM`DHRU)_6(7hq*IDzeH0% zwb<-2)Ie<1FBt2+QC5E}S-m8ROq?)A)+ttMg@A900I`AsYkkE-IJh+#3V?%b2ERiYP0mX zSs_02ejd#e^QbmpWKtl}yUtYm?-kL3K^LX^r2ZD1rm6fkg2qZ1b6H|aB$z$1*d`_~ zw_@GsXv`E49^c+?7oC)}fxcH#hLnnt%YAMT z)3-}t-8T}_yZ({Ly-!8>!t`ylWWk@){+wKYhrzmCJlmHyID%jKr#~9Q+a;#HF`97) zdr%uo7$vB77H$z8$}!pmRM+-2)VH5-ZxQW!Sv ziLpZ4Xp?)=^{yE3=8<;R>4baOwq$HpWmaJAd1-HND~i>AZel5@=(P-BI^eL{R!V|ed ziXpG*o3{$zRr01*PUJT1QinqZFS#RziA~>EMXAz^;<3Lm+xmIozVE+v&d2e`s#j^= zAz}h{^}FWS{YLMFR&Wqa09;t>!Sf0ct)K#7?{d6Lqu4e%mO&3cX z8y<+;EddT;&RvlV&%48q)t($Dq#wN=E>Sqz*l%vqy`CO-tJ)sc4l*d7Y1?1)`g-?> zQo!E>+Fuv>`gD3{Zd2{GV}N{p+uA8wH~}5lcD{biNLi;Z_BwFyBmJ8sJ|VsWIte6w z187Sq-snnqeH%U{;L-VxZP70wei%g{@?QQeaIc+^=qy-z3mGS>|0@;f_e+KF?Dt8x z-Sk=$!K%$0_|>ql)QZtzTH%yLU4S08n96V^!JPLaM3UU(6JZweIV6R2QtXYD5ss;w zWR+kkq3a(Jz}%dVs4~4oRAV^7^0}0R1Jd8leQ+YebE)3l^~-2g#UwOSQE?1N%iC2U zqSGI@s3hnIe^^(=r3o6-%QVQUX2-;o4O4yj#y9kRmOZ+D%ZS;SSWf1cJs$FyM~QPM z`J@VaUQ)Ic2einwM~2LBu5%JmGK5 zPsTSYoq(5AD!(gG*`R1wXP;D_TEM^UI_9Bvk#T%TD|q2LW=B$!F#yaJCmb9PWaoU% zW()aBdoLZVe3?zqLMQpTXd=+0CI|b8PMV%!GCC$Mhvte-RzXWSp^P&R-=k1cn_oHE zcRC-V$4ttRUnNN~u7E*-UI||d)#1E4{h4Zyf46WfmsvHlxrA0FD^#wiiX&wSQB%ZM zJ3n1Yu9~V5Q9u`wteWdKldR>)pfO}VTjr@!^wX+H8)s0>yS28=sfU)U=U%2NyS7v- z%|d4iK3kluS}Lnxsn^vw*Hads?$u+#OZuRpNb^2ltjE%XO<+3KZnl8y)sZRJgUUpp zQ%&-brCEUaLPShLK@pajg)VSvUU<3=Q`PE6M9_36WnC30&C(VhQv-^v4(%hO`xOG5 z>*wbw8k(UNcePkrdCgsFYHVpoZ?OzFziAo{ICeq_)j5?^FK@>$H!ggb-Vk!I;SK87h%FTdQjV+EcftUe9@Mk$!> zz!5$Q__erz@fq4)WLy!9*SxqkGS_s6RpF-cX7%A&y?xTEJk;=T^>)FzcQAm>`(aS; zZV=it(sLH-;l4qNlhh`?!WI^}^owE<+$)1^NAml2*JrH3BM^^jf1$(qSM^Ca4!sv?B8SS@4-KPsAp7r{a)77=BA0J8 zRZ%^R+g}9S2J{0NbBR56zR5LCB(z?FM;^DXl8JgknK=puKJ3efxlN~eau#`|?9e%1 z&Ah^=)9qU@{QO;Os))w1xXJQBrC?!NEuS<0GuD9;Dsbj1t?FzC(Y!m-$vH;A?2viH zs4v`gH$M_sm!21KB+}%%u#{gK*jG9sal<^(B3W1V=g2}oaba0ifv0)fP1jLUR^)EyIXZCvxP&vX=+{lFcowRiy77vxy@zM?Pl##r zCIy1GU!v?RT3cs}rj2({W#ufsx_O)0>u$jC*)DM|b%*_icT~#SKH)}iTO`9}TtV0& z{UmksTSmihtni=LUj=t%gWRUGMjgN^PkX|B_cM7Qhf-6I1LbnJxx{0~82gqz19jK= z5$wxab-^Rx+omZCD_h^y#bb=VruJ+{N) zo7ZF4yHT3~7Wb21%-pM_Uao^*1YC6bpo<~GZnZ(e$9aSvd+XL$Q*A<5q^3gfG3|ZAav*%uF7gc5FMot5>fTicwNB(9 zuHrC1PGqh;R$z^}2GV}Gh-DGE-_<>7v}iqct`}TGmA|>@%Q#E_j(8$0N8F+6RXpx` zC8@h??Ak{0$oF=+!{&g$3p|!-Juvp%4TId9O+DWCd+?e1j3xTuf<1DBd_L}Ze0TPF zx99x<=1nK-!1p%gpP;isnlS_2A$|*sVHSucLaMn|g`{d!@~JJ=A-O zxcI|)JW+UUQNjL-E-n$dF445!J~#eq8va}A0qW8=B9i_(MEMy^%4YK16a?l9+Hts{d2eg0$RnGaqXq{{cgY)YBps`5kZ(jxm2M2+J z!=b^T;9w2!ppm{nMpR_tw=coqkYs2`G8p;ocpx>@mk}+LD-rpHZ72jBS_BOx_73uu z4&jyd1NH}5-gwu*{`k8H5LJXV!@^o`!&-=tVewv?5NWEn$nD_p4(S*2MBd`ifPP5$ zWH53kEI7~H~`dO%pOx*s1nnzS(b zLth?@@9eE`S(gha#>#vcbuuk6L`@rJ9z z;UEEP{#PQ1nVe~lB>wvy*@nNR%CL!KWw=qb>F>jJ$KC8z#&?C{A#d?M$ z@}WC0Hzc|f$MU#>In98KcZs3I2rzF$ESU_rgf1ip3RW&ibeK>490JB@NMhgt1I-e{ zHDeR#V`I_4RkvUsNHSqxY++8k_P*Y=RNOcEgiEx*QH_Kh{{(&X_^Me?tB}aW+my$| z_=!WNJG>QVelaTvAYek{$RiCRsB% zts!PJ#33^&DKH6q$`|`)FR8F0`C2n0%1m$-Jr*S-`Ga~ojcfYBK+ zVJxOr(wUa8-ctV*G;)JoJEGU)v&zM|51<#tG zm+`!h@Wx0RB8KpU$_0iN2Frp%wF>{Z`h#SP^zPG?L-W;>?UC-IjkO9)wDKI__J&2d z82km_-7>IA3bY1`+Zu}%5{qIQili3usa%U4??OsKOHBETD+@~A4dmAj=Cp#_ME-~**~0Ka&{ScLm|5ihb0AaYd?Un`i$`+S0*q0aO4!f094VHGg*-dICHo)^6-Ad<^(`U3Q ze-1|9%j8m^=i?K*;mGB`8>$$E7k>n1TEHur9;zBcOW7^*I7q7~oC~sI3kbE!l?SA6 zio&-Cb12-(8Z{x8vXMD(mtzLYQ@H;HM#@?;1jbb}piy%VkAIm`gSd}HCarmZ=X7eu zqo%~6lP1$Fmb_1_LTjiRgIAGYmSzxF14!}@%&MOoN*%uZ5d`v4X;(=OR#^cnOVH}= zQmXMV5hWAm^|-~=?@3Fr7V9~~s^h;j=)y}0v}&{6YJXvdYsrD67HeP0$z-I|>W6tL z4Ap*@D-sGVN^Yt-FfT9o5_!kanD>yiFI!tURCcUhXX9FD1Q2|u)Z7bf?oVm%D@GpS z)?Z6d7Rp2(6lfV*M49WgoFUMkzP6l{Ygr*h9&T#cBCz&&4e*_{s{Piwp=Pa89H087 z;=sQ)6WGKHZ`>nMuO&?}Ols1bS7>iao8!~%*sMPJQl}#jcNdm=Y0;h++Egy5i2fnG zg)|4N*}7D&?B=Fpfez`#wwvHi^`ws7?v7uF9pWK9b#lF*hkK-pYiNgS)EJEzb$XfQ zJ4inWVvqNEKK6Nk==WkoRxL>l{fR=3gcRD`AHLKdeu&Jq)~lV`*)W6}a@Y?JA0S8S z%`NUwxoe{*Y0s-M5pwT_Yqu7Jcf(Rz7HL``mR&N8KU}$olMWXj3_Pazn=rPoIS;qJ8R;k)p3>>iO&LBk?Z0yA!&(~7(Ls1U zy44^NjZQBO>@){BG7hFTdM#O2z=sVxst1L21}g+x>)pq$+*^xM$8Has?=8okAIFfG zhQ8~J|6mNjwj7QbZe-CKosl1bgCAkS>l*HlrrU;5ntD{y*Bw5(P!hQ>=z^(R zxFTbOsyJ`U?Kjka5ksvV8*zM1%J z<)Sk(ivLu`tJ^{RG|{6or};Gd@y(PU^%QaV%(K8O&YRiv*MjGVgHlgZ4b6z@2OU^w z#5BqiOmZ0(Tsr-t1ty;cOAz{@l*X^zGD&SUWp*@4(K7mKarO*jVfe$GIL4@J%S3L( z0t?fE`N&*R#IWIqNdt|=ij+z3qdDW{Ne!~a4xxqXxjCnn5tj&a-zQgpp|S9^>7Jvqv^{>s|l=?Cc(MSmJ80yi(RF2=sct8WU1b1izk9GLFt*I zheg%paZ`m(Hid;UiJETqe}u~y31Ut z%d-*dQKjn&WGhP^(>0IFgr3VVk9jiB4cyv!$khDl!~8(n_+by)&YqfTx&^uIWQqofUEVRXuzJk#@_b4>{{4~ciI*ZlXH(~{& zJYLqd-XO(Z{(!whcf37Awjr?GEuZ>ZiE#%rZzHQ~8kSR+QefM5@@@p zVT*NTt9oR!FKw3nx#QdMxSH^`Puez#{7Nk2UeL;dy6&DLb6iOJ@LSNPCumaJdWb-8 zW3F^(zSwH{Xno^o$2fhRYGtsn3|>A8|ERcw^=dcq^X`?xYKPVCCH0nH*{;gSuDY=6 z56pdXtHnmWy=KK<0bKjglzlGHtaIvCtkEHk)mB5uniBb@t0H0xbiAm4yFvN1dv>_A z)4FUzzEiUTuL2z(f#B8T2m8!BxzF36mg9owomujQQ0rCZmcyW#!)4Ik>dI#Q$lkk^ zeSL*B|CN)0(Q#z2-PYqnUW}t7hGQ4THQCn9aO`7dy|sAI?!nOsOmVV{IliaWau*wZ z+6q4xKDjL94hdr=K$pRXtC46HmK^ z*P(s~twR$;h+fK%qYmi@ouJdy=SzO(SuB=)l<)hkEu$i*AugM`mleb)Q$Ki4ln6&u3DD-5)likA_IdW z*JFC8nK);!`mZNH&P@Nh|G0WO;B^HA-Kmt_jjdiyAg-w%GGu1>0I*Xj5}_ zV@u25lC66ew6nLqf1r5?F+9>U);BRZIy^fD8=q{LURr2fS>2e~TAbhM9;-Y&IzBl) zUYfeNyjttMZMuhF4emeRBI5zV)fZVGa44HS&0>Fb!I(xSuLp*f8QsJ;$w*37S-QeW zMfns-CjP6pKfkM%zn&=y{xVaj)8hH+U^-f(*tky*_o}*7yPm-U6z4id)!oQo)msZx z-i3m2EHLs15;sPSd@vu_ZtE&H!+{sruS+z_#RuY1P0}9-SvtENSo{PJ`7#W~hslfe z$i7rrOy)f~c}64|BX$NdzT4S-$qw>kJc*naWbp~%KKIm6oT^0MOqJ=&O>IStY2VH zt@sqT)UUcfzC+FMjhmUH@Oy;~&Ghq7kk*&MH#d3bOVLTa9!z4rw;q5j2{8yF%I?qd zqS+zL31FIqttx)n*~7C}KBEh91tZa?I}Pgd5>NM!bPH3XaH->rt*)eaDvb1ZajrxLn`u?kR-Qbg!pduc@%&ylUDqGHwP4xw3n5Dy-<*x+CfK`8q6@b2PwY=Y&k1v4NMmb zB{NXFstOmvOSD@S;6*Ey_l6})S&4a32hj|c3h4lDa>((tIX4ZrQLaEFu2#@>-h( zeaN@ zR{l&KTCQ@DN;ow+O@%qNM25y|&h+IC|ca zbPzGLVZw7(J)sQ;v=m6%_M= z4RLDq`ez2(MG*KE?psgP)XWA-B5`~*R&U>?wd_oFu6^TDw;!L#vAlws4?T_or;58T zoLp+Nmf`&0XHoMkFQOyO4AiX;|}NbQY4seaz;WRuqJa4(++m zd*>a(2qyo*EB;XWHmV*k7Y%)Yn)1a+poqJHssprZGXOV8&>%h9fVEqhq|(&@a|1WT zNtUV?H?;1MoVMPm|Lp3uNZ2qD85uEOpQ;_3oggXo*=_$)8k;s$BInpCDMzoh5`?Hj zqv;F`krr?|`#LCCpS-bAvq|Rd-uF3ssgTbTS(~FGixgU_#rCGZf`pl)#tx zPR)LMP(%r>NG0(A?oMnvlR}ow?|X5_)^VIq`1JT5=rUTVHNTb^&E(DC3P!Ed#vVp^ zG}M5p_*u#VRFXpBNX@Ve66QSdTBGm!bOWq0_8B`4JIdOy=?X=>qJDb7CAM345VlQr ziv<90w{b#K54Ehd7$9jMG8ma%RPMPA!Mnpyt|6By`r2Y3*h9}=qC8W=ID;@}C9;5) z6X5@P`f#v0SOhB|xvmgspyo%f<9D2d!Il563gNx{BKO%9+DJ-O2yzdng#e`R9! zv!M zx05ZEEyYyT-&dwX=Ka41s9WNt46uXNJ}k-XB$$*tg9h9 z%dz4p^>dxr_U_TT2b!NB#^bmhl35<}0jqH^-9Sg9EHVp|6aI$P9mgskOf#z|l9G;5 zjyj;P*@a!>lm>Hc^EaWBb34L;v)&uXk0WdUoyEZeU7qjZSP1Js2lq>c*LBb#A^pG$ zymhWxC%;!K)K+-0i(Gow(4*#4@(ZGodTh?pDJDQ@8~^C9YnJXvvhxT5fu$-ZsHVEG zMGQyp7Li$f!BNXa-6U`(bM(Bau-RUXSV&!R>!u(;5iVInvRhgSO=T0-iU@1oR!?*- ztu50}KA7L>@o^JGBeyFWnqOmKfesO3UDP(E9raz^_fsERRz~W==}|M6h{<+JfFBCI zf_`Ts_S-8Km&y3sZpKqaFMH>J`=Y3~V~7!#!4&>|u$Q3Y)zM{Fo8KXRo#jv|=-Q-< z?8wCQZpK5%g<{HMBHiZwV4l#!R2$hrorufYdjAy^Q7@eN8=-WlkL=c(*t+Ry-L&7e z{GiK|aTQbMv>qVhu1+Pq7r)!QxgO!s|6TYYjJENHY0M+TB7Ju&Bj#FD_X#rBx>op| zXYN+_?3c>Ob_EJn={)Xfa)+fDAXl zoOq{#8~6GGpEoe8#{zG00-sV_5BWMz8amfi9?#_<-+ffi#a)|SR6jVO>q)*jeLw1V zTHoBa-b5N+Xb?bsfs@Fd4=EJr`QF2nK!m>^Ll6u+D{vEb@opgWhtRscKr;j0_)P73 z$$>ExU{2p5P8U-AKl|M!p#fiEKE$*h!43#}UGRsuwSuzKHXG``PEcQFjUb+W-|ZV; zw_s1_TVHMs8$PK2FR<51pQRz-EF!=Mh}YL!Qyd}*8yAcqjX)F^QUVtzA%IsT z3=qxhmZstFW*-8nK>b^>ErErW^@qMvLMrDCtAT`8YJ@>u!l2-=Dri_4u9rTCoLn6+IB2I!Mu7V>E!IAyEk(a!nN3_UiSi}t-2!%Lm5-kdY zIDE$ilou6pMHrY1_;~5;4&p_GWV-}DeY27YX2-sZ!b1zdEwnV!h#r#k&IZ~2i;$QP6)TK=p&9o8h{N2E zdvh0^yA~@W6L;+rr-mMXp%Jf%9;Y!MzjPZf+Ym3M8E=vlsgsnzsTpM{6N^P3|5h^~ z1QJ3Zlb9hLK!O%Rg03yyFGa!o`y+9pt7|lonKwpI3{hdsJNgid+n84B_#DxA1N4M@ zx>&Ko1Yy2}nEiMKnFOBs1o^wTW8Nfjvm^`61c*$y4RLI0LsGg~a@t*TSyEEoUD7Yt zggW}<#=G!yqU5HAI4w;?QY$nWn;r~wmDhAh9FzeQLcn%1F<(_f?(x8%(7n_E?vC@p z%E4f#om35t(04(Btou>bNhwvX=^d`|H3RW*SaKVE>YiroSyFl&dIsDq<6I`AMmlv@ zBfWwzquDhBi6kXyAief3UEeJIJ|qKkAr>V$^UXpeGJk5+JZf)}cPunG+$AP8I7`Jj zHLu=o68#OnTNeF%mMA1GC?q@NE>W=`%(M@#d6o3MpMph_)pD1KR+RI$F^6w3N187Q zN}nZ)k$M-BlT(FyAX%iBnGn!DBw!E{#a|f1Ul_LlNl1poCl`h`7N%kprr#H4h8AWQ73MM&WrY?M zXca{_79}JXWndJiG86~O7AKh(m&1!9LyPmZiki(KK&337{|_(+8R;)n2NlKJ%jf?D z?EL>jmqcPf4*shj;qCQ*1^#%wdjBur&#QQ0vh3WP{5(j(+r*MmKdHB5{{Vk#-b$7I z1^&cU*0r~5cqPX$^z{q;1N<4{!{b7oLZ!r+n`>F{!^P`o7+n__C!a>;*yF%iJUmKR zSsUZ~n?+vXaUvsAK1QzgjUJ9@NBL27U~!7lz04|K7xsKKfi_Ju5pw`B|9(`R_W4Mm zVi${bCs<_<#GY6&)x@NZfX-y;g=iHmgX&ZgEj#pT2q?EU3ODN?EZ4)jLX^JT2DS zlJuZ}@j4nFvf~EfBjn<59N z4uW(SCYiH7&o^-AMaOxo-DUqEZyu06305VNs|2|ImD|Ce@pRzF0ZuI4%=v9RC2^VS zIU|ok8rVCjw>vW!cw$^(KN;@dCmFFdzS<1uBHxbJc~9fIfB>1Oa*>$Hfnt)w{qXBv zges^7BAG|3Xo?szFStIBTh`3?YI`25)@yqq0;noXqao z#0RS?W4ZZIZO5T)UhP{?GPEisD0h2dB8Hg~9SBitVyG)qMzO7rs=YOVIz*RB>&m+LfI)8Pv9k%p^MXV8X-v8f8(KYO`SO1FG4wc$aMNONm8p2ud>KZ)EyAjxO=)>I`a>lVSoMxQ%uT~M zGLbvjA{NG7f*9X*ki22iYPRWnP_N}{8M6F=IIaFuT6oqc-fvMo^nKbraEidHrdGyu z(SFO^Q`>df7pJU2c`sCUb$YriMB$Ad8msR5f^$qapUEFzz|sM^DfMH3iJklWuMXxL z0=_2}n~&Drt9<9l;Cu{2^VcO3Tu44+&MioK zQ~un|WmzW{>9Xl#8hfA{tRq(`L!3p?Y$np#K5D+DtWlkvE*X6*cZv6UlEd{?Ouh;s zGrtY+k|dh4dd4KiB^KU`8H*KlNBow6fBbRm#N3h>Cgu5M4ZW~N3V&86;>b3W>=U#c zo#d5W9a1hCTd)-CJNAH&Mqnfv06yL^4kCd3C+yR^1}s9ow}DuLt-U2wBfr+K;L z>B%_>nUSyVmTxxggXbu;X#RNQE(PL!|5nN?`@`+$VG@41sld_tXkyo0Iw5*o?eF-= z#2G$4SRCQ7-Wxot>EvIRTJiGr-|W0Sr&gD^`xPmTCnHlFGypH}WW;-iq5=;tDtvLpt zy1J{EwCx}r3sLn(#jTe@dqwK?{AyLt`la5g#oE+Ja}BdINJ@_1^se}QS_4%vq9P7I zx{3M@swI?Kaam%0mF+IzvoBm;Oq=N}P)^o;2ZUXc;G{6rb?a@U*!)cSP}w+TqhXU< zk5a1i?Lk5(`MPGD@1uM)s{CB5S`oKg3a_c$Xx-i**b^kcI^VM#HuRjXt~X5}ybbEmS!ebTIz4V{Ng(Y0*mWd2P9Gl+ABc zZH>b%!pe=(Zi6fMO22GERRT3K!X(7-cjI4p($vhlZBVH0B#C2Qfn%XEATqcl?*+E}8bHV74_`qC z28z5s#_N82PYc^$hvPS6S4S7=eI}-Yq!eu)uQ`2=!dAOtG)Y~BId!)&)W0D#s|`Md z|!+|N{iYD-KN4zw2UXQ89DWzHi9dJp$= z?bvk{K?;Wkqz|xu#k#6Q#K@tEz{C7FsII0!;mBP3VF5N;S64rBWDR^+T*I!1_9-0O zr93R{Db_d6jT}2QJuIJs>YH~JPTUqBR&Gb@TW>~AJRcrbk#L~xsEVh)q>pQ8-=Up^ zqo;uakLzzEq5l*3^SD7Y2JPh;Jp%zBHz{x$`lS@lV^SWs=)Sww@_3{sNKlP2i(nVw z6qvp9#&LHrCiqJA!pgE;a7R>Rlv-RDP7GW6@a=d^s%&`CjNjt*H>pMm(M*S{2+jHZ zOi76aEys1n?yu`iUvodVm>#~r9eW}C#RIcM^U2BdFsAGa7KYqv4NKz@8o@P;mPx%n zyB1aGwB4AR5r!C6gCYF^Ro6~W?ew{?FahRXUM0~O6;D)pT7{$<{yIPXNlCl&rb0z#~wBdn@vuV>d-Q0T~J#TC%8L>cDo_eo$L4Uhaea1QX=+g&5qeI|hMN3Q@!0j9YB&qh=?8frJQ zX^Xi6gA!WPz8f8Ve|@Seqn1SPoLcX)ac`cVUq4HFl-62wph^)!RLUt|ymaAaQ&avZ z?OI3p6}pS?W)zd0B6OpX^y&Zh^=FT-w2HfIm&Z!I`q8$A@s+2ys;6(lul>1Shqe9& z+n$P0BkLX~)E$fKKF`~FuO~u%JQEERh!10r&ugeY_ntd1n#bH{5B(iK!F*paIzb6K zZSgt}79u~`luwMh-w3U5si<#zEwGE)N8i)}pU(ZTGDr#*1kms$X7?kPR-B_Y_V33C zgmMK-8>p#!h@z3IyfWGe@R0`l|9+)*Jmas4=B*9zj0p66xC(H873g^ts&%UtJ{!2IYcv2*43#>R6t1y577%U+JIL{g`PmP0V&bd)^2~h{YE=?T~_q ztnAQYqHuMzUyabPW}<)^81`eHo@&3p*H!2lPuRt70Pem<7jIy%hRb^z?*Tx_8bEhr z&tVMJ;LmLcUBAK%6eJD^Ws;6M!Q))yRQq5W^dzAHT>&((8hu<6-`6=9C^9&d z1rYTd5M@N2K;RmEu8yKY7i|_2pa@FF9oPA=(j1iL)IJET3^d|4AMTp_es|!H%|X5yxGU0e{>7 zq|EBBQuB5?3i}KYvhHI1Mj!iv)_jJ~*V9$#@s>`V09V}EvZL>l@)|NT5ku-VY7KgV zJY7QKPSo%|kBj*>F)aua*Xe?@P*=0%5Co+2{jO-vBNiTko z!i=#9;g-gq-wJ5uJcVF~vkGj!B_GSooj?-KamswJOu_jXdyo~G{4^70^64iDw!(q{ z^#oe^yL^LtEWB~E%XtM((Jbw{d`mbo=bBYsBIz7^ft^;tAITT-Z_)nu1Wpnu?Ejf~ z`2zpdagq49<03vSH!tN?c46GBSNUZnMSnXkg8z12cqP>|HAi^-MzcLrLebu zq;0r=aB?cBZ*+8gV0y7~Ze?tC)pIFoeq(KSVe24qeSdHC@I12qDyif;|03Y_uAuIj z_lOjV_PZVUTKAn!@IFPMC$>Nqt~8c`Xc#sv+(4Rj{aa)`?6_``s~aqt;CJ;<0p zar0}8++9pQZ_<(?wx3MQZh1U+%iVjD_q6qytAX4cvP{`uqu#M8FHz6vsd}j=sX6@p z$stra34O*BQ2WSIUY@x#grSzS;08e8*KOyifF~2Ay^b53I9MhC_Ug0S0*t3+jdbs5 zLG^PsGJUX6lKfY9m7@dI{gl(q^Ncg~`5P++VSEI8HHvb58;ov8i+ZIQq12^X)BEG~ zGBb*KM0zJ#qTg|8UfbVPb;M(mqRuRio%YZ9-pDKJoS%Q6J~B~v_K4eo+5XBH89{%P zfGmqdCz7$XP|qIx0OlL(sXU$UEvtwAn$7Z*9TA#)V;g~ok&l!dtO|os7Xg=Yz%uz?sOL3%95-$lgf4pdt2@oS2avz!tLDi#u=XuQULv7+mS7oT^p+ zFIHGR>HVJ^XcY>r;&~YsOHoc=yk%*|rXyvF?0SyQrf#}!nhra8scle+x)e+WT@Ov6 z!kVB7JFIzU2R->TA6ik004;lx9`%y4JlaOmDMOZxYm{7#nkbqD%2Cd%4%7 z&;uV=oa57J3Bkl6uBQ2B&wt?K84R+Qw5Mc7W1InVj`b55~9s^L6Tw1E-V2?jxLP)u)|$`y^zg;Nnem=+e%RPK?o@VnP|;?6 zay>X}!2&NX7Jfujm1^5G?n#sEQv|`ahL7PLH^=8EET}jvXg6Pg)UD{s(*5(*ceox5 zsu^`=vH&1Z>qQU^j}R8tpSQf?b15N~Tl7@6>0V(~X5gOSR7tkl97 zvR?9SL&T75m0qO2%EGwis4xAkg84>!)sBcOhi_aV9Ajgha<7(4wL*n>NgMZxic@SU zN--Se)4C@_0v9yyUn8rJfnN{Rp~m5BH6X9R{>o?g$--ssVreJ8fd5eFi&OsQ%Fxy; zZ+^FSl%`nqn|3bY$jW!XJ(T5vmp@p=0R2cstVz8a^;HQII4+Gq;%E4)fk&BQNes7k zLdPmWP0mO}_!vjdWeH&^yp}D(`cmUVJ&#nv{JY4r$WkNZk$yW&D|A#iW_@>&LB2Oj2DE+Q_4+d7q}x>J zYCwsaDK3jkDTZR29?f1+41ZgmR`9?~$%pbX=Xr}xT0%?7tBNCS`>~6Lx)0pnK}Xi9 z^|phtfK9peEFsC>kU7t^Ty6FplJjuRO*5hJSo}%cu3yf+)BKPG&AC%pxDlKBwKOV) zeM~GuXXo^mHY>ih#>v2FpU*&+aWCy%0Z*+N!zx5b?RYd^G$8LwknE5xnQ8>TY7xbs zul-`YTwo=#{GZ#V^7pz|p7BOIAl7-2^4G&Gldgwb;+p-{MUMIa-GiNlg{k^yPDo|$ zLDjvB`cE2sp#0SVO9otcDbKPZqQsne~y04pMgI8bjehr05}%3W1ox5yC*7_tQ)m6)i@nb>vBf?gGAopEkO^RALpPV zdT>qCX%Dl^_JQ89xU_)6U;tT@Tp$qG-^4@USE#@{t%M<+C@ zQ*^2Qba#Gd9sjARl{X9!abBN%m02FB!}MF5bf!KOT0}Kk6mk+^pil%>PYds%MP^i~ z2A%g>G#_e(H8+w?oy7~MQ%4;!1Z$^XI~FRi2WbxF(DXTZDkYd(e#Unrr)HQI*<*HohmywkB}8Q35kw_JTIH#>l{9jXsukRaW)Vt ze3Klg{c&ub*m@Cv0z3&*e!boO?TJfXpZsgZ&CL5@FuuL-m75wz7k-WXQ>Fg^!c^`; z6AZ)Y!d16w+2Y5X9XKDVKB-8C;UVs{5J__3ERw%}lM=;CcyjD*?P+obsmo9?!8v=T};R9T1`u6Wq(Y#7+V9Nt)7gbMtmXs0#9@ozt ztrBPJOJ}5G&nr$=weVee+-9aQYfj)Nq5W&wxyeNAE1}A!eMR8q6h=JXFdNg@IRkwf zmvnj2+z|Y_^=77`jK#y$WAf4ZaZ8feInpHcwB`kHm~akU34F8;>C^7lD8HdTS=I*~ zEN(h8AD(0{9mU+>z_p@bA4#?->p_I}pHi)O#|z#q}MWx`K6^lzE&n4cO2kuN~%!@N_^yZ{j9MJBx~v&V#dTC@fJg6 zskZ#|eGylxH7T~B`IxX0V1T^5d%txz23%3hn7g=X;dxo)(RRL%fynJC=gLuG!r9_R zG{`2_dZdtF7Mwgk#6aC^O_Z9=dOW3aeu3{PU!tF(&|jd!R+TZ!jZ$CIx3?ZM4p1YU zP9MAopL19?gIoobJ$3RZYD7QH;C|YvcZ)@~elJd)a>KTI?M2|?Lv)LVTkV4v>qCC) zLqX(A$?Hp{;Y;Jp8olxQu7_;wH?pKX~>A$SP%a{}SQOq2m7FAN+OBW9`;`w7|Z*&tsX# zr%#%t7`!!=dNn4KU+8iZyJ1GWt?Ui$1sj%p7Kc;qzs1tuW-v zK&Dk{!)UemHFv{}upmV@U(^s=)l@fxR2S+-7vv{5-PE|dl7t}f$Zs#O1CI9Q&<4nl z#xYNJR-7iONr~cVj>izrtXLNDloo7#F}u_;e~T=@hAnCib-2}17&&8RkDPC*;qF&- zSF!XY-BQVtV`W=oW!e&ZQg{wS+yu(qgf&cA$BcejB)FJ-DwmjP=5WT*YedSeI&^WV zDW%(}aC^N`@oHKG`FlhFV3UqgxU)->&NvbxaMUbZ6C^x5<29n@B+^r_ldjN`gBI-` zjFK}w;{6{}p|H~UEi!&+qzR2_gnO7)t-91$sKt6@ST`p%t|m2Gq|=fmw_B)pwP$v7 z*~Yq}X_*5UV&@g*z?&iP`;`(g*DS|6h0{o-_U9F#BV zGNO?|=#j;fk~R9MAJuLe1M3RL?I*)+7HXCqAC&Fpnw%A$oD&}3bZtnBg1s!L8bq9B zE}BIIn`EGtmw0bKf#WdMo=4Lj7s?$u51XH_pIPJbnZ2r!2sF;`fwg^~qkE%36Xa2a z5?3n3$qSM!fXK)YZ7&d8EkJwBT49AjNH+*xNXXMKbQQ~29?K>R$?qx45AV|o<}hDK zG9MxSwi8sc0+AE*2(8DRVQd)qu(o*A<2_RHC_ZY?n1worZApI(w-!7NY)1QT=Z~F z9V^Z_DaH&>w8JWQkGD#C)k~4m5aY=Lr_F=I$>cN1t2uDsh^){K?b#Y6~>={>pfnMw*Kv-joE-gJ9Q|iYKASx}0c!~$c6j#pVwRRYzZjM|VY@%f0padPGlwsM?88>F>XUqY% zcQRB8!b9D`qIqPCR+D`0rE@BDaPHHJ)6qd{m|&$OsKy#?dEP>G@`FhRZVh6GFCulu z&b;`sl!Iisz!VoaIY})lu4a09=6!?AJ100`rj{zQS_U4QfkIgcKG9N`@U1ADLTY-dsbq706!B{x*u9>M`nm|1oyx@BbFX1t!2+(F_VhVM^C61694*Ui*g^-q8y&u>)b#tF_llOihbSsDWhOzUQd4 z_MVPBVy7o8vV}-2Dl1Kr;8-uYQx<}^J#_-U#;P5o${bki7~~&Q*!e9qEbKtBETOL7 zn5Uh~eX~gXwt$qpKJs}_o~S`wkM1-3-ecjx z4XaiT?-oLsp_AJH3GcR+ie4Fn=JA^V`S(UmzJB!c;Th-t&ba<0t@dzC05zYSv38we zR(DoYCqu}v8a?T+s&4J`03~ajb+1aN$p$&CQ9Jx0oV3yJ4Wp_XL(OaXn%1LNAOQU+ za|=FmLm7B0{E_&s61=DpQ}40b$Whyk(G7(`$BiM9w&CcF5#}fpd;0P2_0i%<;0XvI zXah@y9xfRd7=ZxH)1KIQ0w!n4Ci21eB8;JVj}QaLKq?2ysyZtsC-UqkYR-Xm_}vqH zQ*Bj4-cdlBw^1DP@p^5MspYKk8!5yY?@{XGsVyczPt;^KAM)V)1S4=77eS83y;nwI zdgC2%EHg@-R&4`1#eY1-h)ukVJ05Z)DRkV!j{R?Qm1q!k5M5B?e=W3q0RV6Bf52$J z|1?>|h9?JwgeLxn$s#f(FgQ8yUnYx+u%d$6^vd$O(3;|gw)U*Lme!81xk{PW(2yVG z@INMt5kIeigsBNg;QZL^YU#4q;>Psa?q2)&==Raj*2LQJ>G5#n&i?Mk<<-pH%WFb) zTGMCu$BO|4lfIB&%H!@{u>4Nh4*R0ioOJex_?TkSq~0kT%%49l@#&Don7uJr4PBBi9&Jk2M( zgNm`FAC%T+1_LL=`)`#@bvuTU$+{&{Y)-7d=W+GGWn~>r$KMV{nyum==#~v$_H3r0 zS9I$e#s7!}@#LPi)`4CMPLq6)8k=o-pud!-A2g$R z{_v&?dA1NxX}N))jy!ly2MTlDPfs?pOh{=+ zGtf?34%21!Zu^RorillNNhkBlL7^tE`x27mJZVZZQgCVQUX(RY%b?^fQwm}$Xy`H{ z#b~9=5;Y^rOGxcAEXp(X;HYYi1JR&k!QR;wnpbnIcabq44bm8|MY;46}nfp3QQ-a8Jqd zA@5Mk8Djsr%Zt4cTWwe<;hqJ`A35K!6`EG`yyx+w~)?~CrhS&u&8$js-v&uTw z3erpBC6{P3Tia5s%4`1!^ws`O1FSQ&9i2+i_Fj!lSd|Dm!LfBLONhe^PGFy&%thVf>1I*bGy>ae{oUOI>(CkM)klA=$a&F{6 z+g4P@Ve7IlH7VJ2HfC{r(4G5HRN40(Aijt7gMt4xVk0u6X9kiRo_CF6HJ;m#6P9*4 zB1Ws%@|Vtdo?ipEe<1XHf=0$6_>vuDMm3C|J6#D(P*Z*K7f@>E#8Ke~`@l};pPsuP z$yi~wjhof1>r^|xN@rvkAtaY z5lGQDP=9i)CAEB1N1K71Iw1K5TYnq> zJ0YxT>$PgML9&POGd?M4dkDm1Odvy%M9rI|-SD{`Zd}$`LUae)Y)pix;3h>-8^3nV zeTXfg2CDg-m>}1E6!|b75X4)|(i|@}WuoAsiJpgLg)PlYdzpdAxATn`IK_V1U?t{V z$asM{$-boEHf)@X`P4S)`caUM<82C3$UUfrd!1S`VTLpH+^6k&=epgM$0aL1&CN0$ z?kZ!}81usA#-mI$E}i=fF-CKZp_sH~r9rj2EFr4;D!Ghimh=(O9EGxoDgQ{df6Hu$Xl0SWvlg z?%j7w*y*y}PjuXl7?-AO%PI9O^a{bq3_3WT@5<4~+YGsD$}Yk#74|;mGTmZJ4FzXq zeE@=4rm#ih>}0o$IGSI~!}5J24|QS5qBLsE!S=|G9vHG!##GX)2RX;W3EoD-|l zo{x=rh}Gue(ra^H`|2{;YD=|=wWW^7=DO%=YcuKfwY4vZw)wlJ)5LnOGJ8b~FkMfp zS<7IjqP1<&s`s>Fu}iaA51sF{=wp2PSJ|TZFLruI4c^|H;Ho5gn{tZY$g!WIGMKwQ z*2b-G8h@tQD(8I2WO65R;m^jNEg~H<-E{G}{qbDE#{P75kcDx5>ogau(!785cSJji7oKUX2@&7VaF7ngX zjrGZFcdtpvnK?H+KBtqmMor56n`F;mKtF?1#vId^MFE`|wvV*!Fxt22D6um(!Win2 zV^~9zUMR4Frjyd-v z`MhD0l?xc3aotYNrWepwYM$`9WXo&47-x)`wtX!sMpmOTg>8umN=U=avT7AIQ1lF1 zTAC@@FS+>T*Rj6fOq_qVbfTPy=M0>5>X^t7Q*Z73q5{=K_P^U%J9U_W3~|!igf}5* zYKO>q5rlE(U+Oiy?RUObTD&X}_&mdSjFJ5zR%0*@zwu1UT4{a>nmtc+y*ku|3*0NC>_In7e+QM~m@?vKSq^^$cy_Zhx)T6o`6)uC%-r{$< zEFU<4c7=2=oj=vAfMIwTvG7iB+)G%(5qccotP0k1!*3|ysFKH!KF{UZT`SBH{LFzo zOB&%-AAsV>9jYC@i5B;0CdqfdWb!5LBuqHin|9g-{=L-wv=JRxO8rBB)~Y_coylSM zgueFeo+4ttMZaWVqGs#PNXcilZOdbAi126a=xZ81Jo$F~#`6%F`VF#bUGNy`7mLGp z&En-n&n)%yk&fH~cjLv?{2|Nx`SL{b!z{i{jAmZeYO_GpSVIP9D4BUI#}c#)&>frJfsVeh)^z18SHlDn1x6 zeosd}SJ2MrFx~{_Sny$f`0jwNjW;uIaCb66u0SqqpG5~$=rZqUEP?{|VDlBPUu7XwU>I8F z0c9-6g~Or#>cMy7!Hxioz_gHhf6cICfViP>&~uno8gbeRe)VEV&s~TJ3>(Y^dU~1k!vELe1}H8q zZ}^wREx)O(qU#@vTWe3tzrKBANBgF}EN%n87nWC6tC!X`x3=Rpx~N4(Gem_Ie@N|I zZ0>cyp@_*ICa7~^V}t#;{8+xWr>U#^smCut9fUpGTD#pFIt4aPIv12LCs!yQ7>IgJYv!--aB zJSiUo9{Yw}#S(H<5)>W>H=L=dY*}Z|cPC5AwIZG#j>UM9idcHCLFzNwxa?&#cC*v{ z=62U;SxL9k>vIc{5}Rqe2YHx=1g6zt`nwnh3*Yg4eNT26+$UlcjssbL7Oz)KF}Jg> z+jSCh{Fo+q3jKE@@w%CVgSAm%Xc~nTE^0B6f~yQ9fez&3!J(TT+UTxp&hUJ;L+>?q z6z(OK@H(%iR<9>)<}SBSvYhwl?j6`@&TqP1Z%^JUosQdtzAx4@KR4e!Qf$i(1}V(a z%t7If#RGv+GQzzx49r(jVoM|_JlN)90YEI0+*W)OHB5@E5 zL1dxWP?F`cwUVA+yvR1eKF%C<|pJwKU3x zpzSayEE}L+p#`Nt83t)u!VQLJP(~S{a$Kl~$nyB!FJ2aga8evEemSh`E^p~UpE@FZ zR+OwRMrV_{Y}iRi2ih!u*1D~lFGA&%=D&;zv7#_gOL&PgNL<(;rC&)lUT$e$GJ;bZ zlRX4?*{h)42a9a76+DEI{;^Be3DyCBRS`4|e;#nZ=3e_b#4!GbRTZ}+Itr<>!*kZF zG9xsG2#Icu12#)N&ldKfAXp4TSUSU{np$m06Y8j#w;&1$?I~Ch#RZzH~*+KupAfK#(gbD zo9Pxr`~6QH3BhUAEk?&d z4DAJ}5JCW#!_S9Ak&lm0knmhmU=Z~8rYY0I#PoDzv2Z5kd^jv791yCo7ENs+GV@9x zQOZtmVYC(U3YzW&HE54rJ25O#Q63z>LmNuk1cMQ=NOgk05TYidWfE|+*Bk>9M#?@| zFiJG%bfktNNze@(Ih7=-g!AX|4j2u25R%(-wN^e;n7T1U?A8bq+T^YWILr$9F^E{v z-)kuSBGR@3!9n_tS!6z1!F_CypdU$bsub{}kOOMNm{D}mpqa|>&oB%WJvVWFTneX? z${^WuXf{)Ju6Df3oVrV57)y$0hHcOe z!;|I&2yEAOG~{d>2#;{Oe$V2AHGszBi3y0C5yB2h57Il7$`0K~5sMyWucL|7RC*Qh z?bA0X^N~8qD6jNqry3xjE#ds+uSio4V}D|Umzo&I^=T2~XmCX@l4G&boOo3@ zoVq1Z{Z6e=POH+5;8MZ*-5j!2GpTJ)+DxeURObP(I53xlZud7-kD0QqAi=WE#*scc zDqg)nz0%SU(WxHQV@v&91qWWGoWZcU>2AY1GBc8 zBIXP^LoE;&Qc=0g-#%1&V-NSK4JcdVn3}1LZam)76>U2$>@0us!JbD6H&4?tv3cqF z)VYIL>pm*2i}`%ikSD8}l3<+MASo?Sg{{?6w;Or8_SEw{b>&KD4}RQJ!N5R4pQ#P1 zHQOc9`6^onK(mJbG{oX^%BE+uuzl0F+2})^oT>}qoZQ_n5X7kP8Tw@s83{(zQao^7 zfwvuKXd!_ewqV1|`sF?SZzk0LJBJk@UuPTtyTc8A5M3~#|EWj+yJF<^GjjtZP)4jHP@aOR8_#|lm^q;%^o3``Y zhsUSqm#F*cp`^iAFgSb(d6XZjAQS}jz`A@97kDN9z(aG=;n1(U{S8d!%;5+k&Io)- zR2=?DYNcDHc1Gi|R3@Ww!_o%RDIpFg%A~Qx*(6-SKsW*^XQ|(5Y%X6tdVwe(Qoc|s zTT0~=%9^kUVRgK@a=Av45C1zlXy9t2`Fuq*)1%2;iT$U|aJPg+^=7B}sxFxVCgT2igay(?m;YtUjAr{~haZK7#!LZ!k7!Arx=Y06pC$!szB0FUlW-{DaZ4O;Li>Kg{NsQDE@$LEI5;bnlLa;C(R)2DM~$` zBBK33&jv)%X*>GL>1g}ri%T9Mr6y(zi2mgA~1%P#?J%3nsfswIL`J`j3j)E=nDmfUiUWU;SP zk5oQxdm6^19Kj_el-h>Zlq?PmT%|HDQmJJ@k))0XBtEZ)C8>0}xJo6ivK=%$;JzKx zg_)ui70k()m=vx`H4(om&EZfjX+l3napOKP#`Wp~PT_tBNmXm?&@?3AZdM=vsj%js zTuPH_qLs6vl_?{2ujcEDnWv-Oki%q1-0o?zjo2Siy431TX`>?Rui!Y(9r*~e?mBRV zv+RCYd$T4zfqSYcx=3lVPX%G5vnNqYcs~7<4I!72t?px&nHx+Hqrd5nGW2G4N9T2E zsKixjWojnWq+=09dt0MUx#ysh=qaYao66&ST*s^z`h7la3rRRhCG7w^*T9A!xSBU9 zmb<^&e|I{mF7Znot=X9{7UH%_MQc7@x~`@xsg&1cb8@*nWKJE4F|`8KSGyqFlh^i`+XY9! zz&lSiZe_c_4=ZMVe<2Pe>xZ9qLK%GxpB4FP;#KvEb9>Nd{Q^DSu1Bw?-foxedfD!` zYd=4R4}O1sf@FzLfl2g%-46Ic$>f3Y2lPSmTnAW;XF^cH^uy^|0kB20AUV^@5Ip6X z@ey|81SEt@!mfi*X!GIhF#l~V{*U1Ke_xJlzwW=N{}1H|0QD~nF$Mo$;^zW#DAICL z)^F9-sde$F2?PX*`1}a;>bh@bkQEA;fI2=gy*k4{Mj%x60y*m^c z?Zj&7Y8ya;%_?=HKRD6{xba2A8>-g&;fKlTF0njICY{O%jmifj42I|qixuIHtROF- zO-c6wkKJf4o_-29*Y;w&8_<#87!6_Hqj}Q?R8Gms2eY`b9uUFyMJy&FlwN|U&DQ!ul8epr(@8V}Rf)&#)Z0NGo z>g3C>Z>V`f>)r6}Vv@d8-xIW8cdTZYA>euH!ddhAa`^o&3)NNdXP1~_!Nh63EF`Rx z64f$JzmK+6;xyato~pPjLXZBsyS&J3KHCMa9%YcgEA0YrBRf?^NqVz88yMDe)7PeF zLK(jkH=HIoLq1suza@}(B*-IBDgVnFB*h2`jpnW&Kf2)1TLeApd6CZ+gAOKyy&ZVB zXolzbnhb>T+X(~LQPpz6kdkIX;o37vEe_T&1?_KMiE(5&R))G#NHGDS=BC0NM{*=K^u3S=3hTDL~SR3}l!E=Vy< zCm1+N$dHmPLr(U_1dyiFeJmTgFo&=gq)~CFCb+ql!0u)IQ3xZ^`#=rL0xjq3H~jTV|o%zY!?Y&Fa%M&|Z-Kd~5PH8GxT&D}^#T6F?jcSTHtK37#NQzIqQW5gtN z-}QGx1RMWEi5i)nb(Na?F0P^8rg|r~)%WC!nSS->MJ~TC36nB=QSTg<#yN z423>BKdGZ@w6<^p_C!?-q5b9~YYHh3-q+gua$?7=E+MOT%u>SI8frXAFf1QZRuiol zwhl9dYK0t{Sg;$d=>&2g5#*2B#5|Y$?j$kim8aOIKsMw2auf0O2P|?2Hvjbh;vuWuj z4Ng!iTKc2#<_ZMnDclu_3!`w&{$+bpunu#T z+_&Zc7mReS@z^#Hyq193O9}yIwZpCD%EK{YQ!^D~pE85K!Y&d<8=c&YVhlavM)(mt11CI0 zOsL{KB}ADZ4;hyG=Riz+(%Bvpx9TL*scmrdI~gmuMu(g*YlgIw1~r-o0g9oLYfO_7 z`HlIcT}-@N+PTU{9~xh9Hd=P4<^b4DUFiG(E?V&MP>n&YR{eqQDsD!L-v}V0 zV{#6|i<^J${WhC?q2%q7IuYvd((cLFKmZmR=!-O0Fbq4{d>-Xvv;qocl1K=5Gxp?s=X z99|1DfDikWz&|Ct`&@t09M8^E0SGoiSkc_TpCOgpQQpCoP?==0njv zJgvZ+h>-6HQ@K}mkPM{ylq&;jsV{y6R^{uWB$OakGTR8n-Ku)w|5GFLWA^^n?29?|E5RVi_g{P8vW&{oqPp z4)5lk*7DAQ!HAxblD^R<@1dx%@!#V>zuKv#`tG^%+2-Y~)U}D#-O7!{fy3sLk&W%k z`2B;krmKhPoadKJ=JeN(+`qT%`%jES{eV{_&xTPZ1;HDCqh30^cBL=-Y`ki9n=2FGeJQY1i7}+Ab9^e2nLSXzs|9TSzU5~)4BCcFw zm>Q)cj;W*JYWjPN@oqR;>(OMThQN(~Q>)o(rNLr-gm1gSv?t4C(AM|0opO5@;%LSG zF+hI90fxt4+Yx~l8q60oY-GzaR>FzCNnmm)}24Z!;YmW`>DJ|&{%?Vsfg`w^PbOsdH-N*njR6HpsM_v@4# z^F8jPOjjh$(rm9;VhO}RcSu!wkMQKfTyNEkqhb%m2%4Ov8XR@C1!RtFE*#BP16M*R zsDd!=0(ZlR$CEPBAUcLroIoIGo>i_GsoR+w$JVl8WdmTTAf_IY7zi5ZZZ1eafT*z( z07u!TpD?Dde$ax1eq?N$$p7dc9I~pYV_0fApYK`0f>+=+G%{}3^@7ofQ%$=zUK9^^ zc`8(k#Dh)d!&N{{*F#I$Y2A3r4=>#`HX`voVbL(t<})7Qva`08gJTr0op|1;f)C$v z+14#*_HDKDcEFNx;F=-Ssu$$iGpkkx%qz40p~d^Uo{9E7w9}6Z{=$DwD#~FPU2EfF zZo^jlpcN%#LMuhpfB+>$8wuQ0So@63RNd$=C{>18&8n$hDCTKPdeBIFd9lx4a zIL_0$ed~9#f=G_nMf5mVm8BxLd4A54jOop1y<7o27J24;X_ks}$0yrysHaKC-vYe| z&OI&DRaR4(Z(3(B9PBS1USUGolpZt#riD`{{hM12o0l$aVw{TZei2Fe3{CM3 zp@}U*XTq?zF!#Oeuzt@X8clCrY9(#97z0pEYebf~y3oSGQGUVbwV#>om{!t$AqnOp zjgQP(%-MSqCl?I1L4-{KBaVWE883ydN1j(s^kMIig<7!R1jAhqW9s<5JnM6gA{xXO zr~~1g4hkRE2QgqCuwj*VO2}2GF3IGKE}CM3NeT0?8{^Xvs5hL2Ik2;8=dM6Yh#f!o zL-)PbxEzKHW!FFi=?AmbT8iy#Z+p8;h|Jaj*KaR z-9ZOzjd>ah;E={9b= zlz;^iYCK`@rU-vC3bJ@i(US6_c==N%PNo@WLS*{39M*r+SQ{b0ky&8K+L~*X5&TcO#zih7XaiM{3T$8fKXsFzHuV@?KnH4*H zjGyKu8~4^C+-Xp0v!%FXKBAs0SyOSwP(FdQ%0Db)&ir^}exmATU_U zul0d}qpz|CEH+i_r>wpE8kY>$fJ*s!@-@pC^x|`dW6$^QQ)h1c-ZQE1ogAB&en;nU zpl+O9U3}Methh4I9PWC0A=lcQ(uW$m?3}m7v`~eWhfk0knSqcFet!^5K-O1F;|{6= z<~xp>U0*$+{pWne2pUtRt~k&bntVm0b5Nx$gq5Z4pa`O6|BA`#@!>{0sj?bk_*n!; z$<`aGMWrw&+bD!uHR$eXu(UPJs9~8~Ok_N|wzWh}t?o^t>MGLxFp)5H421Hw(&sq>onS)XKv&D z_bFjCi0c<+M)0rS=Ks=fhWq_zUXou#L~x3C(!cVOa?*oB(*J`ptBm%osR7jfqfM#v zZfUP-Z>sNV>1nJQ?Ck3AZ65lf%*q-^fxTaeNo@=5^Ruf9Q8iP;D@)4_>$TfQ$I}OU z`+Z;j&Fk)~g+IW@rjyvk+t&A&SFjS1Aju>lKoQc{&$zsDe+U8zp(nM2e7-3ZjnqO~ zSRrWuaVUS`^6zf~?o?Ot${H-i%(9aoV}nO5MY1ln+Ll_Y2}kVNXJ4gYQHD|~moh%1 zb8KNAnExwZi~x?Ubk1HL@(aOEvT7>P=On308za+*g#CTXoXLD4){61T$YNEQVj~)8 za3_^P+e|xArFP_UQGOnb$>l`(Ig*=}cT(DAO_#YomAL|=)tFv5f_n`V( zwe3<=MqoYXRot>+;2D7WuFd>VIBI(BUFHBVm!^fE44;I1ah^{X|wV$-;0} zD2pN`q4bobl!IhfWskCKvlZV{9Q$=!un#OE>pL7ZqQHek5VOD-ZL`wAr*sQ5-v{n6 z{LK7ejtytGC3h-U_cgOzBRk<^bB8JZ>rl5chm+XU3x78ngCS+gY**_HNz;tDUCP`b zTU5>ctgs~y^W24#@S`dpfTmX71Ww3$e#jPVX~8!ZU|HdiSXhZclCMR1iH0eUZsVcL ztEOju&gzy|Nys=}oqf&e8E6e*29dAV^cYp$=`Ig-?$aW%Z9zXFlXX4v+ER`Ij^T=d zHeS!qr5=-qb>y(|FmsAHI?9a#P{t8;lW5MpG9xlRDSD&NL@e>u!eJFs4P!ELRT+v{ ztaC}quZmO(1GRm7$=%Q5TJX@EykTJD#Sx@@HeXSb!_mF}o@_jRqhs*W<6A@90+ z!ZPb>3HPe&e$Uj-7C0cY*tn}wQMH_fv`>@H`{Z5y+s1L7p%MX8$2RA1^>0H&f77OpjO*(LQRgRZFj;kn_hJzbZY!_Wr4%CB?Dkd_tKU$zY46ZGpuRa zaIW;??352hIzNv7@M$&$6l&;oLHAS>)R<}w;;c?SwLz6II+hy}t@l0js71qDWMpPe z+b*{2V8A(hp=}mOsPXjWUwZAQX?$01qZlHQ_pT%++Fx?&QOrkFv5cIDAPFeb`D-oI zH2cjf1lTUoPZ=g0x%MHp(pw=!t5`9Q9Ku7|x%#&<0(2hh;?S@&Fk8ZnX`JBd5=;!s z;I|Ypo#G=Hfb>+QRD211E>1OIU{Spy7|PiI4pt&5&Hax*;ALEvZ#b<(mYUopYmz_% zhS0QJp3av(j}WaqRjS3%%b#AG^0Q&gDyxvt%JrR$kzR6~dc+pxRXyxXS$tfjD@HHQ zx|seK%D+t@B|*(V;6cUz$4O%p=NFzJL!X5whK;StC5xgCm;9d*bIF_-nOV(Q8o7BL z*jikexYTw35iu8*+SD?c)A_rLOSlyHwfIlbTyrg47!mc!)iz*EsGm#MdVx=0bwC!f>S2QHD{tYy-%4S;{i2c@;z8!JcCCM=&f`?0;et`W=Vn5&R#TZI{Q)_h)r&d-1oqp$xr_>^!$2chc0&|yN zuKRVfQQdy5^=kc~%K4ppqi3fc)YaAg1_ejDkYcf$hQ8rI4MglH^@j|@iSH*_ z8^$(5UYx{AXM6n-WgGSe8xi+sQo$De4o=tro3 z5Q7+^WQpg8VgXk3!|)B&DSdT)B}+2QY}KPYJzqjhA_gCn_0p_TXiS4WPFIh@ETGzs z{QN?m379>igZHv7EoMaZ@OM2*(s41z4~qz(I=1tkHKdO7zkive3Zgi$gq%$wjbY;1 zKitU^SWZ{=D>>wFC@NbI$Hft3kWh=O+rhl(l48PnP|<3V*Yu&dfWZkxX@Db|(qu8M z{F24jRny!sTrZUJaj?>2Z3f}Ule`67%W4lY&$W}5Jh;R)t(_3~FHw@I{$ej*g$!t&t&Q>P1Kc@&+y+^oz ziDv#te!rMsK{0A}abb#QvzL}$yq`I{wZ2$9Q~X=rVr|AR11h0!Ie@PB$-ywlhzX#p zuJz0c8^L*so*c#dy+RuU-5*T@&bNS=iHtHWy68YGck!qw$mD;5D#6=hs8^xL@Jh^l zj&iXPvu&e_h8%61h3_KT8HDYk6LG@fG zs8@{Rr+u=-pa59jl`{Yk=^qfMs?8}Oy?w4q=jmEvm8iLe!j+M_XgwF`)eo)uGRccJ zAS#lG-7fL>^&?C48ed8;`LL18CzFFDvU;*6*iFN`Pn2hyZ6av5^$Okbv z=fhB}3_?DDhp?mtxucqx34OP=SuiSQB*e&{KH{5kl0s z1|{-f8763P+;8!ViABCCeG<}Ac76~pGezzj?)enRdGR@HIruPry+RQQyb{y{h>Ebi zcvVcXwO!MM=YkXHH4Vw2q>;V{S;CxPOlb7R#!&ECqsA7Bm}&!qc$9C`o$x`4 zu!(^d4E?@kLnS2Q_K;{GPLaPB^ic+AJf)mT__j1Ti3n2cQ2WL9-m%6 zLBQee(ev~A1HTM93ruke-w@%*7|v(w_K3nzS;h3t>kCI?_g=HXCChRo5S}Ft_uWv7 z$G?`{x>m3oYmC6fQW@*OlPlPI?S>|CU z zeC^4M?tD62xrsSlDp_tV*(HmnORef)u}Ub&^yBExQnEZrt+d*XB)D;kn=MY=>N)9f zzdarUa|>@Zxw#LHLks=ocjCE4WL4X(LlpmR{ro_+Bfc5mlIBqu&6!66Q}MV7K;}^1 zv4nOl)ei!&eVudSb2#VPd)$QX`mI2$?1rCuujKq96_Y4XpdMM74hmNZ+YH9GJYEfC zc~nP?4!T)UaG%=6&X384J=ydJ(2LYyI9v3i(YX)L*(N-fz=oZG)kQ%2(n_)1aqMS`LB3go~Y z&$QwoHv>2ehZUK`^Hk7~#${+27sKnGZj=<+Dw_LxyJ~C~P^O&HD)cpL&gKtM&=ObH z9*)zbr^B(G#N;7SpBibkfb&1z-Z61>u2B`E5w_?%qvsP#&B&4wVcl`T6aR; zE?Ai)1;L1R%^tV9rRO{P+YX0pwee3z{1k)%ztB=ou z_1zTLI^W&2>fx8Ap7V~LEkqE@=6*(E-uiy-C)LHhql~!E{k(E3<-?+^TJ?jTx*OvC zl78!-hZTWP8xCdT(&)QYYw+U7b)MGmoRap-o1+`@?qJ9;&?M92sP3}iZ+qTmt(fPg{mKf`DALaS&wIAlE_KY3pX4ZI;mL>MQo&`7M z@SN98AaP$bulw>|R-E~9yHvFz_*6F+|Z4P5tQ|9CkgV*do(ZGVCs zbArl6`}?r!FCnLKLi$DrAUf->5@QR&jzk088}(Px30!Ij9qVTHz;{@~yf(=;A;5Ig)o1^bC z4ZeF8L?yGaT9ej?N5GQ))MjF6=P~U~xmp0*2da4YkK}@*b!^Jk^Dc7-p_FS>a@sND zF6&sPlxKNt+O^{@`#P$W?})n**19j}xvNy*ZtS-YDhVYOvG5P*j^AFWyQu`H{ScU6 zv7$Wp`G%rLu@!w6% zBtTi_1jfGN*Ds1vXvgytL!3%JzrNJdu%%o|*yzK9XjHUm2Mb;6vji7I<)u6Da{;Us zFSKc=vYUxY0+jJTwH{!3zY8rj=#o;#9T#i*q=yG3>{Q!FS85;O&N@MX{r_RH7{8!} z|4mzoV*4kpIw9si)2d@QIPmid^8b<67LezBNo!HjX#XXxMMM48@vY{ot)!yiHw@Aa zj}DE|R%W*F{*zWciv~ryOv>7~9#+>l|AjH^?fyn1gg$}p?D#tF%dV&0xnE#kPM@CR ze*gRN7tI_38+!+&#j0!P;g-^ByVnoz!#@p1IJ+MQyP0MKSRcn{{!Ii7U7abr7?%1g zJ+&S5Pe>}0-Ul~HehGvWJHqk8s=|CYAzffg6wO`iRAz!D^;G-3;P29Pf2eH1rV5!1 z2@86YBF42rNc};El^sxj4G^Tc3JQs9eU7q-PweTt(_i!}VWu{DDrf$rVF@QJmO;|$ z^P>)b2arHJ+iq4Gya#!YBZ8e)D#&?*07cZ>aJ=V>d=r_tWB1cg= zzliKMYajanE|s5yc<+FbzE)tWFaW&#)bb~lACrrvbcO2FLBBr)ruj~QxTO66Y$APe zlJJj*)H!)kSjU}U2&tWZ1$eQ&Jd3kz%4B&JNrSy`zk9)f2s%>r#84^bY#3B0;T`{tOiV~FvUHzJe?JUL_JdUB-D#YsvW3))B-kfFsX+FPM|n!gBV z`U|35cyk=5}7K)?i?J7`J;yx}U!ib(B;P*{pGPT{H=DrQiS*ng^H&*5qf4Qv|5T_cBy@=xI|+KN)V7%D1Tq^#@h+as-85^N{V zivbWf;#IR-Z{`-wS^V_k!O>i4j>iItUaZ+|QRdk-OwE8_C`|Yn8HkT{mAv zz0L7CR#BOV_$iS;z!USoeVY;FbJ?BLS_huxwv#{o z7_XmjOuNHf;o@Zc?Q?BiKt=*`^C;++cWF~$wM2wI!qop$&SpL{Lw#Ji;fz`K`x?07 zRfuTQ73>Xz5@68Zo1cGgdEfZ=w=S=`+rI3c(v!9BQ3Ah^4`v$(t4F7C2uaCdhI1b4S2Tkh@L znYPm(+L=!Ohwqzt-}jvJoDgBVJS-}UT3}RziTPwES5t;GeTOyK&yrQ_eH}2f)?9qo zWJy8Lh)9+DO@d69?ne+OXnA!SXsJkhm5&l@b*Sh^Lbpwxqa_n+hDYqvtVV@~sFXA3 zm`cX8yElATFHxzc5p6)U0T37sk-K$mliZ-!ZJF?GG)Y-MHA58Un6!>kOAqDA>>|~k z)ktQ|fDtKzQ72Z>AFE1F31dQ1Jr#|4?9X6FWWW+!pTmK3&Xt0vB5@3x_NHyfIy>Ly zli&m2{78tW=q&m6xkzT1XC^@pnq!n1bUMh7*}kUNH6rMgh8)OhPr(d%of=h&vTNBC4T zQln;I&Zm8dgjS|iA^k73y?0eQ7umcyDkEM`fTiz*c1`>>DMl9yX7}|rtYOQbpI~5_q9yFf4_C-Q6bmLOKbtzH)eK?vUXNDG z!m0TXc2?NJ1Q6{-79psYA<9De~BnG=Q) zSuH|qBi2U{66E7sh3ZO}23fjy-xHODjzzH2L#6)T zQ=OTPg**ydT`E!LlR{p=t2hqwHSiqI`hoP9__=zh|8l2v=USa9rT$M}B9a@SOEn1cXc3L_7GjLTT z#vnDVOeAA)?qV%!sb`SCrD3Qap4}ntXv_7^LRydfivNUV8XLK#m0k;^lhEa{(j+iS z7jx@YXRYkE@jTwSa--$Hvm}bKHYf$U)G@@ETCvpT-bmZKy>NKDMp7;qCy@FY`q3*L z@2kp9q${|LW{N~z!4h#xnDgO&xa&ZCH4tLdVQzNwpkf1A&SXYj>u|AXe>Mr6|g zUh`?o07nFu@HcF z8=y%;4b}EeU!jhR3`9H*&{+-C&-CJJ@X~9*r#5l;ArQe49E8H;gKia3Mnu;{z}1`K zQ^4tG_lt1y5baV@*i`J>?P9Q~D|YBjm_Vklkj{6JV*}6U(AlA|0Dhb`4{RkdyDivI z(qmE?e9}NwG8Hw_KoydUM?ZZYg!?`Ne9mavdbHME^v{pzcZH;QLG~)N#N5}BFm$jV z3eg*gWALt#32BJCyd2xKF=>Bc!E$1+*2j#LyN7ckVyeY0Tg9nU$E`!+Hdo`epW}9k zGviMo@#m}Ym(TIn#0l^#`1cSzbq>UbReZj*ghyb)gMjN21MwOm zOqG`Bpk){g9Wkvj4uWoC*J0R0D;nC1`;_FnzK5B)YlrsY(fnvDd>bW6%rm!&Q~&4= z86`9Y6H-}!X+d_x^~OL*N+xYg#)p`Pi`p4_VmJ|b@+J|oR`^Ja`p})A*?VGtou})u z@bI;gWU~&vAhdPT_K@F${g8;-)%F=JGYxIk+W^I9Kp<96C&En!&kE>m1i*5E1nD)V z8=?3x)274Ehg($O+gdrm7pGPAqDNrkNG7J2J_RKahvERVMCGEyb+yDE^lf+pKI%dp z%8A40o`CasnQ{P%KE|+!wGhAyTCfiNUY}v?NosUC>1A+$8YDmuF^Ulw$?TOC?#;FC znJBS_lJ_EAz#I^*&9ad7}tTx`vy6Kgf`>xiP~7e3r~&aA9BX1V9?QW z#Gj`Kr}7}Q=L}brU?I9-R^rrb!JJ%`oM@EncINE(cHsE}P*?i>vl%7>eYT^r>PeDq znzMpNqj+3$T-@?mQXCT2ay}gASDu81yikhie6iH_>?(=bYe)GVr497SSBJJj=SHaNsK}ikrj;$n!(}TBoFJ+v|5W&u)#WEwH`Vx6)V~|e!&;kr z0&3s=d09V)!HsQ$9b+Rc%@cF;k=@gkV+(8j%RN&~>jm4(dpo;_^-G(hCjm#7ztV4R zQxflPu3{b@Z(rZio?*D9Zbuvo%Ypzn6**Gz`u!2Z)Itif&e*~ZxSE;>kVH-C*sCvG zUBHHQX}=G7QHx%<1|yoR2~iV-cY5+^%=+_wDCdWt^5y}rXN+!?bD;zAsG54ISPRCY z86${Y*gzC*#)pOiC2* z1fOqW%=niV@GS_9_sJ+2%_X_e^;82g<`Z_nzwSFQ3f}U*x6X`;jEeHgjLLF)&WL{I zB&3ZlofFXYX3K|A#)cfM0z(bF5sQRymk;)7iIgC-&?Hbj{STdka~J(2LCy-<St!`lQXdzF z68>V$&W4QT7YCvY92G=9dSfQKG4q?rola8sgJ_9W24D+u;ZDnuojOj?vEFw!L99&q zQ&`wRQdSI}M3#PZ$aK($kj0Dsld6*C>xvrMav_Yeu0sa%5G(8P3UCQ3?O8J{jIB*W z)xVn{o|=F1EG^ASZ#K0M(vXAjelN7DYU(gFI6y(hd||w(1(cEaY*$P;GG98vBX@z#sRp+iVh+xu*7zMzl7bhe%f2R z`WYyC&NUb+M1DKi!KQdT{Aoah2@$`5`OuqKz3`?N)7P$f8oBPkEE(Yoq&g@o8`Zu% zP=&0wbD=+{rTUlJ8%t}Y!<$@NPLF&I0&acP!;=1lPf&ef@*9j&_i1tMx|MJtKaB6$ zMeqBoi?*>>cw_v%fFpl&4pOJ0{@9|^geJJcQ*H0w)tX#Y#b6d={Mc+&_0hXJscGwB zCD$Xgw$3~>x(y-V?jq9Y^w`>Zj=3O=!m$Q zLiP8Qf9#(ihwqm+!zpxt90L0_G)0<~6h7*(?v4U3(LNbxJb*7djlMAn5LQM9Weio) zx-$J2YEI8Ar;vyu?~AZmI*WPyRfSYNl`=(a#O0+|f{oJ|rji;2X@DrnIfrUY?yXye zOYwk`T&xi`J8h6**1db-gBEg@jjg2FJ??T07mCmRsjw)u;KbMHuMh__4F;^=#AEJ) zg<_b3d-FIVL%l$VB`1;$dL`L$ImO3a?Niul_c>ZIM+bu_a4yrexks@SJHt^e^L=Nr zr2|ZWpY7~oR@YcFTrg~VkQu>OS0D^V`8VWO{_cZs37J)v4d{LH>?mAhdoL1k#{1(0 z#Be}o8C&6`UDb>VC>P#=Tz&}3@LZx(0WSah83;`O!3+h z0oEUDm0CB>=?$^rtnA2d3Q55s0ud#~3Y5`(RzF;enF(n4TPo(Qtru;`AFKpnvFE$v znoH>8Xym_^OIiO7jkAL$l>!&jpGC9JsEbaAGW_vZ>sr5kW%D*C0 z=`myxo;KXk>F7=AjaEBs4sWjA$g|ZHvwOJ55Ywv zGN+Jvi)Opy#ZaST3;L&&@!34xMU#Rp!oyIH=*hCFWCRhU(2^^19ypw}?d9L5ZJpMB5_LO`Ul!z+&AXB<0nrV8aJv^Da$N zeSs|gX^skhXqTOK$@4%i`c1Wj?%BY0M}vm{W9`YNp6~$SOzHx+&w@h#1KjJlE`xPZ zc#M@J%j?94MMf}%zr@j_O2NMt=AbvhErq-LDc{(;)+tp>4wvh$$ae@4(dXf>Xq>Yr zNS7V$xmLJbU3zhZHr*3`j>lTB1)ld$?!0-1!B_@lMgJ&cL#IZK@X#BUhRZ5R+Ff8C zbyiA&?t0)0PFf@373ME5!f;C_6ip)O1MHb|@U4oo(+2e&c8^n`m}gedhx&y_?~&Nt zi!2lhCal@bZVZQQrWKF5+hKtcF#xOoxbOv|eOt;el>u!^s57h)R(R;jctGQ&|kO zAdD_+XqWHnnK71r--z(##}HkEL`&fYMFJ0P3eq;qI$sR#o*nDoI46xC}I1qu24U=1^QoFbG$91**=hc3WEi4gpc zXe2kWJ0~k=|AW&m+Rn-Km7XiNv3M-l!Oep)7vB%pk8n(ZzNOBItvHV>0s*ao z^qxNdjsi`P*-m9x0TAZSM3yClz!c;F9hwklN%%4%C*sUtb|9d4#Aooxw;kT^2s^Zw z2X~$7T~!O+`{FJ0s%zJu&>;*U|0nq3OHhWg_xwUGFz#}S-7|X zW7^BbA#~rZSG;aA0$ALQ_J@KqET#NFQ14A*jfqT*%DvcYMkQN5uToEAHUSOm%usdF z&=yA6hRB3^a6*@i@8na$WSie4*dKuz_Gr)_NjGtfF%hOc@!zOlrwn`#vcEbE8mFv3 zKsS-p+Mh%>k>tdmmpIrG;5 zJyyGSi%+t!;9W|CC3F!ZNxv3=rVoh|Ai`!P31STz6FeGc-H0aCFan-N6Ie168lMs| z1hfA>Cy<@MTDc}N8vB<=C2SofmPYxZXZf`=0(dz9_|{3ojBIbf#G18ivazHpl3Xe| zR#7H()FngJd6ikoZ!*lP=J@6+>B)R2=3}x!j?An@czJ^@8TXobBtue0bo@fiL?X>; zDImwR$22>T(iJYd*NXH;TJCq$EDO2IgjhBIG4;jG{_5e)LW zdF#B)Z59i8#|Nxv$CD^%=B2d{m(-_1>a6sjEU6z^X@Ue|@}>U@v--sWJjBTYOv!67 zehTY8gDa&mxP(cF~yn(UNs+5YCSR zNKipqW3FpDh@!#XDVy9>2uzjDHq&71C0}B7T9V;Zmdso#7hL`ZQ<@P~4nI;ZK~wQq z2o6ulotVqTGXu-*mITLDX7~g>2WoIhd6WqrfdW`nWiFi)>-0< zj_*i&G!{$=1P&UXxgSHt1XqIXG3^z*>g{>2RVEZ1^2C^areSm9K@h zsm(I3=w>%Z`L+^e&)I}73Wc(w`8K_TPCxk%XeV7IHsZAV540x@ zF((OaME2^%mb&=vWhS$p1mXDx?PR8a1~PU}ytH3}gh?=-P>Wu`nZ|-Hae{SbV-fq> z0ei1z|0+`BA_ESF#p{QZc;s5Nn3a3Ilm~-q{j}i&ki8ZkTCPR|krP>M0rhd*K>(j* zXKN4F3TPZ)z1S?Q(D^A(>%^m!mVeGTFn@r(Ii!SIMkU}ae{aotGCD$bylm+xEB_^H zoH3ghEoYddte(slJ*orh)zqL1&pldt$%3Ijp0jk6Q$dm|TIJuuA6h9B`C+VJk;K7e zObxDFkUB>hiA6eMI4^0#97~7AKegy^Hs$msMbKQ5o)xfp7BMDVJQ0%&%;{cf6l|0Q zu$o@mEgXCI{(kSmQm^0l<7>hZ`B{N0Fh6O(%P*1AI&4Q(51Jo$?# zK!NbNsl0A+OdlXLI)&CN#W2y5@Ez3nxg|HxJrQ05Bu7>xYe``!YvyfnvdKox0}zv;jF@JlRN?e0lT(w}J@;ahP0|#-ir$;@r`;<2UzLq|BNlp9 zvZG`+LSgT|Ar~7=zjuDlX`u7^2z6VQ#kGsQA(7a=ra~k87IUSZaHDRCJY# z$Vlh#SPDv3I_hY}68PA-GFp*5M1HjT)w(E~DEJR*L&s=vG^ko+tVVu3(}(#%7i?`k z(fp?(+-EFa9y^)^eaHZ1j2ufDry`Vz$8O}OA}GOc%Eu4f)+JG^IJM#T1GLD1v_lGo`6IGkG5JT0Qxfd~ADj9ENiO zA`*6RKF!z$u8$tCi+<xZKLKhtof-*XCX|EmEj z#y378F){Ytfc0NWd454sNK#3B0Vtxh6frOvTw51akx<<9&cTJG1$1`$`E~X7_4l{O z42<-Q1Po1Pj(7hYpITVV?Hij}>kFA!Snpb0U)m2{`!)0X7DF(f1cw&vT4KlYtemDJO;@%GD6HpBkKRZW<9c-r` zEAiTd{qJksFMpm^z6Xg-SRk^#*lCQLwdq|7U0JV9#V3DJBGPrSn#@)W96}Yk8?>m6 zY5gLct!MPp$r+8cI;CN+w?|O#{34@vdmZAie~6*hMs>Iw0{#BS#z$d)UX?abbKT8$ zt#Tj)N=}(yvzjqCvN3I__Ob+`^9S<>YWQnj#r3NxJ9&6NF7X~3xWS{Alex8j^q{7K zZB5GyU8aEFWgaJZ+I1(wMa~UJOrpYztY3XXkQl}iqW zrzNJ3(236~jT~Ugrp_=YwHZr`XXr35&9Ol#4o|#XJ}J#bwXC4`>hRvrm%yxK(Q>+d zv8htbJC%#CM$NYLX+jOhuBk=R1=qAq^0H(VaGlbZw;R81z|Ts*iT)T|kEjMsGqY5H zjk0M!HgkuOUBZWbc(aEf5ntEvEc&t9w?sVCzQ+ZYg&om3SKkd-&zENtG=BGIbwA;- z6rIr9N~`Mo)fIn;(jZOfJ9VG%b=5;*Un^Z3UVyM`MvXb%h}l5sg!ALTIcO)8Qd8QI=Lp>&p~1QC$PSo1j=vvQLiVNGE*@E5mF%vB;P-Q`hslMzZm5VtYwM2*v63&TjD03w zjzj#u-{3$#Pkrrm{@>z;=I z?c~4v8~XZBVbY>~H+6mhzoD1%^q#x}EZcL1tT(sA?h%X{B*L)G=@4YGQxav%&+xw< zaBqGNqg;m}-fzQxN8&D^zYYiM7>#(tGSl^nC6#7kmVr)4;n;-9< zl1Oo3H!?^egyY%#Mve~libC9;vN=5~9AD>$JZY-fw@mD!bb?z^YuOB6HVR19Ek2|C zqp^(&-Ct6)>J#c$4nN_d+c1zF`a;?tjXPKj-!wXeO*k9!gKeLfNO$-d2P42Xm6ELQ zppoAJ$6!xLALa}oNPzkgnsAp>FgbiA!vd$~#faV!=iCK0P+{_rzd&b-I@@>=f$#%<_m3H3fZv4RfEeEKbXIcnsX z=y&O}40CkQ*U#P~CZr3QY=e5x-!H<}H|Np!OKMwXsFip{E#pZ&!kAyobrd(ZI^R7RDPc(>@NnGM{gk$%FKOHVt zue;;wN|eaxzHspN%-C@X@9*+>X*@GOB>UJ_KHRVK_2y3%X%JLv%DwB+v~cF!$5jBR zyxc#MZ+_!%l!z&(!)32(4YTvCR46}Mfzn!y^PQFEmhZh1acUV$@b;e)i2BoiH5P)t z*;%Sr)#6aXS!emxT$?E9t_!UY$f8#XuINom$nsZL+h#Iv>#iL6c#>u!vaQKl%?%NH zwq(PQN@u-oJ%R=d8y~2Ay6FKmLo40SQ*{AP6FdK2pMJt%)(0Ug?!u8h_X3paLx?7K zk%XW7u;c5Y?|!;psQSE$1GwxO`tge%QH)j1~BKWdq zvKcz8SLaP=R~x{Y9H(z1&xN_7-r~sHlS;1^Rc$t4LAYhNapod0-YOUnRGxG5^B}#G z_`cC#@Hwr{dq#izFv)l5*D#v+3iCM zkaZ7?YMC>zMm>@zL=Jf)qm)l;izsK%Eg$lDO`L6;85lFtBJ3(HJSTpp8xb1Rma%Z{ z0N(d0{adAduT95(%JC~W^Fp+O)ouP)wx7>Wl$*u7^jQZ#S4eMu5^QSpdHRX5eLgi- z8(Dgi`ch-0R3*ih$33%cz2ffBk@Cp0uHAOR3|UwReUw&2Z8-Q1_AaU1{(Fn`bh@+0 zHPuM?We@X{(QG&usPoa`WCR_QF8}nwZ?x&FqI94Vl5M?(tW|TMM63FEq&NbBp#M-H zyww*qJduYmn{f|yXMBO#<)BAPBT@)TTRI=1ju;#y7`9JQyje5LNn$GlQ{D_3aHMU> zA&V1~s%!YctwkzCFf;@-5?sabHFCl4b2(1Pe;}X&awN2n1#pD@K z?%JC~91nC(QEf1{KIw)>wm-*odqt0P;rc>c&-;xd*G6MN$3#35@h8V| zO)I!xapJ3q`J2n)hP`mPaAMsf$*;Z0F4JPii6c|h-DgLm=49xWiJWv8BMynVra}^2 zEt%dR0so{D3PJH0Ywz4i6X@r7@$l?W)m(s|-|YC};B@z7>t^-x-rl=@TJ^529c`ZZou2Qe_Uzo|uFow$ z&&MymzWp)#^=3Iq`N5|bT@6krmUPELwiz>9JMO5tnrQUw7GfEFtk5?i@&Y7IL2C&)^vj~S-E)MZX2Ea zZd18e%61r^ZS~r?Fh1f@QPH7su6rBzvHQ%uT4Tmp>}UAKwX$dz{n71S=gX|syo;14 zgW;>|c4E6u8~Z(}gWbB4c_{CeuZNsLhbQB!ElK&_h#(d}SOk;v;WUJr|GVX9=ci)9 zA17LgQlcqTIWb_u??fN3U0YXnS~qIa55;t=jeDs@C4B1p=Z3xq!NF-`I1dkjjm1ii z3Ht(wiIaj|fSC`JzJPw|T~QLa#qUmcoJ`T7*z&SPjA7lT(;gv1iCao56~uIk?CYg? z!ovQHTd`cU&RH{> zwc(&rT{<{#U?Ps5H~pvPxcmGJm0<6T9JJ_rlr-zwu{727n~E(O|2=@NU_lUn7ONL- z#ZlllXX#`hrGEtxvju4gs5s`d;#;X1sT^Zz>N=5Glno!3IJptbFG)~!tDbBz2~$xJ zE1ValNAe0a9ltEzF1wM2a@c9lqGPUp?j8$&qH$g9 z6Qw|0++goTuhX)o$qpl_?x|(oXk2H* zj(7WE;2}<8i^lw1E!3JuHTTqVl6?;_AHyy@XicBco0jCN*kspj+x|Xov;7hQPUHRj zb2k~jJLQfh#PgSD$6>_KV+)OJ+a32~b7C0g_)W=6I8wj1fwsE`sSP(37Rhao*^GVb zR-o*q_(8g9fErIhJc;FziUrAQO6$&fLmBlIo4|Y+-k#8T2ALWv-UB49&=G!bTQJvT z-14)334ej>bud|t(C_f7sR!s;8Y4~w4O4DE978GxGF}X`^wF1urEf=@rXkB#qklM0 zkMUdr@DO$-54-z*VA$`Sk=~lg04)COAz@c*(k?c2p3Z^TExk1mrNC$-{&ije1F{!3 z&mM7sL=*zqABfSs093UPtS!Q7ITnl+`)-4wNFp0sxB~sZvsC;Q3qw#DHj2sl+#gqbe;qqYv#T99ANIX9=&0~bAlkdP`IQO&F)&mVbWDsc+2~uh$ z#p;mLl5T|)%u+}w@ieC}E(5;ow>S8qlS`5uj8;wch4asp$m5pqR0tzpW5G-2ZMuwh z?5E8{hs2f;9?8;t9ioG(6o^d5INZ~EV58#yiHW?DMHX2=sS8qF{Bib=SXf}WKUQA@ z7#@T-u&AP(;GtMbJ8>@zbWRA;cq_!Wlp9i5$0dOL{*a?`JHfzy0!*skMf-+_=98%! z&O9jIT$%{WmZ=Vp3S)_QP91>@QAZbH3K8kNomJoDh1tpMHN=+;8C|b7rrvc&qeFq7ZXpFOY{4n5=!q0KBZ9ovyuE+xI4r7Bth@q zQ-W^fbYI31SE0D14Xe~4Bhj@_&d1*nw;5HIM3GaW5-G2t4^hc%TVTHMs+6q5W;9CY zDAxeXFIDP2R3eE=1+&CQf0N>CQ3aTnVk|J>lYRqpUYy~Wd(dP-lL!d$-OBkgDq|wq kZ)#@B_c^kBguR`PYrQbojPjUPbrAlKvJn4AQ9a@R1~`BM!vFvP literal 0 HcmV?d00001 diff --git a/docs/ja/images/logo.svg b/docs/ja/images/logo.svg new file mode 100644 index 00000000000..70662da887e --- /dev/null +++ b/docs/ja/images/logo.svg @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/docs/ja/images/row_oriented.gif b/docs/ja/images/row_oriented.gif new file mode 100644 index 0000000000000000000000000000000000000000..53daa20f322d37f67652e5f53c26a433ea8cb5c9 GIT binary patch literal 41571 zcmeFZbzD{3+BS?>NLolq2uO#ZbQmZdib{7k2uOD>Sg`2s?(S|_)S|n)yF=!;0QcT{ zp8Y)MJ?A^`AK&l!{6k^7<{0;!Yg~6+_caEe052<>vKb=cjYWF|pj!q|7uet52Xd7G zl$(IqZJ;Rxa2EvTB7uH?U_BP7LjZd10cJvA#2CnX2aq5FPi_HSY=9I4(CiL4H~{|I zz)?SNwgNCR03irARu-u#Nq`g&P=EmFssUZGz)&)fLI=dg$`_6RA&$T*6rdpiw51~> z!houHfSnFl&zzi`0B|1%cP#?R2mmt*@a7FrTMM-31B(uThbB%-2H33tj(PzE1fT-}fdAP{g#YynKoR}$VL`-zdw5l%{UJjp&-jA&1sgll zOZFFxL_hx!!)G&A(_j~W&--&O_-|aqTBfE(?DX{J=H_(fFX;@8HR&1H*x2Y_Fw!$J z(!xJMYhr0&s%$}PU_$c82=CQRRE@QbOtlRSh%QG|Rx$i;%0&#f^!gF>js6_gz=ZDV zap+VH_315?jp!NZUR*xXA0HGE`JX?kum9(#O-#kr|FZXQ2R8X)X{1gsu5M!Z-B?u} z{&AAasf^fpjMbG*4UNAT8tVP=7C&kkni`sD85$As@Lat$A_@^@Rc(XIip%ayyqf@k3*-et;Wv!{ypypwznTxc-}HGFnnNQV`O{D^YR5N+Y2VP7w>qO z{zDy}GfsnYtRUv7tWE)q}HZ|BWp?jPGB) zV`gE3kN4v5ZMnWz?Qd-Pc`x&SdoMlQ82Zbv_m{8xk4x}1bot|-ix>XOpUYp}0KVpp z;fwnQaPi~(?DXXL=nnGZgysRYI0(HY;Mw4sIRN7sjjN5C@(85DK083$j{5o$mNlr>kh>weniH?el z2oDPl2@VPj@b~lefp~j?Jv~6~ZmurQPL2-tcD6RwR+bj#X5URsjK3Kf8tCik>S$|e zYN)HJswgWdD#**p%1BE|N__qDS^U#SF;Nj=AwdCtKHd-SdEUK!^O~EBlY^a&m4%t< z)k{W(7xZ+rG}Kg-6y#*jNlA#G5fKvLKgGkv!Nz)m`S{Vp2lp}1(NIy4?;+j2bNkj! zM7WE1c5>_XT?Bgs8o0MWxOC(QcL4`mCubK|H+PVSC)mpy;^XV*9}pN6918=)S zwSrF{+0bdyJ)V#6 zP-!sYSYHh}V0E&C^2A@HEz2BWE+pm9Y-soWT7fQpR+Ka4=ROX4Bv1f*eM4_?SUEvL z<7t)?vs~ny+x>_p%E)e0#uP|EB%XM`<#(E4N3yZA6jypx~_)a%te{{(&;zn2AC9>(Au9^2dMR z$$!s}ckr`1;WzC7QQfC%cRp$9px^c3e0R7L8=lj&;vy9EWT6E0@UW##&Fq{zJh38548I}04ASnwsQ}Rm`@9-!X5&MlI z`Ua_$Q@XfzJex>2N|qV1cS34KTMfy(2`tyw^@sFRG4f}Ngc-YJzq7lT1*B|AVUNn6Rb}7t#t7E|W34*}(b&(oh z^szR^=NNn)S&MH(G@=?W%qu>D(|+LWmcJ+IR6l0)v_&2^S=@}B7oG#{ojGaTM&#v7M0<^>Hv z?+Am`3(dJmlJ?d|Kd{&dkGN#Ay&|8hLM)q$kuzwWcliQ2S(?y-2@VC5pjnZ}f36=Q zPK?>B@yg3y)9D@ z-)Gz1A+emQmvC5~9)e9VE`Z}^P?y(9X@c0rX-#U3*I7Y`oWJ#Vlg(L*6f1jEAg#KIzRhFM;IVl(a<>N^-T_}%e;*9$n)Od8$`;EkRKB4Kn zh<_L&^`4Y*$K$mr@#8p(4!4C?4p70(&gTh1- zOl`qWI^rJr`|(t_wS_s@`rWT|oOTGf=+Gvt3JX8kHxq86dJ~g2kS)lRDH=p3-s&lsXV;gm-l-|u z7d~KO%oH1RPPNh9)?W_$o@MWyp_GE#WmO-Pi^!#`G>|^rWWbcC5tOK)I3(_~S(yLE zR!hUuO4>_eFgZ$pMME1`rdOr7kn{(Qmc6G8ES)(e;b28qc1?V6v^e1%zNYdDyF^Pq zQ-1XaJ;fiRQk8d(vdw7ej21k{7a0djTRYc2x}pqE5gg^Ub#5v>YnNX#!>SnarsG=i z9MGx`N?(r5Tsv-4ShF&Z7)3L*i4&Sys5Y-0yPflC2}SY8X-SRGm#vjA;O^n_!-6{+ z6#AIL)3XM)50t64&r|fivZIi4#6v?wrYoGcN zeOx0}Meh)HHi0`g)ZB%(ZMp|m#m-=DX58HIbhVj#HiQLz9i7K1{d7K=xvYeJFv}w> zQjL(LtWAn6Kgc9=f!c(voeq6309>h#YhC7xcqht5w?YHO8oLfl<7MO+jV8OKMNz!> z%jl_dW$v8729v`*M|I@c+dkNt6&q{Hmcolz&L>cL1!Hg*%_5)BX?t+BX&95Y3UBg> z$@65!L_TkAalX<12xgO%mdbtsN!+$fiK0wJd7W2V!vli6Oi6h7Yw|}{gO#vXDW65w zXhT9OmpQjWUjyD{%u{8xR@nt^d*jnus-dvnXfwAo`3-&Rv#vDES8*+92FAMk-PR(e zsbcc$GLOzCGA<6{zlq41HCPQFR-5PAW*PD@j(uB8Hm~{aZK%8z(X(w(Pvre3_chs@svkIBD*DQMBcl(0XYvkAmj)nbVrd&ny6oMfLb5N;jAUUKgp(AmtP(VzC3 zaQphvhmUfI9wl|z9$12`CloQxiiV;Oib8DHW$DdFF`|xehkmS2Y@QVW7n?0D6-_1> zHnVv42jy57+hO+B;|<)Woh7zA0ntBJNEFv5jxLU>V0$}Y&MigtqYFJa*- zb!>N^LSFpIH_2rQ0av|-=Mo<&Tt%fE7RI z@ImtQqVP21_w>;6j9K=u2YCidd1Pw2KSSUoe*rF(0vB6=OGClsrQph8aP={`7Tc?y z)vHm;tJ%T}8tTeVsq1v~cY<_F(E;I#heJ!s)Q9O^w<>ODT}J$dXsjSZP)1t9ZM zkVOl~awufA6tX@H**u19WBcrCc~igf-m~yI4)r-L^*JB*xj6Pg!0|=2^gbBoXb-*7 z0rG85^=*Or)-U@uBKbin{95__8npbXKz`M!el<|P(q+F2B>(bdKVmljM>t;31;CVS z;OA`KRSh2Ash%%RJni@cGAIJvkphxH0g+k(3DAJoYyofBJlM+uvRMOJqytlz1F}*B z`A0nXaDu$B!HS{w+d~c_$8M5oZnBj2dQjPK*lv;^+)l898=2i#<6TtJ>;NoR11VRj z5w|?DkVH58Y>i-fkKhnQXErS-A_^C-Qbz`@@BJ(;4mhDO%}`H)P|i@mX52k=8q0)# zncYS@%w5|hv@EonCA4ZN^es}DPgq#|N~qsTSe#|}d;aj}EMX}L!Lj6S7a!($kw5{1(f#YPi$%EEu}xk{zQ-NKD;8A+0si9|y- zRJO8*wMKN~I7+6w;vh#zg~u75B)cudNwg+uT8OCQ+CP_ZCC4RwtCL90o_M^H_>n4x zpFQalRnn%=KhrR+;& zV3?=ew@OwT%vk=E7D;6n8Sb>V;@B^fWj7M-S)TUQGd%$rNDo=fO0=@;ZOw9F&sw0$ zUQf@Sl1V3APDwk>o}o$_lu5l9$--et^Cyojp~|Qa&dAfwM8I>^ql!eM%Dq>fad0=Q zWhVn{o-4+mX^NcrR3@`dCa;ZM7a7GJM&-UFlRA%_eIS_g{48x-C*5~6n_4%YRyT(k zyMQ7iKZWZ1GwMW2yc|}J9Qur$&G0M|p#t)VH14&0_J{($_5xCl?3J?g?DAwKHn#$u zJlXIdy!JRra7KqSv< zSgXb8cT03oEJcNKKU&ACpp+@dlxu^_qEO=RQJ0~N6=|N71i@qLL$l+EhJ}AD!QK6q+F=bs5bXGbC&SYGyT3Rc&Osh(0uR;OU z{6I+&7plS4t$BB+_Q1XRe2AlAtkzxD;J#ed!{Ey9=1L5&s-CfuhfnKpE9;(mm10Mh zg4=5}QECivQWo)Qi@^1_ys9s#YqU{nkRugH&w-@$wQ3puyvKMAE#Vah5fuw-4U0)N z)zS^q?F|pt(%Cy?5rr$ryo&Bbsv$jfcq#m`cCPl3T;=yQ4Fa1gJFvcubtMi>)!kYS28*OMBTMt{vaG{2tjjWt`!sm@+c-5ae46HfJCuN%yzBfJbYMPR0Q`c)wjcnGd zY*Wx{BRa3kl&e!cZ!4^9)<6GfI3E7Zt3^OK*Ubj10j)RAY>5!I`b^W{o0;M-R}m-- zWj=3Lhc?!GwY)wreJfnU!x{2QF6fq5>*P4l;Hg((UKva|-s&pX5)#o7-Cmn84okCX z%X`|jwceJ6+I3*lg~8meRoS(#*L8y0UR3$943&NTEUD_esykDwm$SA(uCu|W6D_JU zva-`^sRK#2RnVpt#0kss=y@R9VRGKOI-c831M3`bv5f5Ppn>&eCexjF8d`MJBeyAN zecU{6+S6;_UGGYd>|?iWe^b?mkJ`-{rT<>O-wfP!zTW&CtNSmzchiq2f2!&gje^p0_9`Iv^wadT3-qkG_r9>{E(q(s?OmZR0z<(X(CLiT z6X8hA?Bbc|0)(qrCx*Cb+qp!VL9{^s4gG$>s9~WWeZI8aB2_))6IILSg->mJp8V)H zw;ie09o){SCSVvc+~`5d>Y<7nX+|EpgVsw)+bfITCNDqmqH}cVX{USSC~Z|7z5GZL z%GjgM0p_TtcIsh=AH$we;hfVdRXr6yfWoS%7OkiT8T|2l@2Q#dnUlo{)vVDx-LWLvX-MT@ zPb&<`u8&WC#?5vZd2_y(YX)_48iinl-+RW{dqL7*;xSYA4VhU&+C>AZp^B%u*|f>p zqRBU-hjiifiO}L))zpiNiMQNyv9fa+V~cE%d5+0JE`vD)yD{Xbu{zYL%B)#3g~fxc zSrpE3S%dLUn+pNil|l;Resl|B=nKJ<^8;B6r8P@!SYLQxn}{|rN#3O zZ-OxO)hs7zhDcnU~IK4cOfGExi+dK0~74q1McybdLC82s{$Y5bWbma>-un-N|fO9=ij?Ed!#ANgYr^1>O-O@AfQI5>9w&AZNrdm!=`iI ze{$oi=uCL^&Vt>h9Kr4=Y$FKPL@>2LK)*UAx{9Q@{5@)L6EccUxNUN}TBE;phkku! zbD%SOt0jA~e6lB-d;96x+RKe)BlMP2g?UqgBD@!xKYpl?!l z9X#*a8QVM{#@NL*+$0e@jE&w=65AzETtzTnd^fefda?gWakN1@d!2zrgDUQRsr38akK+7#l8267LQ_%b2!#ffulmIGX{c_BKiYK zkpp#){U?M=UiN3ETt`kA+YO4x9aF{mTZ=l;=h+Dd@>?e{hNsFkhhy}ogBJ&rQ#%@6 z3))>PR9&aE_N(vh_AUs3w2R&F&WZuCogtr-Ho|3Z3?#ct!q&yg%FPpO2lDa(Lwo~V z{R6#%L!980tw)qwWI%j)Vo0)cYG8UoOh$G}Zcezi?eBzbb*x{0a~d?MJqpNdX>IHA z?1~-m9SZ6!=<^=y?iq=LGvTvy9fR}BAw`W_+dI2^+n~dv?*a(9E_k%_h=r`NmRlnQ z3N-QzqKyPmO3-D+%s9VqJ`R7hR2JU~(fQ0GLMd+aHsXS2NpJNt|E8YKMBCTUk121w z9jkELBfrMHcNY|~I{Y}6^mT~0X}>nq2rrrj%+HOp2t za3jPZqY0y1XbwESnCP%1XmO)@U9-<(rM~POBNV+TWi_ue6t5^IoHd#R==BwTDmDl^ zGvgbp#c5s_t~}1I1>Gi>U}w%7D^%O!jaE2$FuXP&P7g~yVw9MhZrFM!8eMMLv9yr< z9VU08lD?A8;}*_+aekJikkJj$krvXsb9t7!n;{_zvO zF*wH%EaP@|JUHX})PM1r9@c7|GKdU3G+vS zX^Vk!2A1lfO7&@K5i;!j8ezuG+F@aevmVa_%oq_D!d624I!=T#ZW2^{rovA z+3do_Xz*tFY_TkD@%EO@2XH0m1i`}^8Va8g1+q066oeo06cE&;& za|T0efRwdy9A!<1Nh#A9=S~k}OGkc-01N(07*T>{Vb|*%fNiUq(9G&Z3P2!8Z-{tz z{9u?;U%t3bCg2C92MS)l(LmCj&i(OsEJ%C3^@}0}FgEIqS5tNX+VP~>T^RNFaGS_+ zH*by}>xB2*#__x}4;|}VD9J`?e}%X=E5uVsp=>hV2Yq)U5UZMX0ZHQC_tMCU^^+w` z1=^MB{bLHdZ?UPSS*W=T|B0v-f*-KdkPBAT@Pf?+-yA-I ziUp^R$#c6d#n)U1VUmVcQ>i5|yVc>>6%*&-_VmlgiNQ78$K4Hl`>P1TF$XZonhpF6 z>%>RX7f<++c4ja*i@~v@k9ptD2Z78FPA6`K@Nwo!1G!EF=R`*d17$`J6<3rBz@y$!nQM zmAWLKmu+#!2l{ko=M|Kfd(uy-OE~$JwYe}SZ-5Ihcnd`2E*&Z9nIFRVG=Z#zl1o0@ z*j<@3iM8r(N7}c5*OT+l$rsZ2W$Xe1j~bt!Znlb9odyJsYilk|3h?t>U_^QgFSE(A zcG=T?iNhFJD*d>^_?2BW^j4Gr!v;mXhQwYtp=&b#Tfttv;Kvc)Iz}0T%J|-#ej^&| zSXP{+>^aOb33z3r^|~fhh!1f;dAZ{`uR1$Mcyd9?V!t*8R#;cUW?x)|+U*Y~nmsP) zL8&`0bkJ2<2T~Znq%mR2NEw1Yrz3wclPbFZatOCKM6@pnrk5c*qaz+{W}0MZOQNtB z{Gc% z1yd=sN>}dF=+JC*zhYMsnUZBhLOU^LarQ(eSHao{onx`TXw54SSt;?8l6x;(wj>-JG`+)z0r-b5!${@I5a zSb>~eI~rKjO886IE%cB)HqFWqT-~O|p@lqOAgEkrt9ZC*H;dk()AI{1UKtriur5*& zjeLz&Sv_8~Q7BvH41I`n(vv59!7nq19w~&8@k`|CrO+%5GY3_VHtbP^WJ(dQEp@*1 zcuBF9smW0j+;r3ITilP;T^~x$053sOYOeMd85@M>sW~yb6W$Q8gCD zk2_LCi^5v@AG^dWUGiuZ19}dTyY_qp60qIiit`0p7H1qK(Y^2-xkctKHke8F zUfACG0tdj>;Y?>5@u+H<_lT|6G21keaAHM7@w77mwwHo6vHGrsy}jtdBwI3SmFbAR zFNDr4_~o;OZQs-Gy6nUJzVQuZP3x|K>hEPj_?yxt){|q|M~Qt^n`&FuegEc*R|go!U*&4)#r0vm34;d9QDg^dL~J0_<%3qDIlZ*9-{eci{$2}{gYI0y#S zus){75jIb$Mjz*p*d1EFsGK$Qv91M}(Jqv?XW{K4^7X5BNeUnA)a2Df8N#hMOyghdQxa9O=7-bJj@MP#?@X|MwV>RSp=^sNXz>!sKh)Zd%i`k$Hvb$A? zrq%aBhxDV*_xas!Cb^csa3-+;9N9`8DMFoVha3s`o%t-B-#~55mYuPsTx?R^?O4rw zQ{2jG-QJhFbB2N}N}Ww9+>Jur9hc3)$L=Pq9)YRO=B2irM@}vvMnCiS0VzgEQbhfW zuKOkS@4vWun7Inw(vH$XkwbEAG;>pw(of`bAQ-ZWGFN;!{4&P^#SO{BiNfn0zgIZ6 zmuIL)D+tt3>eUAF2tM}mx9}=v0o8!KI^Q6bB^=Z~=wuc_v3(+UK<#0kK+3?P;cpl%JISP6J|FYq~KAV*pNoak6R}+hFp`{7Gcps>v$Eh44EuuA{Nj|- zvP`%Hqo}5&uD-gmxdvKWpVrvXmJRFbZR&4t9W1Q8{mYDdH~8UpTwB8M+jW6IucQ5P z9S1=GVeHC6p?_kb2LldyczjS&aEf1AKt@PbXpVoLSAlmCq{O|jJh*xTE_C*PNl^UhZe2=H!aM?hN>Cud5vi$BN&ifOQkn)! z^I*G@pbU`P4~-3uj8FBGr*x%{!sR_nt86Q4s~ejWi#zLc^9%dC``c%GC+DZTA~&&p zrkS0`?-T6bH9!nNc%rmnWWsj;O_0ome8P}7Cn-ZRkKKRD7sq<}v;H9a$} zKr}kqx46==yt=x+(X+LUOu~3so}H2mZ^L$15MXOdV$)3acONjnl=6;aa!0$T5s;e_ z-vq{fsgU{6*{8*ah{qFEI$;hPKq?Y1x3a@~+xrE}q*7}>X=en1{%m8K$r3c0Tes3n zz!ZZo=JnUzFP@LJJ|`1LaeoLmPU#Q-n0HFkRy6x5U7&8UW3^9x$O}`$bHU`R-dK_0 zUY36r+dfTmj?q%ISH?h=Lb3hXX%GjKmXhx+IZ}npQiiz}h*&1*ZCU<8dnkQQXk=M| z1}vKAGt3AvU%SOq?A3i*tCwp-f$|M*wAN;eqs5Yeytbo-YGWmD`<|dbHUBmn>P>j} z!m@aG5E|DNtyEHeur^$vI|Zw%INF+S027K;SDx%K;KeF-Rac!IZO^sGh}GB)VIJ* z;Xh&aJN!ic_BsNBD11R(!u7So&FcmA{{6qPpjdu`oFIHmcs(Wwvc;9^3xe$F z8|W7h6bdFHagPEKg@yUYC4v)@l2THA(=(7Gqw^R&GBQKbN{X_|%X}&-i-J>Y;;Wic zALh3?71kBQlAALk+TBQEyDb{K$3`X^dgF5kTstRc0;gKWmq$0&)~9Q;x4a%&!x!n< z`6(uo4I=VW{#>FuTm&>K&QPPc1_u_Z{fOnQ<_32pdL7izZ*$EM0yHO7X&&OH0P73a z$0OUS?Lj2U)il_j7X{r2bX6kDUXr%O9DkWqV+$nd@_pdAo1v6PK^XNRjy))RkAgQq z@aSHY&evxH(Vr@~F;STHzDAI?qDR&iXN{E@0TYnC8!6cn6&8E5MQJwMfZQr=3ulm#3ZIWZkcIc`Ttdq-(0Qud^>f~tV&!jVWk8S9VZy_sOg$`I1Y;dmb^M1eS8@`ocN zKKQr-b6>nDY`Wvop1r4B2>c?!rjEfSm9`N4Nu_lmM8WQ!MzD$mjz*Y7vJh zP3BkMF5?~o3j8TA^a&14B=?B>LtY4Z@W4L5u%M{4q`c5G|59F9S5@Cs+t}P%WPhsz z*4f?FTin(R9T*rI9_b&eA06}Tnd_bJoS3YcZd;pKUfHPL8QvS8JZRWjKW)yR+}h|utmz@L&wACH)*thm_d`1C&{kKVT{f0I0V{t9wL z1Tlo&8#m$4{lUT7-rmW@*%jpOdCU3#SXPe02#Qu(GA?#R9cet_VmDZ>OgTvA{V_uCIFzf}9*uw2 z=s8wuvNqXSQL;5vYpconw!&fqweA}FdrK7_crl{2%=>@DC-Vu4e9!CnbP4^x;#175 z=jkudry%-B{ZSJhpZG4Eik0FzZ2KC8gp?lMRPH zSv5sGm=VIS>P+Q#Pi-t_k1AyncQ;Nl{DhFf$Q33(mR7_Kcs`@Syn7DXQViH@z{}sq8+*ePd;`jeXd3`zM*!aX{qBsjr zau=7DS60{7H#V<${QZN=MA3)g^v6X#)t8Sr{JF7pPE;3M4=r+)8(feVZvW=R#o;~! z1#T+f@V_)&AjQ&c^rQLdGdBJH@EPNzbHfn+*`ck8de97kXru+hTsef0@Lq@6lc^SC z5|Tlh6Gjb~HG}&Td@NIiKC4eLhwDSGYnML5jh08q{mN%#$%6GKYpnrv(WfO|BT#KD z#ZBw7y|K=i*FJPnEC;JH5b~+bc(dq(GZI}3i%Ofl#6YEWy= zn{~0Y1(|bydCup}ofFl2&ylp}{kR@_%)9#w(zUAjiQXAe3w$Rk@Z4WQ!Q*+Lj9_b0 zu(Z>OdWZmIMJ-e^ULYx)J0B-0LNnZwG*ZD#AjRhkQS4H*&EX1p3|Z**Qmo4=j#eCq zz*8&UiEcF|9&&L{JJD5CP&>(6;Z!R*L}rvSB^(!dH9lHgMkg_4Gb}wJ!=IfhF3TWY zJJWjgcQ+;R(@jM=AX{8J1>`@R0@1ZoAo|@YApZ|e;o*(IM?uh*){geT8_o7@y`BBA zZrI>x-+1O^=)}m3{qWep-1x%c>{7?V@<#jC&REaXWY6|`%j#+W-r3*{Ir0}%Wp?}Z zX!|VsGjFX?TX9fRJK{>=xc}9$O8#_X5)SycA33|axVw3P9zAjbd-}TIL;Qk3nv8Up zIZ-5|W^lY$pigpqKtehwEhXF2JHrD%EV{UuF)2T@JjcH(KdZvEKBcK45$cv#-B_E` z(%BPa^oz$tK#+n1_ZA$u693i5{Jy)xnIZP?cXxoFT9M*+gtEp8Jm`; z;359!qV&FOB_BI9JTn^UWR0%T+Vr9gk9bX>N0gR>#XzRwd&p!Ho+Q;smBQW~GF?Z9 znaiq$cRfCDuEm!`C|dZkv4&O(EqsrU5u`FGI=>c{JeergFM|sLxY0@;rf#382h-O z*u=0@w?x0>h^&-^3~)|CUVcb%iBTb>GP5i`P7T-EsmZ>jwYeQ?e=F^BCsSHi=TO@) ztp6Y4>915C9)c{w$W?wK_>;(TwYPT%yLou|xOiNm&EMA-5*qO!JTf9W#@jRA$L&}Vv=8eWL#27d>N>ypdzNmy$;+^U*+>_j{78-AAi+;O|H*X4fFyFbc(>Un|j24h!Lf)4Y%xpHs9*ANUDUVBB75}q&Q@+T*J(D?DAiM^Jlrgw{!^{y9%Dc zKXU$GIH~2n&vJ2paZ>Pd|BdAGPfqF%#k9jUC-pZu|9PBJ?REUd?|pL<9=}WDKxD}v z6;CP%bk?1^8cD^AF&v)rGna!%yx3Qfs?)%d-N|X9MXK$T@ zh#OfYX&+=P>`z`68I?2n2i zX1D5|pBHbBy1eRC~XfJAm*^}R*>mM8#9vV%D6?QjG^>j{T&15gOFL%t& zujXxLOQ`N$Ch7+yv1eZIF93v_i1U28(`1;>A5zIdbK{VmFZpRG<^8|j&-K^%zZ>gy zKe4X!chnxQ5FG0*|A=*FRFX*>6$ABlM{@TIo3Q=m$|Mo{J2Y%ebA3640X=UVYNJO> z^yenCD$EVAYOjOjCDuuDHX_V7mx_XfVnnMh`DZYn77X8NYyD6Dxv@)4{;&GJ=1*$x zbra^r?ccEeCd;AzI#+*Sw|xU1B#&cVam}F0?gR=*vo3@Xeq|3bolC5HLWw=-@=N2r z7^+K^y@_C<@es0Iq&OcE#4r*F4)VP@Fz>tL_o9fx%q#N&AKAm+Kl~)2{XF2aLYjK8 zjDdi9$VcOQ8i6VhOZ6~~=o9sDF zqlp;5Ca`!2s_-~>=_zFX&*EHgN-8|QrNiS}PJA9HE2AI=-Vg#;lo$EjPW#(E!yUf^ zQ|-S4rut7{M*nkQYF-0VhT_IGFex(fit-DK%Q9u~8L6vlYv8-O@ny=(3!v?pZ5>W=` zopZ<+q?x?jXASP?BwnF$`$JzRkC|KG4U8B>dF5$8}bmv$RUirei zLq8sXrk#-Kph}MDcZQW*5JmZ)IXzJTJ#)bogvPnyI2|jy--9eGfw1EF<2*0&u;akw z;TA+*1ZMoI_D_%;YayJ6@JgC1TnW+l{gbi;UtTf~HCBH?^wxPFA?|x>f#SFV>Om6h zmzA`NWm1T|fu&lgu$^>Ln0S0^LbzDIG+Ee(^45i5<6Y?Q5nJQW#e{hsv0w5rU57%x zKO#2QOL)ZQ`aNRv!6P>2-$m@y5C2#XfrrP2;1RnYUJlK7vrJD;AN$*fpUxz`V>z6+5Rl)4;H^fR`#~#@p!7r^+@~8qm`cn%ZE}3_`Uxtf zv6_G{;hMe;a~>&g0I?!7X_$BNyAWngx(?}YQ*G9?V!M7L-b?J>ub*3aX`2%ExV(!I z{Tya0-5n^RJ3D?VLM5Ec(m>E5sP8&bpw*ZD0VS!y1>VJhQ@PY7&a&kea0a&hNdGPc z4k8MHu!YfBwe1G6!n!%VYNIFNoHvbS{ZxJEOTz5T5=M1c3>QZ-p3JMI(xkoQ#R_;A zr&&5hItI<%N{kn_u)77`+zPWPs8zB^aH8u99XFtf591zi= zijKz*KuUJVbjOO$=-d`cF84&SpSfVk@IQ0FeFBPeCZxkwu_Ymvig(99HCJ(coGle| zAF1(*mSIh}M>d=~2Vt0n&iQa(AgMv#4PvYLF%x)@()hq>8B#w{+SKp5g9P)zjaR7(^6y)vy zfdcs_khlL&LY~m;k^;fU4ZP81%UFT`p~ThEE#+f^pm&>6du!rK@Fpi4nyzHZ=LR>X zHp~pkKn9=a4-tpSW6m-_T2`1)2|Rv$Ux3!Nmu$&Z+1TL1)5$^S&yv+YERKe!Jluz(<&Tb+ySrLM3-bR| z7yOysUB!>zvb(Fs?O#kRyn8eAw@fTe@$&CXEFAJz@#D=M%6WMFfHSfDWNGt&|8;i9 zd)2trUI>!XfH!XC%vKgcl-wv6L)C({7sIrY(-y<^idPmRjG8HzB25Rim!d2d;R0?r z6T1}SaEEF+*5!%LGQ5UNUyk=;UR{RACaRUhKnb0dq)?6YmE=gX)s>W3H>%at#9*D( zwAAGE)%48b)zysLW~#N!!a<$2tkT8wwd~5n)wP`3JJjpBjZbvf^PprI>-inbYwHEw zywn?o{SwrfXu}#A8^z-~XN`|0+^Fe_ry#lpWsA`$2Ib@N?FJ>A)f@(uyRZs_s*Oom z`l_>4Fn!rYa(hnY%^F#w;=2VAxwSXI6-L#^+>u%LAAHf<0h+KiZSqj)wAOc8FilW* zp~ODI#w}!;a>i}AIUS5px*F7%ZHyVh1sx1?HYT0Vw)Bi)^fx>6yPrIY%I|r7cFxet zJk(LxbbI~TLH`#?{euB1&8&k#IrEK!AtiU(!(p`${lgKhl&r&1y^@W?vCl0CN8_eL z`bQHMOIb&gHb)!g0}Qu_j;CF)3|MCn@e zIZ(0F#?oc82P4GzvE3ymW^*t|?PGg5D%8bwFlka#MR+`8Ut@F9?GsaVI#_^FdDhnu z!+qM?j&X5xcyo*Ax5EBk^)dNB)9P3UiR-k&{>Hx{I8j5m>W|kEJ!^!v)-<1cNv98i;?30Ciw|*^q3*bgy)+9C#1lLBRqW&93qex#HEzkF_8(sV9w}_bi zXL2TYZIF)6zhRK>?Q4VH$=X~UtkllL)Y;R~)jQCJ`mS-b>G9xT|KyC#^z6+1LSOeX zlxbva>vr?n%G%Pw>fYRO&*9NY>yO0Snam%jxeXs2Qe3hI4X&8j=LttSv8;D633Sw? zX4>A{5=P;p-eXMs;3&qN{sp^d9%@M66h(^tNJ+$mrK+=V1!DrHc25#nUm#ztI;Qsp zvhDB=iFg#-64wG&k+gT9P?(^p%-4}GIr3SH=g7gjWocRsyRH9MGJV0{GyTqwe`WeR zZvQOPueNR2*u2t#{q)o|@h+E{J_V73I6nit&weVY(^quc;Q^YzyMCVs_y{7$Sv*vls|!ZZXj}EKJk*56 zuvlvB3w(0yxorDu8U}G%uF)%j4Wqb1#o`i%yQpFNx2DYnw_c>foY{ILvDj7hYLjYiap@ zlDNHGFJbdtT+`Dd%WJ)>>+73aLp!^Ca|_3q-Mf2}+ZX4{w{K~QOuw<4n8Xq0^PQ<6 zj;89L=DDOqu;~Tcp%38s84>kls!>|Kx9_vp@p5U#&)YrdIEuZbok__&7}#L%%L2$c zA~CIH9VJ+o+|oA zy~J5wsxw<3#}Pv`VR_I$1lIJiJ3lbnnozeF6VdRwDhc_-Vt>sK*Y*l(uGOb-rr+ii zQ)|-UFLlpulhX8`>Yo3SA3hTV&3gWt9}dqVKcBzK4@(msiyu?W`?4Vl%=_^Yc&It~ z3Lb{41#olUd+sL=@6-pt`GusQuf}ES!5{3>)IvV{-%AQ(j~_`4SM_8gjgZl}mjY2B zQo70y$)m|abK&{n=!j;l3%a0IoE<@WN}T6rt5$*|H?nr3hiJK0Qh;DMWpXfb+e%#c zJ6xTFgh}c2_!Os8%Gfjo9qkPB-hj_0h{aktxrP>KbrW2n(9zl4RS)ZJ=pSf>d$69dzKOBXnd!;4 z`L5aFm4UUIjoz*0CFsu73CF29CCyct29c3?rssiRDG&n~Ym%Eb=+$5@S z4Rk7V_L|*n!?$iCU2%}I=8&gV&L}NEtV?%hn|)&VC-K8X4#A%_FlrOXj@pdq!y*2h(MOpzyEvBmCRBzY)$dIiDNJ zxyhdwL3-|%6GZ^0l^esb+D{uRR2Kp#kN8K0KpGz7zgZ!mL-~7!00!nU1|PW-JAn4@ zDgU>`QmnpmaB_gMe*7Pe{zp59&#gkNcj z?!MoyqJP`6|76?Z{&_Nb47$fRNc<0K1AZRAFxlk)+}E@gd^uX3|9*IRGX3p#>i+C5Y5-m{ zdcEEI^a&Wx?E`h^YxRGY1X`hQ`UuX{0XTktS0p|d7-11m&N4bKD#9l=Avq-_J`G61 z_z3joh62+;z@*}g^769E#Lq=FrPZnRWev5_O);g-9Z9X}^NL{w+S_jQZgdihcQ@+cS~lLqHQMNd%q%YY+I^51H+Fff09l(c_->0u;3 z9FG7*f=^7;Z`wPW@z+T$CHoIE*|O)qYO^)G8l1mQYONZ4?cHsiJ^d{j@Nb@ujEzD~ zc;OqN4!m=VoC`~fE33U-8%-Ls+dEtP&FgE2`(HNBI!;b6w!d}X^xWNE^-$pdaZ;=N zT`B$J<~X5}`j7X0oTmSGu7q;o|H(>NPQ+{jUwtuI{~5|PuKnT~O`u#O9xNLKoKzQ0 zePY-A@}uck#-2A)hSsyKiJ&({=J(l$+(a*_ekafuUmCBiMn1|Qzi6EN< z=0rYiyv~hcO$ylxVfndFg@%v+E1UL-P8{O}6SP(;&V{;@06i_4#j7)DnI*}*AD~Zw z${-k$jIE^$fkwHX%u|#qtjtmkItL2UB_>k}GCnlk&}Et@4HU&v{WG)P0i9WohY-Gh zM#8_fRrbdq`CXdK{&N9@MncLz_Uk{N^$y_iYk~L=Et>tGGyZ-S3G07q(OdW3t#Q1Fp(BuV}^rqb~=zn6avKB!)>hOfD; zt)r{Arx(&c&@eRI-8lx`0P07_<|Y?rmd0o5Rt7fOr`NWo=6CiN4_nc}j}|4r(dOGb zs9^SApYr$G^!pb1|JJGJ|I5rdv{*(!#zrht1DeQ_Ah#7A{0^EdR1DXzy5e3-1Sw@1 zuQZCV&Xw!83Fxu7fZ$8izM-LMV%)Qo@JyvmGO+<$_Ik{) zD-a7yIJT~8tA^xB6A%gynhi42yoR27s`nyEIKnALuB#t!My2{0E}fabbPxYov5d?m zV)v1O`m0#R(d6p@_ZJuOPu;^`r=Ev<5&~#G2FgI8@%P{(GJvQ6$Q=8I&|+DLVE}@d zl#wr?KEyES>0>^IzUj6t+k^a@|V~7KW16aA94Tczl{6m z%3{iS1^I==pwa^U?7!Km|C4#`g+A~x!=nh`{e3#1WzqOu<9+*QjrXnotH%p{mi}94 zowop#np|8^RF+WjStBi{wobnkTmr6YY-w(YZfkFh>`Lqd4)k{<{xjwCILPB_RSuy1 zJ&5vK_}_#0{5gm!E$^>EXjOj=5?)kXTv`@W@!2Omr?!?h`Ez6ys2SYQ*qYSX)g959 z(4ROs&<@0M6Q7uxoSB}RwPBlIo?V^xURv8)UR>E*v+;JD`ZD?Tc#`dC`(k}_|7PL( z^2gln&)dK$jrZiu{wKIB*^)72?Llr7NcVkMxi3O@Smeui?P=?x5T4)Sr9_*y{9`A7 zTqA$)l%j%H@hgFF#zK&6xwfh{>XCYdVlCQAL)c*s8YUq~ZwN{~)nPu=HD?_0 za;!AjewNkK+CbB9c6mV4$9Ah+?+iK`lq;L<(9g0YVKt8co*NGYJR2*wCHba3>Lj{G zK(77Gfan(*tbGwmfME=+i#73nA*i%FYR!Pw#pX%4YAC)6be*h0qFJLeJfZmSsNI|B zRx$pwBMyDZ+?WO zU(0CYc>ZMl^vH1eY-|J1yGR|CR(ze(4OX9T-2-_P+0X|)H`&ky@h#)O6oY@omSsO0 z&0p%D$Jh#F&HgjCC_U}}-Pro{KObAdvF*QW&JGN*2+*4IO<{)n<$y(&_bTdsx(}v| zMNZIRsAX;#+91@{jH?lpA1mBwQ2>xn1r-K4q7@bS`$4mdpTf`%ppFp-Z>avYiGPC~ z{&(Z?_wOI~u>XhN9u8*Y4_$!Oe-)@c&td)Ydzg1g0zei0<@fgO(o ziT?Cqv<)9MVrLlnySg&$&*$UvU1;1p%5`=RE34@qieG%8(*MJiw7DRRRsj6k1uKMe zqciA%LeVi9n!Ssmq?CB##4LX^-!oxg9w#>vH0B-vJipJ4l9GSRZ0qnO0h3cx zlAu?}%&dgm{2Zt_IJdYUtSG27vN8)CU&B+ER}oTP-CWYzk}t0GYdh+dl^qxv?H(GN zgeLliXZok-2Nsvcpd!|}jfJg^^}XHg@xz(@)sv;Oo^SIPU%!lA?QGxmc>f{=?6WC^ z5b;@5mMMRJ${7GIEFzETiwvgoXa-9GM!Wq18v+inL(rX7r4u%-n_y^bj zze;iZpZ?SXbJ|8+l{Bklh zxxO8R_~vr?In+Yrj{XTJTqlgAy>2by9>;TRzZ0!CZ~C7oWKh+9XE-s&^X5L=l`gNB z-*>KwVyJ|~+s&D^2EaRmFzSY#$LsXn{%QQsUJmWmMGB}Sq=6xb!)8N3DXX6op(>rD z8L$0iS|k!8Ci2w$C4hFh2EOVt%dQf`q*F~zkG!Q6_Y_psDr2zO-8r=mRKjh8-*wFuvCk8iRr{L6bZlC^>EcTR14#*B?znKJFF0yVM6n=e=?L zO0k*xwKr?Q8PoLnBiz5}# z4e47^^Qh=xX(q+{^NC&)!l4*4%)Wdx;y0_02Ch{N;Fm_3hsW0jYI(J*InT&O2@x)s zf>jyvbbzsmb6V_(?Ha+h3okaKjbb%(42e=dYG{#yYcwBZ(Y@!_b!d9kXjB7pqzsJR zTe^WJF``H&W`z7qMlr&6y-~t9pO8{LzRJ<*i?^o0>j+a^bWx(0>JIoS@IxJ01TtZC%3oau`UISLm4tP<%7hjfu%(9VoXK}{IKB*#lgk<%F666 zsCi7>5?UslQs$wh+Co23wpCPu2nd4QP`AF+%LOrU=;9{m(AB6IHEhIrJCKmz8-!sq zgq$if`=(@`SFfbz?A|bBF(z^|x@WAUR3e=|FJpda5ruW%ExEt2V9E`RV!-Z-%q=Q$ zS(an77C}!=9T&rJ*4}f^me;KsB7T>6IDAVoci1=H!@|;4yIQhc)f5IfjvmzpVpknL zEuA%h`KHaIA~%-yO?;A~C$}}u7u%%H@P}z$BR@+6Eu^v4gQVe2YUu@!q zeY3+@%f-t}mIgBuy>nJxpz-coa|MKxt%Hu7Vm7b{`_L7*x9hAdx*K)P_GXRPNIre+ z^#d}v3_)Gx+}SesRq=LiVDQ0STJNj$qV(Xym!GmtYCd}3IqQNBDlw&AXw^M!aZ!H4 z@`cqP8)^lAxcUTqJ|G3q-J>=u$9+05R3MD({;K>v=+A_in)f^FE z#8#e=2h3wO_R6A??( zJp?Je*twwBb=O(^%z|H5q%`KWIg~b&u%Gl5^<8lD{?m6L#5$XSey&Ar0~z}SI&yNH z(DNzjm_d5Zlbli=s|CuL$$CDRMx5|Dl=LKHG2v(AqXwbqnki)S0twW^foy5yC;I*` zM@0sM+&Yb(ob99GI7TSGenpN(;*53FFZ=K@);ed~RHH-s63#p`pW1;J)P&h<5u31# zh&9($f(tj{HpQGf5k>r3K-ob|_C+ink5j1$eW{Pra43OEO3$g?nN0xe+Wa_?L{@ra zM3^t7|0FRdukcZ%%g}DzE~Mq=Ozyp_X(8+Pej@nB{^)t-bzsPJ`^YQKgkUoI9sHCo z6<$Y^Qd=x!$%{enl6c&t{DM>4VrlBQviK-CGbV%J{TWtgjmi`@cd)~?E@1VzZcMl6w8{(z~dp-EM}2Q96>}i0pEur3@Qm!vp7z;R&-t@O%tAlAt}e zo-f@DTgY>#Ek&5>S<;L;QG7({HuU56hZ!?9uU5t9OqSr&hK=F%!y%J9pl6~=il}hP zqc1;rcSS-QKU)JL|8bz4b~_D{uPxCSIaF`DovDnfEwhk2(q6iqg^bo#xQ-m@-`&o& zW7bs$%N-k&-p%*R*MXBCMvlz|?-s_R>S~JQzF6zrEzXbD)zy!DdGB$zw2oO1>682F zn0B|kFJIp{H}ch`>2BpTs=j$o?!Pn0LSze+Biygl7X+Gs4e-RA%A(BN#Uhl z82LG{?S2uTUe8GeZKi7*Q>31wsUa z@UksT0~7QSRK>llZ#T1R*_9H!UZpB)H?w$tl=PfA;Rdo?4~H{g0y&J=as%NnH`I7( ze#)C0p&kVYLm=HI?2g6oO7TB55-Oj0g??Nz97HRrr=WlS%|K`}Q`F!pw~8x}XkGb~ z_o`vM{NR&+>gv(gWYFu1KQ{~BezY=(z<8{B9w>k?&pofOxKEc<#I~6l0>6(jaj^M*1Q^KxgWtf0z*nRv z*lhnHxQbc{FEA8^!M|}Itr79L97$R2f*Z4>jS9_@CS+ZM0^AR#!w8HnGIN_)uyk5bj5v9K#^6wk*?Q(HBj(|r4Ip6 zHPR%?k2svDIHoZ>Hmh6bl-TyK&ft(y{thh#!z=>;(fNV!e`h=t25-J%I^g@EgwbVR965fbMAzpesU>lP0 z#xzkSrhN1d(^RzhWKDD6AH0z0yx=i&C7q0tR#zMP)5(asHz|oGECQJL^^if~*ERivcRuWi>f@m-*!%fgeEAf*U6-6a!Lqkm9LO2^z>;^Wb|9BriUNPld zr~p2<$@AGgz=deJWeI3##Z@Jx<;oz|E%eT1Z%lyw*;O~Jqx780aXQbCs8tuwIU2t@z z_6^GdFcn%-kX6F+<^6cA9~*G@T0Y?UO`oCz*9@)y=)bx~O=w5(`|i zF(2M9sV$!0jMof|KQFd;OOO62)N`0*mLPXRrvqklp%La9`n?rm$_da>o`GJCPF+uFGT?R3|Hy7s=5HQT4oE zDOB{zcuCjs5QocFg+L=Kb1IUcs(g^or)JI!3rB6KRX{DinoP=}FB1v_hb5a*m^9+a z$lxv+XJO!md#|*Y=!}_Ln4lsvP?QDW__Ukcme62xQ%|!D@Dx$GDE`CwM6jliPl|R>)K8cPxj3sHXtfCYQ2`n!NTFQ-yX_hf5U*1iv zsMmq8Y?wO2vQk$)Auc^`8G1i-7>VvCx$U~%6FQUzONqae82omDF&-URrnu8gqM42s z7k!cbp^Mn!crP0xIW!o7lIU15Ut3tjs(`7e44e|GjA~$xu*Fzu`H{tHkJ1enyq6kw z*|4AB&(gR(Dt1>!5BJW^swg>_;r8Ri8}3l1-nYpI+i|q<%m=-eCC9aMjI2LUYD`~C zd?}jEWRhxQwGJ;u{?OHOvg8on#L?t}W#byYpBl!x(g>$@wR+dev{mzT6jWc_!*Jx* z8k4o?*2=NCZ$x8uhm_6@ALM2iR@t~>*SLoJ{s^4A~`?lR`pY*!0Or&bxt?B+V} zOV1zgwK`1nS|SMOu*xer*=u^x3lO8n%u0g<=*S_au8j=2&mH?G0c4?;&(<8yg4|5D zqLq{R2+QrnvyU_abNN5(s8zXYAgmJl5H(YwqKgfNBxEzI>M0RjPjrPR2JB!E!$x zGA@Zr))_Zrk8XIUaR^Q+d+|09=@YsxS33fjt&q1ibj0owoYvAzPu+T*%-NXtF>&cuJIwgr?tok(LHW)H4-Wut) z{Yuf-k9LB%q}-r7ccXj~RPnTYB5N~sPhx|qn|f0vB<9GHLX>6F0X0F@H@}d{sh(ea z7N?+3EWe64^|OQvju=ik;U>Sv1eg8`Cmz|uecKoOF%{Fn7i`Sr-%^xc{hTV+cvxAg z{0ZSj8NkO!G9wdnE>RqKsT%&8EIC+eU{(q29RI#xoALRLx^!=C1rO&Aj^e16H%evdnAsl!(uL&Qr)badO4+?)MHVE+)dLd4Oh$_BNj}X670%dW;C_)6 zaIV|QRL>y307(4Ul~=EwtyOR)mEMukDOq%hh6LJG>!V2_s$Jr}SRq&2ky2+PtS`c} zQ+LeZmN=440EzLiSdVG06zpO*t*hy&HVCSv7ZSJtOpD#fnOA&y_nVWh%OB*H1a+?& zCX4z56X|CwkzV%uc2Qwgg-E_K#){|fMi6xfe(kx5SH0fztQCgFQ^bVQ_H!?9uS1j- z8`ARtn7-FY;HcnN+Z1>Wy-V-Faq&;!unpq}-nE`ZeYD-CnwlRJU_gj`pbDbJTEOOP z1w?4Un$QGi_bJI&r^*>3@GxpUv6DZKF}lZiJ%2OmR##2&J!VgMs&~vnV2V;U1|UXy zJB~_~n-ePKE6&$AKJV$2Loa0YR^{nL48w~;@)1k4<@G5INC$yLMgX0s_OyoKcMSaU zeHjz(iQv}nh1^0%+N1*G5}abJaFJFtnBS&z2zfvXa<;|~?lW1JFTfujPHiMi1jbvh zqcxJSEfKZ_U{TsH3w4C7q(2HyRu$Hi*<}DU<{GEoB3xpCIIW)opH3Ic^Od&fy%!7j zn5g6BYxq!Zr+bu2V4>|-NA6{7MEYvJht~xwZsgRAZ)qWLl((h;gUEVIdus2Jx4zBJ z&!cZ~a(niwZle6Fl0nXNdsKbVUgU{}hvyp2+EvbvmLu+qR5(N|-cCIIV-wc9y)17? z8KbqeIoc#@YJ!7v&}&freNfxbBLFbTF_e zhR>Ymb22YV3|5Zp`9$ZozB)@Zl=Ng|M zW{9*82VCEC5CLjueeNVaI?{N)Q}pDWvpcWz77q3}uD4F;b4NM#_@3+=V(&yI<*LHx zO0n-u6(UClwDD8)lZbI@AoQcWcA@TfvrV#*=EG(RaqdcR3*LKh)x>sjCi1T+@)g-f z`+SOaDd{PpX!kSOr^Lb&1qStvRKQiA4HAm?mL&d_tKD^>b)%&38$KV=>p)wjfEZ+t z`R;(;;Md|IKJ~o;48%{BfObC^eTYzkoKSETEb-o;cq%}GT{Rz@0t1MDbp+N7333Yw z0fvM@I|9%D>Ih668VBt!p&6Rw777H0rb0r~=R-5EL$is)a{0sZHNy(s!a%^V5=dCt zd>FK)LnU!InE!vJDKK&5B!A?z=D*VvI3nc1l}XcYP7&G^*zeO7#p{@;NX95J{s5DC z0FdybEHRn_1RyCDofZs;c^)7I@~S4Z9EFb#qzZb{AN}*ndc!pukviJC|I^R87&Q<+ zT0dZ;*HWzS142rq2U8F)k#`7hV46j=#XQz6fAl@F*ALCud#Y&p&S(WbuZH@-W(!}$ zn}{!(er)~D9P{z)DE`90sQDLuB320w$o_7USmc)eA@%+*-TW1qox}MOI#A+pm}8dr zJXuo$S_(gq&RKuw1?b#(>Rb8R5kHXt#mw`0z0$C+tB-97eoNvQiPQAaxgGNBBx9$U4}7X@Fsn1!DR(Cft@(x zCBKWAMuu;4rfqVe&7rSriSt@35jS#(VGB&u}^&FE~<>5kw9#=(|#)z%WVRzc>{619R&)s}=cJcPA! zcgTaz_*ZJ$;)$5%0jl4p4IZXdenMR@xzK__6V;Y92SL`22i zL|rsNF}hmiWJllO<4ZpP#G^wOKheO_L)A8qT_*6&`%*)Xc&U;mrO$YV&8!A!V%h=u zf}|Jqm-#CGO#&J_jX?m7R40u|Qt4?@>5pnHJoYWN1sa|OGTu8HJ_PSJj$}69X~m0c zjReR6b~Jr>-o?zc1|}kZGNhd$eKPALw@{;%S=&~+EfWLN6hhr}&7*ybsArAdTxzD5 zXj4ye$x)BHPRs;2?{ zn0*dWxvP_xcKJa~nT~A_EP>C~youY`t!R+PDqbLHn=^ zTV_c6a<*G0r!``ETKx)Ic{|a@yIa>;+cupLE_M2xoZH%C+A?x+q+HKh!Kx< zBg5P4m*k(Ns|A(z?9;(sUe9uVss=JKW2k zi_$Yp+dA`5pJW4>JMB$M=-(ce-euEhH|)zz7(>%*Gd z-qD%r*5ihT*83*?=`86EDQ8k(3-pY0o1a$8v6C>fkt-CZ7@-`m;i@j=_%Tsq!8 zyS!>H_*{FRd-o$u74|!#J$8rkwfV)ZMvP)-@VxeOTi`y_@+t#K{@9`l_pN%R03qFV z)i#^s%A!_Yk)w8@GqW0a`EqZGFFty=$Mz*&k%9t~PQ@~F!c!#^o{2=+{aU-yptqOUVwoV+XvXyGGgbf<Y z2ODoE)}Fc-cgY;BhG)GJ>|ehe*0$NT^p-o=+P$lA!J#a^<2msF>PDtQDSqc8$tYcD zcOCc3uj@rVSkZcdH%D7z8VbU;9_|;LBOiS}{P^^0tu1+(WMTbRjND*KN@#K{1}!kKhmWQlkxU3AQO#b9!=c><3HxBl!H=MCM<9w>Tm}08 zb)bg!zNv^ZizI!Lq;{xjl744F9-D!{wHdIzjJhb@vH$sReRGD(SE&Du%{1zMw(q6% zeuf`Kszq)X2TLCZfNPN{3!rZgo5QaM6HA%ySQ1Q;@yq|l9nBi*e{b_ zXkC^!Dq~abKAhW*3-!OT`7A+P6#!T7VqHqvbQ>m8*+%0Z0Xn0*JHqV{@Q4QW5r&u6 zm6#8e*H`$FmO^U0OG-ccu$>{p9iKOr;h@Mu&2kDTSj(G+!K5Fm$KNiMwjLubaeSUN z_h&C#rby>3*-i?t>_X=wdtU)wKB{|oJhsz?F1q$kHL*)mfPhIZMWj7@QMk_zk+EPV_cGQIjoYIcqW4R7rtlm`RRql)YY8@;8@&&pvyD!NT zLX>KxqSsF7?t^criv*PFAsI&+?z9VsHGM)}1aQs`;ThMK7ENah*dBpW?b`uTZQ(bg zdV>P1ZOwCBH%d|7=if9@wTOjOG2VQfdMAEiJkRxY0^6&ydIPRXviF2tEQF1grrDFJ z^&1ICXr~`_`?!=K9{*WrT@!6D5V4whkVX}7pvqZi2_eb-U}zYThl#@}I8YTE89`<| zG=43_=%S$Y(bSaO0(szON9iFtAU>H=5;YcE9mBuyo{FW~9FEPf(nZDJtkRHqXwkvW zuPWbgkrybcZ48Sp0m@+Gfed0t#U%7MBU9s8NnJF^JA{$ZV7USZWXzQf6KwP7%|QLL zwGJ*=WCpxKsDu4GN?`-C7Q9Kt0+AD^;Q;h4>@LP`+eMYQVZUvDMpx+%5^=Fu)7!#l znxh*T@rH-8`<(L)e9xY8WKPJMz>!Hydu@JIk2N(6PezXxY#2jo(O2FnMgs*=&L znX@RQbl{9CCwC0*um``C#c&)?cnwp?=}@9<9Cw;8!ZXkubH=4V@X;$=Q0oL}IqIWb z?e^pCCT`?YIohWu@zf7OOy_=sYIjd~w36Ovtr|*|cb1PuCWNyLBp{Z(*B{lgXS2MQ zcWJ0t^jP0CCe-z^MkUxI-n zW#)WC3((AFbv8!1rR2NC$<)glkX*U7#?azy<7I7SWVx-yyQRg&%Q{H;muANyo%Z9y z(vGy`&rup0MYW$HEs1s=PbH_TZZF79Gpx0c!_hV{=Q5gQ-rG*8ha`G`DeQf_!`w-# z^~J@d!NZKwU@P(2TTG|Cm4p({P*PM2`r35v;fhZUsT#tCSFOPB7Nk|5#_W(dD!)0~ z7!2+>x6-==Y$Fe9%5G+dBII%K(vN~JRIf7ewKJKWR~UDkYeGy zuVlujME|WJb*_8=YtrJ}JKp5tjd1Ryn}t>Zm7pRv2}1>ogF#*G+XBAK)OQ+asO1Z| zz}1x+IP%C;!6v+6_o)2@Ji7^|AZ0Jf0Q*w}HVenbh|&Dr$l|VhMP78^Fr`E(&Zna3 z%MAg%(=;1VqX&yN-6QS6;a9cXDnE{do66?82){Yp%|p4?s&q3uUQ3I@7a2Y9K8Oj3 z`Lr#@X}m7U9rHW+U4;wd#p77j{bYM!O+Cx{__>}|j#vDNd{3ZFIzkD8>fa6XZ#xn= zO{x$OFb4Gy8FMWfc`r0a1aiib+`D2@(v`nnuy;1D`h@!tq5oLVp?Lsj>)dg9UkvFD z{d?V*O-+86Hyj0y$T3#J4>vI3yiWN9URDSCfy_-qC{A~;Y$WSbgdnB4_`C{hci#HT z+WL*3F`aj`71Z{QH=N%mZ?CGi3G^vNxhS(^9Wm?bv@8l=;WeUF+ha8?X1Jc0RqMJp zo|3Jsc%M8}86^3LKD->q&uZ&Fd3G3S@oGCb2v$!~|2&G(b>ftxv;*Dy#Bi-~?c)7O z`_hjS?hgcea}}5MKCLG`jzULz!0@^F%v0Ak+f~Um_p!N*tEwxZ3fX5a8)HAOa9=mW za4U%HqOaV%4Hh1MKIWFK_0u9gx_J!Tc01nHdS%<@wP=-fG8OJ~#b3{M+lTe@q#pbF zj=}qIZ0%;bwC(4YwWi~_XpiaS*2`UFpZjDjT}6QIDYCU801X0oG7mrmA))RAaQS_8 zVSMr3d$$z?D;r{mgZ zsBPc$Gs&gmoTn$Fp;mV!TeqjN+VrQ$(PGyO5I1I^=_KPIrn;J706S3I;%KvR(LVPJ z{5C<$(;2vZM!WHxLZd2B8zpE$Nn4uV|Ctm6<)nYjDc)O9kfM~fm79NuC^~Tm3PKzy z#u=G|WpGfXwsU{5Y+P_$Ww0I4pXEF_fH>zgdR*u6ss5!LS2g(CJ#a>BgV<^#}ai8qkv_oF9@rY3U4_J3(eL918JC%LJvhr zL_4**q{yRY(Xy$+Mj5c9AQ3Tt2d$-H4WUMzZ~r6MHbB(OO!Ut-YB{;tW`cquDS!GY z_!SftSJl-+8XB{zK~0Y;0!T|O)IwxnFtTT8bZoo}3WvWRJrK%|t zsPwKJPOm{k2sjkC;`B*8Jh-@z?v4vv8%2b%A&gkc*vuSXH>UbNZ-8B@N<@^T>?!hG z;R(E9HY;htAJkYxVnxCEPnalKKv0i>f@Bg-BXwy$6Pg|ml0wdsT>32vI7&8mvRhWt zu{5X!QUS`eaZ7;|swpQkZH^cb0Y6JTt)z}dt`B%52Da8#i%1y|!7r*5gr8>q5SvLH zI~on`*LQX}VKS$p-{kSI`h%XBbFG!U8nGC=-a(Z}_zM5k;q@w7MiANqMj43%ZAZGG zM^M2_r&X`DO4ZN-_aLH=>?S_MT9ek(+5VVdyE1Vm>IqjlcuGB#?;9I#dWpjN5Ylc% zHR1VINh;Pn`*Jg5rJDLC*Arq+PiWIF4>uwI{;X$X6!#_PM@uI&m%6GW5tdVdVjN@* zMiW17_9Ow_qTqA@{Lc;K7Z;(p-FO_s1#8}2IGWTSd#QoRfvL|OvjLsVQ|nA3mNRHR z#D2idAlBPHYWon+edBP%U9MbqcYGxTyv+W34Nu`HVlt@z7)c@U*&(8N5FGB-SU~+-2l^yU8O1KF1Sjd;6f>8k zqKNh{H}rlvEpSSBQuvE+SznvN?&UhL0DS3O+;4KTELUIBW?Q_kqv5q7^g}?+4njLb z&!;&HB@yR*w0%5R>dNy&c)1;4?X6)GSO4^esX}RRu{63GJUNPX zwJsgsbO@k|wdS`fbT=Mi1T6k6&NWP*+$^r@O>IOa8sd1`Lq+roy}V>n*&_!Czqw#n z5a;sFss-p*DkBI-CCXB@-AV;uKx-yp#0=Vw{o3SE!TKTV3f#~B&~iU{wAV`Z;r2sO zILbIe2VJw_Ar_D20SZzWM;LN+y~s^|dE4Zx>Jh%Q`uqx>`sC91HB@viTH4^-PD=Kz zDkNHBNVpk}0X_GJI^q)>hgubbTMfv=3$b%_6sBXAjF9~^`y^97RbqEF4NvFUymCDU zfNu4t{-PW|*pr%|>>*IMp&Xdq4^PE&(PHiWk19xX3>i|XR{jsk&TyG_`}L-5i%5!& zSq#)wI!|9DO5BhU$!6lW9s5JT`cbUJ0DKHd(8tzyu96tg;Ls4M-*2LlS9YCXKY z@I$V!52QWU>6T&Yl$<^d*p{Krak*}z6sQs%)K(=#ap;cY@eK%7q{LMMqQqhh2g}zO zdZ9o-H0{>dg4-MIvJxeJ1T2wxoyNp_kdWKy?1(4$0jTx-9rG#nK~AqG49U*PW;&#O zf&jFXLMXieoYG+5;fOY4y85n2xHsj1ki>O5-vVmm2ahmW8e(*6FbpQHNnn$v^MvW<2=KgR?@Q-w0e%e;)6LBRwV_0PHk5Yd1!wOHM5W}aPMZY#T(;L}-d z8-zAvG#XFyd+OG@g=gmSc<_Y2kR;LaeK|ii+JkkZ3WXg7x`kBBmA-KhEK(i+n}9QJlA47F_04@UjRS#fp(7szow0CE6Dynt`&MYMn=^d&nz>6|V zxl--9!P&aTi}Dl^tI<;u6<`TmAq#ml&*1HB>+Qv7fSe6UkbAc1Hoioely&ev-h8jX zWt9~MoDn;{FUtqv{uKY#q_b QL3NUUYU=-w$V>$P2TZ7nQUCw| literal 0 HcmV?d00001 diff --git a/docs/ja/index.md b/docs/ja/index.md deleted file mode 100644 index 6dea5f6570b..00000000000 --- a/docs/ja/index.md +++ /dev/null @@ -1,142 +0,0 @@ -# ClickHouseとは? - -ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向のデータベース管理システム（DBMS）です。 - -「通常の」行指向のDBMSでは、データは次の順序で保存されます。 - -| Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | -| ------ | ------------------- | ---------- | ------------------ | --------- | ------------------- | -| #0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | -| #1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | -| #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | -| #N | ... | ... | ... | ... | ... | - -つまり、行に関連するすべての値は物理的に隣り合わせに格納されます。 - -行指向のDBMSの例：MySQL, Postgres および MS SQL Server -{: .grey } - -列指向のDBMSでは、データは次のように保存されます： - -| Row: | #0 | #1 | #2 | #N | -| ----------- | ------------------- | ------------------- | ------------------- | ------------------- | -| WatchID: | 89354350662 | 90329509958 | 89953706054 | ... | -| JavaEnable: | 1 | 0 | 1 | ... | -| Title: | Investor Relations | Contact us | Mission | ... | -| GoodEvent: | 1 | 1 | 1 | ... | -| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | ... | - -これらの例は、データが配置される順序のみを示しています。 -異なる列の値は別々に保存され、同じ列のデータは一緒に保存されます。 - -列指向DBMSの例：Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid および kdb+ -{: .grey } - -異なったデータ格納の順序は、異なったシナリオにより適します。 -データアクセスシナリオとは、クエリの実行内容、頻度、割合を指します。クエリで読み取られるの各種データの量（行、列、バイト）。データの読み取りと更新の関係。作業データのサイズとローカルでの使用方法。トランザクションが使用されるかどうか、およびそれらがどの程度分離されているか。データ複製と論理的整合性の要件。クエリの種類ごとの遅延とスループットの要件など。 - -システムの負荷が高いほど、使用シナリオの要件に一致するようにセットアップされたシステムをカスタマイズすることがより重要になり、このカスタマイズはより細かくなります。大きく異なるシナリオに等しく適したシステムはありません。システムがさまざまなシナリオに適応可能である場合、高負荷下では、システムはすべてのシナリオを同等に不十分に処理するか、1つまたはいくつかの可能なシナリオでうまく機能します。 - -## OLAPシナリオの主要なプロパティ - -- リクエストの大部分は読み取りアクセス用である。 -- データは、単一行ではなく、かなり大きなバッチ（> 1000行）で更新されます。または、まったく更新されない。 -- データはDBに追加されるが、変更されない。 -- 読み取りの場合、非常に多くの行がDBから抽出されるが、一部の列のみ。 -- テーブルは「幅が広く」、多数の列が含まれる。 -- クエリは比較的まれ（通常、サーバーあたり毎秒数百あるいはそれ以下の数のクエリ）。 -- 単純なクエリでは、約50ミリ秒の遅延が容認される。 -- 列の値はかなり小さく、数値や短い文字列（たとえば、URLごとに60バイト）。 -- 単一のクエリを処理する場合、高いスループットが必要（サーバーあたり毎秒最大数十億行）。 -- トランザクションは必要ない。 -- データの一貫性の要件が低い。 -- クエリごとに1つの大きなテーブルがある。 1つを除くすべてのテーブルは小さい。 -- クエリ結果は、ソースデータよりも大幅に小さくなる。つまり、データはフィルター処理または集計されるため、結果は単一サーバーのRAMに収まる。 - -OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセスなど）とは非常に異なることが容易にわかります。 したがって、まともなパフォーマンスを得るには、OLTPまたはKey-Value DBを使用して分析クエリを処理しようとするのは無意味です。 たとえば、分析にMongoDBまたはRedisを使用しようとすると、OLAPデータベースに比べてパフォーマンスが非常に低下します。 - -## OLAPシナリオで列指向データベースがよりよく機能する理由 - -列指向データベースは、OLAPシナリオにより適しています。ほとんどのクエリの処理が少なくとも100倍高速です。 理由を以下に詳しく説明しますが、その根拠は視覚的に簡単に説明できます： - -**行指向DBMS** - -![Row-oriented](images/row_oriented.gif#) - -**列指向DBMS** - -![Column-oriented](images/column_oriented.gif#) - -違いがわかりましたか？ - -### Input/output - -1. 分析クエリでは、少数のテーブル列のみを読み取る必要があります。列指向のデータベースでは、必要なデータのみを読み取ることができます。たとえば、100のうち5つの列が必要な場合、I/Oが20倍削減されることが期待できます。 -2. データはパケットで読み取られるため、圧縮が容易です。列のデータも圧縮が簡単です。これにより、I/Oボリュームがさらに削減されます。 -3. I/Oの削減により、より多くのデータがシステムキャッシュに収まります。 - -たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。 - -
Example -``` -$ clickhouse-client -ClickHouse client version 0.0.52053. -Connecting to localhost:9000. -Connected to ClickHouse server version 0.0.52053. - -:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - -SELECT -CounterID, -count() -FROM hits -GROUP BY CounterID -ORDER BY count() DESC -LIMIT 20 - -┌─CounterID─┬──count()─┐ -│ 114208 │ 56057344 │ -│ 115080 │ 51619590 │ -│ 3228 │ 44658301 │ -│ 38230 │ 42045932 │ -│ 145263 │ 42042158 │ -│ 91244 │ 38297270 │ -│ 154139 │ 26647572 │ -│ 150748 │ 24112755 │ -│ 242232 │ 21302571 │ -│ 338158 │ 13507087 │ -│ 62180 │ 12229491 │ -│ 82264 │ 12187441 │ -│ 232261 │ 12148031 │ -│ 146272 │ 11438516 │ -│ 168777 │ 11403636 │ -│ 4120072 │ 11227824 │ -│ 10938808 │ 10519739 │ -│ 74088 │ 9047015 │ -│ 115079 │ 8837972 │ -│ 337234 │ 8205961 │ -└───────────┴──────────┘ - -20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - -:) -``` - -
- -### CPU - -クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 -データを列に格納し、可能な場合は列ごとに処理することは理にかなっています。 - -これを行うには2つの方法があります: - -1. ベクトルエンジン。 すべての操作は、個別の値ではなく、ベクトルに対して記述されます。 これは、オペレーションを頻繁に呼び出す必要がなく、ディスパッチコストが無視できることを意味します。 操作コードには、最適化された内部サイクルが含まれています。 - -2. コード生成。 クエリ用に生成されたコードには、すべての間接的な呼び出しが含まれています。 - -これは、単純なクエリを実行する場合には意味がないため、「通常の」データベースでは実行されません。 ただし、例外があります。 たとえば、MemSQLはコード生成を使用して、SQLクエリを処理する際の遅延を減らします。 （比較のために、分析DBMSではレイテンシではなくスループットの最適化が必要です。） - -CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。 - -[Original article](https://clickhouse.yandex/docs/en/) diff --git a/docs/ja/index.md b/docs/ja/index.md new file mode 120000 index 00000000000..facf2d71348 --- /dev/null +++ b/docs/ja/index.md @@ -0,0 +1 @@ +../en/index.md \ No newline at end of file diff --git a/docs/ja/interfaces/cli.md b/docs/ja/interfaces/cli.md new file mode 120000 index 00000000000..04588066828 --- /dev/null +++ b/docs/ja/interfaces/cli.md @@ -0,0 +1 @@ +../../en/interfaces/cli.md \ No newline at end of file diff --git a/docs/ja/interfaces/cpp.md b/docs/ja/interfaces/cpp.md new file mode 120000 index 00000000000..581e50e774d --- /dev/null +++ b/docs/ja/interfaces/cpp.md @@ -0,0 +1 @@ +../../en/interfaces/cpp.md \ No newline at end of file diff --git a/docs/ja/interfaces/formats.md b/docs/ja/interfaces/formats.md new file mode 120000 index 00000000000..41a65ebe579 --- /dev/null +++ b/docs/ja/interfaces/formats.md @@ -0,0 +1 @@ +../../en/interfaces/formats.md \ No newline at end of file diff --git a/docs/ja/interfaces/http.md b/docs/ja/interfaces/http.md new file mode 120000 index 00000000000..fb293841d8b --- /dev/null +++ b/docs/ja/interfaces/http.md @@ -0,0 +1 @@ +../../en/interfaces/http.md \ No newline at end of file diff --git a/docs/ja/interfaces/index.md b/docs/ja/interfaces/index.md new file mode 120000 index 00000000000..61537763cac --- /dev/null +++ b/docs/ja/interfaces/index.md @@ -0,0 +1 @@ +../../en/interfaces/index.md \ No newline at end of file diff --git a/docs/ja/interfaces/jdbc.md b/docs/ja/interfaces/jdbc.md new file mode 120000 index 00000000000..27dfe0cfa5a --- /dev/null +++ b/docs/ja/interfaces/jdbc.md @@ -0,0 +1 @@ +../../en/interfaces/jdbc.md \ No newline at end of file diff --git a/docs/ja/interfaces/odbc.md b/docs/ja/interfaces/odbc.md new file mode 120000 index 00000000000..5ff7610e061 --- /dev/null +++ b/docs/ja/interfaces/odbc.md @@ -0,0 +1 @@ +../../en/interfaces/odbc.md \ No newline at end of file diff --git a/docs/ja/interfaces/tcp.md b/docs/ja/interfaces/tcp.md new file mode 120000 index 00000000000..a0529a856e4 --- /dev/null +++ b/docs/ja/interfaces/tcp.md @@ -0,0 +1 @@ +../../en/interfaces/tcp.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/client_libraries.md b/docs/ja/interfaces/third-party/client_libraries.md new file mode 120000 index 00000000000..5320bbe1e16 --- /dev/null +++ b/docs/ja/interfaces/third-party/client_libraries.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/client_libraries.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/gui.md b/docs/ja/interfaces/third-party/gui.md new file mode 120000 index 00000000000..ef7bc904197 --- /dev/null +++ b/docs/ja/interfaces/third-party/gui.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/gui.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/integrations.md b/docs/ja/interfaces/third-party/integrations.md new file mode 120000 index 00000000000..9cd0a21e676 --- /dev/null +++ b/docs/ja/interfaces/third-party/integrations.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/integrations.md \ No newline at end of file diff --git a/docs/ja/interfaces/third-party/proxy.md b/docs/ja/interfaces/third-party/proxy.md new file mode 120000 index 00000000000..877f1b51dab --- /dev/null +++ b/docs/ja/interfaces/third-party/proxy.md @@ -0,0 +1 @@ +../../../en/interfaces/third-party/proxy.md \ No newline at end of file diff --git a/docs/ja/introduction/distinctive_features.md b/docs/ja/introduction/distinctive_features.md new file mode 120000 index 00000000000..9cf00a2a00f --- /dev/null +++ b/docs/ja/introduction/distinctive_features.md @@ -0,0 +1 @@ +../../en/introduction/distinctive_features.md \ No newline at end of file diff --git a/docs/ja/introduction/features_considered_disadvantages.md b/docs/ja/introduction/features_considered_disadvantages.md new file mode 120000 index 00000000000..45d3cdf563a --- /dev/null +++ b/docs/ja/introduction/features_considered_disadvantages.md @@ -0,0 +1 @@ +../../en/introduction/features_considered_disadvantages.md \ No newline at end of file diff --git a/docs/ja/introduction/history.md b/docs/ja/introduction/history.md new file mode 120000 index 00000000000..7004e990a59 --- /dev/null +++ b/docs/ja/introduction/history.md @@ -0,0 +1 @@ +../../en/introduction/history.md \ No newline at end of file diff --git a/docs/ja/introduction/performance.md b/docs/ja/introduction/performance.md new file mode 120000 index 00000000000..cb2912bcb81 --- /dev/null +++ b/docs/ja/introduction/performance.md @@ -0,0 +1 @@ +../../en/introduction/performance.md \ No newline at end of file diff --git a/docs/ja/operations/access_rights.md b/docs/ja/operations/access_rights.md new file mode 120000 index 00000000000..73463029569 --- /dev/null +++ b/docs/ja/operations/access_rights.md @@ -0,0 +1 @@ +../../en/operations/access_rights.md \ No newline at end of file diff --git a/docs/ja/operations/backup.md b/docs/ja/operations/backup.md new file mode 120000 index 00000000000..1003fb30e61 --- /dev/null +++ b/docs/ja/operations/backup.md @@ -0,0 +1 @@ +../../en/operations/backup.md \ No newline at end of file diff --git a/docs/ja/operations/configuration_files.md b/docs/ja/operations/configuration_files.md new file mode 120000 index 00000000000..a2d73dbaa25 --- /dev/null +++ b/docs/ja/operations/configuration_files.md @@ -0,0 +1 @@ +../../en/operations/configuration_files.md \ No newline at end of file diff --git a/docs/ja/operations/index.md b/docs/ja/operations/index.md new file mode 120000 index 00000000000..ce854687b86 --- /dev/null +++ b/docs/ja/operations/index.md @@ -0,0 +1 @@ +../../en/operations/index.md \ No newline at end of file diff --git a/docs/ja/operations/monitoring.md b/docs/ja/operations/monitoring.md new file mode 120000 index 00000000000..515ae8b4fff --- /dev/null +++ b/docs/ja/operations/monitoring.md @@ -0,0 +1 @@ +../../en/operations/monitoring.md \ No newline at end of file diff --git a/docs/ja/operations/quotas.md b/docs/ja/operations/quotas.md new file mode 120000 index 00000000000..1c52cdf1e91 --- /dev/null +++ b/docs/ja/operations/quotas.md @@ -0,0 +1 @@ +../../en/operations/quotas.md \ No newline at end of file diff --git a/docs/ja/operations/requirements.md b/docs/ja/operations/requirements.md new file mode 120000 index 00000000000..a71283af25c --- /dev/null +++ b/docs/ja/operations/requirements.md @@ -0,0 +1 @@ +../../en/operations/requirements.md \ No newline at end of file diff --git a/docs/ja/operations/server_settings/index.md b/docs/ja/operations/server_settings/index.md new file mode 120000 index 00000000000..1d1a0585a42 --- /dev/null +++ b/docs/ja/operations/server_settings/index.md @@ -0,0 +1 @@ +../../../en/operations/server_settings/index.md \ No newline at end of file diff --git a/docs/ja/operations/server_settings/settings.md b/docs/ja/operations/server_settings/settings.md new file mode 120000 index 00000000000..19cd2e82ce7 --- /dev/null +++ b/docs/ja/operations/server_settings/settings.md @@ -0,0 +1 @@ +../../../en/operations/server_settings/settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/constraints_on_settings.md b/docs/ja/operations/settings/constraints_on_settings.md new file mode 120000 index 00000000000..4dacf908662 --- /dev/null +++ b/docs/ja/operations/settings/constraints_on_settings.md @@ -0,0 +1 @@ +../../../en/operations/settings/constraints_on_settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/index.md b/docs/ja/operations/settings/index.md new file mode 120000 index 00000000000..fc3968d1f1e --- /dev/null +++ b/docs/ja/operations/settings/index.md @@ -0,0 +1 @@ +../../../en/operations/settings/index.md \ No newline at end of file diff --git a/docs/ja/operations/settings/permissions_for_queries.md b/docs/ja/operations/settings/permissions_for_queries.md new file mode 120000 index 00000000000..ce8473bf01c --- /dev/null +++ b/docs/ja/operations/settings/permissions_for_queries.md @@ -0,0 +1 @@ +../../../en/operations/settings/permissions_for_queries.md \ No newline at end of file diff --git a/docs/ja/operations/settings/query_complexity.md b/docs/ja/operations/settings/query_complexity.md new file mode 120000 index 00000000000..9a9c6d975a9 --- /dev/null +++ b/docs/ja/operations/settings/query_complexity.md @@ -0,0 +1 @@ +../../../en/operations/settings/query_complexity.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings.md b/docs/ja/operations/settings/settings.md new file mode 120000 index 00000000000..0c8df3cfc90 --- /dev/null +++ b/docs/ja/operations/settings/settings.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings_profiles.md b/docs/ja/operations/settings/settings_profiles.md new file mode 120000 index 00000000000..35d9747ad56 --- /dev/null +++ b/docs/ja/operations/settings/settings_profiles.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings_profiles.md \ No newline at end of file diff --git a/docs/ja/operations/settings/settings_users.md b/docs/ja/operations/settings/settings_users.md new file mode 120000 index 00000000000..3a6a7cf6948 --- /dev/null +++ b/docs/ja/operations/settings/settings_users.md @@ -0,0 +1 @@ +../../../en/operations/settings/settings_users.md \ No newline at end of file diff --git a/docs/ja/operations/system_tables.md b/docs/ja/operations/system_tables.md new file mode 120000 index 00000000000..c5701190dca --- /dev/null +++ b/docs/ja/operations/system_tables.md @@ -0,0 +1 @@ +../../en/operations/system_tables.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/aggregatingmergetree.md b/docs/ja/operations/table_engines/aggregatingmergetree.md new file mode 120000 index 00000000000..907a073e0c8 --- /dev/null +++ b/docs/ja/operations/table_engines/aggregatingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/aggregatingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/buffer.md b/docs/ja/operations/table_engines/buffer.md new file mode 120000 index 00000000000..0a3c372fa67 --- /dev/null +++ b/docs/ja/operations/table_engines/buffer.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/buffer.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/collapsingmergetree.md b/docs/ja/operations/table_engines/collapsingmergetree.md new file mode 120000 index 00000000000..ef5cebb48d8 --- /dev/null +++ b/docs/ja/operations/table_engines/collapsingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/collapsingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/custom_partitioning_key.md b/docs/ja/operations/table_engines/custom_partitioning_key.md new file mode 120000 index 00000000000..a9d18cacb25 --- /dev/null +++ b/docs/ja/operations/table_engines/custom_partitioning_key.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/custom_partitioning_key.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/dictionary.md b/docs/ja/operations/table_engines/dictionary.md new file mode 120000 index 00000000000..2a95f4a669b --- /dev/null +++ b/docs/ja/operations/table_engines/dictionary.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/dictionary.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/distributed.md b/docs/ja/operations/table_engines/distributed.md new file mode 120000 index 00000000000..46994303c35 --- /dev/null +++ b/docs/ja/operations/table_engines/distributed.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/distributed.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/external_data.md b/docs/ja/operations/table_engines/external_data.md new file mode 120000 index 00000000000..27a7b6acec2 --- /dev/null +++ b/docs/ja/operations/table_engines/external_data.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/external_data.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/file.md b/docs/ja/operations/table_engines/file.md new file mode 120000 index 00000000000..27dffc8d78f --- /dev/null +++ b/docs/ja/operations/table_engines/file.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/file.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/graphitemergetree.md b/docs/ja/operations/table_engines/graphitemergetree.md new file mode 120000 index 00000000000..654425d050a --- /dev/null +++ b/docs/ja/operations/table_engines/graphitemergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/graphitemergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/hdfs.md b/docs/ja/operations/table_engines/hdfs.md new file mode 120000 index 00000000000..d4dbfa46e68 --- /dev/null +++ b/docs/ja/operations/table_engines/hdfs.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/hdfs.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/index.md b/docs/ja/operations/table_engines/index.md new file mode 120000 index 00000000000..994dff9b516 --- /dev/null +++ b/docs/ja/operations/table_engines/index.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/index.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/jdbc.md b/docs/ja/operations/table_engines/jdbc.md new file mode 120000 index 00000000000..5165d704b9a --- /dev/null +++ b/docs/ja/operations/table_engines/jdbc.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/jdbc.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/join.md b/docs/ja/operations/table_engines/join.md new file mode 120000 index 00000000000..0914ab950ed --- /dev/null +++ b/docs/ja/operations/table_engines/join.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/join.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/kafka.md b/docs/ja/operations/table_engines/kafka.md new file mode 120000 index 00000000000..cb7bd5dd0f8 --- /dev/null +++ b/docs/ja/operations/table_engines/kafka.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/kafka.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/log.md b/docs/ja/operations/table_engines/log.md new file mode 120000 index 00000000000..2c39ba68522 --- /dev/null +++ b/docs/ja/operations/table_engines/log.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/log.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/log_family.md b/docs/ja/operations/table_engines/log_family.md new file mode 120000 index 00000000000..8c5b5f0365b --- /dev/null +++ b/docs/ja/operations/table_engines/log_family.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/log_family.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/materializedview.md b/docs/ja/operations/table_engines/materializedview.md new file mode 120000 index 00000000000..e3b5deb73dc --- /dev/null +++ b/docs/ja/operations/table_engines/materializedview.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/materializedview.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/memory.md b/docs/ja/operations/table_engines/memory.md new file mode 120000 index 00000000000..eee940c7bd3 --- /dev/null +++ b/docs/ja/operations/table_engines/memory.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/memory.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/merge.md b/docs/ja/operations/table_engines/merge.md new file mode 120000 index 00000000000..9e17d9bb939 --- /dev/null +++ b/docs/ja/operations/table_engines/merge.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/merge.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/mergetree.md b/docs/ja/operations/table_engines/mergetree.md new file mode 120000 index 00000000000..cc6ac1e5297 --- /dev/null +++ b/docs/ja/operations/table_engines/mergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/mergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/mysql.md b/docs/ja/operations/table_engines/mysql.md new file mode 120000 index 00000000000..e4c268658cf --- /dev/null +++ b/docs/ja/operations/table_engines/mysql.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/mysql.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/null.md b/docs/ja/operations/table_engines/null.md new file mode 120000 index 00000000000..c7d9264571e --- /dev/null +++ b/docs/ja/operations/table_engines/null.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/null.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/odbc.md b/docs/ja/operations/table_engines/odbc.md new file mode 120000 index 00000000000..06091fd5377 --- /dev/null +++ b/docs/ja/operations/table_engines/odbc.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/odbc.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/replacingmergetree.md b/docs/ja/operations/table_engines/replacingmergetree.md new file mode 120000 index 00000000000..63ff25a4dd6 --- /dev/null +++ b/docs/ja/operations/table_engines/replacingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/replacingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/replication.md b/docs/ja/operations/table_engines/replication.md new file mode 120000 index 00000000000..b4b22ac708b --- /dev/null +++ b/docs/ja/operations/table_engines/replication.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/replication.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/set.md b/docs/ja/operations/table_engines/set.md new file mode 120000 index 00000000000..d37e659badd --- /dev/null +++ b/docs/ja/operations/table_engines/set.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/set.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/stripelog.md b/docs/ja/operations/table_engines/stripelog.md new file mode 120000 index 00000000000..f6521a41e3e --- /dev/null +++ b/docs/ja/operations/table_engines/stripelog.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/stripelog.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/summingmergetree.md b/docs/ja/operations/table_engines/summingmergetree.md new file mode 120000 index 00000000000..2b67e953d8a --- /dev/null +++ b/docs/ja/operations/table_engines/summingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/summingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/tinylog.md b/docs/ja/operations/table_engines/tinylog.md new file mode 120000 index 00000000000..bda90c7d5ce --- /dev/null +++ b/docs/ja/operations/table_engines/tinylog.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/tinylog.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/url.md b/docs/ja/operations/table_engines/url.md new file mode 120000 index 00000000000..d0de71dcf40 --- /dev/null +++ b/docs/ja/operations/table_engines/url.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/url.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/versionedcollapsingmergetree.md b/docs/ja/operations/table_engines/versionedcollapsingmergetree.md new file mode 120000 index 00000000000..5843fba70b8 --- /dev/null +++ b/docs/ja/operations/table_engines/versionedcollapsingmergetree.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/versionedcollapsingmergetree.md \ No newline at end of file diff --git a/docs/ja/operations/table_engines/view.md b/docs/ja/operations/table_engines/view.md new file mode 120000 index 00000000000..3f2164181a7 --- /dev/null +++ b/docs/ja/operations/table_engines/view.md @@ -0,0 +1 @@ +../../../en/operations/table_engines/view.md \ No newline at end of file diff --git a/docs/ja/operations/tips.md b/docs/ja/operations/tips.md new file mode 120000 index 00000000000..9b3413bdbc3 --- /dev/null +++ b/docs/ja/operations/tips.md @@ -0,0 +1 @@ +../../en/operations/tips.md \ No newline at end of file diff --git a/docs/ja/operations/troubleshooting.md b/docs/ja/operations/troubleshooting.md new file mode 120000 index 00000000000..84f0ff34f41 --- /dev/null +++ b/docs/ja/operations/troubleshooting.md @@ -0,0 +1 @@ +../../en/operations/troubleshooting.md \ No newline at end of file diff --git a/docs/ja/operations/update.md b/docs/ja/operations/update.md new file mode 120000 index 00000000000..88a092c0dff --- /dev/null +++ b/docs/ja/operations/update.md @@ -0,0 +1 @@ +../../en/operations/update.md \ No newline at end of file diff --git a/docs/ja/operations/utils/clickhouse-copier.md b/docs/ja/operations/utils/clickhouse-copier.md new file mode 120000 index 00000000000..c9e89e33c7b --- /dev/null +++ b/docs/ja/operations/utils/clickhouse-copier.md @@ -0,0 +1 @@ +../../../en/operations/utils/clickhouse-copier.md \ No newline at end of file diff --git a/docs/ja/operations/utils/clickhouse-local.md b/docs/ja/operations/utils/clickhouse-local.md new file mode 120000 index 00000000000..032aaaa2b84 --- /dev/null +++ b/docs/ja/operations/utils/clickhouse-local.md @@ -0,0 +1 @@ +../../../en/operations/utils/clickhouse-local.md \ No newline at end of file diff --git a/docs/ja/operations/utils/index.md b/docs/ja/operations/utils/index.md new file mode 120000 index 00000000000..dd089d1ef4b --- /dev/null +++ b/docs/ja/operations/utils/index.md @@ -0,0 +1 @@ +../../../en/operations/utils/index.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/combinators.md b/docs/ja/query_language/agg_functions/combinators.md new file mode 120000 index 00000000000..2b914cebd15 --- /dev/null +++ b/docs/ja/query_language/agg_functions/combinators.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/combinators.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/index.md b/docs/ja/query_language/agg_functions/index.md new file mode 120000 index 00000000000..2fcf67abdeb --- /dev/null +++ b/docs/ja/query_language/agg_functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/parametric_functions.md b/docs/ja/query_language/agg_functions/parametric_functions.md new file mode 120000 index 00000000000..fd3ffafcc5b --- /dev/null +++ b/docs/ja/query_language/agg_functions/parametric_functions.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/parametric_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/agg_functions/reference.md b/docs/ja/query_language/agg_functions/reference.md new file mode 120000 index 00000000000..c5651cb0793 --- /dev/null +++ b/docs/ja/query_language/agg_functions/reference.md @@ -0,0 +1 @@ +../../../en/query_language/agg_functions/reference.md \ No newline at end of file diff --git a/docs/ja/query_language/alter.md b/docs/ja/query_language/alter.md new file mode 120000 index 00000000000..44f4ecf9737 --- /dev/null +++ b/docs/ja/query_language/alter.md @@ -0,0 +1 @@ +../../en/query_language/alter.md \ No newline at end of file diff --git a/docs/ja/query_language/create.md b/docs/ja/query_language/create.md new file mode 120000 index 00000000000..a13304d176e --- /dev/null +++ b/docs/ja/query_language/create.md @@ -0,0 +1 @@ +../../en/query_language/create.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts.md b/docs/ja/query_language/dicts/external_dicts.md new file mode 120000 index 00000000000..491b94bffe6 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict.md b/docs/ja/query_language/dicts/external_dicts_dict.md new file mode 120000 index 00000000000..e27820fee60 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_layout.md b/docs/ja/query_language/dicts/external_dicts_dict_layout.md new file mode 120000 index 00000000000..e391c5be723 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_layout.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_layout.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md b/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md new file mode 120000 index 00000000000..03b53c09077 --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_lifetime.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_lifetime.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_sources.md b/docs/ja/query_language/dicts/external_dicts_dict_sources.md new file mode 120000 index 00000000000..d4f4bf8ef3e --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_sources.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_sources.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/external_dicts_dict_structure.md b/docs/ja/query_language/dicts/external_dicts_dict_structure.md new file mode 120000 index 00000000000..69ff759caea --- /dev/null +++ b/docs/ja/query_language/dicts/external_dicts_dict_structure.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/external_dicts_dict_structure.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/index.md b/docs/ja/query_language/dicts/index.md new file mode 120000 index 00000000000..fdc188ca2a2 --- /dev/null +++ b/docs/ja/query_language/dicts/index.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/index.md \ No newline at end of file diff --git a/docs/ja/query_language/dicts/internal_dicts.md b/docs/ja/query_language/dicts/internal_dicts.md new file mode 120000 index 00000000000..3f9408dcd45 --- /dev/null +++ b/docs/ja/query_language/dicts/internal_dicts.md @@ -0,0 +1 @@ +../../../en/query_language/dicts/internal_dicts.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/arithmetic_functions.md b/docs/ja/query_language/functions/arithmetic_functions.md new file mode 120000 index 00000000000..c22acb8c7f5 --- /dev/null +++ b/docs/ja/query_language/functions/arithmetic_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/arithmetic_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/array_functions.md b/docs/ja/query_language/functions/array_functions.md new file mode 120000 index 00000000000..268b2295a97 --- /dev/null +++ b/docs/ja/query_language/functions/array_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/array_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/array_join.md b/docs/ja/query_language/functions/array_join.md new file mode 120000 index 00000000000..b100dac784d --- /dev/null +++ b/docs/ja/query_language/functions/array_join.md @@ -0,0 +1 @@ +../../../en/query_language/functions/array_join.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/bit_functions.md b/docs/ja/query_language/functions/bit_functions.md new file mode 120000 index 00000000000..b5cccd0c56c --- /dev/null +++ b/docs/ja/query_language/functions/bit_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bit_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/bitmap_functions.md b/docs/ja/query_language/functions/bitmap_functions.md new file mode 120000 index 00000000000..0a31d3d71d8 --- /dev/null +++ b/docs/ja/query_language/functions/bitmap_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/bitmap_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/comparison_functions.md b/docs/ja/query_language/functions/comparison_functions.md new file mode 120000 index 00000000000..417c589867c --- /dev/null +++ b/docs/ja/query_language/functions/comparison_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/comparison_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/conditional_functions.md b/docs/ja/query_language/functions/conditional_functions.md new file mode 120000 index 00000000000..ad0d775dbb5 --- /dev/null +++ b/docs/ja/query_language/functions/conditional_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/conditional_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/date_time_functions.md b/docs/ja/query_language/functions/date_time_functions.md new file mode 120000 index 00000000000..d11b9b8bb6b --- /dev/null +++ b/docs/ja/query_language/functions/date_time_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/date_time_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/encoding_functions.md b/docs/ja/query_language/functions/encoding_functions.md new file mode 120000 index 00000000000..b2e6be1405b --- /dev/null +++ b/docs/ja/query_language/functions/encoding_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/encoding_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ext_dict_functions.md b/docs/ja/query_language/functions/ext_dict_functions.md new file mode 120000 index 00000000000..6318f900e4b --- /dev/null +++ b/docs/ja/query_language/functions/ext_dict_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ext_dict_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/functions_for_nulls.md b/docs/ja/query_language/functions/functions_for_nulls.md new file mode 120000 index 00000000000..fa57e10ad15 --- /dev/null +++ b/docs/ja/query_language/functions/functions_for_nulls.md @@ -0,0 +1 @@ +../../../en/query_language/functions/functions_for_nulls.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/geo.md b/docs/ja/query_language/functions/geo.md new file mode 120000 index 00000000000..86fa3a85d34 --- /dev/null +++ b/docs/ja/query_language/functions/geo.md @@ -0,0 +1 @@ +../../../en/query_language/functions/geo.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/hash_functions.md b/docs/ja/query_language/functions/hash_functions.md new file mode 120000 index 00000000000..90de8ba97e7 --- /dev/null +++ b/docs/ja/query_language/functions/hash_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/hash_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/higher_order_functions.md b/docs/ja/query_language/functions/higher_order_functions.md new file mode 120000 index 00000000000..077feba2a3e --- /dev/null +++ b/docs/ja/query_language/functions/higher_order_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/higher_order_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/in_functions.md b/docs/ja/query_language/functions/in_functions.md new file mode 120000 index 00000000000..3ae5f24dbca --- /dev/null +++ b/docs/ja/query_language/functions/in_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/in_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/index.md b/docs/ja/query_language/functions/index.md new file mode 120000 index 00000000000..a4e9d619cc0 --- /dev/null +++ b/docs/ja/query_language/functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ip_address_functions.md b/docs/ja/query_language/functions/ip_address_functions.md new file mode 120000 index 00000000000..b58175a7cdf --- /dev/null +++ b/docs/ja/query_language/functions/ip_address_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ip_address_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/json_functions.md b/docs/ja/query_language/functions/json_functions.md new file mode 120000 index 00000000000..1b37184e006 --- /dev/null +++ b/docs/ja/query_language/functions/json_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/json_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/logical_functions.md b/docs/ja/query_language/functions/logical_functions.md new file mode 120000 index 00000000000..32015440e09 --- /dev/null +++ b/docs/ja/query_language/functions/logical_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/logical_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/machine_learning_functions.md b/docs/ja/query_language/functions/machine_learning_functions.md new file mode 120000 index 00000000000..4509602717e --- /dev/null +++ b/docs/ja/query_language/functions/machine_learning_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/machine_learning_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/math_functions.md b/docs/ja/query_language/functions/math_functions.md new file mode 120000 index 00000000000..e01674eca4d --- /dev/null +++ b/docs/ja/query_language/functions/math_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/math_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/other_functions.md b/docs/ja/query_language/functions/other_functions.md new file mode 120000 index 00000000000..65164784ced --- /dev/null +++ b/docs/ja/query_language/functions/other_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/other_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/random_functions.md b/docs/ja/query_language/functions/random_functions.md new file mode 120000 index 00000000000..b873e0c86ac --- /dev/null +++ b/docs/ja/query_language/functions/random_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/random_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/rounding_functions.md b/docs/ja/query_language/functions/rounding_functions.md new file mode 120000 index 00000000000..e1217e3b25a --- /dev/null +++ b/docs/ja/query_language/functions/rounding_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/rounding_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/splitting_merging_functions.md b/docs/ja/query_language/functions/splitting_merging_functions.md new file mode 120000 index 00000000000..5f8771abdec --- /dev/null +++ b/docs/ja/query_language/functions/splitting_merging_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/splitting_merging_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_functions.md b/docs/ja/query_language/functions/string_functions.md new file mode 120000 index 00000000000..cc4104aaf53 --- /dev/null +++ b/docs/ja/query_language/functions/string_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_replace_functions.md b/docs/ja/query_language/functions/string_replace_functions.md new file mode 120000 index 00000000000..4ec963ffd0f --- /dev/null +++ b/docs/ja/query_language/functions/string_replace_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_replace_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/string_search_functions.md b/docs/ja/query_language/functions/string_search_functions.md new file mode 120000 index 00000000000..0a2c7f4c4f1 --- /dev/null +++ b/docs/ja/query_language/functions/string_search_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/string_search_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/type_conversion_functions.md b/docs/ja/query_language/functions/type_conversion_functions.md new file mode 120000 index 00000000000..fcf51570d15 --- /dev/null +++ b/docs/ja/query_language/functions/type_conversion_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/type_conversion_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/url_functions.md b/docs/ja/query_language/functions/url_functions.md new file mode 120000 index 00000000000..529e4ffdd53 --- /dev/null +++ b/docs/ja/query_language/functions/url_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/url_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/uuid_functions.md b/docs/ja/query_language/functions/uuid_functions.md new file mode 120000 index 00000000000..95e3ded0477 --- /dev/null +++ b/docs/ja/query_language/functions/uuid_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/uuid_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/functions/ym_dict_functions.md b/docs/ja/query_language/functions/ym_dict_functions.md new file mode 120000 index 00000000000..ec5ddc84479 --- /dev/null +++ b/docs/ja/query_language/functions/ym_dict_functions.md @@ -0,0 +1 @@ +../../../en/query_language/functions/ym_dict_functions.md \ No newline at end of file diff --git a/docs/ja/query_language/index.md b/docs/ja/query_language/index.md new file mode 120000 index 00000000000..44dfff9bb18 --- /dev/null +++ b/docs/ja/query_language/index.md @@ -0,0 +1 @@ +../../en/query_language/index.md \ No newline at end of file diff --git a/docs/ja/query_language/insert_into.md b/docs/ja/query_language/insert_into.md new file mode 120000 index 00000000000..29b47662b0d --- /dev/null +++ b/docs/ja/query_language/insert_into.md @@ -0,0 +1 @@ +../../en/query_language/insert_into.md \ No newline at end of file diff --git a/docs/ja/query_language/misc.md b/docs/ja/query_language/misc.md new file mode 120000 index 00000000000..3bd814f3568 --- /dev/null +++ b/docs/ja/query_language/misc.md @@ -0,0 +1 @@ +../../en/query_language/misc.md \ No newline at end of file diff --git a/docs/ja/query_language/operators.md b/docs/ja/query_language/operators.md new file mode 120000 index 00000000000..f94df928a82 --- /dev/null +++ b/docs/ja/query_language/operators.md @@ -0,0 +1 @@ +../../en/query_language/operators.md \ No newline at end of file diff --git a/docs/ja/query_language/select.md b/docs/ja/query_language/select.md new file mode 120000 index 00000000000..c8ec8369383 --- /dev/null +++ b/docs/ja/query_language/select.md @@ -0,0 +1 @@ +../../en/query_language/select.md \ No newline at end of file diff --git a/docs/ja/query_language/show.md b/docs/ja/query_language/show.md new file mode 120000 index 00000000000..4c2f4cf2c4f --- /dev/null +++ b/docs/ja/query_language/show.md @@ -0,0 +1 @@ +../../en/query_language/show.md \ No newline at end of file diff --git a/docs/ja/query_language/syntax.md b/docs/ja/query_language/syntax.md new file mode 120000 index 00000000000..5307fd51ae8 --- /dev/null +++ b/docs/ja/query_language/syntax.md @@ -0,0 +1 @@ +../../en/query_language/syntax.md \ No newline at end of file diff --git a/docs/ja/query_language/system.md b/docs/ja/query_language/system.md new file mode 120000 index 00000000000..6061858c3f2 --- /dev/null +++ b/docs/ja/query_language/system.md @@ -0,0 +1 @@ +../../en/query_language/system.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/file.md b/docs/ja/query_language/table_functions/file.md new file mode 120000 index 00000000000..a514547109a --- /dev/null +++ b/docs/ja/query_language/table_functions/file.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/file.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/hdfs.md b/docs/ja/query_language/table_functions/hdfs.md new file mode 120000 index 00000000000..2616e737eb6 --- /dev/null +++ b/docs/ja/query_language/table_functions/hdfs.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/hdfs.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/index.md b/docs/ja/query_language/table_functions/index.md new file mode 120000 index 00000000000..89b22522859 --- /dev/null +++ b/docs/ja/query_language/table_functions/index.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/index.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/input.md b/docs/ja/query_language/table_functions/input.md new file mode 120000 index 00000000000..f23cc8ee673 --- /dev/null +++ b/docs/ja/query_language/table_functions/input.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/input.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/jdbc.md b/docs/ja/query_language/table_functions/jdbc.md new file mode 120000 index 00000000000..73bec80ca58 --- /dev/null +++ b/docs/ja/query_language/table_functions/jdbc.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/jdbc.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/merge.md b/docs/ja/query_language/table_functions/merge.md new file mode 120000 index 00000000000..383f6c88331 --- /dev/null +++ b/docs/ja/query_language/table_functions/merge.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/merge.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/mysql.md b/docs/ja/query_language/table_functions/mysql.md new file mode 120000 index 00000000000..75c032cc63f --- /dev/null +++ b/docs/ja/query_language/table_functions/mysql.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/mysql.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/numbers.md b/docs/ja/query_language/table_functions/numbers.md new file mode 120000 index 00000000000..a679b915669 --- /dev/null +++ b/docs/ja/query_language/table_functions/numbers.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/numbers.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/odbc.md b/docs/ja/query_language/table_functions/odbc.md new file mode 120000 index 00000000000..7620f920494 --- /dev/null +++ b/docs/ja/query_language/table_functions/odbc.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/odbc.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/remote.md b/docs/ja/query_language/table_functions/remote.md new file mode 120000 index 00000000000..b157c4076d3 --- /dev/null +++ b/docs/ja/query_language/table_functions/remote.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/remote.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/url.md b/docs/ja/query_language/table_functions/url.md new file mode 120000 index 00000000000..038e08f7ba9 --- /dev/null +++ b/docs/ja/query_language/table_functions/url.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/url.md \ No newline at end of file diff --git a/docs/ja/roadmap.md b/docs/ja/roadmap.md new file mode 120000 index 00000000000..24df86352b3 --- /dev/null +++ b/docs/ja/roadmap.md @@ -0,0 +1 @@ +../en/roadmap.md \ No newline at end of file diff --git a/docs/ja/security_changelog.md b/docs/ja/security_changelog.md new file mode 120000 index 00000000000..101a4f4e48c --- /dev/null +++ b/docs/ja/security_changelog.md @@ -0,0 +1 @@ +../en/security_changelog.md \ No newline at end of file diff --git a/docs/ru/getting_started/install.md b/docs/ru/getting_started/install.md index fe8b3b190f1..29ccd2b14f4 100644 --- a/docs/ru/getting_started/install.md +++ b/docs/ru/getting_started/install.md @@ -14,7 +14,7 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not ## Доступные варианты установки -### Из DEB пакетов +### Из DEB пакетов {#from-deb-packages} Яндекс рекомендует использовать официальные скомпилированные `deb` пакеты для Debian или Ubuntu. @@ -37,7 +37,7 @@ sudo apt-get install clickhouse-client clickhouse-server Также эти пакеты можно скачать и установить вручную отсюда: . -### Из RPM пакетов +### Из RPM пакетов {#from-rpm-packages} Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHad и всех остальных дистрибутивов Linux, основанных на rpm. @@ -59,7 +59,7 @@ sudo yum install clickhouse-server clickhouse-client Также есть возможность установить пакеты вручную, скачав отсюда: . -### Из Docker образа +### Из Docker образа {#from-docker-image} Для запуска ClickHouse в Docker нужно следовать инструкции на [Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/). Внутри образов используются официальные `deb` пакеты. diff --git a/docs/ru/getting_started/tutorial.md b/docs/ru/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/ru/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 02837bc3e97..d96963953fe 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -217,7 +217,8 @@ nav: - 'Overview of ClickHouse Architecture': 'development/architecture.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross.md' + - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' + - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index 3c4a5506a06..d96963953fe 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -5,10 +5,12 @@ nav: - 'Distinctive Features of ClickHouse': 'introduction/distinctive_features.md' - 'ClickHouse Features that Can Be Considered Disadvantages': 'introduction/features_considered_disadvantages.md' - 'Performance': 'introduction/performance.md' - - 'The Yandex.Metrica Task': 'introduction/ya_metrika_task.md' + - 'History': 'introduction/history.md' - 'Getting Started': - - 'Deploying and Running': 'getting_started/index.md' + - 'hidden': 'getting_started/index.md' + - 'Installation': 'getting_started/install.md' + - 'Tutorial': 'getting_started/tutorial.md' - 'Example Datasets': - 'OnTime': 'getting_started/example_datasets/ontime.md' - 'New York Taxi Data': 'getting_started/example_datasets/nyc_taxi.md' @@ -215,8 +217,9 @@ nav: - 'Overview of ClickHouse Architecture': 'development/architecture.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross.md' - - 'How to Write C++ code': 'development/style.md' + - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' + - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' + - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 931518e7f06..9e4b948cc02 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -9,7 +9,7 @@ nav: - 'Информационная поддержка': 'introduction/info.md' - 'Начало работы': - - 'hidden': 'getting_started/index.html' + - 'hidden': 'getting_started/index.md' - 'Установка': 'getting_started/install.md' - 'Руководство для начинающих': 'getting_started/tutorial.md' - 'Тестовые наборы данных': @@ -216,7 +216,7 @@ nav: - 'Обзор архитектуры ClickHouse': 'development/architecture.md' - 'Как собрать ClickHouse на Linux': 'development/build.md' - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross.md' + - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross_osx.md' - 'Как писать код на C++': 'development/style.md' - 'Как запустить тесты': 'development/tests.md' - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index 03062fa76df..3f19fe3f72b 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -18,6 +18,7 @@ nav: - '维基访问数据': 'getting_started/example_datasets/wikistat.md' - 'Criteo TB级别点击日志': 'getting_started/example_datasets/criteo.md' - 'Star Schema基准测试': 'getting_started/example_datasets/star_schema.md' + - 'Yandex.Metrica': 'getting_started/example_datasets/metrica.md' - '客户端': - '介绍': 'interfaces/index.md' @@ -214,7 +215,7 @@ nav: - 'ClickHouse架构概述': 'development/architecture.md' - '如何在Linux中编译ClickHouse': 'development/build.md' - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' - - '如何在Linux中编译Mac OS X ClickHouse': 'development/build_cross.md' + - '如何在Linux中编译Mac OS X ClickHouse': 'development/build_cross_osx.md' - '如何编写C++代码': 'development/style.md' - '如何运行ClickHouse测试': 'development/tests.md' - '开发者指南': 'development/developer_instruction.md' diff --git a/docs/tools/make_links.sh b/docs/tools/make_links.sh index 084f8b9d97b..04c51424ec8 100755 --- a/docs/tools/make_links.sh +++ b/docs/tools/make_links.sh @@ -11,7 +11,7 @@ function do_make_links() for lang in "${langs[@]}" do # replacing "/./" with / - dst_file="../${lang}/${src_file}" + dst_file="../${lang}${src_file}" dst_file="${dst_file/\/\.\//\/}" mkdir -p $(dirname "${dst_file}") diff --git a/docs/tools/mkdocs-material-theme/assets/flags/ja.svg b/docs/tools/mkdocs-material-theme/assets/flags/ja.svg index 177d0e78819..a666c272523 100644 --- a/docs/tools/mkdocs-material-theme/assets/flags/ja.svg +++ b/docs/tools/mkdocs-material-theme/assets/flags/ja.svg @@ -1,5 +1,8 @@ - - - - + + + + + + + diff --git a/docs/tools/mkdocs-material-theme/partials/language/ja.html b/docs/tools/mkdocs-material-theme/partials/language/ja.html index 09c3b291fed..47341ab06ee 100644 --- a/docs/tools/mkdocs-material-theme/partials/language/ja.html +++ b/docs/tools/mkdocs-material-theme/partials/language/ja.html @@ -7,6 +7,12 @@ "footer.next": "次", "meta.comments": "コメント", "meta.source": "ソース", + "nav.latest": "master", + "nav.multi_page": "マルチページ版", + "nav.pdf": "PDF版", + "nav.release": "リリース", + "nav.single_page": "シングルページ版", + "nav.source": "ClickHouseソースコード", "search.placeholder": "検索", "search.result.placeholder": "検索キーワードを入力してください", "search.result.none": "何も見つかりませんでした", diff --git a/docs/zh/getting_started/example_datasets/metrica.md b/docs/zh/getting_started/example_datasets/metrica.md new file mode 120000 index 00000000000..984023973eb --- /dev/null +++ b/docs/zh/getting_started/example_datasets/metrica.md @@ -0,0 +1 @@ +../../../en/getting_started/example_datasets/metrica.md \ No newline at end of file diff --git a/docs/zh/getting_started/install.md b/docs/zh/getting_started/install.md index 91ec0ea6c94..aa3cb816218 100644 --- a/docs/zh/getting_started/install.md +++ b/docs/zh/getting_started/install.md @@ -12,7 +12,7 @@ $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not ##可用的安装选项 -### 为Debian/Ubuntu安装 +### 为Debian/Ubuntu安装 {#from-deb-packages} 在`/etc/apt/sources.list` (或创建`/etc/apt/sources.list.d/clickhouse.list`文件)中添加仓库： @@ -37,7 +37,7 @@ ClickHouse包含访问控制配置，它们位于`users.xml`文件中(与'config 默认情况下，允许从任何地方使用默认的‘default’用户无密码的访问ClickHouse。参考‘user/default/networks’。 有关更多信息，请参考"Configuration files"部分。 -###来自RPM包 +###来自RPM包 {#from-rpm-packages} Yandex ClickHouse团队建议使用官方预编译的`rpm`软件包，用于CentOS，RedHat和所有其他基于rpm的Linux发行版。 @@ -59,7 +59,7 @@ sudo yum install clickhouse-server clickhouse-client 您也可以从此处手动下载和安装软件包：。 -###来自Docker +###来自Docker {#from-docker-image} 要在Docker中运行ClickHouse，请遵循[Docker Hub](https://hub.docker.com/r/yandex/clickhouse-server/)上的指南。那些图像使用官方的`deb`包。 diff --git a/docs/zh/getting_started/tutorial.md b/docs/zh/getting_started/tutorial.md new file mode 120000 index 00000000000..8bc40816ab2 --- /dev/null +++ b/docs/zh/getting_started/tutorial.md @@ -0,0 +1 @@ +../../en/getting_started/tutorial.md \ No newline at end of file From 64ddef8724392670e00dd4b00b43f0a9d646d317 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 3 Dec 2019 19:40:47 +0300 Subject: [PATCH 015/312] partially revert c136bee4ce9cfbec249aa1d729b5f88d34b90d2f --- docs/en/index.md | 108 +++++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 59 deletions(-) diff --git a/docs/en/index.md b/docs/en/index.md index 6dea5f6570b..40158f524ec 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -1,8 +1,8 @@ -# ClickHouseとは? +# What is ClickHouse? -ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向のデータベース管理システム（DBMS）です。 +ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP). -「通常の」行指向のDBMSでは、データは次の順序で保存されます。 +In a "normal" row-oriented DBMS, data is stored in this order: | Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | | ------ | ------------------- | ---------- | ------------------ | --------- | ------------------- | @@ -11,12 +11,12 @@ ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向 | #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | | #N | ... | ... | ... | ... | ... | -つまり、行に関連するすべての値は物理的に隣り合わせに格納されます。 +In other words, all the values related to a row are physically stored next to each other. -行指向のDBMSの例：MySQL, Postgres および MS SQL Server +Examples of a row-oriented DBMS are MySQL, Postgres, and MS SQL Server. {: .grey } -列指向のDBMSでは、データは次のように保存されます： +In a column-oriented DBMS, data is stored like this: | Row: | #0 | #1 | #2 | #N | | ----------- | ------------------- | ------------------- | ------------------- | ------------------- | @@ -26,74 +26,68 @@ ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向 | GoodEvent: | 1 | 1 | 1 | ... | | EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | ... | -これらの例は、データが配置される順序のみを示しています。 -異なる列の値は別々に保存され、同じ列のデータは一緒に保存されます。 +These examples only show the order that data is arranged in. +The values from different columns are stored separately, and data from the same column is stored together. -列指向DBMSの例：Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid および kdb+ +Examples of a column-oriented DBMS: Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, and kdb+. {: .grey } -異なったデータ格納の順序は、異なったシナリオにより適します。 -データアクセスシナリオとは、クエリの実行内容、頻度、割合を指します。クエリで読み取られるの各種データの量（行、列、バイト）。データの読み取りと更新の関係。作業データのサイズとローカルでの使用方法。トランザクションが使用されるかどうか、およびそれらがどの程度分離されているか。データ複製と論理的整合性の要件。クエリの種類ごとの遅延とスループットの要件など。 +Different orders for storing data are better suited to different scenarios. +The data access scenario refers to what queries are made, how often, and in what proportion; how much data is read for each type of query – rows, columns, and bytes; the relationship between reading and updating data; the working size of the data and how locally it is used; whether transactions are used, and how isolated they are; requirements for data replication and logical integrity; requirements for latency and throughput for each type of query, and so on. -システムの負荷が高いほど、使用シナリオの要件に一致するようにセットアップされたシステムをカスタマイズすることがより重要になり、このカスタマイズはより細かくなります。大きく異なるシナリオに等しく適したシステムはありません。システムがさまざまなシナリオに適応可能である場合、高負荷下では、システムはすべてのシナリオを同等に不十分に処理するか、1つまたはいくつかの可能なシナリオでうまく機能します。 +The higher the load on the system, the more important it is to customize the system set up to match the requirements of the usage scenario, and the more fine grained this customization becomes. There is no system that is equally well-suited to significantly different scenarios. If a system is adaptable to a wide set of scenarios, under a high load, the system will handle all the scenarios equally poorly, or will work well for just one or few of possible scenarios. -## OLAPシナリオの主要なプロパティ +## Key Properties of the OLAP scenario -- リクエストの大部分は読み取りアクセス用である。 -- データは、単一行ではなく、かなり大きなバッチ（> 1000行）で更新されます。または、まったく更新されない。 -- データはDBに追加されるが、変更されない。 -- 読み取りの場合、非常に多くの行がDBから抽出されるが、一部の列のみ。 -- テーブルは「幅が広く」、多数の列が含まれる。 -- クエリは比較的まれ（通常、サーバーあたり毎秒数百あるいはそれ以下の数のクエリ）。 -- 単純なクエリでは、約50ミリ秒の遅延が容認される。 -- 列の値はかなり小さく、数値や短い文字列（たとえば、URLごとに60バイト）。 -- 単一のクエリを処理する場合、高いスループットが必要（サーバーあたり毎秒最大数十億行）。 -- トランザクションは必要ない。 -- データの一貫性の要件が低い。 -- クエリごとに1つの大きなテーブルがある。 1つを除くすべてのテーブルは小さい。 -- クエリ結果は、ソースデータよりも大幅に小さくなる。つまり、データはフィルター処理または集計されるため、結果は単一サーバーのRAMに収まる。 +- The vast majority of requests are for read access. +- Data is updated in fairly large batches (> 1000 rows), not by single rows; or it is not updated at all. +- Data is added to the DB but is not modified. +- For reads, quite a large number of rows are extracted from the DB, but only a small subset of columns. +- Tables are "wide," meaning they contain a large number of columns. +- Queries are relatively rare (usually hundreds of queries per server or less per second). +- For simple queries, latencies around 50 ms are allowed. +- Column values are fairly small: numbers and short strings (for example, 60 bytes per URL). +- Requires high throughput when processing a single query (up to billions of rows per second per server). +- Transactions are not necessary. +- Low requirements for data consistency. +- There is one large table per query. All tables are small, except for one. +- A query result is significantly smaller than the source data. In other words, data is filtered or aggregated, so the result fits in a single server's RAM. -OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセスなど）とは非常に異なることが容易にわかります。 したがって、まともなパフォーマンスを得るには、OLTPまたはKey-Value DBを使用して分析クエリを処理しようとするのは無意味です。 たとえば、分析にMongoDBまたはRedisを使用しようとすると、OLAPデータベースに比べてパフォーマンスが非常に低下します。 +It is easy to see that the OLAP scenario is very different from other popular scenarios (such as OLTP or Key-Value access). So it doesn't make sense to try to use OLTP or a Key-Value DB for processing analytical queries if you want to get decent performance. For example, if you try to use MongoDB or Redis for analytics, you will get very poor performance compared to OLAP databases. -## OLAPシナリオで列指向データベースがよりよく機能する理由 +## Why Column-Oriented Databases Work Better in the OLAP Scenario -列指向データベースは、OLAPシナリオにより適しています。ほとんどのクエリの処理が少なくとも100倍高速です。 理由を以下に詳しく説明しますが、その根拠は視覚的に簡単に説明できます： +Column-oriented databases are better suited to OLAP scenarios: they are at least 100 times faster in processing most queries. The reasons are explained in detail below, but the fact is easier to demonstrate visually: -**行指向DBMS** +**Row-oriented DBMS** ![Row-oriented](images/row_oriented.gif#) -**列指向DBMS** +**Column-oriented DBMS** ![Column-oriented](images/column_oriented.gif#) -違いがわかりましたか？ +See the difference? ### Input/output -1. 分析クエリでは、少数のテーブル列のみを読み取る必要があります。列指向のデータベースでは、必要なデータのみを読み取ることができます。たとえば、100のうち5つの列が必要な場合、I/Oが20倍削減されることが期待できます。 -2. データはパケットで読み取られるため、圧縮が容易です。列のデータも圧縮が簡単です。これにより、I/Oボリュームがさらに削減されます。 -3. I/Oの削減により、より多くのデータがシステムキャッシュに収まります。 +1. For an analytical query, only a small number of table columns need to be read. In a column-oriented database, you can read just the data you need. For example, if you need 5 columns out of 100, you can expect a 20-fold reduction in I/O. +2. Since data is read in packets, it is easier to compress. Data in columns is also easier to compress. This further reduces the I/O volume. +3. Due to the reduced I/O, more data fits in the system cache. -たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。 +For example, the query "count the number of records for each advertising platform" requires reading one "advertising platform ID" column, which takes up 1 byte uncompressed. If most of the traffic was not from advertising platforms, you can expect at least 10-fold compression of this column. When using a quick compression algorithm, data decompression is possible at a speed of at least several gigabytes of uncompressed data per second. In other words, this query can be processed at a speed of approximately several billion rows per second on a single server. This speed is actually achieved in practice.
Example -``` +```bash $ clickhouse-client ClickHouse client version 0.0.52053. Connecting to localhost:9000. Connected to ClickHouse server version 0.0.52053. - -:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - -SELECT -CounterID, -count() -FROM hits -GROUP BY CounterID -ORDER BY count() DESC -LIMIT 20 - +``` +```sql +SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 +``` +```text ┌─CounterID─┬──count()─┐ │ 114208 │ 56057344 │ │ 115080 │ 51619590 │ @@ -116,27 +110,23 @@ LIMIT 20 │ 115079 │ 8837972 │ │ 337234 │ 8205961 │ └───────────┴──────────┘ - -20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - -:) ```
### CPU -クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 -データを列に格納し、可能な場合は列ごとに処理することは理にかなっています。 +Since executing a query requires processing a large number of rows, it helps to dispatch all operations for entire vectors instead of for separate rows, or to implement the query engine so that there is almost no dispatching cost. If you don't do this, with any half-decent disk subsystem, the query interpreter inevitably stalls the CPU. +It makes sense to both store data in columns and process it, when possible, by columns. -これを行うには2つの方法があります: +There are two ways to do this: -1. ベクトルエンジン。 すべての操作は、個別の値ではなく、ベクトルに対して記述されます。 これは、オペレーションを頻繁に呼び出す必要がなく、ディスパッチコストが無視できることを意味します。 操作コードには、最適化された内部サイクルが含まれています。 +1. A vector engine. All operations are written for vectors, instead of for separate values. This means you don't need to call operations very often, and dispatching costs are negligible. Operation code contains an optimized internal cycle. -2. コード生成。 クエリ用に生成されたコードには、すべての間接的な呼び出しが含まれています。 +2. Code generation. The code generated for the query has all the indirect calls in it. -これは、単純なクエリを実行する場合には意味がないため、「通常の」データベースでは実行されません。 ただし、例外があります。 たとえば、MemSQLはコード生成を使用して、SQLクエリを処理する際の遅延を減らします。 （比較のために、分析DBMSではレイテンシではなくスループットの最適化が必要です。） +This is not done in "normal" databases, because it doesn't make sense when running simple queries. However, there are exceptions. For example, MemSQL uses code generation to reduce latency when processing SQL queries. (For comparison, analytical DBMSs require optimization of throughput, not latency.) -CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。 +Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization. [Original article](https://clickhouse.yandex/docs/en/) From 89a53b1103873a18fb9595dcb94bca08a25a94eb Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 3 Dec 2019 19:57:51 +0300 Subject: [PATCH 016/312] try to fix docs build in CI --- docs/ja/development/tests/developer_instruction_ru.md | 1 - 1 file changed, 1 deletion(-) delete mode 120000 docs/ja/development/tests/developer_instruction_ru.md diff --git a/docs/ja/development/tests/developer_instruction_ru.md b/docs/ja/development/tests/developer_instruction_ru.md deleted file mode 120000 index c053faed45e..00000000000 --- a/docs/ja/development/tests/developer_instruction_ru.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/development/tests/developer_instruction_ru.md \ No newline at end of file From d9b70b766f6f6ce9da910221a5f162c9f4dae9b1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 4 Dec 2019 09:36:47 +0300 Subject: [PATCH 017/312] try to fix docs build in CI --- docs/ja/development/tests/easy_tasks_sorted_ru.md | 1 - docs/ja/development/tests/sanitizers.md | 1 - 2 files changed, 2 deletions(-) delete mode 120000 docs/ja/development/tests/easy_tasks_sorted_ru.md delete mode 120000 docs/ja/development/tests/sanitizers.md diff --git a/docs/ja/development/tests/easy_tasks_sorted_ru.md b/docs/ja/development/tests/easy_tasks_sorted_ru.md deleted file mode 120000 index fb8630a5e8b..00000000000 --- a/docs/ja/development/tests/easy_tasks_sorted_ru.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/development/tests/easy_tasks_sorted_ru.md \ No newline at end of file diff --git a/docs/ja/development/tests/sanitizers.md b/docs/ja/development/tests/sanitizers.md deleted file mode 120000 index c71a3d7aa25..00000000000 --- a/docs/ja/development/tests/sanitizers.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/development/tests/sanitizers.md \ No newline at end of file From 3ce1d7f397cd80574ff6ff973083c44f35c90638 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 4 Dec 2019 17:47:31 +0300 Subject: [PATCH 018/312] few minor improvements --- docs/en/getting_started/install.md | 6 +++--- docs/en/getting_started/tutorial.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/getting_started/install.md b/docs/en/getting_started/install.md index 6bb7a17d340..e47500fa22f 100644 --- a/docs/en/getting_started/install.md +++ b/docs/en/getting_started/install.md @@ -4,7 +4,7 @@ ClickHouse can run on any Linux, FreeBSD or Mac OS X with x86\_64, AArch64 or PowerPC64LE CPU architecture. -Though pre-built binaries are typically compiled for x86\_64 and leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: +Official pre-built binaries are typically compiled for x86\_64 and leverage SSE 4.2 instruction set, so unless otherwise stated usage of CPU that supports it becomes an additional system requirement. Here's the command to check if current CPU has support for SSE 4.2: ``` bash $ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not supported" @@ -16,7 +16,7 @@ To run ClickHouse on processors that do not support SSE 4.2 or have AArch64 or P ### From DEB Packages -Yandex ClickHouse team recommends using official pre-compiled `deb` packages for Debian or Ubuntu. +It is recommended to use official pre-compiled `deb` packages for Debian or Ubuntu. To install official packages add the Yandex repository in `/etc/apt/sources.list` or in a separate `/etc/apt/sources.list.d/clickhouse.list` file: @@ -39,7 +39,7 @@ You can also download and install packages manually from here: ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure automatically. It's recommended to deploy ZooKeeper cluster to separate servers. -ZooKeeper is not a requirement — in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is not recommended — in this case ClickHouse is not able to +ZooKeeper is not a requirement: in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is not recommended, in this case ClickHouse is not able to guarantee data consistency on all replicas. This remains the responsibility of your application.
Specify ZooKeeper locations in configuration file From 533d09bfc6187d9633c827616746f75e17ddb572 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 4 Dec 2019 18:34:39 +0300 Subject: [PATCH 019/312] Quick refactoring of last portion of tutorial (not thoroughly tested though) --- docs/en/getting_started/tutorial.md | 54 +++++++++++++---------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/docs/en/getting_started/tutorial.md b/docs/en/getting_started/tutorial.md index a892828557e..acdd9074beb 100644 --- a/docs/en/getting_started/tutorial.md +++ b/docs/en/getting_started/tutorial.md @@ -497,7 +497,6 @@ ORDER BY AvgDuration DESC LIMIT 10 ``` - ``` sql SELECT sum(Sign) AS visits, @@ -512,13 +511,13 @@ WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartU ClickHouse cluster is a homogenous cluster. Steps to set up: 1. Install ClickHouse server on all machines of the cluster -2. Set up cluster configs in configuration file +2. Set up cluster configs in configuration files 3. Create local tables on each instance 4. Create a [Distributed table](../operations/table_engines/distributed.md) [Distributed table](../operations/table_engines/distributed.md) is actually a kind of "view" to local tables of ClickHouse cluster. SELECT query from a distributed table will be executed using resources of all cluster's shards. You may specify configs for multiple clusters and create multiple distributed tables providing views to different clusters. -
Config for cluster with three shards, one replica each +Example config for cluster with three shards, one replica each: ``` xml @@ -543,37 +542,36 @@ ClickHouse cluster is a homogenous cluster. Steps to set up: ``` -
-Creating a local table: +For further demonstration let's create new local table with exactly the same `CREATE TABLE` query that we used for `hits_v1`, but different table name: ``` sql -CREATE TABLE ontime_local (...) ENGINE = MergeTree(FlightDate, (Year, FlightDate), 8192); +CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... ``` Creating a distributed table providing a view into local tables of the cluster: ``` sql -CREATE TABLE ontime_all AS ontime_local -ENGINE = Distributed(perftest_3shards_1replicas, default, ontime_local, rand()); +CREATE TABLE tutorial.hits_all AS tutorial.hits_local +ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); ``` -You can create a Distributed table on all machines in the cluster. This would allow to run distributed queries on any machine of the cluster. Besides distributed table you can also use [remote](../query_language/table_functions/remote.md) table function. +Common practice is to create similar Distributed tables on all machines of the cluster. This would allow to run distributed queries on any machine of the cluster. Also there's an alternative option to create temporary distributed table for a given SELECT query using [remote](../query_language/table_functions/remote.md) table function. Let's run [INSERT SELECT](../query_language/insert_into.md) into Distributed table to spread the table to multiple servers. ``` sql -INSERT INTO ontime_all SELECT * FROM ontime; +INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; ``` !!! warning "Notice" - The approach given above is not suitable for sharding of large tables. + This approach is not suitable for sharding of large tables. There's a separate tool [clickhouse-copier](../operations/utils/clickhouse-copier.md) that can re-shard arbitrary large tables. -As you could expect heavy queries are executed N times faster being launched on 3 servers instead of one.< +As you could expect computationally heavy queries are executed N times faster being launched on 3 servers instead of one. In this case we have used a cluster with 3 shards each contains a single replica. -To provide for resilience in production environment we recommend that each shard should contain 2-3 replicas distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas. +To provide resilience in production environment we recommend that each shard should contain 2-3 replicas distributed between multiple data-centers. Note that ClickHouse supports unlimited number of replicas. -
Config for cluster of one shard containing three replicas +Example config for cluster of one shard containing three replicas: ``` xml ... @@ -595,15 +593,14 @@ To provide for resilience in production environment we recommend that each shard ``` -
-To enable replication ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure +To enable native replication ZooKeeper is required. ClickHouse will take care of data consistency on all replicas and run restore procedure after failure automatically. It's recommended to deploy ZooKeeper cluster to separate servers. -ZooKeeper is not a requirement: in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is not recommended, in this case ClickHouse is not able to +ZooKeeper is not a strict requirement: in some simple cases you can duplicate the data by writing it into all the replicas from your application code. This approach is **not** recommended, in this case ClickHouse won't be able to guarantee data consistency on all replicas. This remains the responsibility of your application. -
Specify ZooKeeper locations in configuration file +ZooKeeper locations need to be specified in configuration file: ``` xml @@ -620,30 +617,29 @@ ZooKeeper is not a requirement: in some simple cases you can duplicate the data ``` -
-Also we need to set macros for identifying shard and replica — it will be used on table creation: +Also we need to set macros for identifying each shard and replica, it will be used on table creation: ``` xml 01 01 ``` -If there are no replicas at the moment on replicated table creation — a new first replica will be instantiated. If there are already live replicas — new replica will clone the data from existing ones. You have an option to create all replicated tables first and that insert data to it. Another option is to create some replicas and add the others after or during data insertion. + +If there are no replicas at the moment on replicated table creation, a new first replica will be instantiated. If there are already live replicas, new replica will clone the data from existing ones. You have an option to create all replicated tables first and that insert data to it. Another option is to create some replicas and add the others after or during data insertion. ``` sql -CREATE TABLE ontime_replica (...) +CREATE TABLE tutorial.hits_replica (...) ENGINE = ReplcatedMergeTree( - '/clickhouse_perftest/tables/{shard}/ontime', - '{replica}', - FlightDate, - (Year, FlightDate), - 8192); + '/clickhouse_perftest/tables/{shard}/hits', + '{replica}' +) +... ``` Here we use [ReplicatedMergeTree](../operations/table_engines/replication.md) table engine. In parameters we specify ZooKeeper path containing shard and replica identifiers. ``` sql -INSERT INTO ontime_replica SELECT * FROM ontime; +INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; ``` -Replication operates in multi-master mode. Data can be loaded into any replica — it will be synced with other instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and repair consistency once they will become active again. Please notice that such scheme allows for the possibility of just appended data loss. +Replication operates in multi-master mode. Data can be loaded into any replica and it will be synced with other instances automatically. Replication is asynchronous so at a given moment of time not all replicas may contain recently inserted data. To allow data insertion at least one replica should be up. Others will sync up data and repair consistency once they will become active again. Please notice that such approach allows for the low possibility of loss of just appended data. From 6b25cfe207d387e988d739ca844e59eb2fb2189c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 5 Dec 2019 10:31:53 +0300 Subject: [PATCH 020/312] fix link --- website/nginx/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/nginx/default.conf b/website/nginx/default.conf index 5250310f496..fc029323fe2 100644 --- a/website/nginx/default.conf +++ b/website/nginx/default.conf @@ -15,7 +15,7 @@ server { rewrite ^/docs/$ https://clickhouse.yandex/docs/en/ permanent; rewrite ^/tutorial.html$ https://clickhouse.yandex/docs/en/getting_started/tutorial/ permanent; - rewrite ^/presentations/(.*)$ https://yandex.github.io/clickhouse-presentations/$1 permanent; + rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent; rewrite ^/reference_en.html$ https://clickhouse.yandex/docs/en/single/ permanent; rewrite ^/reference_ru.html$ https://clickhouse.yandex/docs/ru/single/ permanent; rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent; From 6a609f36b570f316c069fc9e8d38b80c526d6c68 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 6 Dec 2019 16:26:28 +0300 Subject: [PATCH 021/312] add lunr.ja.js --- .../mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js | 1 + 1 file changed, 1 insertion(+) create mode 120000 docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js diff --git a/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js b/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js new file mode 120000 index 00000000000..c20586be8a3 --- /dev/null +++ b/docs/tools/mkdocs-material-theme/assets/javascripts/lunr/lunr.ja.js @@ -0,0 +1 @@ +lunr.jp.js \ No newline at end of file From 16ce798345ab4daeb8d93bd3138c1d514f789323 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 6 Dec 2019 21:59:48 +0300 Subject: [PATCH 022/312] Suffix for integral values --- dbms/src/IO/ReadHelpers.h | 84 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 19c33f5a83d..e06d70bd81b 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -877,6 +877,88 @@ inline T parse(const char * data, size_t size) return res; } +template +inline std::enable_if_t, void> +readTextWithSuffix(T & x, ReadBuffer & buf) +{ + readIntText(x, buf); + if (buf.eof()) + return; + switch (*buf.position()) + { + case 'k': + { + ++buf.position(); + if (buf.eof()) + { + x *= 1000; + } + else if (*buf.position() == 'i') + { + x = (x << 10); + ++buf.position(); + } + else + { + assertEOF(buf); + } + } + case 'M': + { + ++buf.position(); + if (buf.eof()) + { + x *= 1000000; + } + else if (*buf.position() == 'i') + { + x = (x << 20); + ++buf.position(); + } + else + { + assertEOF(buf); + } + } + case 'G': + { + if (buf.eof()) + { + x *= 1000000000; + } + else if (*buf.position() == 'i') + { + x = (x << 30); + ++buf.position(); + } + else + { + assertEOF(buf); + } + } + case 'T': + { + if (buf.eof()) + { + x *= 1000000000000; + } + else if (*buf.position() == 'i') + { + x = (x << 40); + ++buf.position(); + } + else + { + assertEOF(buf); + } + } + } + return; +} + +template +inline void readTextWithSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } + /// Read something from text format, but expect complete parse of given text /// For example: 723145 -- ok, 213MB -- not ok template @@ -884,7 +966,7 @@ inline T completeParse(const char * data, size_t size) { T res; ReadBufferFromMemory buf(data, size); - readText(res, buf); + readTextWithSuffix(res, buf); assertEOF(buf); return res; } From 11e0b333dc6ef9a3c17fffb9c40b4d2230f2f715 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 6 Dec 2019 22:27:34 +0300 Subject: [PATCH 023/312] Fixed bugs --- dbms/src/IO/ReadHelpers.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index e06d70bd81b..bf92e73722a 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -877,6 +877,9 @@ inline T parse(const char * data, size_t size) return res; } +template +inline void readTextWithSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } + template inline std::enable_if_t, void> readTextWithSuffix(T & x, ReadBuffer & buf) @@ -956,9 +959,6 @@ readTextWithSuffix(T & x, ReadBuffer & buf) return; } -template -inline void readTextWithSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } - /// Read something from text format, but expect complete parse of given text /// For example: 723145 -- ok, 213MB -- not ok template From baf1bb4b580a79b72163c8c117574f357e6b49b9 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 6 Dec 2019 22:29:58 +0300 Subject: [PATCH 024/312] Fixed bugs --- dbms/src/IO/ReadHelpers.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index bf92e73722a..86f02395bfe 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -878,7 +878,8 @@ inline T parse(const char * data, size_t size) } template -inline void readTextWithSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } +inline std::enable_if_t, void> +readTextWithSuffix(T & x, ReadBuffer & buf) { readText(x, buf); } template inline std::enable_if_t, void> From 6a5762d86cb47638c8ecba9b421bede38dd9ce19 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 6 Dec 2019 22:31:33 +0300 Subject: [PATCH 025/312] Fixed bugs --- dbms/src/IO/ReadHelpers.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 86f02395bfe..41eb95ce4aa 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -906,6 +906,7 @@ readTextWithSuffix(T & x, ReadBuffer & buf) { assertEOF(buf); } + break; } case 'M': { @@ -923,6 +924,7 @@ readTextWithSuffix(T & x, ReadBuffer & buf) { assertEOF(buf); } + break; } case 'G': { @@ -939,6 +941,7 @@ readTextWithSuffix(T & x, ReadBuffer & buf) { assertEOF(buf); } + break; } case 'T': { @@ -955,6 +958,11 @@ readTextWithSuffix(T & x, ReadBuffer & buf) { assertEOF(buf); } + break; + } + default: + { + assertEOF(buf); } } return; From eb26a00c4e6fd240a13f20ccd72b357a484bcb20 Mon Sep 17 00:00:00 2001 From: millb Date: Sat, 7 Dec 2019 02:29:06 +0300 Subject: [PATCH 026/312] Fixed bugs --- dbms/src/IO/ReadHelpers.h | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 41eb95ce4aa..552f81cdbef 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -891,80 +891,73 @@ readTextWithSuffix(T & x, ReadBuffer & buf) switch (*buf.position()) { case 'k': - { ++buf.position(); if (buf.eof()) { x *= 1000; + return; } else if (*buf.position() == 'i') { x = (x << 10); - ++buf.position(); } else { - assertEOF(buf); + return; } break; - } case 'M': - { ++buf.position(); if (buf.eof()) { - x *= 1000000; + x *= 1000000; /// 1e+6 + return; } else if (*buf.position() == 'i') { x = (x << 20); - ++buf.position(); } else { - assertEOF(buf); + return; } break; - } case 'G': - { + ++buf.position(); if (buf.eof()) { - x *= 1000000000; + x *= 1000000000; /// 1e+9 + return; } else if (*buf.position() == 'i') { x = (x << 30); - ++buf.position(); } else { - assertEOF(buf); + return; } break; - } case 'T': - { + ++buf.position(); if (buf.eof()) { - x *= 1000000000000; + x *= 1000000000000; /// 1e+12 + return; } else if (*buf.position() == 'i') { x = (x << 40); - ++buf.position(); } else { - assertEOF(buf); + return; } break; - } default: - { - assertEOF(buf); - } + return; } + ++buf.position(); return; } From f9fe0577c49f76e07633d0ba555614e89bd0d85a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 10 Dec 2019 11:15:38 +0300 Subject: [PATCH 027/312] docs build fixes --- docs/tools/build.py | 6 ++++-- docs/tools/github.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index a76ac845d3d..0e855ce9f1e 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -45,6 +45,9 @@ def build_for_lang(lang, args): os.environ['SINGLE_PAGE'] = '0' config_path = os.path.join(args.docs_dir, 'toc_%s.yml' % lang) + if args.is_stable_release and not os.path.exists(config_path): + logging.warn('Skipping %s docs, because %s does not exist' % (lang, config_path)) + return try: theme_cfg = { @@ -249,6 +252,7 @@ if __name__ == '__main__': arg_parser.add_argument('--output-dir', default='build') arg_parser.add_argument('--enable-stable-releases', action='store_true') arg_parser.add_argument('--version-prefix', type=str, default='') + arg_parser.add_argument('--is-stable-release', action='store_true') arg_parser.add_argument('--skip-single-page', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') @@ -260,8 +264,6 @@ if __name__ == '__main__': from github import choose_latest_releases args.stable_releases = choose_latest_releases() if args.enable_stable_releases else [] - - logging.basicConfig( level=logging.DEBUG if args.verbose else logging.INFO, diff --git a/docs/tools/github.py b/docs/tools/github.py index e07d8a0683a..d92dfe7435b 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -15,7 +15,7 @@ def choose_latest_releases(): candidates = requests.get('https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100').json() for tag in candidates: name = tag.get('name', '') - if 'v18' in name or 'stable' not in name: + if ('v18' in name) or ('stable' not in name) or ('prestable' in name): continue major_version = '.'.join((name.split('.', 2))[:2]) if major_version not in seen: @@ -33,6 +33,7 @@ def process_release(args, callback, release): tar.extractall(base_dir) args = copy.deepcopy(args) args.version_prefix = name + args.is_stable_release = True args.docs_dir = os.path.join(base_dir, os.listdir(base_dir)[0], 'docs') callback(args) From 9f4b9116ad5fdfa43d657d76a739af6b335fb331 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 10 Dec 2019 12:34:59 +0300 Subject: [PATCH 028/312] git rm ja/index.md --- docs/ja/index.md | 1 - 1 file changed, 1 deletion(-) delete mode 120000 docs/ja/index.md diff --git a/docs/ja/index.md b/docs/ja/index.md deleted file mode 120000 index facf2d71348..00000000000 --- a/docs/ja/index.md +++ /dev/null @@ -1 +0,0 @@ -../en/index.md \ No newline at end of file From 5868c6cd7414ad00b54006b49654ff9311f74819 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 10 Dec 2019 12:36:32 +0300 Subject: [PATCH 029/312] restore ja/index.md from a7f016f159daade64a5a793dc133221726aef10d --- docs/ja/index.md | 142 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 docs/ja/index.md diff --git a/docs/ja/index.md b/docs/ja/index.md new file mode 100644 index 00000000000..a7f8681a2bf --- /dev/null +++ b/docs/ja/index.md @@ -0,0 +1,142 @@ +# ClickHouseとは? + +ClickHouseは、クエリのオンライン分析処理（OLAP）用の列指向のデータベース管理システム（DBMS）です。 + +「通常の」行指向のDBMSでは、データは次の順序で保存されます。 + +| Row | WatchID | JavaEnable | Title | GoodEvent | EventTime | +| ------ | ------------------- | ---------- | ------------------ | --------- | ------------------- | +| #0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | +| #1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | +| #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | +| #N | ... | ... | ... | ... | ... | + +つまり、行に関連するすべての値は物理的に隣り合わせに格納されます。 + +行指向のDBMSの例：MySQL, Postgres および MS SQL Server +{: .grey } + +列指向のDBMSでは、データは次のように保存されます： + +| Row: | #0 | #1 | #2 | #N | +| ----------- | ------------------- | ------------------- | ------------------- | ------------------- | +| WatchID: | 89354350662 | 90329509958 | 89953706054 | ... | +| JavaEnable: | 1 | 0 | 1 | ... | +| Title: | Investor Relations | Contact us | Mission | ... | +| GoodEvent: | 1 | 1 | 1 | ... | +| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | ... | + +これらの例は、データが配置される順序のみを示しています。 +異なる列の値は別々に保存され、同じ列のデータは一緒に保存されます。 + +列指向DBMSの例：Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid および kdb+ +{: .grey } + +異なったデータ格納の順序は、異なったシナリオにより適します。 +データアクセスシナリオとは、クエリの実行内容、頻度、割合を指します。クエリで読み取られるの各種データの量（行、列、バイト）。データの読み取りと更新の関係。作業データのサイズとローカルでの使用方法。トランザクションが使用されるかどうか、およびそれらがどの程度分離されているか。データ複製と論理的整合性の要件。クエリの種類ごとの遅延とスループットの要件など。 + +システムの負荷が高いほど、使用シナリオの要件に一致するようにセットアップされたシステムをカスタマイズすることがより重要になり、このカスタマイズはより細かくなります。大きく異なるシナリオに等しく適したシステムはありません。システムがさまざまなシナリオに適応可能である場合、高負荷下では、システムはすべてのシナリオを同等に不十分に処理するか、1つまたはいくつかの可能なシナリオでうまく機能します。 + +## OLAPシナリオの主要なプロパティ + +- リクエストの大部分は読み取りアクセス用である。 +- データは、単一行ではなく、かなり大きなバッチ（> 1000行）で更新されます。または、まったく更新されない。 +- データはDBに追加されるが、変更されない。 +- 読み取りの場合、非常に多くの行がDBから抽出されるが、一部の列のみ。 +- テーブルは「幅が広く」、多数の列が含まれる。 +- クエリは比較的まれ（通常、サーバーあたり毎秒数百あるいはそれ以下の数のクエリ）。 +- 単純なクエリでは、約50ミリ秒の遅延が容認される。 +- 列の値はかなり小さく、数値や短い文字列（たとえば、URLごとに60バイト）。 +- 単一のクエリを処理する場合、高いスループットが必要（サーバーあたり毎秒最大数十億行）。 +- トランザクションは必要ない。 +- データの一貫性の要件が低い。 +- クエリごとに1つの大きなテーブルがある。 1つを除くすべてのテーブルは小さい。 +- クエリ結果は、ソースデータよりも大幅に小さくなる。つまり、データはフィルター処理または集計されるため、結果は単一サーバーのRAMに収まる。 + +OLAPシナリオは、他の一般的なシナリオ（OLTPやKey-Valueアクセスなど）とは非常に異なることが容易にわかります。 したがって、まともなパフォーマンスを得るには、OLTPまたはKey-Value DBを使用して分析クエリを処理しようとするのは無意味です。 たとえば、分析にMongoDBまたはRedisを使用しようとすると、OLAPデータベースに比べてパフォーマンスが非常に低下します。 + +## OLAPシナリオで列指向データベースがよりよく機能する理由 + +列指向データベースは、OLAPシナリオにより適しています。ほとんどのクエリの処理が少なくとも100倍高速です。 理由を以下に詳しく説明しますが、その根拠は視覚的に簡単に説明できます： + +**行指向DBMS** + +![Row-oriented](images/row_oriented.gif#) + +**列指向DBMS** + +![Column-oriented](images/column_oriented.gif#) + +違いがわかりましたか？ + +### Input/output + +1. 分析クエリでは、少数のテーブル列のみを読み取る必要があります。列指向のデータベースでは、必要なデータのみを読み取ることができます。たとえば、100のうち5つの列が必要な場合、I/Oが20倍削減されることが期待できます。 +2. データはパケットで読み取られるため、圧縮が容易です。列のデータも圧縮が簡単です。これにより、I/Oボリュームがさらに削減されます。 +3. I/Oの削減により、より多くのデータがシステムキャッシュに収まります。 + +たとえば、「各広告プラットフォームのレコード数をカウントする」クエリでは、1つの「広告プラットフォームID」列を読み取る必要がありますが、これは非圧縮では1バイトの領域を要します。トラフィックのほとんどが広告プラットフォームからのものではない場合、この列は少なくとも10倍の圧縮が期待できます。高速な圧縮アルゴリズムを使用すれば、1秒あたり少なくとも非圧縮データに換算して数ギガバイトの速度でデータを展開できます。つまり、このクエリは、単一のサーバーで1秒あたり約数十億行の速度で処理できます。この速度はまさに実際に達成されます。 + +
Example +``` +$ clickhouse-client +ClickHouse client version 0.0.52053. +Connecting to localhost:9000. +Connected to ClickHouse server version 0.0.52053. + +:) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 + +SELECT +CounterID, +count() +FROM hits +GROUP BY CounterID +ORDER BY count() DESC +LIMIT 20 + +┌─CounterID─┬──count()─┐ +│ 114208 │ 56057344 │ +│ 115080 │ 51619590 │ +│ 3228 │ 44658301 │ +│ 38230 │ 42045932 │ +│ 145263 │ 42042158 │ +│ 91244 │ 38297270 │ +│ 154139 │ 26647572 │ +│ 150748 │ 24112755 │ +│ 242232 │ 21302571 │ +│ 338158 │ 13507087 │ +│ 62180 │ 12229491 │ +│ 82264 │ 12187441 │ +│ 232261 │ 12148031 │ +│ 146272 │ 11438516 │ +│ 168777 │ 11403636 │ +│ 4120072 │ 11227824 │ +│ 10938808 │ 10519739 │ +│ 74088 │ 9047015 │ +│ 115079 │ 8837972 │ +│ 337234 │ 8205961 │ +└───────────┴──────────┘ + +20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) + +:) +``` + +
+ +### CPU + +クエリを実行するには大量の行を処理する必要があるため、個別の行ではなくベクター全体のすべての操作をディスパッチするか、ディスパッチコストがほとんどないようにクエリエンジンを実装すると効率的です。 適切なディスクサブシステムでこれを行わないと、クエリインタープリターが必然的にCPUを失速させます。 +データを列に格納し、可能な場合は列ごとに処理することは理にかなっています。 + +これを行うには2つの方法があります: + +1. ベクトルエンジン。 すべての操作は、個別の値ではなく、ベクトルに対して記述されます。 これは、オペレーションを頻繁に呼び出す必要がなく、ディスパッチコストが無視できることを意味します。 操作コードには、最適化された内部サイクルが含まれています。 + +2. コード生成。 クエリ用に生成されたコードには、すべての間接的な呼び出しが含まれています。 + +これは、単純なクエリを実行する場合には意味がないため、「通常の」データベースでは実行されません。 ただし、例外があります。 たとえば、MemSQLはコード生成を使用して、SQLクエリを処理する際の遅延を減らします。 （比較のために、分析DBMSではレイテンシではなくスループットの最適化が必要です。） + +CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。 + +[Original article](https://clickhouse.yandex/docs/ja/) From 2a94832ef6409d3522631377f361ab66a56200e2 Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 12 Dec 2019 18:49:15 +0800 Subject: [PATCH 030/312] parallel INSERT in INSERT SELECT query --- dbms/src/Core/Settings.h | 1 + dbms/src/DataStreams/IBlockStream_fwd.h | 1 + .../NullAndDoCopyBlockInputStream.h | 23 +++- dbms/src/DataStreams/copyData.cpp | 73 ++++++++++++ dbms/src/DataStreams/copyData.h | 2 + .../Interpreters/InterpreterInsertQuery.cpp | 110 +++++++++++------- dbms/src/Storages/IStorage.h | 3 + dbms/src/Storages/StorageMergeTree.h | 2 + 8 files changed, 169 insertions(+), 46 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 5a8af895610..e469e31c386 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -52,6 +52,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ + M(SettingMaxThreads, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ diff --git a/dbms/src/DataStreams/IBlockStream_fwd.h b/dbms/src/DataStreams/IBlockStream_fwd.h index 06319a55262..d74a9528ed9 100644 --- a/dbms/src/DataStreams/IBlockStream_fwd.h +++ b/dbms/src/DataStreams/IBlockStream_fwd.h @@ -12,5 +12,6 @@ class IBlockOutputStream; using BlockInputStreamPtr = std::shared_ptr; using BlockInputStreams = std::vector; using BlockOutputStreamPtr = std::shared_ptr; +using BlockOutputStreams = std::vector; } diff --git a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h index 8ef64cc5e05..e3a0d923747 100644 --- a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h +++ b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h @@ -21,9 +21,19 @@ class NullAndDoCopyBlockInputStream : public IBlockInputStream { public: NullAndDoCopyBlockInputStream(const BlockInputStreamPtr & input_, BlockOutputStreamPtr output_) - : input(input_), output(output_) { - children.push_back(input_); + input_streams.push_back(input_); + output_streams.push_back(output_); + + for (auto & input_stream : input_streams) + children.push_back(input_stream); + } + + NullAndDoCopyBlockInputStream(const BlockInputStreams & input_, BlockOutputStreams & output_) + : input_streams(input_), output_streams(output_) + { + for (auto & input_stream : input_) + children.push_back(input_stream); } /// Suppress readPrefix and readSuffix, because they are called by copyData. @@ -39,13 +49,16 @@ public: protected: Block readImpl() override { - copyData(*input, *output); + if (input_streams.size() == 1 && output_streams.size() == 1) + copyData(*input_streams.at(0), *output_streams.at(0)); + else + copyData(input_streams, output_streams); return Block(); } private: - BlockInputStreamPtr input; - BlockOutputStreamPtr output; + BlockInputStreams input_streams; + BlockOutputStreams output_streams; }; } diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 9d17596fc8d..f3bf6fb1123 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -1,6 +1,8 @@ +#include #include #include #include +#include namespace DB @@ -51,6 +53,73 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall inline void doNothing(const Block &) {} + +void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) +{ + if (froms.size() == tos.size()) + { + std::vector threads(froms.size()); + for (size_t i = 0; i < froms.size(); i++) + { + threads[i] = std::thread( + [&](BlockInputStreamPtr from, BlockOutputStreamPtr to) { + from->readPrefix(); + to->writePrefix(); + while (Block block = from->read()) + { + to->write(block); + } + from->readSuffix(); + to->writeSuffix(); + }, froms.at(i), tos.at(i)); + } + for (auto & thread : threads) + thread.join(); + } + else + { + ConcurrentBoundedQueue queue(froms.size()); + std::thread from_threads([&]() { + std::vector _from_threads; + for (auto & _from : froms) + { + _from_threads.emplace_back([&](BlockInputStreamPtr from) { + from->readPrefix(); + while (Block block = from->read()) + { + queue.push(block); + } + from->readSuffix(); + }, _from); + } + for (auto & thread : _from_threads) + thread.join(); + for (size_t i = 0; i < tos.size(); i++) + queue.push({}); + }); + std::vector _to_threads; + for (auto & _to : tos) + { + _to_threads.emplace_back([&](BlockOutputStreamPtr to) { + to->writePrefix(); + Block block; + while (true) + { + queue.pop(block); + if (!block) + break; + to->write(block); + } + to->writeSuffix(); + }, _to); + } + + from_threads.join(); + for (auto & thread : _to_threads) + thread.join(); + } +} + void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled) { auto is_cancelled_pred = [is_cancelled] () @@ -61,6 +130,10 @@ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic & is_cancelled) { diff --git a/dbms/src/DataStreams/copyData.h b/dbms/src/DataStreams/copyData.h index f2bce8f411b..5cc296d51b5 100644 --- a/dbms/src/DataStreams/copyData.h +++ b/dbms/src/DataStreams/copyData.h @@ -16,6 +16,8 @@ class Block; */ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled = nullptr); +void copyData(BlockInputStreams & froms, BlockOutputStreams & tos); + void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled); void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled, diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index e01e6cd1c44..37c164fb5d7 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -103,56 +103,84 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); - /// We create a pipeline of several streams, into which we will write data. - BlockOutputStreamPtr out; - - /// NOTE: we explicitly ignore bound materialized views when inserting into Kafka Storage. - /// Otherwise we'll get duplicates when MV reads same rows again from Kafka. - if (table->noPushingToViews() && !no_destination) - out = table->write(query_ptr, context); - else - out = std::make_shared(query.database, query.table, table, context, query_ptr, no_destination); - - /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. - /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). - if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash) - { - out = std::make_shared( - out, out->getHeader(), context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); - } - auto query_sample_block = getSampleBlock(query, table); - - /// Actually we don't know structure of input blocks from query/table, - /// because some clients break insertion protocol (columns != header) - out = std::make_shared( - out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); - - if (const auto & constraints = table->getConstraints(); !constraints.empty()) - out = std::make_shared(query.table, - out, query_sample_block, table->getConstraints(), context); - - auto out_wrapper = std::make_shared(out); - out_wrapper->setProcessListElement(context.getProcessListElement()); - out = std::move(out_wrapper); - - BlockIO res; - - /// What type of query: INSERT or INSERT SELECT? + BlockInputStreams in_streams; + size_t out_streams_size = 1; if (query.select) { /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - /// BlockIO may hold StoragePtrs to temporary tables - res = interpreter_select.execute(); - res.out = nullptr; + if (table->supportsParallelInsert()) + { + in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); + const Settings & settings = context.getSettingsRef(); + out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); + } + else + in_streams.emplace_back(interpreter_select.execute().in); + } + + BlockOutputStreams out_streams; + auto query_sample_block = getSampleBlock(query, table); + + while (out_streams_size-- > 0) + { + /// We create a pipeline of several streams, into which we will write data. + BlockOutputStreamPtr out; + + /// NOTE: we explicitly ignore bound materialized views when inserting into Kafka Storage. + /// Otherwise we'll get duplicates when MV reads same rows again from Kafka. + if (table->noPushingToViews() && !no_destination) + out = table->write(query_ptr, context); + else + out = std::make_shared(query.database, query.table, table, context, query_ptr, no_destination); + + /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. + /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). + if (!(context.getSettingsRef().insert_distributed_sync && table->isRemote()) && !no_squash) + { + out = std::make_shared( + out, out->getHeader(), context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); + } + + /// Actually we don't know structure of input blocks from query/table, + /// because some clients break insertion protocol (columns != header) + out = std::make_shared( + out, query_sample_block, out->getHeader(), table->getColumns().getDefaults(), context); + + if (const auto & constraints = table->getConstraints(); !constraints.empty()) + out = std::make_shared(query.table, + out, query_sample_block, table->getConstraints(), context); + + auto out_wrapper = std::make_shared(out); + out_wrapper->setProcessListElement(context.getProcessListElement()); + out = std::move(out_wrapper); + out_streams.emplace_back(std::move(out)); + } + + /// What type of query: INSERT or INSERT SELECT? + if (query.select) + { + for (auto & in_stream : in_streams) + { + in_stream = std::make_shared( + context, in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); + } + + Block in_header = in_streams.at(0)->getHeader(); + if(in_streams.size() > 1) + { + for (size_t i = 1; i < in_streams.size(); ++i) + assertBlocksHaveEqualStructure(in_streams[i]->getHeader(), in_header, "INSERT SELECT"); + } + + res.in = std::make_shared(in_streams, out_streams); res.in = std::make_shared(context, res.in, out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); res.in = std::make_shared(res.in, out); if (!allow_materialized) { - Block in_header = res.in->getHeader(); for (const auto & column : table->getColumns()) if (column.default_desc.kind == ColumnDefaultKind::Materialized && in_header.has(column.name)) throw Exception("Cannot insert column " + column.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); @@ -160,12 +188,12 @@ BlockIO InterpreterInsertQuery::execute() } else if (query.data && !query.has_tail) /// can execute without additional data { + res.out = std::move(out_streams.at(0)); res.in = std::make_shared(query_ptr, nullptr, query_sample_block, context, nullptr); res.in = std::make_shared(res.in, out); } else - res.out = std::move(out); - + res.out = std::move(out_streams.at(0)); res.pipeline.addStorageHolder(table); return res; diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index d99a9b95026..b8da78c988f 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -104,6 +104,9 @@ public: /// Returns true if the storage replicates SELECT, INSERT and ALTER commands among replicas. virtual bool supportsReplication() const { return false; } + /// Returns true if the storage supports parallel insert. + virtual bool supportsParallelInsert() const { return false; } + /// Returns true if the storage supports deduplication of inserted data blocks. virtual bool supportsDeduplication() const { return false; } diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index dfef9261145..00da29bd4ae 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -35,6 +35,8 @@ public: std::string getTableName() const override { return table_name; } std::string getDatabaseName() const override { return database_name; } + bool supportsParallelInsert() const override { return true; } + bool supportsIndexForIn() const override { return true; } Pipes readWithProcessors( From f564d524e5c709289cdfbdf3773b0e286cbbe70c Mon Sep 17 00:00:00 2001 From: Vxider Date: Fri, 13 Dec 2019 11:09:22 +0800 Subject: [PATCH 031/312] parallel insert support for ReplicatedMergeTree --- dbms/src/Storages/StorageReplicatedMergeTree.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 3727d5de0d8..0dc8c80a113 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -86,6 +86,7 @@ public: std::string getTableName() const override { return table_name; } std::string getDatabaseName() const override { return database_name; } + bool supportsParallelInsert() const override { return true; } bool supportsReplication() const override { return true; } bool supportsDeduplication() const override { return true; } From 49065a63483852dced09beaf7b57eda7382d8383 Mon Sep 17 00:00:00 2001 From: Vxider Date: Fri, 13 Dec 2019 11:13:11 +0800 Subject: [PATCH 032/312] use threadPool and code style update --- dbms/src/DataStreams/copyData.cpp | 52 ++++++++++--------- .../Interpreters/InterpreterInsertQuery.cpp | 4 +- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index f3bf6fb1123..30a7854cbcc 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -58,20 +59,19 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) { if (froms.size() == tos.size()) { - std::vector threads(froms.size()); + std::vector threads; + threads.reserve(froms.size()); for (size_t i = 0; i < froms.size(); i++) { - threads[i] = std::thread( - [&](BlockInputStreamPtr from, BlockOutputStreamPtr to) { - from->readPrefix(); - to->writePrefix(); - while (Block block = from->read()) - { - to->write(block); - } - from->readSuffix(); - to->writeSuffix(); - }, froms.at(i), tos.at(i)); + threads.emplace_back([from = froms.at(i), to = tos.at(i)]() + { + from->readPrefix(); + to->writePrefix(); + while (Block block = from->read()) + to->write(block); + from->readSuffix(); + to->writeSuffix(); + }); } for (auto & thread : threads) thread.join(); @@ -79,28 +79,30 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) else { ConcurrentBoundedQueue queue(froms.size()); - std::thread from_threads([&]() { - std::vector _from_threads; - for (auto & _from : froms) + ThreadFromGlobalPool from_threads([&]() + { + std::vector from_threads_; + from_threads_.reserve(froms.size()); + for (auto & from : froms) { - _from_threads.emplace_back([&](BlockInputStreamPtr from) { + from_threads_.emplace_back([&queue, from]() + { from->readPrefix(); while (Block block = from->read()) - { queue.push(block); - } from->readSuffix(); - }, _from); + }); } - for (auto & thread : _from_threads) + for (auto & thread : from_threads_) thread.join(); for (size_t i = 0; i < tos.size(); i++) queue.push({}); }); - std::vector _to_threads; - for (auto & _to : tos) + std::vector to_threads; + for (auto & to : tos) { - _to_threads.emplace_back([&](BlockOutputStreamPtr to) { + to_threads.emplace_back([&queue, to]() + { to->writePrefix(); Block block; while (true) @@ -111,11 +113,11 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) to->write(block); } to->writeSuffix(); - }, _to); + }); } from_threads.join(); - for (auto & thread : _to_threads) + for (auto & thread : to_threads) thread.join(); } } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 37c164fb5d7..8b30da0c149 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -123,7 +123,7 @@ BlockIO InterpreterInsertQuery::execute() BlockOutputStreams out_streams; auto query_sample_block = getSampleBlock(query, table); - while (out_streams_size-- > 0) + for (size_t i = 0; i < out_streams_size; i++) { /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; @@ -168,7 +168,7 @@ BlockIO InterpreterInsertQuery::execute() } Block in_header = in_streams.at(0)->getHeader(); - if(in_streams.size() > 1) + if (in_streams.size() > 1) { for (size_t i = 1; i < in_streams.size(); ++i) assertBlocksHaveEqualStructure(in_streams[i]->getHeader(), in_header, "INSERT SELECT"); From ec784150b3e8332e801694508f875135f35650f2 Mon Sep 17 00:00:00 2001 From: Vxider Date: Fri, 13 Dec 2019 16:42:08 +0800 Subject: [PATCH 033/312] fix code style --- dbms/src/DataStreams/copyData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 30a7854cbcc..333f07d2836 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -63,7 +63,7 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) threads.reserve(froms.size()); for (size_t i = 0; i < froms.size(); i++) { - threads.emplace_back([from = froms.at(i), to = tos.at(i)]() + threads.emplace_back([from = froms.at(i), to = tos.at(i)]() { from->readPrefix(); to->writePrefix(); @@ -79,13 +79,13 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) else { ConcurrentBoundedQueue queue(froms.size()); - ThreadFromGlobalPool from_threads([&]() + ThreadFromGlobalPool from_threads([&]() { std::vector from_threads_; from_threads_.reserve(froms.size()); for (auto & from : froms) { - from_threads_.emplace_back([&queue, from]() + from_threads_.emplace_back([&queue, from]() { from->readPrefix(); while (Block block = from->read()) @@ -101,7 +101,7 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) std::vector to_threads; for (auto & to : tos) { - to_threads.emplace_back([&queue, to]() + to_threads.emplace_back([&queue, to]() { to->writePrefix(); Block block; From f31d61e450f534659e2e75e6488d696a556b7634 Mon Sep 17 00:00:00 2001 From: Vxider Date: Fri, 13 Dec 2019 18:50:14 +0800 Subject: [PATCH 034/312] add performance test --- dbms/tests/performance/parallel_insert.xml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 dbms/tests/performance/parallel_insert.xml diff --git a/dbms/tests/performance/parallel_insert.xml b/dbms/tests/performance/parallel_insert.xml new file mode 100644 index 00000000000..e1091950742 --- /dev/null +++ b/dbms/tests/performance/parallel_insert.xml @@ -0,0 +1,23 @@ + + loop + + + + 2 + + + + + + + + + default.hits_10m_single + + + CREATE TABLE hits2 AS hits_10m_single + + INSERT INTO hits2 SELECT * FROM hits_10m_single + + DROP TABLE IF EXISTS hits2 + From 0490510b0b02bf3a41d652776e4ee9c67d6d47dd Mon Sep 17 00:00:00 2001 From: Vxider Date: Tue, 17 Dec 2019 16:18:17 +0800 Subject: [PATCH 035/312] resolve conflicts --- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 8b30da0c149..a639b02fe2f 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -99,6 +99,7 @@ BlockIO InterpreterInsertQuery::execute() const auto & query = query_ptr->as(); checkAccess(query); + BlockIO res; StoragePtr table = getTable(query); auto table_lock = table->lockStructureForShare(true, context.getInitialQueryId()); @@ -176,9 +177,6 @@ BlockIO InterpreterInsertQuery::execute() res.in = std::make_shared(in_streams, out_streams); - res.in = std::make_shared(context, res.in, out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); - res.in = std::make_shared(res.in, out); - if (!allow_materialized) { for (const auto & column : table->getColumns()) @@ -190,7 +188,7 @@ BlockIO InterpreterInsertQuery::execute() { res.out = std::move(out_streams.at(0)); res.in = std::make_shared(query_ptr, nullptr, query_sample_block, context, nullptr); - res.in = std::make_shared(res.in, out); + res.in = std::make_shared(res.in, res.out); } else res.out = std::move(out_streams.at(0)); From 326e418434f3285ecb2feadcae973824eb3f9213 Mon Sep 17 00:00:00 2001 From: Mikhail Korotov Date: Wed, 18 Dec 2019 15:01:28 +0300 Subject: [PATCH 036/312] Test added --- .../01039_test_setting_parse.reference | 10 ++++++++-- .../0_stateless/01039_test_setting_parse.sql | 19 ++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference b/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference index 30237035c2c..9fbf4df175f 100644 --- a/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference +++ b/dbms/tests/queries/0_stateless/01039_test_setting_parse.reference @@ -1,2 +1,8 @@ -10000000001 -10000000001 +1000000000 +3221225472 +1567000 +125952 +12000000 +32505856 +1000000000000 +1099511627776 diff --git a/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql b/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql index 494e43b001f..a5c1b46e033 100644 --- a/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql +++ b/dbms/tests/queries/0_stateless/01039_test_setting_parse.sql @@ -1,7 +1,16 @@ -SET max_memory_usage = 10000000001; - +SET max_memory_usage = '1G'; SELECT value FROM system.settings WHERE name = 'max_memory_usage'; - -SET max_memory_usage = '1G'; -- { serverError 27 } - +SET max_memory_usage = '3Gi'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1567k'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '123ki'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '12M'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '31Mi'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1T'; +SELECT value FROM system.settings WHERE name = 'max_memory_usage'; +SET max_memory_usage = '1Ti'; SELECT value FROM system.settings WHERE name = 'max_memory_usage'; From 3725b6cca3eb82b3bd9ed72e4a42360984ee675b Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 19 Dec 2019 11:26:23 +0800 Subject: [PATCH 037/312] remove unnessary thread --- dbms/src/DataStreams/copyData.cpp | 39 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 333f07d2836..b3820828977 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -55,7 +55,7 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall inline void doNothing(const Block &) {} -void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) +static void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) { if (froms.size() == tos.size()) { @@ -79,25 +79,6 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) else { ConcurrentBoundedQueue queue(froms.size()); - ThreadFromGlobalPool from_threads([&]() - { - std::vector from_threads_; - from_threads_.reserve(froms.size()); - for (auto & from : froms) - { - from_threads_.emplace_back([&queue, from]() - { - from->readPrefix(); - while (Block block = from->read()) - queue.push(block); - from->readSuffix(); - }); - } - for (auto & thread : from_threads_) - thread.join(); - for (size_t i = 0; i < tos.size(); i++) - queue.push({}); - }); std::vector to_threads; for (auto & to : tos) { @@ -116,7 +97,23 @@ void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) }); } - from_threads.join(); + std::vector from_threads_; + from_threads_.reserve(froms.size()); + for (auto & from : froms) + { + from_threads_.emplace_back([&queue, from]() + { + from->readPrefix(); + while (Block block = from->read()) + queue.push(block); + from->readSuffix(); + }); + } + + for (auto & thread : from_threads_) + thread.join(); + for (size_t i = 0; i < tos.size(); i++) + queue.push({}); for (auto & thread : to_threads) thread.join(); } From c61e2aef3f45c461b01226439c311fed7f3ebab6 Mon Sep 17 00:00:00 2001 From: Vxider Date: Thu, 19 Dec 2019 11:28:00 +0800 Subject: [PATCH 038/312] set at least 1 even if settings.max_insert_threads is zero --- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index a639b02fe2f..b326c13401f 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -116,6 +116,8 @@ BlockIO InterpreterInsertQuery::execute() in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); const Settings & settings = context.getSettingsRef(); out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); + if (out_streams_size == 0) + out_streams_size = 1; } else in_streams.emplace_back(interpreter_select.execute().in); From 1fd3bbc0920dd4947d5f4eb53390194883aea8d9 Mon Sep 17 00:00:00 2001 From: Mikhail Korotov <55493615+millb@users.noreply.github.com> Date: Thu, 19 Dec 2019 17:34:16 +0300 Subject: [PATCH 039/312] Update comments --- dbms/src/IO/ReadHelpers.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index a0c65e230f6..358b8530d9c 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -1050,6 +1050,8 @@ readTextWithSuffix(T & x, ReadBuffer & buf) /// Read something from text format, but expect complete parse of given text /// For example: 723145 -- ok, 213MB -- not ok +/// Integral values parsing with suffix (k, ki, M, Mi, G, Gi, T, Ti) +/// For example: 133M = 133000000 template inline T completeParse(const char * data, size_t size) { From 7e398f4731fdb43bda2c7be891d5bf0c7d23b2a8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 17:44:58 +0300 Subject: [PATCH 040/312] Change implementation of copyData for multiple streams. --- .../NullAndDoCopyBlockInputStream.h | 4 + dbms/src/DataStreams/copyData.cpp | 125 ++++++++++-------- 2 files changed, 71 insertions(+), 58 deletions(-) diff --git a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h index e3a0d923747..8fe05c387a3 100644 --- a/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h +++ b/dbms/src/DataStreams/NullAndDoCopyBlockInputStream.h @@ -49,6 +49,10 @@ public: protected: Block readImpl() override { + /// We do not use cancel flag here. + /// If query was cancelled, it will be processed by child streams. + /// Part of the data will be processed. + if (input_streams.size() == 1 && output_streams.size() == 1) copyData(*input_streams.at(0), *output_streams.at(0)); else diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index b3820828977..0a8e0521950 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -54,69 +55,77 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall inline void doNothing(const Block &) {} +namespace +{ + + +struct ParallelInsertsHandler +{ + using CencellationHook = std::function; + + explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_) + : outputs(output_streams.size()), cancellation_hook(std::move(cancellation_hook_)) + { + exceptions.resize(output_streams.size()); + + for (auto & output : output_streams) + outputs.push(output.get()); + } + + void onBlock(Block & block, size_t /*thread_num*/) + { + IBlockOutputStream * out = nullptr; + + outputs.pop(out); + out->write(block); + outputs.push(out); + } + + void onFinishThread(size_t /*thread_num*/) {} + void onFinish() {} + + void onException(std::exception_ptr & exception, size_t thread_num) + { + exceptions[thread_num] = std::move(exception); + cancellation_hook(); + } + + void rethrowFirstException() + { + for (auto & exception : exceptions) + if (exception) + std::rethrow_exception(exception); + } + + ConcurrentBoundedQueue outputs; + std::vector exceptions; + CencellationHook cancellation_hook; +}; + +} static void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) { - if (froms.size() == tos.size()) - { - std::vector threads; - threads.reserve(froms.size()); - for (size_t i = 0; i < froms.size(); i++) - { - threads.emplace_back([from = froms.at(i), to = tos.at(i)]() - { - from->readPrefix(); - to->writePrefix(); - while (Block block = from->read()) - to->write(block); - from->readSuffix(); - to->writeSuffix(); - }); - } - for (auto & thread : threads) - thread.join(); - } - else - { - ConcurrentBoundedQueue queue(froms.size()); - std::vector to_threads; - for (auto & to : tos) - { - to_threads.emplace_back([&queue, to]() - { - to->writePrefix(); - Block block; - while (true) - { - queue.pop(block); - if (!block) - break; - to->write(block); - } - to->writeSuffix(); - }); - } + for (auto & output : tos) + output->writePrefix(); - std::vector from_threads_; - from_threads_.reserve(froms.size()); - for (auto & from : froms) - { - from_threads_.emplace_back([&queue, from]() - { - from->readPrefix(); - while (Block block = from->read()) - queue.push(block); - from->readSuffix(); - }); - } + using Processor = ParallelInputsProcessor; + Processor * processor_ptr = nullptr; - for (auto & thread : from_threads_) - thread.join(); - for (size_t i = 0; i < tos.size(); i++) - queue.push({}); - for (auto & thread : to_threads) - thread.join(); - } + ParallelInsertsHandler handler(tos, [&processor_ptr]() { processor_ptr->cancel(false); }); + ParallelInputsProcessor processor(froms, nullptr, froms.size(), handler); + processor_ptr = &processor; + + processor.process(); + processor.wait(); + handler.rethrowFirstException(); + + /// readPrefix is called in ParallelInputsProcessor. + for (auto & input : froms) + input->readSuffix(); + + for (auto & output : tos) + output->writeSuffix(); } void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled) From 720c1987edc82b809c848e53ee4153ca30df8bc9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 18:37:11 +0300 Subject: [PATCH 041/312] Disable parallel inserting if max_insert_threads is zero,] --- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index b326c13401f..37083ca66e1 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -96,6 +96,8 @@ Block InterpreterInsertQuery::getSampleBlock(const ASTInsertQuery & query, const BlockIO InterpreterInsertQuery::execute() { + const Settings & settings = context.getSettingsRef(); + const auto & query = query_ptr->as(); checkAccess(query); @@ -106,7 +108,7 @@ BlockIO InterpreterInsertQuery::execute() BlockInputStreams in_streams; size_t out_streams_size = 1; - if (query.select) + if (query.select && settings.max_insert_threads > 0) { /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; @@ -114,10 +116,7 @@ BlockIO InterpreterInsertQuery::execute() if (table->supportsParallelInsert()) { in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); - const Settings & settings = context.getSettingsRef(); out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); - if (out_streams_size == 0) - out_streams_size = 1; } else in_streams.emplace_back(interpreter_select.execute().in); From 7a78bdc9741cc1756cb04dab45c8cad531ece57f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 18:54:32 +0300 Subject: [PATCH 042/312] Set 0 as default for max_insert_threads. --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index e469e31c386..f64d06d8c02 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -52,7 +52,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ - M(SettingMaxThreads, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. By default, it is determined automatically.", 0) \ + M(SettingUInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ From 9a4b5992e9fab852a68c5ced7b1206766a61176e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 19:09:05 +0300 Subject: [PATCH 043/312] Fix insert select. --- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 37083ca66e1..ec69674f2b9 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -108,12 +108,12 @@ BlockIO InterpreterInsertQuery::execute() BlockInputStreams in_streams; size_t out_streams_size = 1; - if (query.select && settings.max_insert_threads > 0) + if (query.select) { /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - if (table->supportsParallelInsert()) + if (table->supportsParallelInsert() && settings.max_insert_threads > 0) { in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); @@ -187,9 +187,9 @@ BlockIO InterpreterInsertQuery::execute() } else if (query.data && !query.has_tail) /// can execute without additional data { - res.out = std::move(out_streams.at(0)); + // res.out = std::move(out_streams.at(0)); res.in = std::make_shared(query_ptr, nullptr, query_sample_block, context, nullptr); - res.in = std::make_shared(res.in, res.out); + res.in = std::make_shared(res.in, out_streams.at(0)); } else res.out = std::move(out_streams.at(0)); From 725ac1255b5ee4294b7b661e5d8ba926bc39e12b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 19:27:35 +0300 Subject: [PATCH 044/312] Fix possible segfault on exception. --- dbms/src/DataStreams/copyData.cpp | 18 +++++++++--------- dbms/src/DataStreams/copyData.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 0a8e0521950..4d22ecffded 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -86,7 +86,7 @@ struct ParallelInsertsHandler void onException(std::exception_ptr & exception, size_t thread_num) { - exceptions[thread_num] = std::move(exception); + exceptions[thread_num] = exception; cancellation_hook(); } @@ -104,16 +104,16 @@ struct ParallelInsertsHandler } -static void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) +static void copyDataImpl(BlockInputStreams & inputs, BlockOutputStreams & outputs) { - for (auto & output : tos) + for (auto & output : outputs) output->writePrefix(); using Processor = ParallelInputsProcessor; Processor * processor_ptr = nullptr; - ParallelInsertsHandler handler(tos, [&processor_ptr]() { processor_ptr->cancel(false); }); - ParallelInputsProcessor processor(froms, nullptr, froms.size(), handler); + ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }); + ParallelInputsProcessor processor(inputs, nullptr, outputs.size(), handler); processor_ptr = &processor; processor.process(); @@ -121,10 +121,10 @@ static void copyDataImpl(BlockInputStreams & froms, BlockOutputStreams & tos) handler.rethrowFirstException(); /// readPrefix is called in ParallelInputsProcessor. - for (auto & input : froms) + for (auto & input : inputs) input->readSuffix(); - for (auto & output : tos) + for (auto & output : outputs) output->writeSuffix(); } @@ -138,9 +138,9 @@ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic & is_cancelled) diff --git a/dbms/src/DataStreams/copyData.h b/dbms/src/DataStreams/copyData.h index 5cc296d51b5..ae72dbd2421 100644 --- a/dbms/src/DataStreams/copyData.h +++ b/dbms/src/DataStreams/copyData.h @@ -16,7 +16,7 @@ class Block; */ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled = nullptr); -void copyData(BlockInputStreams & froms, BlockOutputStreams & tos); +void copyData(BlockInputStreams & inputs, BlockOutputStreams & outputs); void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled); From 8a1ec4c379757f4f8d24b1ac5459687befb72204 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 22:20:13 +0300 Subject: [PATCH 045/312] Chenge number of threads for insert select. --- dbms/src/DataStreams/copyData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 4d22ecffded..fd4bfab28d8 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -63,10 +63,10 @@ struct ParallelInsertsHandler { using CencellationHook = std::function; - explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_) + explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_, size_t num_threads) : outputs(output_streams.size()), cancellation_hook(std::move(cancellation_hook_)) { - exceptions.resize(output_streams.size()); + exceptions.resize(num_threads); for (auto & output : output_streams) outputs.push(output.get()); @@ -112,8 +112,8 @@ static void copyDataImpl(BlockInputStreams & inputs, BlockOutputStreams & output using Processor = ParallelInputsProcessor; Processor * processor_ptr = nullptr; - ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }); - ParallelInputsProcessor processor(inputs, nullptr, outputs.size(), handler); + ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }, inputs.size()); + ParallelInputsProcessor processor(inputs, nullptr, inputs.size(), handler); processor_ptr = &processor; processor.process(); From 54b04eb8a91b0d4f7fd30b09f6b9e10ee86ac417 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 19 Dec 2019 22:31:06 +0300 Subject: [PATCH 046/312] updated perftest. --- dbms/tests/performance/parallel_insert.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/tests/performance/parallel_insert.xml b/dbms/tests/performance/parallel_insert.xml index e1091950742..44a2964f881 100644 --- a/dbms/tests/performance/parallel_insert.xml +++ b/dbms/tests/performance/parallel_insert.xml @@ -16,6 +16,7 @@ CREATE TABLE hits2 AS hits_10m_single + set max_insert_threads=8 INSERT INTO hits2 SELECT * FROM hits_10m_single From 7320447f9285e64aa694a13253139912a78a0dc2 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Wed, 8 Jan 2020 03:13:12 -0600 Subject: [PATCH 047/312] Add Avro formats Add Avro file input/output formats Add AvroConfluent input format (for Kafka) --- .gitmodules | 6 +- CMakeLists.txt | 1 + cmake/find/avro.cmake | 43 ++ cmake/find/boost.cmake | 3 +- cmake/sanitize.cmake | 2 + contrib/CMakeLists.txt | 16 + contrib/avro | 1 + contrib/avro-cmake/CMakeLists.txt | 76 +++ contrib/boost | 2 +- contrib/boost-cmake/CMakeLists.txt | 5 + dbms/CMakeLists.txt | 5 + dbms/src/Core/Settings.h | 1 + dbms/src/Formats/FormatFactory.cpp | 3 + dbms/src/Formats/FormatFactory.h | 2 + dbms/src/Formats/FormatSettings.h | 6 + dbms/src/Formats/config_formats.h.in | 1 + .../Formats/Impl/AvroRowInputFormat.cpp | 620 ++++++++++++++++++ .../Formats/Impl/AvroRowInputFormat.h | 70 ++ .../Formats/Impl/AvroRowOutputFormat.cpp | 326 +++++++++ .../Formats/Impl/AvroRowOutputFormat.h | 60 ++ 20 files changed, 1246 insertions(+), 3 deletions(-) create mode 100644 cmake/find/avro.cmake create mode 160000 contrib/avro create mode 100644 contrib/avro-cmake/CMakeLists.txt create mode 100644 dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp create mode 100644 dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h create mode 100644 dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp create mode 100644 dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h diff --git a/.gitmodules b/.gitmodules index a35e4ba36e0..8147eb31799 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,7 +46,7 @@ url = https://github.com/ClickHouse-Extras/protobuf.git [submodule "contrib/boost"] path = contrib/boost - url = https://github.com/ClickHouse-Extras/boost.git + url = https://github.com/oandrew/clickhouse-boost [submodule "contrib/base64"] path = contrib/base64 url = https://github.com/aklomp/base64.git @@ -137,3 +137,7 @@ [submodule "contrib/ryu"] path = contrib/ryu url = https://github.com/ClickHouse-Extras/ryu.git +[submodule "contrib/avro"] + path = contrib/avro + url = https://github.com/apache/avro.git + ignore = untracked diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c32baa569d..949879cd29a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,6 +351,7 @@ include (cmake/find/simdjson.cmake) include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) include (cmake/find/orc.cmake) +include (cmake/find/avro.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake new file mode 100644 index 00000000000..7eb5c187cf8 --- /dev/null +++ b/cmake/find/avro.cmake @@ -0,0 +1,43 @@ +option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) + +if (ENABLE_AVRO) + +option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) + +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") + if(USE_INTERNAL_AVRO_LIBRARY) + message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_AVRO_LIBRARY 1) + set(USE_INTERNAL_AVRO_LIBRARY 0) +endif() + +if (NOT USE_INTERNAL_AVRO_LIBRARY) + find_package(Snappy REQUIRED) + find_library(AVROCPP avrocpp) +elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY) + include(cmake/find/snappy.cmake) + add_subdirectory(contrib/avro-cmake) + set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") + set(AVROCPP_LIBRARY avrocpp_s) +endif () + +if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) + set(USE_AVRO 1) +endif() + + +# if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) +# set(USE_AVROCPP 1) +# elseif (Boost_INCLUDE_DIRS AND SNAPPY_LIBRARY) +# set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") +# set(AVROCPP_LIBRARY avrocpp_s) +# set(USE_AVROCPP 1) +# else() +# set(USE_INTERNAL_AVROCPP_LIBRARY 0) +# message(STATUS "avro deps: ${Boost_INCLUDE_DIRS}; ${SNAPPY_LIBRARY}; ${ZLIB_LIBRARY}") +# endif() + +endif() + +message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_LIBRARY} ${AVROCPP_INCLUDE_DIR}") diff --git a/cmake/find/boost.cmake b/cmake/find/boost.cmake index 6776d0cea06..ec10a34d839 100644 --- a/cmake/find/boost.cmake +++ b/cmake/find/boost.cmake @@ -31,6 +31,7 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY) set (Boost_SYSTEM_LIBRARY boost_system_internal) set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal) set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY}) + set (Boost_IOSTREAMS_LIBRARY boost_iostreams_internal) set (Boost_REGEX_LIBRARY boost_regex_internal) set (Boost_INCLUDE_DIRS) @@ -48,4 +49,4 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY) list (APPEND Boost_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/boost") endif () -message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_REGEX_LIBRARY}") +message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_IOSTREAMS_LIBRARY},${Boost_REGEX_LIBRARY}") diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index cb099ade7f5..089ec81691a 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -51,9 +51,11 @@ if (SANITIZE) set (ENABLE_READLINE 0 CACHE BOOL "") set (ENABLE_ORC 0 CACHE BOOL "") set (ENABLE_PARQUET 0 CACHE BOOL "") + set (ENABLE_AVRO 0 CACHE BOOL "") set (USE_CAPNP 0 CACHE BOOL "") set (USE_INTERNAL_ORC_LIBRARY 0 CACHE BOOL "") set (USE_ORC 0 CACHE BOOL "") + set (USE_AVRO 0 CACHE BOOL "") set (ENABLE_SSL 0 CACHE BOOL "") elseif (SANITIZE STREQUAL "thread") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 53ad9a0c138..f155940c32a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -212,6 +212,22 @@ else() endif() endif() +if (USE_INTERNAL_AVRO_LIBRARY) + if(USE_INTERNAL_SNAPPY_LIBRARY) + set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") + if (NOT MAKE_STATIC_LIBRARIES) + set(BUILD_SHARED_LIBS 1) # TODO: set at root dir + endif() + + add_subdirectory(snappy) + + set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") + if(SANITIZE STREQUAL "undefined") + target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) + endif() + endif() +endif() + if (USE_INTERNAL_POCO_LIBRARY) set (POCO_VERBOSE_MESSAGES 0 CACHE INTERNAL "") set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) diff --git a/contrib/avro b/contrib/avro new file mode 160000 index 00000000000..89218262cde --- /dev/null +++ b/contrib/avro @@ -0,0 +1 @@ +Subproject commit 89218262cde62e98fcb3778b86cd3f03056c54f3 diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt new file mode 100644 index 00000000000..643c68c54c5 --- /dev/null +++ b/contrib/avro-cmake/CMakeLists.txt @@ -0,0 +1,76 @@ +# project and source dir +set(AVROCPP_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++) +set(AVROCPP_INCLUDE_DIR ${AVROCPP_ROOT_DIR}/api) +set(AVROCPP_SOURCE_DIR ${AVROCPP_ROOT_DIR}/impl) + +#set(AVROCPP_COMMON_DIR ${HDFS3_SOURCE_DIR}/common) + +if (EXISTS ${AVROCPP_ROOT_DIR}/../../share/VERSION.txt) + file(READ "${AVROCPP_ROOT_DIR}/../../share/VERSION.txt" + AVRO_VERSION) +endif() + +string(REPLACE "\n" "" AVRO_VERSION ${AVRO_VERSION}) +set (AVRO_VERSION_MAJOR ${AVRO_VERSION}) +set (AVRO_VERSION_MINOR "0") + +set (AVROCPP_SOURCE_FILES + ${AVROCPP_SOURCE_DIR}/Compiler.cc + ${AVROCPP_SOURCE_DIR}/Node.cc + ${AVROCPP_SOURCE_DIR}/LogicalType.cc + ${AVROCPP_SOURCE_DIR}/NodeImpl.cc + ${AVROCPP_SOURCE_DIR}/ResolverSchema.cc + ${AVROCPP_SOURCE_DIR}/Schema.cc + ${AVROCPP_SOURCE_DIR}/Types.cc + ${AVROCPP_SOURCE_DIR}/ValidSchema.cc + ${AVROCPP_SOURCE_DIR}/Zigzag.cc + ${AVROCPP_SOURCE_DIR}/BinaryEncoder.cc + ${AVROCPP_SOURCE_DIR}/BinaryDecoder.cc + ${AVROCPP_SOURCE_DIR}/Stream.cc + ${AVROCPP_SOURCE_DIR}/FileStream.cc + ${AVROCPP_SOURCE_DIR}/Generic.cc + ${AVROCPP_SOURCE_DIR}/GenericDatum.cc + ${AVROCPP_SOURCE_DIR}/DataFile.cc + ${AVROCPP_SOURCE_DIR}/parsing/Symbol.cc + ${AVROCPP_SOURCE_DIR}/parsing/ValidatingCodec.cc + ${AVROCPP_SOURCE_DIR}/parsing/JsonCodec.cc + ${AVROCPP_SOURCE_DIR}/parsing/ResolvingDecoder.cc + ${AVROCPP_SOURCE_DIR}/json/JsonIO.cc + ${AVROCPP_SOURCE_DIR}/json/JsonDom.cc + ${AVROCPP_SOURCE_DIR}/Resolver.cc + ${AVROCPP_SOURCE_DIR}/Validator.cc + ) + + +add_definitions(-std=c++17 -fPIC) + +add_library (avrocpp SHARED ${AVROCPP_SOURCE_FILES}) + +set_property (TARGET avrocpp + APPEND PROPERTY COMPILE_DEFINITIONS AVRO_DYN_LINK) + +add_library (avrocpp_s STATIC ${AVROCPP_SOURCE_FILES}) + +set_property (TARGET avrocpp avrocpp_s + APPEND PROPERTY COMPILE_DEFINITIONS AVRO_SOURCE) + +set_target_properties (avrocpp PROPERTIES + VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) + +set_target_properties (avrocpp_s PROPERTIES + VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) + +target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY} ${SNAPPY_LIBRARY}) +target_link_libraries (avrocpp_s ${Boost_IOSTREAMS_LIBRARY} ${SNAPPY_LIBRARY}) + +target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) +target_compile_definitions (avrocpp_s PUBLIC SNAPPY_CODEC_AVAILABLE) + +include_directories(${AVROCPP_INCLUDE_DIR}) +include_directories(${Boost_INCLUDE_DIRS}) +include_directories(${SNAPPY_INCLUDE_DIR}) + +ADD_CUSTOM_TARGET(symlink_headers ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${AVROCPP_ROOT_DIR}/include + COMMAND ${CMAKE_COMMAND} -E create_symlink ${AVROCPP_ROOT_DIR}/api ${AVROCPP_ROOT_DIR}/include/avro +) \ No newline at end of file diff --git a/contrib/boost b/contrib/boost index 830e51edb59..a2cfeb63eaf 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 830e51edb59c4f37a8638138581e1e56c29ac44f +Subproject commit a2cfeb63eaf3b32cf233105b1a40f4a5f26b8495 diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index d9a8a70ef17..54dcd750320 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -37,3 +37,8 @@ target_link_libraries(boost_filesystem_internal PRIVATE boost_system_internal) if (USE_INTERNAL_PARQUET_LIBRARY) add_boost_lib(regex) endif() + +if (USE_INTERNAL_AVRO_LIBRARY) + add_boost_lib(iostreams) + target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES}) +endif() diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index e0c8b7da37a..4f9ca404cdf 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -483,6 +483,11 @@ if (USE_PARQUET) endif () endif () +if (USE_AVRO) + dbms_target_link_libraries(PRIVATE ${AVROCPP_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${AVROCPP_INCLUDE_DIR}) +endif () + if (OPENSSL_CRYPTO_LIBRARY) dbms_target_link_libraries (PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 724b31ca642..26684153832 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -186,6 +186,7 @@ struct Settings : public SettingsCollection M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ + M(SettingString, input_format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \ \ M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \ \ diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp index 240e591123f..ade91c5a391 100644 --- a/dbms/src/Formats/FormatFactory.cpp +++ b/dbms/src/Formats/FormatFactory.cpp @@ -68,6 +68,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; + format_settings.avro.schema_registry_url = settings.input_format_avro_schema_registry_url; return format_settings; } @@ -325,6 +326,8 @@ FormatFactory::FormatFactory() registerInputFormatProcessorORC(*this); registerInputFormatProcessorParquet(*this); registerOutputFormatProcessorParquet(*this); + registerInputFormatProcessorAvro(*this); + registerOutputFormatProcessorAvro(*this); registerInputFormatProcessorTemplate(*this); registerOutputFormatProcessorTemplate(*this); diff --git a/dbms/src/Formats/FormatFactory.h b/dbms/src/Formats/FormatFactory.h index cbf64afeaec..345ceaee690 100644 --- a/dbms/src/Formats/FormatFactory.h +++ b/dbms/src/Formats/FormatFactory.h @@ -166,6 +166,8 @@ void registerInputFormatProcessorORC(FormatFactory & factory); void registerOutputFormatProcessorParquet(FormatFactory & factory); void registerInputFormatProcessorProtobuf(FormatFactory & factory); void registerOutputFormatProcessorProtobuf(FormatFactory & factory); +void registerInputFormatProcessorAvro(FormatFactory & factory); +void registerOutputFormatProcessorAvro(FormatFactory & factory); void registerInputFormatProcessorTemplate(FormatFactory & factory); void registerOutputFormatProcessorTemplate(FormatFactory &factory); diff --git a/dbms/src/Formats/FormatSettings.h b/dbms/src/Formats/FormatSettings.h index 6219edf6e6d..6ca54c12265 100644 --- a/dbms/src/Formats/FormatSettings.h +++ b/dbms/src/Formats/FormatSettings.h @@ -110,6 +110,12 @@ struct FormatSettings }; Custom custom; + + struct Avro + { + String schema_registry_url; + } avro; + }; } diff --git a/dbms/src/Formats/config_formats.h.in b/dbms/src/Formats/config_formats.h.in index 1ddd0e18aa9..308ded92b5d 100644 --- a/dbms/src/Formats/config_formats.h.in +++ b/dbms/src/Formats/config_formats.h.in @@ -2,6 +2,7 @@ // .h autogenerated by cmake! +#cmakedefine01 USE_AVRO #cmakedefine01 USE_CAPNP #cmakedefine01 USE_SNAPPY #cmakedefine01 USE_PARQUET diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp new file mode 100644 index 00000000000..eca22670a87 --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -0,0 +1,620 @@ +#include "AvroRowInputFormat.h" +#if USE_AVRO + +#include + +#include +#include +#include +#include + + +#include + +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; + extern const int ILLEGAL_COLUMN; + extern const int TYPE_MISMATCH; +} + +class InputStreamReadBufferAdapter : public avro::InputStream +{ +public: + InputStreamReadBufferAdapter(ReadBuffer & in_) : in(in_) {} + + bool next(const uint8_t ** data, size_t * len) + { + if (in.eof()) + { + *len = 0; + return false; + } + + *data = (const uint8_t *)in.position(); + *len = in.available(); + + in.position() += in.available(); + return true; + } + + void backup(size_t len) { in.position() -= len; } + + void skip(size_t len) { in.tryIgnore(len); } + + size_t byteCount() const { return in.count(); } + +private: + ReadBuffer & in; +}; + +static void deserializeNoop(IColumn &, avro::Decoder &) +{ +} + +AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type) +{ + auto logical_type = root_node->logicalType().type(); + WhichDataType target(target_type); + switch (root_node->type()) + { + case avro::AVRO_STRING: + if (target.isString()) + { + return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable { + decoder.decodeString(tmp); + column.insertData(tmp.c_str(), tmp.length()); + }; + } + case avro::AVRO_BYTES: + if (target.isString()) + { + return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable { + decoder.decodeString(tmp); + column.insertData(tmp.c_str(), tmp.length()); + }; + } + break; + case avro::AVRO_INT: + if (target.isInt32()) + { + return + [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeInt()); }; + } + if (target.isDate() && logical_type == avro::LogicalType::DATE) + { + return [](IColumn & column, avro::Decoder & decoder) { + assert_cast(column).insertValue(decoder.decodeInt()); + }; + } + break; + case avro::AVRO_LONG: + if (target.isInt64()) + { + return + [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeLong()); }; + } + if (target.isDateTime64()) + { + auto date_time_scale = assert_cast(*target_type).getScale(); + if ((logical_type == avro::LogicalType::TIMESTAMP_MILLIS && date_time_scale == 3) + || (logical_type == avro::LogicalType::TIMESTAMP_MICROS && date_time_scale == 6)) + { + return [](IColumn & column, avro::Decoder & decoder) { + assert_cast(column).insertValue(decoder.decodeLong()); + }; + } + } + break; + case avro::AVRO_FLOAT: + if (target.isFloat32()) + { + return [](IColumn & column, avro::Decoder & decoder) { + assert_cast(column).insertValue(decoder.decodeFloat()); + }; + } + break; + case avro::AVRO_DOUBLE: + if (target.isFloat64()) + { + return [](IColumn & column, avro::Decoder & decoder) { + assert_cast(column).insertValue(decoder.decodeDouble()); + }; + } + break; + case avro::AVRO_BOOL: + if (target.isUInt8()) + { + return + [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeBool()); }; + } + break; + case avro::AVRO_ARRAY: { + if (target.isArray()) + { + auto nested_source_type = root_node->leafAt(0); + auto nested_target_type = assert_cast(*target_type).getNestedType(); + auto nested_deserialize = createDeserializeFn(nested_source_type, nested_target_type); + return [nested_deserialize](IColumn & column, avro::Decoder & decoder) { + ColumnArray & column_array = assert_cast(column); + ColumnArray::Offsets & offsets = column_array.getOffsets(); + IColumn & nested_column = column_array.getData(); + size_t total = 0; + for (size_t n = decoder.arrayStart(); n != 0; n = decoder.arrayNext()) + { + total += n; + for (size_t i = 0; i < n; i++) + { + nested_deserialize(nested_column, decoder); + } + } + offsets.push_back(offsets.back() + total); + }; + } + break; + } + case avro::AVRO_UNION: { + auto nullable_deserializer = [root_node, target_type](size_t non_null_union_index) { + auto nested_deserialize = createDeserializeFn(root_node->leafAt(non_null_union_index), removeNullable(target_type)); + return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder) { + ColumnNullable & col = assert_cast(column); + size_t union_index = decoder.decodeUnionIndex(); + if (union_index == non_null_union_index) + { + nested_deserialize(col.getNestedColumn(), decoder); + col.getNullMapData().push_back(0); + } + else + { + col.insertDefault(); + } + }; + }; + if (root_node->leaves() == 2 && target.isNullable()) + { + if (root_node->leafAt(0)->type() == avro::AVRO_NULL) + return nullable_deserializer(1); + if (root_node->leafAt(1)->type() == avro::AVRO_NULL) + return nullable_deserializer(0); + } + break; + } + case avro::AVRO_NULL: + if (target.isNullable()) + { + auto nested_type = removeNullable(target_type); + if (nested_type->getTypeId() == TypeIndex::Nothing) + { + return [](IColumn & column, avro::Decoder & decoder) { + (void)column; + decoder.decodeNull(); + }; + } + else + { + return [](IColumn & column, avro::Decoder & decoder) { + ColumnNullable & col = assert_cast(column); + decoder.decodeNull(); + col.insertDefault(); + }; + } + } + break; + case avro::AVRO_ENUM: + if (target.isString()) + { + std::vector symbols; + for (size_t i = 0; i < root_node->names(); i++) + { + symbols.push_back(root_node->nameAt(i)); + } + return [symbols](IColumn & column, avro::Decoder & decoder) { + size_t enum_index = decoder.decodeEnum(); + const auto & enum_symbol = symbols[enum_index]; + column.insertData(enum_symbol.c_str(), enum_symbol.length()); + }; + } + if (target.isEnum()) + { + const auto & enum_type = assert_cast(*target_type); + std::vector symbol_mapping; + for (size_t i = 0; i < root_node->names(); i++) + { + symbol_mapping.push_back(enum_type.castToValue(root_node->nameAt(i))); + } + return [symbol_mapping](IColumn & column, avro::Decoder & decoder) { + size_t enum_index = decoder.decodeEnum(); + column.insert(symbol_mapping[enum_index]); + }; + } + break; + case avro::AVRO_FIXED: { + size_t fixed_size = root_node->fixedSize(); + if (target.isFixedString() && target_type->getSizeOfValueInMemory() == fixed_size) + { + return [tmp_fixed = std::vector(fixed_size)](IColumn & column, avro::Decoder & decoder) mutable { + decoder.decodeFixed(tmp_fixed.size(), tmp_fixed); + column.insertData(reinterpret_cast(tmp_fixed.data()), tmp_fixed.size()); + }; + } + break; + } + case avro::AVRO_MAP: + case avro::AVRO_RECORD: + default: + break; + } + + throw Exception( + "Type " + target_type->getName() + " is not compatible" + " with Avro " + avro::ValidSchema(root_node).toJson(false), + ErrorCodes::ILLEGAL_COLUMN); +} + +AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) +{ + switch (root_node->type()) + { + case avro::AVRO_STRING: + return [](avro::Decoder & decoder) { decoder.skipString(); }; + case avro::AVRO_BYTES: + return [](avro::Decoder & decoder) { decoder.skipBytes(); }; + case avro::AVRO_INT: + return [](avro::Decoder & decoder) { decoder.decodeInt(); }; + case avro::AVRO_LONG: + return [](avro::Decoder & decoder) { decoder.decodeLong(); }; + case avro::AVRO_FLOAT: + return [](avro::Decoder & decoder) { decoder.decodeFloat(); }; + case avro::AVRO_DOUBLE: + return [](avro::Decoder & decoder) { decoder.decodeDouble(); }; + case avro::AVRO_BOOL: + return [](avro::Decoder & decoder) { decoder.decodeBool(); }; + case avro::AVRO_ARRAY: { + auto nested_skip_fn = createSkipFn(root_node->leafAt(0)); + return [nested_skip_fn](avro::Decoder & decoder) { + for (size_t n = decoder.arrayStart(); n != 0; n = decoder.arrayNext()) + { + for (size_t i = 0; i < n; ++i) + { + nested_skip_fn(decoder); + } + } + }; + } + case avro::AVRO_UNION: { + std::vector union_skip_fns; + for (size_t i = 0; i < root_node->leaves(); i++) + { + union_skip_fns.push_back(createSkipFn(root_node->leafAt(i))); + } + return [union_skip_fns](avro::Decoder & decoder) { union_skip_fns[decoder.decodeUnionIndex()](decoder); }; + } + case avro::AVRO_NULL: + return [](avro::Decoder & decoder) { decoder.decodeNull(); }; + case avro::AVRO_ENUM: + return [](avro::Decoder & decoder) { decoder.decodeEnum(); }; + case avro::AVRO_FIXED: { + auto fixed_size = root_node->fixedSize(); + return [fixed_size](avro::Decoder & decoder) { decoder.skipFixed(fixed_size); }; + } + + case avro::AVRO_MAP: { + auto value_skip_fn = createSkipFn(root_node->leafAt(1)); + return [value_skip_fn](avro::Decoder & decoder) { + for (size_t n = decoder.mapStart(); n != 0; n = decoder.mapNext()) + { + for (size_t i = 0; i < n; ++i) + { + decoder.skipString(); + value_skip_fn(decoder); + } + } + }; + } + case avro::AVRO_RECORD: { + std::vector field_skip_fns; + for (size_t i = 0; i < root_node->leaves(); i++) + { + field_skip_fns.push_back(createSkipFn(root_node->leafAt(i))); + } + return [field_skip_fns](avro::Decoder & decoder) { + for (auto & skip_fn : field_skip_fns) + skip_fn(decoder); + }; + } + default: + throw Exception("Unsupported Avro type", ErrorCodes::ILLEGAL_COLUMN); + } +} + + +AvroDeserializer::AvroDeserializer(const DB::ColumnsWithTypeAndName & columns, avro::ValidSchema schema) +{ + auto schema_root = schema.root(); + if (schema_root->type() != avro::AVRO_RECORD) + { + throw Exception("Root schema must be a record", ErrorCodes::TYPE_MISMATCH); + } + field_mapping.resize(schema_root->leaves(), -1); + for (size_t i = 0; i < schema_root->leaves(); ++i) + { + skip_fns.push_back(createSkipFn(schema_root->leafAt(i))); + deserialize_fns.push_back(&deserializeNoop); + } + for (size_t i = 0; i < columns.size(); ++i) + { + const auto & column = columns[i]; + size_t field_index; + if (!schema_root->nameIndex(column.name, field_index)) + { + throw Exception("Field " + column.name + " not found in Avro schema", ErrorCodes::THERE_IS_NO_COLUMN); + } + auto field_schema = schema_root->leafAt(field_index); + try + { + deserialize_fns[field_index] = createDeserializeFn(field_schema, column.type); + } + catch (Exception & e) + { + e.addMessage("column " + column.name); + e.rethrow(); + } + field_mapping[field_index] = i; + } +} + +void AvroDeserializer::deserializeRow(MutableColumns & columns, avro::Decoder & decoder) +{ + for (size_t i = 0; i < field_mapping.size(); i++) + { + if (field_mapping[i] >= 0) + { + deserialize_fns[i](*columns[field_mapping[i]], decoder); + } + else + { + skip_fns[i](decoder); + } + } +} + + +AvroRowInputFormat::AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) + : IRowInputFormat(header_, in_, params_) + , file_reader(std::make_unique(in_)) + , deserializer(header_.getColumnsWithTypeAndName(), file_reader.dataSchema()) +{ + file_reader.init(); +} + +bool AvroRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (file_reader.hasMore()) + { + file_reader.decr(); + deserializer.deserializeRow(columns, file_reader.decoder()); + return true; + } + return false; +} + + +class AvroConfluentRowInputFormat::SchemaRegistry +{ +public: + SchemaRegistry(const std::string & base_url_) + { + if (base_url_.empty()) + { + throw Exception("Empty Schema Registry URL", ErrorCodes::BAD_ARGUMENTS); + } + try + { + base_url = base_url_; + } + catch (Poco::SyntaxException & e) + { + throw Exception("Invalid Schema Registry URL", Exception(Exception::CreateFromPoco, e), ErrorCodes::BAD_ARGUMENTS); + } + } + + avro::ValidSchema getSchema(uint32_t id) + { + try + { + try + { + Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); + Poco::Net::HTTPClientSession session(url.getHost(), url.getPort()); + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, url.getPathAndQuery()); + session.sendRequest(request); + Poco::Net::HTTPResponse response; + auto & response_body = session.receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + { + throw Exception("http code " + std::to_string(response.getStatus()), ErrorCodes::INCORRECT_DATA); + } + Poco::JSON::Parser parser; + auto json_body = parser.parse(response_body).extract(); + auto schema = json_body->getValue("schema"); + return avro::compileJsonSchemaFromString(schema); + } + catch (const Exception & e) + { + throw e; + } + catch (const Poco::Exception & e) + { + throw Exception(Exception::CreateFromPoco, e); + } + catch (const avro::Exception & e) + { + throw Exception(e.what(), ErrorCodes::INCORRECT_DATA); + } + } + catch (Exception & e) + { + e.addMessage("while fetching schema id=" + std::to_string(id)); + throw; + } + } + +private: + Poco::URI base_url; +}; + +static uint32_t readConfluentSchemaId(ReadBuffer & in) +{ + Poco::Buffer buf(5); + in.readStrict(buf.begin(), buf.capacity()); + Poco::MemoryBinaryReader binary_reader(buf, Poco::BinaryReader::BIG_ENDIAN_BYTE_ORDER); + + uint8_t magic; + uint32_t schema_id; + binary_reader >> magic >> schema_id; + if (magic != 0x00) + { + throw Exception("Invalid magic byte", ErrorCodes::INCORRECT_DATA); + } + + return schema_id; +} + +AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( + const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) + : IRowInputFormat(header_.cloneEmpty(), in_, params_) + , columns(header_.getColumnsWithTypeAndName()) + , schema_registry(std::make_unique(format_settings_.avro.schema_registry_url)) + , input_stream(std::make_unique(in)) + , decoder(avro::binaryDecoder()) + +{ + (void)format_settings_; + decoder->init(*input_stream); +} + +bool AvroConfluentRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) +{ + if (in.eof()) + { + return false; + } + SchemaId schema_id = readConfluentSchemaId(in); + auto & deserializer = getOrCreateDeserializer(schema_id); + deserializer.deserializeRow(columns, *decoder); + decoder->drain(); + return true; +} + +AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId schema_id) +{ + auto it = deserializer_cache.find(schema_id); + if (it == deserializer_cache.end()) + { + auto schema = schema_registry->getSchema(schema_id); + AvroDeserializer deserializer(columns, schema); + it = deserializer_cache.emplace(schema_id, deserializer).first; + } + return it->second; +} + +void registerInputFormatProcessorAvro(FormatFactory & factory) +{ + factory.registerInputFormatProcessor( + "Avro", + [=](ReadBuffer & buf, + const Block & sample, + const Context & context, + const RowInputFormatParams & params, + const FormatSettings & settings) { + (void)(params); + (void)(settings); + (void)(context); + return std::make_shared(sample, buf, params); + }); + + factory.registerInputFormatProcessor( + "AvroConfluent", + [=](ReadBuffer & buf, + const Block & sample, + const Context & context, + const RowInputFormatParams & params, + const FormatSettings & settings) { + (void)(params); + (void)(settings); + (void)(context); + return std::make_shared(sample, buf, params, settings); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerInputFormatProcessorAvro(FormatFactory &) +{ +} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h new file mode 100644 index 00000000000..65b46074d7e --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -0,0 +1,70 @@ +#pragma once +#include "config_formats.h" +#if USE_AVRO + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ +class AvroDeserializer +{ +public: + AvroDeserializer(const DB::ColumnsWithTypeAndName & columns, avro::ValidSchema schema); + void deserializeRow(MutableColumns & columns, avro::Decoder & decoder); + +private: + using DeserializeFn = std::function; + using SkipFn = std::function; + static DeserializeFn createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type); + static SkipFn createSkipFn(avro::NodePtr root_node); + + std::vector field_mapping; + std::vector skip_fns; + std::vector deserialize_fns; +}; + +class AvroRowInputFormat : public IRowInputFormat +{ +public: + AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); + virtual bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + String getName() const override { return "AvroRowInputFormat"; } + +private: + avro::DataFileReaderBase file_reader; + AvroDeserializer deserializer; +}; + +class AvroConfluentRowInputFormat : public IRowInputFormat +{ +public: + AvroConfluentRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_); + virtual bool readRow(MutableColumns & columns, RowReadExtension & ext) override; + String getName() const override { return "AvroConfluentRowInputFormat"; } + +private: + const DB::ColumnsWithTypeAndName columns; + + class SchemaRegistry; + std::unique_ptr schema_registry; + + using SchemaId = uint32_t; + std::unordered_map deserializer_cache; + AvroDeserializer & getOrCreateDeserializer(SchemaId schema_id); + + avro::InputStreamPtr input_stream; + avro::DecoderPtr decoder; +}; +} +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp new file mode 100644 index 00000000000..0fd40a6e36c --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -0,0 +1,326 @@ +#include "AvroRowOutputFormat.h" +#if USE_AVRO + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_TYPE_OF_FIELD; + extern const int BAD_ARGUMENTS; + extern const int THERE_IS_NO_COLUMN; + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; + extern const int CANNOT_READ_ALL_DATA; +} + +class OutputStreamWriteBufferAdapter : public avro::OutputStream +{ +public: + OutputStreamWriteBufferAdapter(WriteBuffer & out_) : out(out_) {} + + virtual bool next(uint8_t ** data, size_t * len) override + { + out.nextIfAtEnd(); + *data = (uint8_t *)out.position(); + *len = out.available(); + out.position() += out.available(); + + return true; + } + + virtual void backup(size_t len) override { out.position() -= len; } + + virtual uint64_t byteCount() const override { return out.count(); } + virtual void flush() override { out.next(); } + +private: + WriteBuffer & out; +}; + + +AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeFn(DataTypePtr data_type) +{ + switch (data_type->getTypeId()) + { + case TypeIndex::UInt8: + return {avro::BoolSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + encoder.encodeBool(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int32: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int64: + return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + encoder.encodeLong(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Float32: + return {avro::FloatSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + encoder.encodeFloat(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Float64: + return {avro::DoubleSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + encoder.encodeDouble(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Date: { + auto schema = avro::IntSchema(); + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::DATE)); + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + UInt16 date = assert_cast(column).getElement(row_num); + encoder.encodeInt(date); + }}; + } + case TypeIndex::DateTime: + throw Exception("Unsupported Avro type", ErrorCodes::BAD_TYPE_OF_FIELD); + case TypeIndex::DateTime64: { + auto schema = avro::LongSchema(); + const auto & provided_type = assert_cast(*data_type); + + if (provided_type.getScale() == 3) + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MILLIS)); + else if (provided_type.getScale() == 6) + schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MICROS)); + else + throw Exception("Unsupported Avro type", ErrorCodes::BAD_TYPE_OF_FIELD); + + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const auto & col = assert_cast(column); + encoder.encodeLong(col.getElement(row_num)); + }}; + } + case TypeIndex::String: + return {avro::StringSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeBytes(reinterpret_cast(s.data), s.size); + }}; + case TypeIndex::FixedString: { + return {avro::FixedSchema(data_type->getSizeOfValueInMemory(), "fixed"), + [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeFixed(reinterpret_cast(s.data), s.size); + }}; + } + case TypeIndex::Enum8: { + auto schema = avro::EnumSchema("enum"); + std::unordered_map enum_mapping; + const auto & enum_values = assert_cast(*data_type).getValues(); + for (size_t i = 0; i < enum_values.size(); ++i) + { + schema.addSymbol(enum_values[i].first); + enum_mapping.emplace(enum_values[i].second, i); + } + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; + } + case TypeIndex::Enum16: { + auto schema = avro::EnumSchema("enum"); + std::unordered_map enum_mapping; + const auto & enum_values = assert_cast(*data_type).getValues(); + for (size_t i = 0; i < enum_values.size(); ++i) + { + schema.addSymbol(enum_values[i].first); + enum_mapping.emplace(enum_values[i].second, i); + } + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; + } + case TypeIndex::Array: { + const auto & array_type = assert_cast(*data_type); + auto nested_mapping = createSchemaWithSerializeFn(array_type.getNestedType()); + return {avro::ArraySchema(nested_mapping.schema), + [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const ColumnArray & column_array = assert_cast(column); + const ColumnArray::Offsets & offsets = column_array.getOffsets(); + size_t offset = offsets[row_num - 1]; + size_t next_offset = offsets[row_num]; + size_t row_count = next_offset - offset; + const IColumn & nested_column = column_array.getData(); + + encoder.arrayStart(); + if (row_count > 0) + { + encoder.setItemCount(row_count); + } + for (size_t i = offset; i < next_offset; ++i) + { + nested_mapping.serialize(nested_column, i, encoder); + } + encoder.arrayEnd(); + }}; + } + case TypeIndex::Nullable: { + auto nested_type = removeNullable(data_type); + auto nested_mapping = createSchemaWithSerializeFn(nested_type); + if (nested_type->getTypeId() == TypeIndex::Nothing) + { + return nested_mapping; + } + else + { + avro::UnionSchema union_schema; + union_schema.addType(avro::NullSchema()); + union_schema.addType(nested_mapping.schema); + return {union_schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const ColumnNullable & col = assert_cast(column); + if (!col.isNullAt(row_num)) + { + encoder.encodeUnionIndex(1); + nested_mapping.serialize(col.getNestedColumn(), row_num, encoder); + } + else + { + encoder.encodeUnionIndex(0); + encoder.encodeNull(); + } + }}; + } + } + case TypeIndex::LowCardinality: { + const auto & nested_type = removeLowCardinality(data_type); + auto nested_mapping = createSchemaWithSerializeFn(nested_type); + return {nested_mapping.schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { + const auto & col = assert_cast(column); + nested_mapping.serialize(*col.getDictionary().getNestedColumn(), col.getIndexAt(row_num), encoder); + }}; + } + case TypeIndex::Nothing: + return {avro::NullSchema(), [](const IColumn &, size_t, avro::Encoder & encoder) { encoder.encodeNull(); }}; + default: + break; + } + throw Exception("Type " + data_type->getName() + " is not supported for Avro output", ErrorCodes::ILLEGAL_COLUMN); +} + + +AvroSerializer::AvroSerializer(const ColumnsWithTypeAndName & columns) +{ + avro::RecordSchema record_schema("row"); + for (auto & column : columns) + { + try + { + auto field_mapping = createSchemaWithSerializeFn(column.type); + serialize_fns.push_back(field_mapping.serialize); + //TODO: verify name starts with A-Za-z_ + record_schema.addField(column.name, field_mapping.schema); + } + catch (Exception & e) + { + e.addMessage("column " + column.name); + e.rethrow(); + } + } + schema.setSchema(record_schema); +} + +void AvroSerializer::serializeRow(const Columns & columns, size_t row_num, avro::Encoder & encoder) +{ + size_t num_columns = columns.size(); + for (size_t i = 0; i < num_columns; ++i) + { + serialize_fns[i](*columns[i], row_num, encoder); + } +} + +AvroRowOutputFormat::AvroRowOutputFormat( + WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_) + : IRowOutputFormat(header_, out_, callback) + , settings(settings_) + , serializer(header_.getColumnsWithTypeAndName()) + , file_writer(std::make_unique(out_), serializer.getSchema(), 16 * 1024, avro::Codec::SNAPPY_CODEC) +{ +} + +AvroRowOutputFormat::~AvroRowOutputFormat() = default; + +void AvroRowOutputFormat::writePrefix() +{ + file_writer.syncIfNeeded(); +} + +void AvroRowOutputFormat::write(const Columns & columns, size_t row_num) +{ + file_writer.syncIfNeeded(); + serializer.serializeRow(columns, row_num, file_writer.encoder()); + file_writer.incr(); +} + +void AvroRowOutputFormat::writeSuffix() +{ + file_writer.close(); +} + +void registerOutputFormatProcessorAvro(FormatFactory & factory) +{ + factory.registerOutputFormatProcessor( + "Avro", + [=](WriteBuffer & buf, + const Block & sample, + const Context & context, + FormatFactory::WriteCallback callback, + const FormatSettings & settings) { + (void)(context); + (void)(callback); + return std::make_shared(buf, sample, callback, settings); + }); +} + +} + +#else + +namespace DB +{ +class FormatFactory; +void registerOutputFormatProcessorAvro(FormatFactory &) +{ +} +} + +#endif diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h new file mode 100644 index 00000000000..efe63c1a72f --- /dev/null +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.h @@ -0,0 +1,60 @@ +#pragma once +#include "config_formats.h" +#if USE_AVRO +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ +class WriteBuffer; + +class AvroSerializer +{ +public: + AvroSerializer(const ColumnsWithTypeAndName & columns); + const avro::ValidSchema & getSchema() const { return schema; } + void serializeRow(const Columns & columns, size_t row_num, avro::Encoder & encoder); + +private: + using SerializeFn = std::function; + struct SchemaWithSerializeFn + { + avro::Schema schema; + SerializeFn serialize; + }; + static SchemaWithSerializeFn createSchemaWithSerializeFn(DataTypePtr data_type); + + std::vector serialize_fns; + avro::ValidSchema schema; +}; + +class AvroRowOutputFormat : public IRowOutputFormat +{ +public: + AvroRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_); + virtual ~AvroRowOutputFormat() override; + + String getName() const override { return "AvroRowOutputFormat"; } + void write(const Columns & columns, size_t row_num) override; + void writeField(const IColumn &, const IDataType &, size_t) override {} + virtual void writePrefix() override; + virtual void writeSuffix() override; + +private: + FormatSettings settings; + AvroSerializer serializer; + avro::DataFileWriterBase file_writer; +}; + +} +#endif From 38ab600a087ec688bdff26940f7dc5fc5e60338c Mon Sep 17 00:00:00 2001 From: oandrew Date: Wed, 8 Jan 2020 22:22:49 -0600 Subject: [PATCH 048/312] Fix compilation errors --- .../Processors/Formats/Impl/AvroRowInputFormat.cpp | 13 +++++-------- .../Processors/Formats/Impl/AvroRowInputFormat.h | 2 +- .../Processors/Formats/Impl/AvroRowOutputFormat.cpp | 4 +--- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index eca22670a87..f7b34a076d3 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -90,7 +90,7 @@ public: return false; } - *data = (const uint8_t *)in.position(); + *data = reinterpret_cast(in.position()); *len = in.available(); in.position() += in.available(); @@ -125,6 +125,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node column.insertData(tmp.c_str(), tmp.length()); }; } + break; case avro::AVRO_BYTES: if (target.isString()) { @@ -472,7 +473,7 @@ public: } catch (Poco::SyntaxException & e) { - throw Exception("Invalid Schema Registry URL", Exception(Exception::CreateFromPoco, e), ErrorCodes::BAD_ARGUMENTS); + throw Exception("Invalid Schema Registry URL: " + e.displayText(), ErrorCodes::BAD_ARGUMENTS); } } @@ -541,7 +542,7 @@ static uint32_t readConfluentSchemaId(ReadBuffer & in) AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) : IRowInputFormat(header_.cloneEmpty(), in_, params_) - , columns(header_.getColumnsWithTypeAndName()) + , header_columns(header_.getColumnsWithTypeAndName()) , schema_registry(std::make_unique(format_settings_.avro.schema_registry_url)) , input_stream(std::make_unique(in)) , decoder(avro::binaryDecoder()) @@ -570,7 +571,7 @@ AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId if (it == deserializer_cache.end()) { auto schema = schema_registry->getSchema(schema_id); - AvroDeserializer deserializer(columns, schema); + AvroDeserializer deserializer(header_columns, schema); it = deserializer_cache.emplace(schema_id, deserializer).first; } return it->second; @@ -582,12 +583,10 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) "Avro", [=](ReadBuffer & buf, const Block & sample, - const Context & context, const RowInputFormatParams & params, const FormatSettings & settings) { (void)(params); (void)(settings); - (void)(context); return std::make_shared(sample, buf, params); }); @@ -595,12 +594,10 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) "AvroConfluent", [=](ReadBuffer & buf, const Block & sample, - const Context & context, const RowInputFormatParams & params, const FormatSettings & settings) { (void)(params); (void)(settings); - (void)(context); return std::make_shared(sample, buf, params, settings); }); } diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h index 65b46074d7e..ef5e01973dd 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -54,7 +54,7 @@ public: String getName() const override { return "AvroConfluentRowInputFormat"; } private: - const DB::ColumnsWithTypeAndName columns; + const DB::ColumnsWithTypeAndName header_columns; class SchemaRegistry; std::unique_ptr schema_registry; diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 0fd40a6e36c..786626170fc 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -63,7 +63,7 @@ public: virtual bool next(uint8_t ** data, size_t * len) override { out.nextIfAtEnd(); - *data = (uint8_t *)out.position(); + *data = reinterpret_cast(out.position()); *len = out.available(); out.position() += out.available(); @@ -302,10 +302,8 @@ void registerOutputFormatProcessorAvro(FormatFactory & factory) "Avro", [=](WriteBuffer & buf, const Block & sample, - const Context & context, FormatFactory::WriteCallback callback, const FormatSettings & settings) { - (void)(context); (void)(callback); return std::make_shared(buf, sample, callback, settings); }); From e96b3059c04eecaa06ce4343424e8d3588d5d028 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Thu, 9 Jan 2020 23:08:29 -0600 Subject: [PATCH 049/312] Refactor Avro CMake --- cmake/find/avro.cmake | 21 ++------------ contrib/CMakeLists.txt | 42 ++++++++++----------------- contrib/avro-cmake/CMakeLists.txt | 48 ++++++++++++++----------------- contrib/boost | 2 +- 4 files changed, 40 insertions(+), 73 deletions(-) diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake index 7eb5c187cf8..39ad2e31e54 100644 --- a/cmake/find/avro.cmake +++ b/cmake/find/avro.cmake @@ -2,7 +2,7 @@ option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) if (ENABLE_AVRO) -option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ON) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") if(USE_INTERNAL_AVRO_LIBRARY) @@ -13,31 +13,16 @@ if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") endif() if (NOT USE_INTERNAL_AVRO_LIBRARY) - find_package(Snappy REQUIRED) - find_library(AVROCPP avrocpp) elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY) include(cmake/find/snappy.cmake) - add_subdirectory(contrib/avro-cmake) set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") - set(AVROCPP_LIBRARY avrocpp_s) + set(AVROCPP_LIBRARY avrocpp) endif () if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) set(USE_AVRO 1) endif() - -# if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) -# set(USE_AVROCPP 1) -# elseif (Boost_INCLUDE_DIRS AND SNAPPY_LIBRARY) -# set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") -# set(AVROCPP_LIBRARY avrocpp_s) -# set(USE_AVROCPP 1) -# else() -# set(USE_INTERNAL_AVROCPP_LIBRARY 0) -# message(STATUS "avro deps: ${Boost_INCLUDE_DIRS}; ${SNAPPY_LIBRARY}; ${ZLIB_LIBRARY}") -# endif() - endif() -message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_LIBRARY} ${AVROCPP_INCLUDE_DIR}") +message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index f155940c32a..c8b8a7ed7a1 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -146,6 +146,20 @@ if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) add_subdirectory (icu-cmake) endif () +if(USE_INTERNAL_SNAPPY_LIBRARY) + set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") + if (NOT MAKE_STATIC_LIBRARIES) + set(BUILD_SHARED_LIBS 1) # TODO: set at root dir + endif() + + add_subdirectory(snappy) + + set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") + if(SANITIZE STREQUAL "undefined") + target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) + endif() +endif() + if (USE_INTERNAL_PARQUET_LIBRARY) if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time @@ -189,20 +203,6 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) endif() else() - if(USE_INTERNAL_SNAPPY_LIBRARY) - set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") - if (NOT MAKE_STATIC_LIBRARIES) - set(BUILD_SHARED_LIBS 1) # TODO: set at root dir - endif() - - add_subdirectory(snappy) - - set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") - if(SANITIZE STREQUAL "undefined") - target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) - endif() - endif() - add_subdirectory(arrow-cmake) # The library is large - avoid bloat. @@ -213,19 +213,7 @@ endif() endif() if (USE_INTERNAL_AVRO_LIBRARY) - if(USE_INTERNAL_SNAPPY_LIBRARY) - set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") - if (NOT MAKE_STATIC_LIBRARIES) - set(BUILD_SHARED_LIBS 1) # TODO: set at root dir - endif() - - add_subdirectory(snappy) - - set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") - if(SANITIZE STREQUAL "undefined") - target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined) - endif() - endif() + add_subdirectory(avro-cmake) endif() if (USE_INTERNAL_POCO_LIBRARY) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index 643c68c54c5..f544b3c50cd 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -1,9 +1,8 @@ -# project and source dir set(AVROCPP_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++) set(AVROCPP_INCLUDE_DIR ${AVROCPP_ROOT_DIR}/api) set(AVROCPP_SOURCE_DIR ${AVROCPP_ROOT_DIR}/impl) -#set(AVROCPP_COMMON_DIR ${HDFS3_SOURCE_DIR}/common) +set (CMAKE_CXX_STANDARD 17) if (EXISTS ${AVROCPP_ROOT_DIR}/../../share/VERSION.txt) file(READ "${AVROCPP_ROOT_DIR}/../../share/VERSION.txt" @@ -41,36 +40,31 @@ set (AVROCPP_SOURCE_FILES ${AVROCPP_SOURCE_DIR}/Validator.cc ) +add_library (avrocpp ${AVROCPP_SOURCE_FILES}) +set_target_properties (avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) -add_definitions(-std=c++17 -fPIC) +target_include_directories(avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR}) -add_library (avrocpp SHARED ${AVROCPP_SOURCE_FILES}) +target_include_directories(avrocpp SYSTEM PUBLIC ${Boost_INCLUDE_DIRS}) +target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY}) -set_property (TARGET avrocpp - APPEND PROPERTY COMPILE_DEFINITIONS AVRO_DYN_LINK) +if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) + target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) + target_include_directories (avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) + target_link_libraries (avrocpp ${SNAPPY_LIBRARY}) +endif () -add_library (avrocpp_s STATIC ${AVROCPP_SOURCE_FILES}) +if (COMPILER_GCC) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) +elseif (COMPILER_CLANG) + set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) +endif () -set_property (TARGET avrocpp avrocpp_s - APPEND PROPERTY COMPILE_DEFINITIONS AVRO_SOURCE) +target_compile_options(avrocpp PRIVATE ${SUPPRESS_WARNINGS}) -set_target_properties (avrocpp PROPERTIES - VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) - -set_target_properties (avrocpp_s PROPERTIES - VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) - -target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY} ${SNAPPY_LIBRARY}) -target_link_libraries (avrocpp_s ${Boost_IOSTREAMS_LIBRARY} ${SNAPPY_LIBRARY}) - -target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) -target_compile_definitions (avrocpp_s PUBLIC SNAPPY_CODEC_AVAILABLE) - -include_directories(${AVROCPP_INCLUDE_DIR}) -include_directories(${Boost_INCLUDE_DIRS}) -include_directories(${SNAPPY_INCLUDE_DIR}) - -ADD_CUSTOM_TARGET(symlink_headers ALL +# create a symlink to include headers with +ADD_CUSTOM_TARGET(avro_symlink_headers ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${AVROCPP_ROOT_DIR}/include COMMAND ${CMAKE_COMMAND} -E create_symlink ${AVROCPP_ROOT_DIR}/api ${AVROCPP_ROOT_DIR}/include/avro -) \ No newline at end of file +) +add_dependencies(avrocpp avro_symlink_headers) \ No newline at end of file diff --git a/contrib/boost b/contrib/boost index a2cfeb63eaf..86be2aef20b 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit a2cfeb63eaf3b32cf233105b1a40f4a5f26b8495 +Subproject commit 86be2aef20bee2356b744e5569eed6eaded85dbe From 6e26211758c2b1feec19f6953468813354568a92 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Thu, 9 Jan 2020 23:59:01 -0600 Subject: [PATCH 050/312] Fix code style --- .../Formats/Impl/AvroRowInputFormat.cpp | 131 ++++++----- .../Formats/Impl/AvroRowOutputFormat.cpp | 215 ++++++++++-------- 2 files changed, 194 insertions(+), 152 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index f7b34a076d3..a0eba94bfdb 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -120,7 +120,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_STRING: if (target.isString()) { - return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable { + return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable + { decoder.decodeString(tmp); column.insertData(tmp.c_str(), tmp.length()); }; @@ -129,7 +130,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_BYTES: if (target.isString()) { - return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable { + return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable + { decoder.decodeString(tmp); column.insertData(tmp.c_str(), tmp.length()); }; @@ -138,12 +140,15 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_INT: if (target.isInt32()) { - return - [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeInt()); }; + return [](IColumn & column, avro::Decoder & decoder) + { + assert_cast(column).insertValue(decoder.decodeInt()); + }; } if (target.isDate() && logical_type == avro::LogicalType::DATE) { - return [](IColumn & column, avro::Decoder & decoder) { + return [](IColumn & column, avro::Decoder & decoder) + { assert_cast(column).insertValue(decoder.decodeInt()); }; } @@ -151,8 +156,10 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_LONG: if (target.isInt64()) { - return - [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeLong()); }; + return [](IColumn & column, avro::Decoder & decoder) + { + assert_cast(column).insertValue(decoder.decodeLong()); + }; } if (target.isDateTime64()) { @@ -160,7 +167,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node if ((logical_type == avro::LogicalType::TIMESTAMP_MILLIS && date_time_scale == 3) || (logical_type == avro::LogicalType::TIMESTAMP_MICROS && date_time_scale == 6)) { - return [](IColumn & column, avro::Decoder & decoder) { + return [](IColumn & column, avro::Decoder & decoder) + { assert_cast(column).insertValue(decoder.decodeLong()); }; } @@ -169,7 +177,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_FLOAT: if (target.isFloat32()) { - return [](IColumn & column, avro::Decoder & decoder) { + return [](IColumn & column, avro::Decoder & decoder) + { assert_cast(column).insertValue(decoder.decodeFloat()); }; } @@ -177,7 +186,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_DOUBLE: if (target.isFloat64()) { - return [](IColumn & column, avro::Decoder & decoder) { + return [](IColumn & column, avro::Decoder & decoder) + { assert_cast(column).insertValue(decoder.decodeDouble()); }; } @@ -185,17 +195,20 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node case avro::AVRO_BOOL: if (target.isUInt8()) { - return - [](IColumn & column, avro::Decoder & decoder) { assert_cast(column).insertValue(decoder.decodeBool()); }; + return [](IColumn & column, avro::Decoder & decoder) + { + assert_cast(column).insertValue(decoder.decodeBool()); + }; } break; - case avro::AVRO_ARRAY: { + case avro::AVRO_ARRAY: if (target.isArray()) { auto nested_source_type = root_node->leafAt(0); auto nested_target_type = assert_cast(*target_type).getNestedType(); auto nested_deserialize = createDeserializeFn(nested_source_type, nested_target_type); - return [nested_deserialize](IColumn & column, avro::Decoder & decoder) { + return [nested_deserialize](IColumn & column, avro::Decoder & decoder) + { ColumnArray & column_array = assert_cast(column); ColumnArray::Offsets & offsets = column_array.getOffsets(); IColumn & nested_column = column_array.getData(); @@ -212,11 +225,13 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node }; } break; - } - case avro::AVRO_UNION: { - auto nullable_deserializer = [root_node, target_type](size_t non_null_union_index) { + case avro::AVRO_UNION: + { + auto nullable_deserializer = [root_node, target_type](size_t non_null_union_index) + { auto nested_deserialize = createDeserializeFn(root_node->leafAt(non_null_union_index), removeNullable(target_type)); - return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder) { + return [non_null_union_index, nested_deserialize](IColumn & column, avro::Decoder & decoder) + { ColumnNullable & col = assert_cast(column); size_t union_index = decoder.decodeUnionIndex(); if (union_index == non_null_union_index) @@ -245,14 +260,15 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node auto nested_type = removeNullable(target_type); if (nested_type->getTypeId() == TypeIndex::Nothing) { - return [](IColumn & column, avro::Decoder & decoder) { - (void)column; + return [](IColumn &, avro::Decoder & decoder) + { decoder.decodeNull(); }; } else { - return [](IColumn & column, avro::Decoder & decoder) { + return [](IColumn & column, avro::Decoder & decoder) + { ColumnNullable & col = assert_cast(column); decoder.decodeNull(); col.insertDefault(); @@ -268,7 +284,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { symbols.push_back(root_node->nameAt(i)); } - return [symbols](IColumn & column, avro::Decoder & decoder) { + return [symbols](IColumn & column, avro::Decoder & decoder) + { size_t enum_index = decoder.decodeEnum(); const auto & enum_symbol = symbols[enum_index]; column.insertData(enum_symbol.c_str(), enum_symbol.length()); @@ -282,17 +299,20 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node { symbol_mapping.push_back(enum_type.castToValue(root_node->nameAt(i))); } - return [symbol_mapping](IColumn & column, avro::Decoder & decoder) { + return [symbol_mapping](IColumn & column, avro::Decoder & decoder) + { size_t enum_index = decoder.decodeEnum(); column.insert(symbol_mapping[enum_index]); }; } break; - case avro::AVRO_FIXED: { + case avro::AVRO_FIXED: + { size_t fixed_size = root_node->fixedSize(); if (target.isFixedString() && target_type->getSizeOfValueInMemory() == fixed_size) { - return [tmp_fixed = std::vector(fixed_size)](IColumn & column, avro::Decoder & decoder) mutable { + return [tmp_fixed = std::vector(fixed_size)](IColumn & column, avro::Decoder & decoder) mutable + { decoder.decodeFixed(tmp_fixed.size(), tmp_fixed); column.insertData(reinterpret_cast(tmp_fixed.data()), tmp_fixed.size()); }; @@ -328,9 +348,11 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) return [](avro::Decoder & decoder) { decoder.decodeDouble(); }; case avro::AVRO_BOOL: return [](avro::Decoder & decoder) { decoder.decodeBool(); }; - case avro::AVRO_ARRAY: { + case avro::AVRO_ARRAY: + { auto nested_skip_fn = createSkipFn(root_node->leafAt(0)); - return [nested_skip_fn](avro::Decoder & decoder) { + return [nested_skip_fn](avro::Decoder & decoder) + { for (size_t n = decoder.arrayStart(); n != 0; n = decoder.arrayNext()) { for (size_t i = 0; i < n; ++i) @@ -340,7 +362,8 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) } }; } - case avro::AVRO_UNION: { + case avro::AVRO_UNION: + { std::vector union_skip_fns; for (size_t i = 0; i < root_node->leaves(); i++) { @@ -352,14 +375,16 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) return [](avro::Decoder & decoder) { decoder.decodeNull(); }; case avro::AVRO_ENUM: return [](avro::Decoder & decoder) { decoder.decodeEnum(); }; - case avro::AVRO_FIXED: { + case avro::AVRO_FIXED: + { auto fixed_size = root_node->fixedSize(); return [fixed_size](avro::Decoder & decoder) { decoder.skipFixed(fixed_size); }; } - - case avro::AVRO_MAP: { + case avro::AVRO_MAP: + { auto value_skip_fn = createSkipFn(root_node->leafAt(1)); - return [value_skip_fn](avro::Decoder & decoder) { + return [value_skip_fn](avro::Decoder & decoder) + { for (size_t n = decoder.mapStart(); n != 0; n = decoder.mapNext()) { for (size_t i = 0; i < n; ++i) @@ -370,13 +395,15 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) } }; } - case avro::AVRO_RECORD: { + case avro::AVRO_RECORD: + { std::vector field_skip_fns; for (size_t i = 0; i < root_node->leaves(); i++) { field_skip_fns.push_back(createSkipFn(root_node->leafAt(i))); } - return [field_skip_fns](avro::Decoder & decoder) { + return [field_skip_fns](avro::Decoder & decoder) + { for (auto & skip_fn : field_skip_fns) skip_fn(decoder); }; @@ -579,27 +606,23 @@ AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId void registerInputFormatProcessorAvro(FormatFactory & factory) { - factory.registerInputFormatProcessor( - "Avro", - [=](ReadBuffer & buf, - const Block & sample, - const RowInputFormatParams & params, - const FormatSettings & settings) { - (void)(params); - (void)(settings); - return std::make_shared(sample, buf, params); - }); + factory.registerInputFormatProcessor("Avro", [=]( + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings &) + { + return std::make_shared(sample, buf, params); + }); - factory.registerInputFormatProcessor( - "AvroConfluent", - [=](ReadBuffer & buf, - const Block & sample, - const RowInputFormatParams & params, - const FormatSettings & settings) { - (void)(params); - (void)(settings); - return std::make_shared(sample, buf, params, settings); - }); + factory.registerInputFormatProcessor("AvroConfluent",[=]( + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings & settings) + { + return std::make_shared(sample, buf, params, settings); + }); } } diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 786626170fc..56aee6930dc 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -85,36 +85,42 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF switch (data_type->getTypeId()) { case TypeIndex::UInt8: - return {avro::BoolSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeBool(assert_cast(column).getElement(row_num)); - }}; + return {avro::BoolSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeBool(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Int32: - return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeInt(assert_cast(column).getElement(row_num)); - }}; + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Int64: - return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeLong(assert_cast(column).getElement(row_num)); - }}; + return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeLong(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Float32: - return {avro::FloatSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeFloat(assert_cast(column).getElement(row_num)); - }}; + return {avro::FloatSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeFloat(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Float64: - return {avro::DoubleSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeDouble(assert_cast(column).getElement(row_num)); - }}; - case TypeIndex::Date: { + return {avro::DoubleSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeDouble(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Date: + { auto schema = avro::IntSchema(); schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::DATE)); - return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - UInt16 date = assert_cast(column).getElement(row_num); - encoder.encodeInt(date); - }}; + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + UInt16 date = assert_cast(column).getElement(row_num); + encoder.encodeInt(date); + }}; } - case TypeIndex::DateTime: - throw Exception("Unsupported Avro type", ErrorCodes::BAD_TYPE_OF_FIELD); - case TypeIndex::DateTime64: { + case TypeIndex::DateTime64: + { auto schema = avro::LongSchema(); const auto & provided_type = assert_cast(*data_type); @@ -123,27 +129,32 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF else if (provided_type.getScale() == 6) schema.root()->setLogicalType(avro::LogicalType(avro::LogicalType::TIMESTAMP_MICROS)); else - throw Exception("Unsupported Avro type", ErrorCodes::BAD_TYPE_OF_FIELD); + break; - return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const auto & col = assert_cast(column); - encoder.encodeLong(col.getElement(row_num)); - }}; + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const auto & col = assert_cast(column); + encoder.encodeLong(col.getElement(row_num)); + }}; } case TypeIndex::String: - return {avro::StringSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const StringRef & s = assert_cast(column).getDataAt(row_num); - encoder.encodeBytes(reinterpret_cast(s.data), s.size); - }}; - case TypeIndex::FixedString: { - return {avro::FixedSchema(data_type->getSizeOfValueInMemory(), "fixed"), - [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const StringRef & s = assert_cast(column).getDataAt(row_num); - encoder.encodeFixed(reinterpret_cast(s.data), s.size); - }}; + return {avro::StringSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeBytes(reinterpret_cast(s.data), s.size); + }}; + case TypeIndex::FixedString: + { + auto schema = avro::FixedSchema(data_type->getSizeOfValueInMemory(), "fixed"); + return {schema, [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const StringRef & s = assert_cast(column).getDataAt(row_num); + encoder.encodeFixed(reinterpret_cast(s.data), s.size); + }}; } - case TypeIndex::Enum8: { - auto schema = avro::EnumSchema("enum"); + case TypeIndex::Enum8: + { + auto schema = avro::EnumSchema("enum8"); std::unordered_map enum_mapping; const auto & enum_values = assert_cast(*data_type).getValues(); for (size_t i = 0; i < enum_values.size(); ++i) @@ -151,13 +162,15 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF schema.addSymbol(enum_values[i].first); enum_mapping.emplace(enum_values[i].second, i); } - return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - auto enum_value = assert_cast(column).getElement(row_num); - encoder.encodeEnum(enum_mapping.at(enum_value)); - }}; + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; } - case TypeIndex::Enum16: { - auto schema = avro::EnumSchema("enum"); + case TypeIndex::Enum16: + { + auto schema = avro::EnumSchema("enum16"); std::unordered_map enum_mapping; const auto & enum_values = assert_cast(*data_type).getValues(); for (size_t i = 0; i < enum_values.size(); ++i) @@ -165,36 +178,40 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF schema.addSymbol(enum_values[i].first); enum_mapping.emplace(enum_values[i].second, i); } - return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - auto enum_value = assert_cast(column).getElement(row_num); - encoder.encodeEnum(enum_mapping.at(enum_value)); - }}; + return {schema, [enum_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + auto enum_value = assert_cast(column).getElement(row_num); + encoder.encodeEnum(enum_mapping.at(enum_value)); + }}; } - case TypeIndex::Array: { + case TypeIndex::Array: + { const auto & array_type = assert_cast(*data_type); auto nested_mapping = createSchemaWithSerializeFn(array_type.getNestedType()); - return {avro::ArraySchema(nested_mapping.schema), - [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const ColumnArray & column_array = assert_cast(column); - const ColumnArray::Offsets & offsets = column_array.getOffsets(); - size_t offset = offsets[row_num - 1]; - size_t next_offset = offsets[row_num]; - size_t row_count = next_offset - offset; - const IColumn & nested_column = column_array.getData(); + auto schema = avro::ArraySchema(nested_mapping.schema); + return {schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const ColumnArray & column_array = assert_cast(column); + const ColumnArray::Offsets & offsets = column_array.getOffsets(); + size_t offset = offsets[row_num - 1]; + size_t next_offset = offsets[row_num]; + size_t row_count = next_offset - offset; + const IColumn & nested_column = column_array.getData(); - encoder.arrayStart(); - if (row_count > 0) - { - encoder.setItemCount(row_count); - } - for (size_t i = offset; i < next_offset; ++i) - { - nested_mapping.serialize(nested_column, i, encoder); - } - encoder.arrayEnd(); - }}; + encoder.arrayStart(); + if (row_count > 0) + { + encoder.setItemCount(row_count); + } + for (size_t i = offset; i < next_offset; ++i) + { + nested_mapping.serialize(nested_column, i, encoder); + } + encoder.arrayEnd(); + }}; } - case TypeIndex::Nullable: { + case TypeIndex::Nullable: + { auto nested_type = removeNullable(data_type); auto nested_mapping = createSchemaWithSerializeFn(nested_type); if (nested_type->getTypeId() == TypeIndex::Nothing) @@ -206,28 +223,31 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF avro::UnionSchema union_schema; union_schema.addType(avro::NullSchema()); union_schema.addType(nested_mapping.schema); - return {union_schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const ColumnNullable & col = assert_cast(column); - if (!col.isNullAt(row_num)) - { - encoder.encodeUnionIndex(1); - nested_mapping.serialize(col.getNestedColumn(), row_num, encoder); - } - else - { - encoder.encodeUnionIndex(0); - encoder.encodeNull(); - } - }}; + return {union_schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const ColumnNullable & col = assert_cast(column); + if (!col.isNullAt(row_num)) + { + encoder.encodeUnionIndex(1); + nested_mapping.serialize(col.getNestedColumn(), row_num, encoder); + } + else + { + encoder.encodeUnionIndex(0); + encoder.encodeNull(); + } + }}; } } - case TypeIndex::LowCardinality: { + case TypeIndex::LowCardinality: + { const auto & nested_type = removeLowCardinality(data_type); auto nested_mapping = createSchemaWithSerializeFn(nested_type); - return {nested_mapping.schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - const auto & col = assert_cast(column); - nested_mapping.serialize(*col.getDictionary().getNestedColumn(), col.getIndexAt(row_num), encoder); - }}; + return {nested_mapping.schema, [nested_mapping](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + const auto & col = assert_cast(column); + nested_mapping.serialize(*col.getDictionary().getNestedColumn(), col.getIndexAt(row_num), encoder); + }}; } case TypeIndex::Nothing: return {avro::NullSchema(), [](const IColumn &, size_t, avro::Encoder & encoder) { encoder.encodeNull(); }}; @@ -298,15 +318,14 @@ void AvroRowOutputFormat::writeSuffix() void registerOutputFormatProcessorAvro(FormatFactory & factory) { - factory.registerOutputFormatProcessor( - "Avro", - [=](WriteBuffer & buf, - const Block & sample, - FormatFactory::WriteCallback callback, - const FormatSettings & settings) { - (void)(callback); - return std::make_shared(buf, sample, callback, settings); - }); + factory.registerOutputFormatProcessor("Avro",[=]( + WriteBuffer & buf, + const Block & sample, + FormatFactory::WriteCallback callback, + const FormatSettings & settings) + { + return std::make_shared(buf, sample, callback, settings); + }); } } From d15e820e9fec9c47dc55dc80e6a0b5b6ceed89c0 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Fri, 10 Jan 2020 03:05:20 -0600 Subject: [PATCH 051/312] Add Avro tests --- .../queries/0_stateless/01060_avro.reference | 37 + dbms/tests/queries/0_stateless/01060_avro.sh | 68 ++ .../0_stateless/data_avro/complex.avro | Bin 0 -> 843 bytes .../0_stateless/data_avro/complex.avsc | 20 + .../0_stateless/data_avro/complex.json | 2 + .../queries/0_stateless/data_avro/empty.avro | Bin 0 -> 120 bytes .../queries/0_stateless/data_avro/empty.avsc | 7 + .../queries/0_stateless/data_avro/empty.json | 0 .../0_stateless/data_avro/generate_avro.sh | 14 + .../0_stateless/data_avro/logical_types.avro | Bin 0 -> 361 bytes .../0_stateless/data_avro/logical_types.avsc | 9 + .../0_stateless/data_avro/logical_types.json | 1 + .../0_stateless/data_avro/primitive.avro | Bin 0 -> 427 bytes .../0_stateless/data_avro/primitive.avsc | 14 + .../0_stateless/data_avro/primitive.json | 2 + .../queries/0_stateless/data_avro/simple.avsc | 7 + .../0_stateless/data_avro/simple.deflate.avro | Bin 0 -> 1698 bytes .../queries/0_stateless/data_avro/simple.json | 1000 +++++++++++++++++ .../0_stateless/data_avro/simple.null.avro | Bin 0 -> 2077 bytes .../0_stateless/data_avro/simple.snappy.avro | Bin 0 -> 2088 bytes 20 files changed, 1181 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01060_avro.reference create mode 100755 dbms/tests/queries/0_stateless/01060_avro.sh create mode 100644 dbms/tests/queries/0_stateless/data_avro/complex.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/complex.avsc create mode 100644 dbms/tests/queries/0_stateless/data_avro/complex.json create mode 100644 dbms/tests/queries/0_stateless/data_avro/empty.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/empty.avsc create mode 100644 dbms/tests/queries/0_stateless/data_avro/empty.json create mode 100755 dbms/tests/queries/0_stateless/data_avro/generate_avro.sh create mode 100644 dbms/tests/queries/0_stateless/data_avro/logical_types.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/logical_types.avsc create mode 100644 dbms/tests/queries/0_stateless/data_avro/logical_types.json create mode 100644 dbms/tests/queries/0_stateless/data_avro/primitive.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/primitive.avsc create mode 100644 dbms/tests/queries/0_stateless/data_avro/primitive.json create mode 100644 dbms/tests/queries/0_stateless/data_avro/simple.avsc create mode 100644 dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/simple.json create mode 100644 dbms/tests/queries/0_stateless/data_avro/simple.null.avro create mode 100644 dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro diff --git a/dbms/tests/queries/0_stateless/01060_avro.reference b/dbms/tests/queries/0_stateless/01060_avro.reference new file mode 100644 index 00000000000..d8ee426a337 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01060_avro.reference @@ -0,0 +1,37 @@ +=== input += primitive +1,1,2,3.4,5.6,"b1","s1" +0,-1,9223372036854775807,3.00004,0.00001,"","" +1,2,"s1" +0,9223372036854775807,"" +"s1",2,1 +"",9223372036854775807,0 +"s1" +"" += complex +"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba" +"C","f","[]","[]",\N,123,"79cd909892d7e7ade1987cc7422628ba" +"79cd909892d7e7ade1987cc7422628ba" +"79cd909892d7e7ade1987cc7422628ba" += logical_types +"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000" +18250,1578641516227,1578641516227000 += compression +1000 +1000 +1000 += other +0 +not compatible +not found +=== output += primitive +1,1,2,3.4,5.6,"b1","s1" += complex +"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba" += logical_types +"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000" += other +0 +1000 +not supported diff --git a/dbms/tests/queries/0_stateless/01060_avro.sh b/dbms/tests/queries/0_stateless/01060_avro.sh new file mode 100755 index 00000000000..fbde59e58fa --- /dev/null +++ b/dbms/tests/queries/0_stateless/01060_avro.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + +DATA_DIR=$CUR_DIR/data_avro + +# input +echo === input +echo = primitive + +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_bool UInt8, c_long Int64, g_string String' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'g_string String, c_long Int64, a_bool UInt8' -q 'select * from table' +cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'g_string String' -q 'select * from table' + +echo = complex +cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)" -q 'select * from table' +cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "g_fixed FixedString(32)" -q 'select * from table' + +echo = logical_types +cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)' -q 'select * from table' +cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_date Int32, b_timestamp_millis Int64, c_timestamp_micros Int64' -q 'select * from table' + + + +echo = compression +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +cat $DATA_DIR/simple.deflate.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' + +echo = other +#no data +cat $DATA_DIR/empty.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' +# type mismatch +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int32' -q 'select count() from table' 2>&1 | grep -i 'not compatible' -o +# field not found +cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'b Int64' -q 'select count() from table' 2>&1 | grep -i 'not found' -o + + + + + + +# output +echo === output + +echo = primitive +S1="a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String" +echo '1,1,2,3.4,5.6,"b1","s1"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S1" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S1" -q 'select * from table' + +echo = complex +S2="a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)" +echo "\"A\",\"t\",\"['s1','s2']\",\"[['a1'],['a2']]\",\"s1\",\N,\"79cd909892d7e7ade1987cc7422628ba\"" | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S2" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S2" -q 'select * from table' + +echo = logical_types +S3="a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)" +echo '"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S3" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S3" -q 'select * from table' + +echo = other +S4="a Int64" +${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(0) format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table' +${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(1000) format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table' + +# type not supported +${CLICKHOUSE_LOCAL} -q "select toInt16(123) as a format Avro" 2>&1 | grep -i 'not supported' -o \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.avro b/dbms/tests/queries/0_stateless/data_avro/complex.avro new file mode 100644 index 0000000000000000000000000000000000000000..0880f5818829114622dc789466eead14fca4dba7 GIT binary patch literal 843 zcmb7?u};G<5Qfvp127{bR?ih829c&4@BlC{QB{`XIB6h>t2hA)$}=$VHt-ItEG*2d zh!Jsd(vSwKib%2apM9V2&i{#f(L-D8DwR8mPejZfH|}sYn+iO{so*m8a33cu)>>YJ z@Iw)K3J*tj^Ejg-$zz&HsUHt_`3i)h`#`f+O}w zdnOZF8HTFMaQd6ob(6~7CiI7=8o<2C0*o;(Hnml`^aNik`!McLzqZewUS6}e^N)x1 z_W(Wj3*iB4!z|5Cc|anYP0{XV)1x#IuFXJYhKln{#`gkOMMsnaPl8!1vi( IeX@J=H!X4%b^rhX literal 0 HcmV?d00001 diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.avsc b/dbms/tests/queries/0_stateless/data_avro/complex.avsc new file mode 100644 index 00000000000..325169aeb57 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/complex.avsc @@ -0,0 +1,20 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a_enum_to_string", "type": { "type": "enum", "name": "enum_1", "symbols" : ["A", "B", "C"]}}, + {"name": "b_enum_to_enum", "type": { "type": "enum", "name": "enum_2", "symbols" : ["t", "f"]}}, + {"name": "c_array_string", "type": { "type": "array", "items": "string"}}, + {"name": "d_array_array_string", "type": { "type": "array", "items": {"type": "array", "items": "string"}}}, + {"name": "e_union_null_string", "type": ["null", "string"]}, + {"name": "f_union_long_null", "type": ["long", "null"]}, + {"name": "g_fixed", "type": {"type":"fixed", "size": 32, "name": "fixed_1"}}, + {"name": "h_record_skip", "type": { + "type": "record", + "name": "subrecord", + "fields": [ + {"name": "a", "type": "string"} + ] + }} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/complex.json b/dbms/tests/queries/0_stateless/data_avro/complex.json new file mode 100644 index 00000000000..d05e09c72fc --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/complex.json @@ -0,0 +1,2 @@ +{"a_enum_to_string":"A","b_enum_to_enum":"t","c_array_string":["s1", "s2"],"d_array_array_string":[["a1"], ["a2"]],"e_union_null_string":{"string": "s1"},"f_union_long_null":null,"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}} +{"a_enum_to_string":"C","b_enum_to_enum":"f","c_array_string":[],"d_array_array_string":[],"e_union_null_string":null,"f_union_long_null":{"long": 123},"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}} \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/empty.avro b/dbms/tests/queries/0_stateless/data_avro/empty.avro new file mode 100644 index 0000000000000000000000000000000000000000..7cfae81758cf61917da409cd2a9ad03d3af2130c GIT binary patch literal 120 zcmeZI%3@>@ODrqO*DFrWNX<>`V60XusVqoUvQjEaP0lY$QPNS$OUwoH^UHypw9M3; zlwu{T=xUgFB2W~nB_}^GU8y# primitive.avro +avro-tools fromjson --schema-file complex.avsc complex.json > complex.avro +avro-tools fromjson --schema-file logical_types.avsc logical_types.json > logical_types.avro +avro-tools fromjson --schema-file empty.avsc empty.json > empty.avro + +#compression +avro-tools fromjson --codec null --schema-file simple.avsc simple.json > simple.null.avro +avro-tools fromjson --codec deflate --schema-file simple.avsc simple.json > simple.deflate.avro +avro-tools fromjson --codec snappy --schema-file simple.avsc simple.json > simple.snappy.avro \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/logical_types.avro b/dbms/tests/queries/0_stateless/data_avro/logical_types.avro new file mode 100644 index 0000000000000000000000000000000000000000..7b8a3f60b7a078a39b5bf6b2f7d841ccde80c056 GIT binary patch literal 361 zcmeZI%3@>@ODrqO*DFrWNX<=L!cwhNQdy9yWTjM;nw(#hqNJmgmzWFY=a&OHX_=`x zDaA@w(bX{V#Q2oNl2o7+L?2v#W?l(UFeg7fGdVFQ1Y#ywWo@kvTys)@ODrqO*DFrWNX<=r!&0qOQdy9yWTjM;nw(#hqNJmgmzWFY=a&OHX_=`x zDaA@w(bX{V#Q3EA{2ZVZR3C_!nwY0ls{>b(6rY(_f=~hCAju}j=j7+5Bc#DRB>9y1 zw4D4zB>fOBl8V&$l>E}9oK%DvP(G5HwD_dTlGI{^5(pPbMS6U3Nl_-UOQ3utH5u`F zr8!8R1@mfSYem3en4F)In#=(bV_3N`eD0eSbLGoYi6WF8W@N< hup}9>6dN)y{`(IFj4K!%96n||?%4X{i5&xm0RZw(n9l$J literal 0 HcmV?d00001 diff --git a/dbms/tests/queries/0_stateless/data_avro/primitive.avsc b/dbms/tests/queries/0_stateless/data_avro/primitive.avsc new file mode 100644 index 00000000000..a4f06d02b01 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/primitive.avsc @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a_bool", "type": "boolean"}, + {"name": "b_int", "type": "int"}, + {"name": "c_long", "type": "long"}, + {"name": "d_float", "type": "float"}, + {"name": "e_double", "type": "double"}, + {"name": "f_bytes", "type": "bytes"}, + {"name": "g_string", "type": "string"}, + {"name": "h_null", "type": "null"} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/primitive.json b/dbms/tests/queries/0_stateless/data_avro/primitive.json new file mode 100644 index 00000000000..fc521c8829c --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/primitive.json @@ -0,0 +1,2 @@ +{"a_bool":true,"b_int":1,"c_long":2,"d_float":3.4,"e_double":5.6,"f_bytes":"b1","g_string":"s1","h_null": null} +{"a_bool":false,"b_int":-1,"c_long":9223372036854775807,"d_float":3.00004,"e_double":0.00001,"f_bytes":"","g_string":"","h_null": null} \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.avsc b/dbms/tests/queries/0_stateless/data_avro/simple.avsc new file mode 100644 index 00000000000..923eda71054 --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/simple.avsc @@ -0,0 +1,7 @@ +{ + "type": "record", + "name": "row", + "fields": [ + {"name": "a", "type": "long"} + ] + } \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro b/dbms/tests/queries/0_stateless/data_avro/simple.deflate.avro new file mode 100644 index 0000000000000000000000000000000000000000..d4ba226b44753945fc570014d43472800f3a9af4 GIT binary patch literal 1698 zcmZuyTS#LE0`0`EkHqAH7F`rnt|2Bj@kSC!>^1I(g65$KC@3ha@h0YRb8~OF3DM@! z`Hjy&3%e})p@|4^l*O8fph*2 zM}uMy-mIH;QY38GnP#lhz6Wm5@s4cDTW6A~;DHLGRzaZ~fM&BlR;nmVU;7PdjGb(Ej5{d?Ydn^*i1c{~wImw~gxe*or@6>JfmZKS->;D%C$Kejt4KV|eP-t@P2) zQS#x_55IeL2mG!0z5m_P`>S__r;m$2#^3$r-{0N6{OMuwht<2%KOf({xVSGqeSG)J zDscbmi~p>8ZXf4lA2U231GJXP-j-^xg^sqcYFes>TbQC22Gs)4*D953)zBIpTVpk> zRgJGPku`>E4Jg-E_G_y-TDn2Ys?%1DYMBx(!>I*W$Cc{iYWSE=9J88^t0s?`vLlA~ z2q^2Q?CYp*?V#&AShXEhBOOd}2gBY0Ft#dHTh;t6x@C*ixK%Z=#q8cFUy`RdY|~qx_o_IORZfCx~`SluKAI!WpUS{z01$o^{IB3 z_`3_1-Id1O`H9`-?%hTAuD@c=H!!!N?f6or;mpA#tb5-l<{i$W=Ro{2irbN8PxinAjQW-ch-CG!+i{fMc-Dp)@+w z^$x|DV@T>y%{nx|wOn&OD7;pZ*XrhL#nkmsPv*dr$LIy6R zi9)uikeVzU%Dz4Dew!w9d20Xjjy*O>wMR`upH&~qXG^J8Bn+m z<&C245|r;mg{%{<`a}SqK*R~$bi$iFX_tM;_kJlX6LI@Qf>sfv6T!72-iWAOEaKZm zLdG^%wJqRpLzZp0aho@>-QK;;cW(&${BkZBTO?5#EN#+gS7M=G1Mp=W~nabCSj}^>T2KgP|OvhGQP)SVbHQ#Ubed zlQMuq0SpTe4FU6bz={Mct^iqXF!dX7jsY_mh&qFL)L@kuEKUQ-Iy0%yaQF-(&WNTn z^W>RT_O-?PHCZMx^-1tn38s?}wG#7)#446p>=KfZFsTwaKY>{iL}S7{k+60rEbatZ z;WQ06@ir%BbQ1MW^O(~rby{YfByeNW+~C3+jJzS5Z_HCS)}HKhPxgWa=X>D-2%n?y y%NqFEF#JLUKd0aedMvMu6`fyAD>Vi)B^zMm*?Uf%!lCVmBbUD7}R literal 0 HcmV?d00001 diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.json b/dbms/tests/queries/0_stateless/data_avro/simple.json new file mode 100644 index 00000000000..c09fc0b732f --- /dev/null +++ b/dbms/tests/queries/0_stateless/data_avro/simple.json @@ -0,0 +1,1000 @@ +{"a":1} +{"a":2} +{"a":3} +{"a":4} +{"a":5} +{"a":6} +{"a":7} +{"a":8} +{"a":9} +{"a":10} +{"a":11} +{"a":12} +{"a":13} +{"a":14} +{"a":15} +{"a":16} +{"a":17} +{"a":18} +{"a":19} +{"a":20} +{"a":21} +{"a":22} +{"a":23} +{"a":24} +{"a":25} +{"a":26} +{"a":27} +{"a":28} +{"a":29} +{"a":30} +{"a":31} +{"a":32} +{"a":33} +{"a":34} +{"a":35} +{"a":36} +{"a":37} +{"a":38} +{"a":39} +{"a":40} +{"a":41} +{"a":42} +{"a":43} +{"a":44} +{"a":45} +{"a":46} +{"a":47} +{"a":48} +{"a":49} +{"a":50} +{"a":51} +{"a":52} +{"a":53} +{"a":54} +{"a":55} +{"a":56} +{"a":57} +{"a":58} +{"a":59} +{"a":60} +{"a":61} +{"a":62} +{"a":63} +{"a":64} +{"a":65} +{"a":66} +{"a":67} +{"a":68} +{"a":69} +{"a":70} +{"a":71} +{"a":72} +{"a":73} +{"a":74} +{"a":75} +{"a":76} +{"a":77} +{"a":78} +{"a":79} +{"a":80} +{"a":81} +{"a":82} +{"a":83} +{"a":84} +{"a":85} +{"a":86} +{"a":87} +{"a":88} +{"a":89} +{"a":90} +{"a":91} +{"a":92} +{"a":93} +{"a":94} +{"a":95} +{"a":96} +{"a":97} +{"a":98} +{"a":99} +{"a":100} +{"a":101} +{"a":102} +{"a":103} +{"a":104} +{"a":105} +{"a":106} +{"a":107} +{"a":108} +{"a":109} +{"a":110} +{"a":111} +{"a":112} +{"a":113} +{"a":114} +{"a":115} +{"a":116} +{"a":117} +{"a":118} +{"a":119} +{"a":120} +{"a":121} +{"a":122} +{"a":123} +{"a":124} +{"a":125} +{"a":126} +{"a":127} +{"a":128} +{"a":129} +{"a":130} +{"a":131} +{"a":132} +{"a":133} +{"a":134} +{"a":135} +{"a":136} +{"a":137} +{"a":138} +{"a":139} +{"a":140} +{"a":141} +{"a":142} +{"a":143} +{"a":144} +{"a":145} +{"a":146} +{"a":147} +{"a":148} +{"a":149} +{"a":150} +{"a":151} +{"a":152} +{"a":153} +{"a":154} +{"a":155} +{"a":156} +{"a":157} +{"a":158} +{"a":159} +{"a":160} +{"a":161} +{"a":162} +{"a":163} +{"a":164} +{"a":165} +{"a":166} +{"a":167} +{"a":168} +{"a":169} +{"a":170} +{"a":171} +{"a":172} +{"a":173} +{"a":174} +{"a":175} +{"a":176} +{"a":177} +{"a":178} +{"a":179} +{"a":180} +{"a":181} +{"a":182} +{"a":183} +{"a":184} +{"a":185} +{"a":186} +{"a":187} +{"a":188} +{"a":189} +{"a":190} +{"a":191} +{"a":192} +{"a":193} +{"a":194} +{"a":195} +{"a":196} +{"a":197} +{"a":198} +{"a":199} +{"a":200} +{"a":201} +{"a":202} +{"a":203} +{"a":204} +{"a":205} +{"a":206} +{"a":207} +{"a":208} +{"a":209} +{"a":210} +{"a":211} +{"a":212} +{"a":213} +{"a":214} +{"a":215} +{"a":216} +{"a":217} +{"a":218} +{"a":219} +{"a":220} +{"a":221} +{"a":222} +{"a":223} +{"a":224} +{"a":225} +{"a":226} +{"a":227} +{"a":228} +{"a":229} +{"a":230} +{"a":231} +{"a":232} +{"a":233} +{"a":234} +{"a":235} +{"a":236} +{"a":237} +{"a":238} +{"a":239} +{"a":240} +{"a":241} +{"a":242} +{"a":243} +{"a":244} +{"a":245} +{"a":246} +{"a":247} +{"a":248} +{"a":249} +{"a":250} +{"a":251} +{"a":252} +{"a":253} +{"a":254} +{"a":255} +{"a":256} +{"a":257} +{"a":258} +{"a":259} +{"a":260} +{"a":261} +{"a":262} +{"a":263} +{"a":264} +{"a":265} +{"a":266} +{"a":267} +{"a":268} +{"a":269} +{"a":270} +{"a":271} +{"a":272} +{"a":273} +{"a":274} +{"a":275} +{"a":276} +{"a":277} +{"a":278} +{"a":279} +{"a":280} +{"a":281} +{"a":282} +{"a":283} +{"a":284} +{"a":285} +{"a":286} +{"a":287} +{"a":288} +{"a":289} +{"a":290} +{"a":291} +{"a":292} +{"a":293} +{"a":294} +{"a":295} +{"a":296} +{"a":297} +{"a":298} +{"a":299} +{"a":300} +{"a":301} +{"a":302} +{"a":303} +{"a":304} +{"a":305} +{"a":306} +{"a":307} +{"a":308} +{"a":309} +{"a":310} +{"a":311} +{"a":312} +{"a":313} +{"a":314} +{"a":315} +{"a":316} +{"a":317} +{"a":318} +{"a":319} +{"a":320} +{"a":321} +{"a":322} +{"a":323} +{"a":324} +{"a":325} +{"a":326} +{"a":327} +{"a":328} +{"a":329} +{"a":330} +{"a":331} +{"a":332} +{"a":333} +{"a":334} +{"a":335} +{"a":336} +{"a":337} +{"a":338} +{"a":339} +{"a":340} +{"a":341} +{"a":342} +{"a":343} +{"a":344} +{"a":345} +{"a":346} +{"a":347} +{"a":348} +{"a":349} +{"a":350} +{"a":351} +{"a":352} +{"a":353} +{"a":354} +{"a":355} +{"a":356} +{"a":357} +{"a":358} +{"a":359} +{"a":360} +{"a":361} +{"a":362} +{"a":363} +{"a":364} +{"a":365} +{"a":366} +{"a":367} +{"a":368} +{"a":369} +{"a":370} +{"a":371} +{"a":372} +{"a":373} +{"a":374} +{"a":375} +{"a":376} +{"a":377} +{"a":378} +{"a":379} +{"a":380} +{"a":381} +{"a":382} +{"a":383} +{"a":384} +{"a":385} +{"a":386} +{"a":387} +{"a":388} +{"a":389} +{"a":390} +{"a":391} +{"a":392} +{"a":393} +{"a":394} +{"a":395} +{"a":396} +{"a":397} +{"a":398} +{"a":399} +{"a":400} +{"a":401} +{"a":402} +{"a":403} +{"a":404} +{"a":405} +{"a":406} +{"a":407} +{"a":408} +{"a":409} +{"a":410} +{"a":411} +{"a":412} +{"a":413} +{"a":414} +{"a":415} +{"a":416} +{"a":417} +{"a":418} +{"a":419} +{"a":420} +{"a":421} +{"a":422} +{"a":423} +{"a":424} +{"a":425} +{"a":426} +{"a":427} +{"a":428} +{"a":429} +{"a":430} +{"a":431} +{"a":432} +{"a":433} +{"a":434} +{"a":435} +{"a":436} +{"a":437} +{"a":438} +{"a":439} +{"a":440} +{"a":441} +{"a":442} +{"a":443} +{"a":444} +{"a":445} +{"a":446} +{"a":447} +{"a":448} +{"a":449} +{"a":450} +{"a":451} +{"a":452} +{"a":453} +{"a":454} +{"a":455} +{"a":456} +{"a":457} +{"a":458} +{"a":459} +{"a":460} +{"a":461} +{"a":462} +{"a":463} +{"a":464} +{"a":465} +{"a":466} +{"a":467} +{"a":468} +{"a":469} +{"a":470} +{"a":471} +{"a":472} +{"a":473} +{"a":474} +{"a":475} +{"a":476} +{"a":477} +{"a":478} +{"a":479} +{"a":480} +{"a":481} +{"a":482} +{"a":483} +{"a":484} +{"a":485} +{"a":486} +{"a":487} +{"a":488} +{"a":489} +{"a":490} +{"a":491} +{"a":492} +{"a":493} +{"a":494} +{"a":495} +{"a":496} +{"a":497} +{"a":498} +{"a":499} +{"a":500} +{"a":501} +{"a":502} +{"a":503} +{"a":504} +{"a":505} +{"a":506} +{"a":507} +{"a":508} +{"a":509} +{"a":510} +{"a":511} +{"a":512} +{"a":513} +{"a":514} +{"a":515} +{"a":516} +{"a":517} +{"a":518} +{"a":519} +{"a":520} +{"a":521} +{"a":522} +{"a":523} +{"a":524} +{"a":525} +{"a":526} +{"a":527} +{"a":528} +{"a":529} +{"a":530} +{"a":531} +{"a":532} +{"a":533} +{"a":534} +{"a":535} +{"a":536} +{"a":537} +{"a":538} +{"a":539} +{"a":540} +{"a":541} +{"a":542} +{"a":543} +{"a":544} +{"a":545} +{"a":546} +{"a":547} +{"a":548} +{"a":549} +{"a":550} +{"a":551} +{"a":552} +{"a":553} +{"a":554} +{"a":555} +{"a":556} +{"a":557} +{"a":558} +{"a":559} +{"a":560} +{"a":561} +{"a":562} +{"a":563} +{"a":564} +{"a":565} +{"a":566} +{"a":567} +{"a":568} +{"a":569} +{"a":570} +{"a":571} +{"a":572} +{"a":573} +{"a":574} +{"a":575} +{"a":576} +{"a":577} +{"a":578} +{"a":579} +{"a":580} +{"a":581} +{"a":582} +{"a":583} +{"a":584} +{"a":585} +{"a":586} +{"a":587} +{"a":588} +{"a":589} +{"a":590} +{"a":591} +{"a":592} +{"a":593} +{"a":594} +{"a":595} +{"a":596} +{"a":597} +{"a":598} +{"a":599} +{"a":600} +{"a":601} +{"a":602} +{"a":603} +{"a":604} +{"a":605} +{"a":606} +{"a":607} +{"a":608} +{"a":609} +{"a":610} +{"a":611} +{"a":612} +{"a":613} +{"a":614} +{"a":615} +{"a":616} +{"a":617} +{"a":618} +{"a":619} +{"a":620} +{"a":621} +{"a":622} +{"a":623} +{"a":624} +{"a":625} +{"a":626} +{"a":627} +{"a":628} +{"a":629} +{"a":630} +{"a":631} +{"a":632} +{"a":633} +{"a":634} +{"a":635} +{"a":636} +{"a":637} +{"a":638} +{"a":639} +{"a":640} +{"a":641} +{"a":642} +{"a":643} +{"a":644} +{"a":645} +{"a":646} +{"a":647} +{"a":648} +{"a":649} +{"a":650} +{"a":651} +{"a":652} +{"a":653} +{"a":654} +{"a":655} +{"a":656} +{"a":657} +{"a":658} +{"a":659} +{"a":660} +{"a":661} +{"a":662} +{"a":663} +{"a":664} +{"a":665} +{"a":666} +{"a":667} +{"a":668} +{"a":669} +{"a":670} +{"a":671} +{"a":672} +{"a":673} +{"a":674} +{"a":675} +{"a":676} +{"a":677} +{"a":678} +{"a":679} +{"a":680} +{"a":681} +{"a":682} +{"a":683} +{"a":684} +{"a":685} +{"a":686} +{"a":687} +{"a":688} +{"a":689} +{"a":690} +{"a":691} +{"a":692} +{"a":693} +{"a":694} +{"a":695} +{"a":696} +{"a":697} +{"a":698} +{"a":699} +{"a":700} +{"a":701} +{"a":702} +{"a":703} +{"a":704} +{"a":705} +{"a":706} +{"a":707} +{"a":708} +{"a":709} +{"a":710} +{"a":711} +{"a":712} +{"a":713} +{"a":714} +{"a":715} +{"a":716} +{"a":717} +{"a":718} +{"a":719} +{"a":720} +{"a":721} +{"a":722} +{"a":723} +{"a":724} +{"a":725} +{"a":726} +{"a":727} +{"a":728} +{"a":729} +{"a":730} +{"a":731} +{"a":732} +{"a":733} +{"a":734} +{"a":735} +{"a":736} +{"a":737} +{"a":738} +{"a":739} +{"a":740} +{"a":741} +{"a":742} +{"a":743} +{"a":744} +{"a":745} +{"a":746} +{"a":747} +{"a":748} +{"a":749} +{"a":750} +{"a":751} +{"a":752} +{"a":753} +{"a":754} +{"a":755} +{"a":756} +{"a":757} +{"a":758} +{"a":759} +{"a":760} +{"a":761} +{"a":762} +{"a":763} +{"a":764} +{"a":765} +{"a":766} +{"a":767} +{"a":768} +{"a":769} +{"a":770} +{"a":771} +{"a":772} +{"a":773} +{"a":774} +{"a":775} +{"a":776} +{"a":777} +{"a":778} +{"a":779} +{"a":780} +{"a":781} +{"a":782} +{"a":783} +{"a":784} +{"a":785} +{"a":786} +{"a":787} +{"a":788} +{"a":789} +{"a":790} +{"a":791} +{"a":792} +{"a":793} +{"a":794} +{"a":795} +{"a":796} +{"a":797} +{"a":798} +{"a":799} +{"a":800} +{"a":801} +{"a":802} +{"a":803} +{"a":804} +{"a":805} +{"a":806} +{"a":807} +{"a":808} +{"a":809} +{"a":810} +{"a":811} +{"a":812} +{"a":813} +{"a":814} +{"a":815} +{"a":816} +{"a":817} +{"a":818} +{"a":819} +{"a":820} +{"a":821} +{"a":822} +{"a":823} +{"a":824} +{"a":825} +{"a":826} +{"a":827} +{"a":828} +{"a":829} +{"a":830} +{"a":831} +{"a":832} +{"a":833} +{"a":834} +{"a":835} +{"a":836} +{"a":837} +{"a":838} +{"a":839} +{"a":840} +{"a":841} +{"a":842} +{"a":843} +{"a":844} +{"a":845} +{"a":846} +{"a":847} +{"a":848} +{"a":849} +{"a":850} +{"a":851} +{"a":852} +{"a":853} +{"a":854} +{"a":855} +{"a":856} +{"a":857} +{"a":858} +{"a":859} +{"a":860} +{"a":861} +{"a":862} +{"a":863} +{"a":864} +{"a":865} +{"a":866} +{"a":867} +{"a":868} +{"a":869} +{"a":870} +{"a":871} +{"a":872} +{"a":873} +{"a":874} +{"a":875} +{"a":876} +{"a":877} +{"a":878} +{"a":879} +{"a":880} +{"a":881} +{"a":882} +{"a":883} +{"a":884} +{"a":885} +{"a":886} +{"a":887} +{"a":888} +{"a":889} +{"a":890} +{"a":891} +{"a":892} +{"a":893} +{"a":894} +{"a":895} +{"a":896} +{"a":897} +{"a":898} +{"a":899} +{"a":900} +{"a":901} +{"a":902} +{"a":903} +{"a":904} +{"a":905} +{"a":906} +{"a":907} +{"a":908} +{"a":909} +{"a":910} +{"a":911} +{"a":912} +{"a":913} +{"a":914} +{"a":915} +{"a":916} +{"a":917} +{"a":918} +{"a":919} +{"a":920} +{"a":921} +{"a":922} +{"a":923} +{"a":924} +{"a":925} +{"a":926} +{"a":927} +{"a":928} +{"a":929} +{"a":930} +{"a":931} +{"a":932} +{"a":933} +{"a":934} +{"a":935} +{"a":936} +{"a":937} +{"a":938} +{"a":939} +{"a":940} +{"a":941} +{"a":942} +{"a":943} +{"a":944} +{"a":945} +{"a":946} +{"a":947} +{"a":948} +{"a":949} +{"a":950} +{"a":951} +{"a":952} +{"a":953} +{"a":954} +{"a":955} +{"a":956} +{"a":957} +{"a":958} +{"a":959} +{"a":960} +{"a":961} +{"a":962} +{"a":963} +{"a":964} +{"a":965} +{"a":966} +{"a":967} +{"a":968} +{"a":969} +{"a":970} +{"a":971} +{"a":972} +{"a":973} +{"a":974} +{"a":975} +{"a":976} +{"a":977} +{"a":978} +{"a":979} +{"a":980} +{"a":981} +{"a":982} +{"a":983} +{"a":984} +{"a":985} +{"a":986} +{"a":987} +{"a":988} +{"a":989} +{"a":990} +{"a":991} +{"a":992} +{"a":993} +{"a":994} +{"a":995} +{"a":996} +{"a":997} +{"a":998} +{"a":999} +{"a":1000} diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.null.avro b/dbms/tests/queries/0_stateless/data_avro/simple.null.avro new file mode 100644 index 0000000000000000000000000000000000000000..789ab45101fd0076e5fd966c654d2abf61826f36 GIT binary patch literal 2077 zcmZ9Nf8_f40mt|H{rmNvIcH|he9Sg;&dki5nKNg$nQdlf_RGHa{eEw^^S$r)d~dtk zcDqSSk|arzBuSDaNs=TgB7a58nBNpMCH#y!pq+PaeH@_3E!b z{GWgDvHkDl$+M?_xcca~AHDEN@!_*ak00`eKmO^@ ze*TMJ{^~cs`~4e_-hAurCr{sf{-^i<^0&YL!v);H1H8Zo{2%~=AOyl70-_)W;vfN% zAO+GO1F|3o@}K~UpajaG0;-?}>YxFdpat5X1G=CG`d|QtUqi35~g4pW?&ZP zU>+7=5td*XR$vv@U>!DK6SiO*c3>CwU>^?P5RTv&Uc(!B3n%anPT>sB;Q}t<3a;S> zZs88@;XORSBRs(~yg(OnBM$k7>c6=N}?1>qYTQT9Ll2tDxwl9 zqYA2`8mglPYN8fuqYmn#9_ph38ln*zqib}7ZqWqYp(&c7Ia;74TA?-Cpe@>=J-SB+ zbVMg~Mi=D5ZtTHc?8AN>z(E|sVI09x9K&&(z)76KX`I1XoWprsz(ribWn95kT*GzT zz)jr3ZQQ|K+{1l5z(YL3V|3|OD zh>qzsy`i^sLhtC5&gh&j=#sAJnr`Tp?&zN0(*r%y6Ft)lbul;dFfa2lKMSxR3$ZYZ zuqcbMI7_f3OR+S|uq?~5JS(swE3q=GuqvyuI%}{dYq2)#urBMdJ{zzh8?iCFW;g7X zP1qfqvKgDR1zWNeTeA(@vK`yAdv;()c4B9CVJ`0G9`5Bn?&kp>E!Xv!GC;TEHf+8fsA|j$9CgLI?k|HJ2A|tXQ zC-R~oilQXSq9UrIChDRgnxZAzq9eMZC;DO_hGHbf;#%B@TQL!LVk%~0E*4@bR$?tS zVk>rHFYd)b9K}hT#YMQJTY98d`lMe5WKf1=SVm-2#$;S3WKyPNT4rQc=44(LWKou6 zSyp6K)?{5aWK*_eTXtku_GDiU}SYFE;c`GOKPEO@a&gDWbTv%E-`ax0JWDxdPJfC{RR3af~Us+fwagi5NEN~?^@s+`KJf-0(#DyxdBs+y{+ zhH9#oYO9Xws-Ehrff}lj8mnt{qi)qi-KnXXskvIHrCO=A+NiDCslB>a2X#~@bygSU z(r)e1UhUI<9ne7?(qSFZQ61B9ozO|0(rKO1S)J2)UC>2c(q&!IRbA6{-Ox?l(rw+* zUER}tJL`fb1lZODdg z#71q*#%;nTZOW!?#%68K=54_iZON8x#a3<2)@{Q!ZOgW8$98Sc_U*t9?Z}SpwY{;o zc4F`B)XwbOF6`2-?AmVZ*6!@y-rIvc+LJxoi*-3}$K!Y%pW}A|PS6QCVJG55otP7M z5>C=dIcX>3WSyLocM49?DLG}Q;#8fQQ+FCp(`h+vr{i>;p3`>*&d?b-|9{f|0mqXU A_y7O^ literal 0 HcmV?d00001 diff --git a/dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro b/dbms/tests/queries/0_stateless/data_avro/simple.snappy.avro new file mode 100644 index 0000000000000000000000000000000000000000..b812ed6c7ea976a74e536d1997d543f1e9962376 GIT binary patch literal 2088 zcmZ9NeW>bp0f+m(zkh!t#)ya!hfxt@L`B4ih=_=YiWm{S(cQgwt#Nnn3`Z6On_+K@M^-5jly8h=?4dG3uZGd>;5b|9*b*`da|MaPPfm zPhNTF{+kcqzUKoUT)g}K(}x$YUOanv|H-ol7cXBtzW4Sg{N#@x!yA8i_~^kq7q9;6 zga7$^AKU*<9zA*d`-=~M^WpQK6z@NI@bJEV=kdL#Pv8I4^YefI>uaZ%g3pQ9{{D|I zCO;ZEl^6f!>_ZG<%B8l?efs$qKJ(enz4-Zym%i|&FTecCSHAl7Z+!FBZ+-ha-~HbA zfAGT}{rIOpd+q1H_~oyE`@7d4yz%B+j~+jL_U?Os`tx7@>H==y0bbw(eh>gb5CUNk z0Z|YGagYE>kOFCt0a=g(c~AgFPy%I80aZ`~bqi3 z5~g4pW?&ZPU>+7=5td*XR$vv@U>!DK6SiO*c3>CwU>^?P5RTv_9K#8`f>U@6XYdBj z;Q}t<3f{ss+`ui|!96^{BRs)7c!n=HB?6p)I=@RMjg~eJ=8}7G(;nGiN{ZXq3iioF-_Jrf8aGXqM(^o)&14mS~w)XqDDzoi=EbwrHDn zXqWbApAP7dj_4&F(+RzzQ+iEj^oGvqf-dQb-qJPQ&@J83Jw4DPJ<&UQrY`1Y9_D2} z=4SyGWFZ!25f)`J7H0{TWGR+r8J1-^mS+W4WF=N+6;@?6R%Z>?WG&Wa9oA(%)@K7Y zWFvOT#%#i_*pywf8M|R~wqQ%PVz+F~Hf+mwY|jqt$WH8zotcZfxrckXkNbIm2YHBx zd4xxKjK_I`CwYped4^|sj^}xS7kP=7d4*Sbjn{dDH+hSJVkh?EAdccB?!;NRq+5EVSNf!124ql%WLQRIRK{dnCS+2kWLjoq zR_0`07GzPDWLZ{ZRn}x(He^$_WLtJ*SN3FI4&+ddX)m1&!R|7RvBXz09YND>xR9&l?x>0krP)oH^w`#36YO8i? zuMX;{PU=pbl}o#|M|-tT`*lDEbx4PGL`QW@$8|y{bxNmoMrU9IFZ5Ec^sQd&jo#{=-s^)t>XW|H zXYDd><1t?2Gky~=K@&1z6ERT}GjWqJNs}^ZlQCJ7GkH@mMN=|mQ!!OjGj-E2P17=M z(=lDsGkr5KLo+g$W^5+r%1q6*nVB0iHw&{gD|2hsW@ENyXZGe`j^;Y|)l%*;Z`T)@dZWSp#%bMj8XDLN&m>{OhpQ*-K0!)ZD#r|oo{uG4e+&cGQu NBj=x|eE9$0`aema9i{*P literal 0 HcmV?d00001 From 7d9a6130b61e3eb072fb35b17cfd439a5120efc4 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Fri, 10 Jan 2020 03:28:58 -0600 Subject: [PATCH 052/312] Default to deflate if snappy is not enabled --- dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 8 +++++++- dbms/tests/queries/0_stateless/01060_avro.reference | 1 - dbms/tests/queries/0_stateless/01060_avro.sh | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 56aee6930dc..603eb05528d 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -42,6 +42,12 @@ #include #include +#define DEFAULT_SYNC_INTERVAL 16*1024 +#ifdef SNAPPY_CODEC_AVAILABLE +#define DEFAULT_CODEC avro::Codec::SNAPPY_CODEC +#else +#define DEFAULT_CODEC avro::Codec::DEFLATE_CODEC +#endif namespace DB { @@ -293,7 +299,7 @@ AvroRowOutputFormat::AvroRowOutputFormat( : IRowOutputFormat(header_, out_, callback) , settings(settings_) , serializer(header_.getColumnsWithTypeAndName()) - , file_writer(std::make_unique(out_), serializer.getSchema(), 16 * 1024, avro::Codec::SNAPPY_CODEC) + , file_writer(std::make_unique(out_), serializer.getSchema(), DEFAULT_SYNC_INTERVAL, DEFAULT_CODEC) { } diff --git a/dbms/tests/queries/0_stateless/01060_avro.reference b/dbms/tests/queries/0_stateless/01060_avro.reference index d8ee426a337..a304a2b1f75 100644 --- a/dbms/tests/queries/0_stateless/01060_avro.reference +++ b/dbms/tests/queries/0_stateless/01060_avro.reference @@ -19,7 +19,6 @@ = compression 1000 1000 -1000 = other 0 not compatible diff --git a/dbms/tests/queries/0_stateless/01060_avro.sh b/dbms/tests/queries/0_stateless/01060_avro.sh index fbde59e58fa..1d37db93f6a 100755 --- a/dbms/tests/queries/0_stateless/01060_avro.sh +++ b/dbms/tests/queries/0_stateless/01060_avro.sh @@ -29,7 +29,9 @@ cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --ou echo = compression cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' cat $DATA_DIR/simple.deflate.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' -cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' + +#snappy is optional +#cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table' echo = other #no data From 872f759b077304a4a8fc94a23b603eba234629de Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Fri, 10 Jan 2020 16:44:33 -0600 Subject: [PATCH 053/312] Fix tests - timezone --- dbms/tests/queries/0_stateless/01060_avro.reference | 4 ++-- dbms/tests/queries/0_stateless/01060_avro.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01060_avro.reference b/dbms/tests/queries/0_stateless/01060_avro.reference index a304a2b1f75..f8b3434177d 100644 --- a/dbms/tests/queries/0_stateless/01060_avro.reference +++ b/dbms/tests/queries/0_stateless/01060_avro.reference @@ -14,7 +14,7 @@ "79cd909892d7e7ade1987cc7422628ba" "79cd909892d7e7ade1987cc7422628ba" = logical_types -"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000" +"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000" 18250,1578641516227,1578641516227000 = compression 1000 @@ -29,7 +29,7 @@ not found = complex "A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba" = logical_types -"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000" +"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000" = other 0 1000 diff --git a/dbms/tests/queries/0_stateless/01060_avro.sh b/dbms/tests/queries/0_stateless/01060_avro.sh index 1d37db93f6a..c92cba188d7 100755 --- a/dbms/tests/queries/0_stateless/01060_avro.sh +++ b/dbms/tests/queries/0_stateless/01060_avro.sh @@ -21,7 +21,7 @@ cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-f cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "g_fixed FixedString(32)" -q 'select * from table' echo = logical_types -cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)' -q 'select * from table' +cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "a_date Date, b_timestamp_millis DateTime64(3, 'UTC'), c_timestamp_micros DateTime64(6, 'UTC')" -q 'select * from table' cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S 'a_date Int32, b_timestamp_millis Int64, c_timestamp_micros Int64' -q 'select * from table' @@ -58,8 +58,8 @@ S2="a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_stri echo "\"A\",\"t\",\"['s1','s2']\",\"[['a1'],['a2']]\",\"s1\",\N,\"79cd909892d7e7ade1987cc7422628ba\"" | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S2" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S2" -q 'select * from table' echo = logical_types -S3="a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)" -echo '"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S3" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S3" -q 'select * from table' +S3="a_date Date, b_timestamp_millis DateTime64(3, 'UTC'), c_timestamp_micros DateTime64(6, 'UTC')" +echo '"2019-12-20","2020-01-10 07:31:56.227","2020-01-10 07:31:56.227000"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S3" -q "select * from table format Avro" | ${CLICKHOUSE_LOCAL} --input-format Avro --output-format CSV -S "$S3" -q 'select * from table' echo = other S4="a Int64" From 4c13317fba17c02851b1610bc7915807094e681b Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Fri, 10 Jan 2020 16:46:48 -0600 Subject: [PATCH 054/312] refactor avro cmake --- cmake/find/avro.cmake | 2 +- cmake/find/poco.cmake | 14 +++++++++++++- dbms/CMakeLists.txt | 4 ++++ dbms/src/Core/config_core.h.in | 1 + .../Processors/Formats/Impl/AvroRowInputFormat.cpp | 6 +++++- .../Processors/Formats/Impl/AvroRowInputFormat.h | 3 +++ 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake index 39ad2e31e54..cdb3fc84d3d 100644 --- a/cmake/find/avro.cmake +++ b/cmake/find/avro.cmake @@ -2,7 +2,7 @@ option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) if (ENABLE_AVRO) -option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ON) +option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") if(USE_INTERNAL_AVRO_LIBRARY) diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake index b44d2932276..0c676d374f1 100644 --- a/cmake/find/poco.cmake +++ b/cmake/find/poco.cmake @@ -14,6 +14,7 @@ if (NOT ENABLE_LIBRARIES) set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "") set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "") set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "") + set (ENABLE_POCO_JSON ${ENABLE_LIBRARIES} CACHE BOOL "") endif () set (POCO_COMPONENTS Net XML SQL Data) @@ -34,6 +35,9 @@ if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) list (APPEND POCO_COMPONENTS DataODBC) list (APPEND POCO_COMPONENTS SQLODBC) endif () +if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON) + list (APPEND POCO_COMPONENTS JSON) +endif () if (NOT USE_INTERNAL_POCO_LIBRARY) find_package (Poco COMPONENTS ${POCO_COMPONENTS}) @@ -112,6 +116,11 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) endif () endif () + if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON) + set (Poco_JSON_LIBRARY PocoJSON) + set (Poco_JSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/JSON/include/") + endif () + if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)) set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES}) set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES}) @@ -145,8 +154,11 @@ endif () if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND) set (USE_POCO_SQLODBC 1) endif () +if (Poco_JSON_LIBRARY) + set (USE_POCO_JSON 1) +endif () -message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}") +message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY},${Poco_JSON_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}, JSON=${USE_POCO_JSON}") # How to make sutable poco: # use branch: diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 4f9ca404cdf..07677309380 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -465,6 +465,10 @@ if (USE_POCO_NETSSL) dbms_target_link_libraries (PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) endif() +if (USE_POCO_JSON) + dbms_target_link_libraries (PRIVATE ${Poco_JSON_LIBRARY}) +endif() + dbms_target_link_libraries (PRIVATE ${Poco_Foundation_LIBRARY}) if (USE_ICU) diff --git a/dbms/src/Core/config_core.h.in b/dbms/src/Core/config_core.h.in index fdbd69decd3..2365340cf33 100644 --- a/dbms/src/Core/config_core.h.in +++ b/dbms/src/Core/config_core.h.in @@ -10,5 +10,6 @@ #cmakedefine01 USE_POCO_DATAODBC #cmakedefine01 USE_POCO_MONGODB #cmakedefine01 USE_POCO_REDIS +#cmakedefine01 USE_POCO_JSON #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 USE_SSL diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index a0eba94bfdb..9e63c76f4cb 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -484,7 +484,7 @@ bool AvroRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) return false; } - +#ifdef USE_POCO_JSON class AvroConfluentRowInputFormat::SchemaRegistry { public: @@ -603,6 +603,7 @@ AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId } return it->second; } +#endif void registerInputFormatProcessorAvro(FormatFactory & factory) { @@ -615,6 +616,7 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) return std::make_shared(sample, buf, params); }); +#ifdef USE_POCO_JSON factory.registerInputFormatProcessor("AvroConfluent",[=]( ReadBuffer & buf, const Block & sample, @@ -623,6 +625,8 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) { return std::make_shared(sample, buf, params, settings); }); +#endif + } } diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h index ef5e01973dd..1941fb14300 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -46,6 +46,7 @@ private: AvroDeserializer deserializer; }; +#ifdef USE_POCO_JSON class AvroConfluentRowInputFormat : public IRowInputFormat { public: @@ -66,5 +67,7 @@ private: avro::InputStreamPtr input_stream; avro::DecoderPtr decoder; }; +#endif + } #endif From bfc610275d00d04a87f1f6b6ca05e44efa97a93a Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Fri, 10 Jan 2020 22:26:12 -0600 Subject: [PATCH 055/312] fix bad cast --- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 9e63c76f4cb..6f2faaf09f0 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -293,7 +293,7 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node } if (target.isEnum()) { - const auto & enum_type = assert_cast(*target_type); + const auto & enum_type = dynamic_cast(*target_type); std::vector symbol_mapping; for (size_t i = 0; i < root_node->names(); i++) { From 9da0df4f0397f3e10156dbcb0a08dc8516be17c8 Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Sat, 11 Jan 2020 01:01:20 -0600 Subject: [PATCH 056/312] Add codec and sync interval settings --- dbms/src/Core/Settings.h | 2 ++ dbms/src/Formats/FormatFactory.cpp | 2 ++ dbms/src/Formats/FormatSettings.h | 6 +++- .../Formats/Impl/AvroRowOutputFormat.cpp | 32 +++++++++++++++---- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 26684153832..ab192289811 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -198,6 +198,8 @@ struct Settings : public SettingsCollection M(SettingUInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(SettingBool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ M(SettingUInt64, output_format_parquet_row_group_size, 1000000, "Row group size in rows.", 0) \ + M(SettingString, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \ + M(SettingUInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ \ M(SettingBool, use_client_time_zone, false, "Use client timezone for interpreting DateTime string values, instead of adopting server timezone.", 0) \ \ diff --git a/dbms/src/Formats/FormatFactory.cpp b/dbms/src/Formats/FormatFactory.cpp index ade91c5a391..f812b56aa5d 100644 --- a/dbms/src/Formats/FormatFactory.cpp +++ b/dbms/src/Formats/FormatFactory.cpp @@ -100,6 +100,8 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; + format_settings.avro.output_codec = settings.output_format_avro_codec; + format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; return format_settings; } diff --git a/dbms/src/Formats/FormatSettings.h b/dbms/src/Formats/FormatSettings.h index 6ca54c12265..cc6f7f4dbb3 100644 --- a/dbms/src/Formats/FormatSettings.h +++ b/dbms/src/Formats/FormatSettings.h @@ -114,7 +114,11 @@ struct FormatSettings struct Avro { String schema_registry_url; - } avro; + String output_codec; + UInt64 output_sync_interval = 16 * 1024; + }; + + Avro avro; }; diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 603eb05528d..9f6233303ad 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -42,12 +42,6 @@ #include #include -#define DEFAULT_SYNC_INTERVAL 16*1024 -#ifdef SNAPPY_CODEC_AVAILABLE -#define DEFAULT_CODEC avro::Codec::SNAPPY_CODEC -#else -#define DEFAULT_CODEC avro::Codec::DEFLATE_CODEC -#endif namespace DB { @@ -294,12 +288,36 @@ void AvroSerializer::serializeRow(const Columns & columns, size_t row_num, avro: } } +static avro::Codec getCodec(const std::string& codec_name) +{ + if (codec_name == "") + { +#ifdef SNAPPY_CODEC_AVAILABLE + return avro::Codec::SNAPPY_CODEC; +#else + return avro::Codec::DEFLATE_CODEC; +#endif + } + + if (codec_name == "null") return avro::Codec::NULL_CODEC; + if (codec_name == "deflate") return avro::Codec::DEFLATE_CODEC; +#ifdef SNAPPY_CODEC_AVAILABLE + if (codec_name == "snappy") return avro::Codec::SNAPPY_CODEC; +#endif + + throw Exception("Avro codec " + codec_name + " is not available", ErrorCodes::BAD_ARGUMENTS); +} + AvroRowOutputFormat::AvroRowOutputFormat( WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_) : IRowOutputFormat(header_, out_, callback) , settings(settings_) , serializer(header_.getColumnsWithTypeAndName()) - , file_writer(std::make_unique(out_), serializer.getSchema(), DEFAULT_SYNC_INTERVAL, DEFAULT_CODEC) + , file_writer( + std::make_unique(out_), + serializer.getSchema(), + settings.avro.output_sync_interval, + getCodec(settings.avro.output_codec)) { } From 8ab68e2c770ff428861b3a3c6e266ffd1d9637b9 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 13 Jan 2020 21:00:32 +0300 Subject: [PATCH 057/312] change iface for one-to-many blocks expressions calculation --- .../ExpressionBlockInputStream.cpp | 24 ++++++ .../DataStreams/ExpressionBlockInputStream.h | 24 +++++- dbms/src/Interpreters/ExpressionActions.cpp | 23 +++++- dbms/src/Interpreters/ExpressionActions.h | 9 ++- dbms/src/Interpreters/IJoin.h | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 2 +- dbms/src/Interpreters/Join.cpp | 2 +- dbms/src/Interpreters/Join.h | 2 +- dbms/src/Interpreters/MergeJoin.cpp | 80 ++++++++++++------- dbms/src/Interpreters/MergeJoin.h | 8 +- 10 files changed, 131 insertions(+), 45 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index 9673395a21a..c986078386d 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -44,4 +44,28 @@ Block ExpressionBlockInputStream::readImpl() return res; } +Block SplittingExpressionBlockInputStream::readImpl() +{ + if (!initialized) + { + if (expression->resultIsAlwaysEmpty()) + return {}; + + initialized = true; + } + + Block res; + if (likely(!not_processed)) + { + res = children.back()->read(); + if (!res) + return res; + } + else + res.swap(not_processed); + + action_number = expression->execute(res, action_number, not_processed); + return res; +} + } diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index af0e2db589b..6c021691bbb 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -15,10 +15,9 @@ class ExpressionActions; */ class ExpressionBlockInputStream : public IBlockInputStream { -private: +public: using ExpressionActionsPtr = std::shared_ptr; -public: ExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_); String getName() const override; @@ -26,12 +25,29 @@ public: Block getHeader() const override; protected: + bool initialized = false; + ExpressionActionsPtr expression; + Block readImpl() override; private: - ExpressionActionsPtr expression; Block cached_header; - bool initialized = false; +}; + +/// ExpressionBlockInputStream that could generate many out blocks for single input block. +class SplittingExpressionBlockInputStream : public ExpressionBlockInputStream +{ +public: + SplittingExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) + : ExpressionBlockInputStream(input, expression_) + {} + +protected: + Block readImpl() override; + +private: + Block not_processed; + size_t action_number = 0; }; } diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 793dc7a8a3d..26c3e25608f 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -346,7 +346,7 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, } -void ExpressionAction::execute(Block & block, bool dry_run) const +Block ExpressionAction::execute(Block & block, bool dry_run) const { size_t input_rows_count = block.rows(); @@ -477,8 +477,9 @@ void ExpressionAction::execute(Block & block, bool dry_run) const case JOIN: { - join->joinBlock(block); - break; + Block not_processed; + join->joinBlock(block, not_processed); + return not_processed; } case PROJECT: @@ -537,6 +538,8 @@ void ExpressionAction::execute(Block & block, bool dry_run) const break; } + + return {}; } @@ -762,6 +765,20 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } } +size_t ExpressionActions::execute(Block & block, size_t start_action, Block & not_processed) const +{ + for (size_t i = start_action; i < actions.size(); ++i) + { + not_processed = actions[i].execute(block, false); + checkLimits(block); + + if (not_processed) + return i; + } + + return 0; +} + bool ExpressionActions::hasTotalsInJoin() const { for (const auto & action : actions) diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 90c01191a74..7130b84806d 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -138,8 +138,11 @@ private: friend class ExpressionActions; void prepare(Block & sample_block, const Settings & settings, NameSet & names_not_for_constant_folding); - void execute(Block & block, bool dry_run) const; void executeOnTotals(Block & block) const; + + /// Executes action on block (modify it). If block is splitted @returns block of not processed rows, empty block otherwise. + /// Block could be splitted in case of JOIN (or another row multiplying action). + Block execute(Block & block, bool dry_run) const; }; @@ -221,6 +224,10 @@ public: /// Execute the expression on the block. The block must contain all the columns returned by getRequiredColumns. void execute(Block & block, bool dry_run = false) const; + /// Execute the expression on the block. + /// @returns starting action to continue with and block with not processed rows if any. + size_t execute(Block & block, size_t start_action, Block & not_processed) const; + /// Check if joined subquery has totals. bool hasTotalsInJoin() const; diff --git a/dbms/src/Interpreters/IJoin.h b/dbms/src/Interpreters/IJoin.h index c62f39e9edf..b5781a0dbb1 100644 --- a/dbms/src/Interpreters/IJoin.h +++ b/dbms/src/Interpreters/IJoin.h @@ -23,7 +23,7 @@ public: /// Join the block with data from left hand of JOIN to the right hand data (that was previously built by calls to addJoinedBlock). /// Could be called from different threads in parallel. - virtual void joinBlock(Block & block) = 0; + virtual void joinBlock(Block & block, Block & not_processed) = 0; virtual bool hasTotals() const = 0; virtual void setTotals(const Block & block) = 0; diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index b147c5f4887..b1eb85f58b0 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1113,7 +1113,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS header_before_join = pipeline.firstStream()->getHeader(); /// Applies to all sources except stream_with_non_joined_data. for (auto & stream : pipeline.streams) - stream = std::make_shared(stream, expressions.before_join); + stream = std::make_shared(stream, expressions.before_join); if (isMergeJoin(expressions.before_join->getTableJoinAlgo()) && settings.partial_merge_join_optimizations) { diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 2cce5d7f51d..783cf8e5974 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -1091,7 +1091,7 @@ void Join::joinGet(Block & block, const String & column_name) const } -void Join::joinBlock(Block & block) +void Join::joinBlock(Block & block, Block &) { std::shared_lock lock(data->rwlock); diff --git a/dbms/src/Interpreters/Join.h b/dbms/src/Interpreters/Join.h index c8c9376b926..be8132ba4dc 100644 --- a/dbms/src/Interpreters/Join.h +++ b/dbms/src/Interpreters/Join.h @@ -158,7 +158,7 @@ public: /** Join data from the map (that was previously built by calls to addJoinedBlock) to the block with data from "left" table. * Could be called from different threads in parallel. */ - void joinBlock(Block & block) override; + void joinBlock(Block & block, Block & not_processed) override; /// Infer the return type for joinGet function DataTypePtr joinGetReturnType(const String & column_name) const; diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index f301de17bc5..8df2d6629c5 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -294,11 +294,14 @@ void joinEqualsAnyLeft(const Block & right_block, const Block & right_columns_to copyRightRange(right_block, right_columns_to_add, right_columns, range.right_start, range.left_length); } +template void joinEquals(const Block & left_block, const Block & right_block, const Block & right_columns_to_add, - MutableColumns & left_columns, MutableColumns & right_columns, const Range & range, bool is_all) + MutableColumns & left_columns, MutableColumns & right_columns, const Range & range) { size_t left_rows_to_add = range.left_length; - size_t right_rows_to_add = is_all ? range.right_length : 1; + size_t right_rows_to_add = 1; + if constexpr (is_all) + right_rows_to_add = range.right_length; size_t row_position = range.right_start; for (size_t right_row = 0; right_row < right_rows_to_add; ++right_row, ++row_position) @@ -308,22 +311,20 @@ void joinEquals(const Block & left_block, const Block & right_block, const Block } } -void appendNulls(MutableColumns & right_columns, size_t rows_to_add) -{ - for (auto & column : right_columns) - column->insertManyDefaults(rows_to_add); -} - +template void joinInequalsLeft(const Block & left_block, MutableColumns & left_columns, MutableColumns & right_columns, - size_t start, size_t end, bool copy_left) + size_t start, size_t end) { if (end <= start) return; size_t rows_to_add = end - start; - if (copy_left) + if constexpr (copy_left) copyLeftRange(left_block, left_columns, start, rows_to_add); - appendNulls(right_columns, rows_to_add); + + /// append nulls + for (auto & column : right_columns) + column->insertManyDefaults(rows_to_add); } Blocks blocksListToBlocks(const BlocksList & in_blocks) @@ -427,7 +428,7 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & ri , size_limits(table_join->sizeLimits()) , right_sample_block(right_sample_block_) , nullable_right_side(table_join->forceNullableRight()) - , is_all(table_join->strictness() == ASTTableJoin::Strictness::All) + , is_all_join(table_join->strictness() == ASTTableJoin::Strictness::All) , is_inner(isInner(table_join->kind())) , is_left(isLeft(table_join->kind())) , skip_not_intersected(table_join->enablePartialMergeJoinOptimizations()) @@ -567,7 +568,8 @@ bool MergeJoin::addJoinedBlock(const Block & src_block) return saveRightBlock(std::move(block)); } -void MergeJoin::joinBlock(Block & block) +/// TODO: not processed +void MergeJoin::joinBlock(Block & block, Block &) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); @@ -575,12 +577,22 @@ void MergeJoin::joinBlock(Block & block) sortBlock(block, left_sort_description); if (is_in_memory) - joinSortedBlock(block); + { + if (is_all_join) + joinSortedBlock(block); + else + joinSortedBlock(block); + } else - joinSortedBlock(block); + { + if (is_all_join) + joinSortedBlock(block); + else + joinSortedBlock(block); + } } -template +template void MergeJoin::joinSortedBlock(Block & block) { std::shared_lock lock(rwlock); @@ -610,11 +622,11 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); - leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); } left_cursor.nextN(left_key_tail); - joinInequalsLeft(block, left_columns, right_columns, left_cursor.position(), left_cursor.end(), is_all); + joinInequalsLeft(block, left_columns, right_columns, left_cursor.position(), left_cursor.end()); //left_cursor.nextN(left_cursor.end() - left_cursor.position()); changeLeftColumns(block, std::move(left_columns)); @@ -638,7 +650,7 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); - innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); } left_cursor.nextN(left_key_tail); @@ -647,6 +659,7 @@ void MergeJoin::joinSortedBlock(Block & block) } } +template void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) { @@ -662,28 +675,32 @@ void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block Range range = left_cursor.getNextEqualRange(right_cursor); - joinInequalsLeft(left_block, left_columns, right_columns, left_unequal_position, range.left_start, is_all); + joinInequalsLeft(left_block, left_columns, right_columns, left_unequal_position, range.left_start); if (range.empty()) break; - if (is_all) - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, is_all); + if constexpr (is_all) + joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range); else joinEqualsAnyLeft(right_block, right_columns_to_add, right_columns, range); right_cursor.nextN(range.right_length); /// Do not run over last left keys for ALL JOIN (cause of possible duplicates in next right block) - if (is_all && right_cursor.atEnd()) + if constexpr (is_all) { - left_key_tail = range.left_length; - break; + if (right_cursor.atEnd()) + { + left_key_tail = range.left_length; + break; + } } left_cursor.nextN(range.left_length); } } +template void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) { @@ -696,14 +713,17 @@ void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc if (range.empty()) break; - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, is_all); + joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range); right_cursor.nextN(range.right_length); /// Do not run over last left keys for ALL JOIN (cause of possible duplicates in next right block) - if (is_all && right_cursor.atEnd()) + if constexpr (is_all) { - left_key_tail = range.left_length; - break; + if (right_cursor.atEnd()) + { + left_key_tail = range.left_length; + break; + } } left_cursor.nextN(range.left_length); } @@ -711,7 +731,7 @@ void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc void MergeJoin::changeLeftColumns(Block & block, MutableColumns && columns) { - if (is_left && !is_all) + if (is_left && !is_all_join) return; block.setColumns(std::move(columns)); } diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index 960ca31153d..fd5ef3f1ffe 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -48,7 +48,7 @@ public: MergeJoin(std::shared_ptr table_join_, const Block & right_sample_block); bool addJoinedBlock(const Block & block) override; - void joinBlock(Block &) override; + void joinBlock(Block &, Block & not_processed) override; void joinTotals(Block &) const override; void setTotals(const Block &) override; bool hasTotals() const override { return totals; } @@ -85,7 +85,7 @@ private: size_t right_blocks_bytes = 0; bool is_in_memory = true; const bool nullable_right_side; - const bool is_all; + const bool is_all_join; const bool is_inner; const bool is_left; const bool skip_not_intersected; @@ -98,13 +98,15 @@ private: template size_t rightBlocksCount(); - template + template void joinSortedBlock(Block & block); template std::shared_ptr loadRightBlock(size_t pos); + template void leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); + template void innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); From f0023dc4b1dfa2b9842f1f34cf14c4eb7e4642f6 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 14 Jan 2020 22:02:22 +0300 Subject: [PATCH 058/312] split in progress --- dbms/src/Core/Settings.h | 1 + dbms/src/Interpreters/AnalyzedJoin.cpp | 1 + dbms/src/Interpreters/AnalyzedJoin.h | 2 + dbms/src/Interpreters/MergeJoin.cpp | 56 ++++++++++++++++++++++---- dbms/src/Interpreters/MergeJoin.h | 5 ++- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 724b31ca642..f3380fcf743 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -52,6 +52,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ + M(SettingUInt64, max_joined_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algo support it). 0 means unlimited.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index 5e4bf1fe53b..bffb058d081 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -23,6 +23,7 @@ AnalyzedJoin::AnalyzedJoin(const Settings & settings, const String & tmp_path_) : size_limits(SizeLimits{settings.max_rows_in_join, settings.max_bytes_in_join, settings.join_overflow_mode}) , default_max_bytes(settings.default_max_bytes_in_join) , join_use_nulls(settings.join_use_nulls) + , max_joined_block_size(settings.max_joined_block_size) , partial_merge_join(settings.partial_merge_join) , partial_merge_join_optimizations(settings.partial_merge_join_optimizations) , partial_merge_join_rows_in_right_blocks(settings.partial_merge_join_rows_in_right_blocks) diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index 677662d949c..f944d1278ee 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -40,6 +40,7 @@ class AnalyzedJoin const SizeLimits size_limits; const size_t default_max_bytes; const bool join_use_nulls; + const size_t max_joined_block_size = 0; const bool partial_merge_join = false; const bool partial_merge_join_optimizations = false; const size_t partial_merge_join_rows_in_right_blocks = 0; @@ -86,6 +87,7 @@ public: bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); } bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); } size_t defaultMaxBytes() const { return default_max_bytes; } + size_t maxJoinedBlockSize() const { return max_joined_block_size; } size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; } bool enablePartialMergeJoinOptimizations() const { return partial_merge_join_optimizations; } diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index 8df2d6629c5..2774ddfcb7d 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -432,6 +432,7 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & ri , is_inner(isInner(table_join->kind())) , is_left(isLeft(table_join->kind())) , skip_not_intersected(table_join->enablePartialMergeJoinOptimizations()) + , max_joined_block_size(table_join->maxJoinedBlockSize()) , max_rows_in_right_block(table_join->maxRowsInRightBlock()) { if (!isLeft(table_join->kind()) && !isInner(table_join->kind())) @@ -568,8 +569,7 @@ bool MergeJoin::addJoinedBlock(const Block & src_block) return saveRightBlock(std::move(block)); } -/// TODO: not processed -void MergeJoin::joinBlock(Block & block, Block &) +void MergeJoin::joinBlock(Block & block, Block & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); @@ -579,21 +579,21 @@ void MergeJoin::joinBlock(Block & block, Block &) if (is_in_memory) { if (is_all_join) - joinSortedBlock(block); + joinSortedBlock(block, not_processed); else - joinSortedBlock(block); + joinSortedBlock(block, not_processed); } else { if (is_all_join) - joinSortedBlock(block); + joinSortedBlock(block, not_processed); else - joinSortedBlock(block); + joinSortedBlock(block, not_processed); } } template -void MergeJoin::joinSortedBlock(Block & block) +void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) { std::shared_lock lock(rwlock); @@ -623,6 +623,15 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + + if constexpr (is_all) + { + if (max_joined_block_size && left_columns.size() > max_joined_block_size) + { + splitResultBlock(block, not_processed, left_cursor.position(), std::move(left_columns), std::move(right_columns)); + return; + } + } } left_cursor.nextN(left_key_tail); @@ -651,6 +660,15 @@ void MergeJoin::joinSortedBlock(Block & block) std::shared_ptr right_block = loadRightBlock(i); innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); + + if constexpr (is_all) + { + if (max_joined_block_size && left_columns.size() > max_joined_block_size) + { + splitResultBlock(block, not_processed, left_cursor.position(), std::move(left_columns), std::move(right_columns)); + return; + } + } } left_cursor.nextN(left_key_tail); @@ -745,6 +763,30 @@ void MergeJoin::addRightColumns(Block & block, MutableColumns && right_columns) } } +/// Split block into processed (result) and not processed. Not processed block would be joined next time. +void MergeJoin::splitResultBlock(Block & processed, Block & not_processed, size_t position, + MutableColumns && left_columns, MutableColumns && right_columns) +{ + if (position < processed.rows()) + { + not_processed = processed.cloneEmpty(); + + /// @note it's not OK for ANY JOIN. @sa changeLeftColumns() + MutableColumns columns = processed.mutateColumns(); + for (auto & col : columns) + { + auto tmp = col->cloneEmpty(); + tmp->insertRangeFrom(*col, position, processed.rows() - position); + col = std::move(tmp); + } + + not_processed.setColumns(std::move(columns)); + } + + changeLeftColumns(processed, std::move(left_columns)); + addRightColumns(processed, std::move(right_columns)); +} + template size_t MergeJoin::rightBlocksCount() { diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index fd5ef3f1ffe..2b09bbbcdae 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -89,17 +89,20 @@ private: const bool is_inner; const bool is_left; const bool skip_not_intersected; + const size_t max_joined_block_size; const size_t max_rows_in_right_block; void changeLeftColumns(Block & block, MutableColumns && columns); void addRightColumns(Block & block, MutableColumns && columns); + void splitResultBlock(Block & processed, Block & not_processed, size_t position, + MutableColumns && left_columns, MutableColumns && right_columns); void mergeRightBlocks(); template size_t rightBlocksCount(); template - void joinSortedBlock(Block & block); + void joinSortedBlock(Block & block, Block & not_processed); template std::shared_ptr loadRightBlock(size_t pos); From 39f26536635885f0291300442bdee33d4733fdd3 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 15 Jan 2020 23:33:29 +0300 Subject: [PATCH 059/312] rewrite with cursor position --- dbms/src/Core/Block.h | 8 + .../ExpressionBlockInputStream.cpp | 11 +- .../DataStreams/ExpressionBlockInputStream.h | 2 +- dbms/src/Interpreters/ExpressionActions.cpp | 20 ++- dbms/src/Interpreters/ExpressionActions.h | 16 +- dbms/src/Interpreters/IJoin.h | 3 +- dbms/src/Interpreters/Join.cpp | 2 +- dbms/src/Interpreters/Join.h | 2 +- dbms/src/Interpreters/MergeJoin.cpp | 138 +++++++++++------- dbms/src/Interpreters/MergeJoin.h | 20 ++- 10 files changed, 140 insertions(+), 82 deletions(-) diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index ae8b07718dd..668c9e6a401 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -147,6 +147,14 @@ using BlocksList = std::list; using BlocksPtr = std::shared_ptr; using BlocksPtrs = std::shared_ptr>; +/// Extends block with extra data in derived classes +struct ExtraBlock +{ + Block block; +}; + +using ExtraBlockPtr = std::shared_ptr; + /// Compare number of columns, data types, column types, column names, and values of constant columns. bool blocksHaveEqualStructure(const Block & lhs, const Block & rhs); diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index c986078386d..d41d21915e1 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -58,13 +58,14 @@ Block SplittingExpressionBlockInputStream::readImpl() if (likely(!not_processed)) { res = children.back()->read(); - if (!res) - return res; + if (res) + expression->execute(res, not_processed, action_number); } else - res.swap(not_processed); - - action_number = expression->execute(res, action_number, not_processed); + { + res = std::move(not_processed->block); + expression->execute(res, not_processed, action_number); + } return res; } diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index 6c021691bbb..b1a1f9674fd 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -46,7 +46,7 @@ protected: Block readImpl() override; private: - Block not_processed; + ExtraBlockPtr not_processed; size_t action_number = 0; }; diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 26c3e25608f..f10d4501d2d 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -346,7 +346,7 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings, } -Block ExpressionAction::execute(Block & block, bool dry_run) const +void ExpressionAction::execute(Block & block, bool dry_run, ExtraBlockPtr & not_processed) const { size_t input_rows_count = block.rows(); @@ -477,9 +477,8 @@ Block ExpressionAction::execute(Block & block, bool dry_run) const case JOIN: { - Block not_processed; join->joinBlock(block, not_processed); - return not_processed; + break; } case PROJECT: @@ -538,8 +537,6 @@ Block ExpressionAction::execute(Block & block, bool dry_run) const break; } - - return {}; } @@ -765,18 +762,19 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } } -size_t ExpressionActions::execute(Block & block, size_t start_action, Block & not_processed) const +/// @warning It's a tricky method that allows to continue ONLY ONE action in reason of one-to-many ALL JOIN logic. +void ExpressionActions::execute(Block & block, ExtraBlockPtr & not_processed, size_t & start_action) const { - for (size_t i = start_action; i < actions.size(); ++i) + size_t i = start_action; + start_action = 0; + for (; i < actions.size(); ++i) { - not_processed = actions[i].execute(block, false); + actions[i].execute(block, false, not_processed); checkLimits(block); if (not_processed) - return i; + start_action = i; } - - return 0; } bool ExpressionActions::hasTotalsInJoin() const diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 7130b84806d..fbb3071e236 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -140,9 +140,14 @@ private: void prepare(Block & sample_block, const Settings & settings, NameSet & names_not_for_constant_folding); void executeOnTotals(Block & block) const; - /// Executes action on block (modify it). If block is splitted @returns block of not processed rows, empty block otherwise. - /// Block could be splitted in case of JOIN (or another row multiplying action). - Block execute(Block & block, bool dry_run) const; + /// Executes action on block (modify it). Block could be splitted in case of JOIN. Then not_processed block is created. + void execute(Block & block, bool dry_run, ExtraBlockPtr & not_processed) const; + + void execute(Block & block, bool dry_run) const + { + ExtraBlockPtr extra; + execute(block, dry_run, extra); + } }; @@ -224,9 +229,8 @@ public: /// Execute the expression on the block. The block must contain all the columns returned by getRequiredColumns. void execute(Block & block, bool dry_run = false) const; - /// Execute the expression on the block. - /// @returns starting action to continue with and block with not processed rows if any. - size_t execute(Block & block, size_t start_action, Block & not_processed) const; + /// Execute the expression on the block with continuation. + void execute(Block & block, ExtraBlockPtr & not_processed, size_t & start_action) const; /// Check if joined subquery has totals. bool hasTotalsInJoin() const; diff --git a/dbms/src/Interpreters/IJoin.h b/dbms/src/Interpreters/IJoin.h index b5781a0dbb1..6d290d143da 100644 --- a/dbms/src/Interpreters/IJoin.h +++ b/dbms/src/Interpreters/IJoin.h @@ -11,6 +11,7 @@ namespace DB { class Block; +struct ExtraBlock; class IJoin { @@ -23,7 +24,7 @@ public: /// Join the block with data from left hand of JOIN to the right hand data (that was previously built by calls to addJoinedBlock). /// Could be called from different threads in parallel. - virtual void joinBlock(Block & block, Block & not_processed) = 0; + virtual void joinBlock(Block & block, std::shared_ptr & not_processed) = 0; virtual bool hasTotals() const = 0; virtual void setTotals(const Block & block) = 0; diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 783cf8e5974..7e6f66f1f7f 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -1091,7 +1091,7 @@ void Join::joinGet(Block & block, const String & column_name) const } -void Join::joinBlock(Block & block, Block &) +void Join::joinBlock(Block & block, ExtraBlockPtr &) { std::shared_lock lock(data->rwlock); diff --git a/dbms/src/Interpreters/Join.h b/dbms/src/Interpreters/Join.h index be8132ba4dc..337c18a5980 100644 --- a/dbms/src/Interpreters/Join.h +++ b/dbms/src/Interpreters/Join.h @@ -158,7 +158,7 @@ public: /** Join data from the map (that was previously built by calls to addJoinedBlock) to the block with data from "left" table. * Could be called from different threads in parallel. */ - void joinBlock(Block & block, Block & not_processed) override; + void joinBlock(Block & block, ExtraBlockPtr & not_processed) override; /// Infer the return type for joinGet function DataTypePtr joinGetReturnType(const String & column_name) const; diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index 2774ddfcb7d..995c20cfad9 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -295,20 +295,33 @@ void joinEqualsAnyLeft(const Block & right_block, const Block & right_columns_to } template -void joinEquals(const Block & left_block, const Block & right_block, const Block & right_columns_to_add, - MutableColumns & left_columns, MutableColumns & right_columns, const Range & range) +bool joinEquals(const Block & left_block, const Block & right_block, const Block & right_columns_to_add, + MutableColumns & left_columns, MutableColumns & right_columns, Range & range, size_t max_rows [[maybe_unused]]) { - size_t left_rows_to_add = range.left_length; - size_t right_rows_to_add = 1; - if constexpr (is_all) - right_rows_to_add = range.right_length; + bool one_more = true; - size_t row_position = range.right_start; - for (size_t right_row = 0; right_row < right_rows_to_add; ++right_row, ++row_position) + if constexpr (is_all) { - copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); - copyRightRange(right_block, right_columns_to_add, right_columns, row_position, left_rows_to_add); + size_t range_rows = range.left_length * range.right_length; + if (range_rows > max_rows) + one_more = false; + + size_t left_rows_to_add = range.left_length; + size_t row_position = range.right_start; + for (size_t right_row = 0; right_row < range.right_length; ++right_row, ++row_position) + { + copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); + copyRightRange(right_block, right_columns_to_add, right_columns, row_position, left_rows_to_add); + } } + else + { + size_t left_rows_to_add = range.left_length; + copyLeftRange(left_block, left_columns, range.left_start, left_rows_to_add); + copyRightRange(right_block, right_columns_to_add, right_columns, range.right_start, left_rows_to_add); + } + + return one_more; } template @@ -569,7 +582,7 @@ bool MergeJoin::addJoinedBlock(const Block & src_block) return saveRightBlock(std::move(block)); } -void MergeJoin::joinBlock(Block & block, Block & not_processed) +void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) { JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight()); materializeBlockInplace(block); @@ -593,7 +606,7 @@ void MergeJoin::joinBlock(Block & block, Block & not_processed) } template -void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) +void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) { std::shared_lock lock(rwlock); @@ -604,9 +617,18 @@ void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) size_t left_key_tail = 0; size_t right_blocks_count = rightBlocksCount(); + size_t starting_right_block = 0; + if (not_processed) + { + auto & continuation = static_cast(*not_processed); + starting_right_block = continuation.next_right_block; + left_cursor.nextN(continuation.next_left_position); + not_processed.reset(); + } + if (is_left) { - for (size_t i = 0; i < right_blocks_count; ++i) + for (size_t i = starting_right_block; i < right_blocks_count; ++i) { if (left_cursor.atEnd()) break; @@ -622,15 +644,10 @@ void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) std::shared_ptr right_block = loadRightBlock(i); - leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); - - if constexpr (is_all) + if (!leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail)) { - if (max_joined_block_size && left_columns.size() > max_joined_block_size) - { - splitResultBlock(block, not_processed, left_cursor.position(), std::move(left_columns), std::move(right_columns)); - return; - } + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), left_cursor.position(), i); + return; } } @@ -643,7 +660,7 @@ void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) } else if (is_inner) { - for (size_t i = 0; i < right_blocks_count; ++i) + for (size_t i = starting_right_block; i < right_blocks_count; ++i) { if (left_cursor.atEnd()) break; @@ -659,15 +676,10 @@ void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) std::shared_ptr right_block = loadRightBlock(i); - innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail); - - if constexpr (is_all) + if (!innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail)) { - if (max_joined_block_size && left_columns.size() > max_joined_block_size) - { - splitResultBlock(block, not_processed, left_cursor.position(), std::move(left_columns), std::move(right_columns)); - return; - } + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), left_cursor.position(), i); + return; } } @@ -677,8 +689,17 @@ void MergeJoin::joinSortedBlock(Block & block, Block & not_processed) } } +static size_t maxRangeRows(size_t current_rows, size_t max_rows) +{ + if (!max_rows) + return std::numeric_limits::max(); + if (current_rows >= max_rows) + return 0; + return max_rows - current_rows; +} + template -void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, +bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) { MergeJoinCursor right_cursor(right_block, right_merge_description); @@ -699,7 +720,14 @@ void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block break; if constexpr (is_all) - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range); + { + size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_size); + if (!max_rows) + return false; + + if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + return false; + } else joinEqualsAnyLeft(right_block, right_columns_to_add, right_columns, range); @@ -716,10 +744,12 @@ void MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block } left_cursor.nextN(range.left_length); } + + return true; } template -void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, +bool MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) { MergeJoinCursor right_cursor(right_block, right_merge_description); @@ -731,7 +761,18 @@ void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc if (range.empty()) break; - joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range); + if constexpr (is_all) + { + size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_size); + if (!max_rows) + return false; + + if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + return false; + } + else + joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, 0); + right_cursor.nextN(range.right_length); /// Do not run over last left keys for ALL JOIN (cause of possible duplicates in next right block) @@ -745,6 +786,8 @@ void MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc } left_cursor.nextN(range.left_length); } + + return true; } void MergeJoin::changeLeftColumns(Block & block, MutableColumns && columns) @@ -764,27 +807,22 @@ void MergeJoin::addRightColumns(Block & block, MutableColumns && right_columns) } /// Split block into processed (result) and not processed. Not processed block would be joined next time. -void MergeJoin::splitResultBlock(Block & processed, Block & not_processed, size_t position, - MutableColumns && left_columns, MutableColumns && right_columns) +template +ExtraBlockPtr MergeJoin::extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, + size_t left_position [[maybe_unused]], size_t right_block_number [[maybe_unused]]) { - if (position < processed.rows()) + ExtraBlockPtr not_processed; + + if constexpr (is_all) { - not_processed = processed.cloneEmpty(); + not_processed = std::make_shared(NotProcessed{{processed.cloneEmpty()}, left_position, right_block_number + 1}); + not_processed->block.swap(processed); - /// @note it's not OK for ANY JOIN. @sa changeLeftColumns() - MutableColumns columns = processed.mutateColumns(); - for (auto & col : columns) - { - auto tmp = col->cloneEmpty(); - tmp->insertRangeFrom(*col, position, processed.rows() - position); - col = std::move(tmp); - } - - not_processed.setColumns(std::move(columns)); + changeLeftColumns(processed, std::move(left_columns)); + addRightColumns(processed, std::move(right_columns)); } - changeLeftColumns(processed, std::move(left_columns)); - addRightColumns(processed, std::move(right_columns)); + return not_processed; } template diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index 2b09bbbcdae..256d2af8808 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -48,13 +48,19 @@ public: MergeJoin(std::shared_ptr table_join_, const Block & right_sample_block); bool addJoinedBlock(const Block & block) override; - void joinBlock(Block &, Block & not_processed) override; + void joinBlock(Block &, ExtraBlockPtr & not_processed) override; void joinTotals(Block &) const override; void setTotals(const Block &) override; bool hasTotals() const override { return totals; } size_t getTotalRowCount() const override { return right_blocks_row_count; } private: + struct NotProcessed : public ExtraBlock + { + size_t next_left_position; + size_t next_right_block; + }; + /// There're two size limits for right-hand table: max_rows_in_join, max_bytes_in_join. /// max_bytes is prefered. If it isn't set we approximate it as (max_rows * bytes/row). struct BlockByteWeight @@ -94,23 +100,25 @@ private: void changeLeftColumns(Block & block, MutableColumns && columns); void addRightColumns(Block & block, MutableColumns && columns); - void splitResultBlock(Block & processed, Block & not_processed, size_t position, - MutableColumns && left_columns, MutableColumns && right_columns); + + template + ExtraBlockPtr extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, + size_t position, size_t right_block_number); void mergeRightBlocks(); template size_t rightBlocksCount(); template - void joinSortedBlock(Block & block, Block & not_processed); + void joinSortedBlock(Block & block, ExtraBlockPtr & not_processed); template std::shared_ptr loadRightBlock(size_t pos); template - void leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + bool leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); template - void innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, + bool innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); bool saveRightBlock(Block && block); From e06b9da71d32c03876d81f60778d99d78f47ad02 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 16 Jan 2020 00:33:21 +0300 Subject: [PATCH 060/312] pm join with splitted blocks (working) --- dbms/src/Interpreters/MergeJoin.cpp | 55 ++++++++++++--- dbms/src/Interpreters/MergeJoin.h | 11 +-- ...all_join_with_block_continuation.reference | 13 ++++ ...62_pm_all_join_with_block_continuation.sql | 68 +++++++++++++++++++ 4 files changed, 132 insertions(+), 15 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference create mode 100644 dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index 995c20cfad9..ae35ce037a5 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -304,7 +304,12 @@ bool joinEquals(const Block & left_block, const Block & right_block, const Block { size_t range_rows = range.left_length * range.right_length; if (range_rows > max_rows) + { + range.right_length = max_rows / range.left_length; + if (!range.right_length) + range.right_length = 1; one_more = false; + } size_t left_rows_to_add = range.left_length; size_t row_position = range.right_start; @@ -615,14 +620,16 @@ void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) MutableColumns right_columns = makeMutableColumns(right_columns_to_add, rows_to_reserve); MergeJoinCursor left_cursor(block, left_merge_description); size_t left_key_tail = 0; + size_t skip_right = 0; size_t right_blocks_count = rightBlocksCount(); size_t starting_right_block = 0; if (not_processed) { auto & continuation = static_cast(*not_processed); - starting_right_block = continuation.next_right_block; - left_cursor.nextN(continuation.next_left_position); + left_cursor.nextN(continuation.left_position); + skip_right = continuation.right_position; + starting_right_block = continuation.right_block; not_processed.reset(); } @@ -644,9 +651,10 @@ void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) std::shared_ptr right_block = loadRightBlock(i); - if (!leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail)) + if (!leftJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail, skip_right)) { - not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), left_cursor.position(), i); + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), + left_cursor.position(), skip_right, i); return; } } @@ -676,9 +684,10 @@ void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed) std::shared_ptr right_block = loadRightBlock(i); - if (!innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail)) + if (!innerJoin(left_cursor, block, *right_block, left_columns, right_columns, left_key_tail, skip_right)) { - not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), left_cursor.position(), i); + not_processed = extraBlock(block, std::move(left_columns), std::move(right_columns), + left_cursor.position(), skip_right, i); return; } } @@ -700,11 +709,19 @@ static size_t maxRangeRows(size_t current_rows, size_t max_rows) template bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, + size_t & skip_right [[maybe_unused]]) { MergeJoinCursor right_cursor(right_block, right_merge_description); left_cursor.setCompareNullability(right_cursor); + /// Set right cursor position in first continuation right block + if constexpr (is_all) + { + right_cursor.nextN(skip_right); + skip_right = 0; + } + while (!left_cursor.atEnd() && !right_cursor.atEnd()) { /// Not zero left_key_tail means there were equality for the last left key in previous leftJoin() call. @@ -726,7 +743,11 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block return false; if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + { + right_cursor.nextN(range.right_length); + skip_right = right_cursor.position(); return false; + } } else joinEqualsAnyLeft(right_block, right_columns_to_add, right_columns, range); @@ -750,11 +771,19 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block template bool MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail) + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, + size_t & skip_right [[maybe_unused]]) { MergeJoinCursor right_cursor(right_block, right_merge_description); left_cursor.setCompareNullability(right_cursor); + /// Set right cursor position in first continuation right block + if constexpr (is_all) + { + right_cursor.nextN(skip_right); + skip_right = 0; + } + while (!left_cursor.atEnd() && !right_cursor.atEnd()) { Range range = left_cursor.getNextEqualRange(right_cursor); @@ -768,7 +797,11 @@ bool MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc return false; if (!joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, max_rows)) + { + right_cursor.nextN(range.right_length); + skip_right = right_cursor.position(); return false; + } } else joinEquals(left_block, right_block, right_columns_to_add, left_columns, right_columns, range, 0); @@ -809,13 +842,15 @@ void MergeJoin::addRightColumns(Block & block, MutableColumns && right_columns) /// Split block into processed (result) and not processed. Not processed block would be joined next time. template ExtraBlockPtr MergeJoin::extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, - size_t left_position [[maybe_unused]], size_t right_block_number [[maybe_unused]]) + size_t left_position [[maybe_unused]], size_t right_position [[maybe_unused]], + size_t right_block_number [[maybe_unused]]) { ExtraBlockPtr not_processed; if constexpr (is_all) { - not_processed = std::make_shared(NotProcessed{{processed.cloneEmpty()}, left_position, right_block_number + 1}); + not_processed = std::make_shared( + NotProcessed{{processed.cloneEmpty()}, left_position, right_position, right_block_number}); not_processed->block.swap(processed); changeLeftColumns(processed, std::move(left_columns)); diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index 256d2af8808..f3a06e16f91 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -57,8 +57,9 @@ public: private: struct NotProcessed : public ExtraBlock { - size_t next_left_position; - size_t next_right_block; + size_t left_position; + size_t right_position; + size_t right_block; }; /// There're two size limits for right-hand table: max_rows_in_join, max_bytes_in_join. @@ -103,7 +104,7 @@ private: template ExtraBlockPtr extraBlock(Block & processed, MutableColumns && left_columns, MutableColumns && right_columns, - size_t position, size_t right_block_number); + size_t left_position, size_t right_position, size_t right_block_number); void mergeRightBlocks(); @@ -116,10 +117,10 @@ private: template bool leftJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, size_t & skip_right); template bool innerJoin(MergeJoinCursor & left_cursor, const Block & left_block, const Block & right_block, - MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail); + MutableColumns & left_columns, MutableColumns & right_columns, size_t & left_key_tail, size_t & skip_right); bool saveRightBlock(Block && block); void flushRightBlocks(); diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference new file mode 100644 index 00000000000..b376600bdaf --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference @@ -0,0 +1,13 @@ +defaults +10000000 +10000000 +10000000 10 +errors +max_joined_block_size = 2000 +10000000 +10000000 10000 +10000000 10 +max_rows_in_join = 1000 +10000000 +10000000 10000 +10000000 10 diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql new file mode 100644 index 00000000000..6c2754fc6cb --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql @@ -0,0 +1,68 @@ +SET max_memory_usage = 50000000; +SET partial_merge_join = 1; + +SELECT 'defaults'; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); + +SELECT 'errors'; +SET max_joined_block_size = 0; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); -- { serverError 241 } + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); -- { serverError 241 } + +SELECT 'max_joined_block_size = 2000'; +SET max_joined_block_size = 2000; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); + +SELECT 'max_rows_in_join = 1000'; +SET max_rows_in_join = 1000; + +SELECT count(1) FROM ( + SELECT materialize(1) as k, n FROM numbers(10) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(1000000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j + USING k); + +SELECT count(1), uniqExact(n) FROM ( + SELECT materialize(1) as k, n FROM numbers(1000000) + JOIN (SELECT materialize(1) AS k, number n FROM numbers(10)) j + USING k); From d9835979eb2ef4ada2e2bb70a01357686a5a6627 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 17 Jan 2020 14:45:43 +0300 Subject: [PATCH 061/312] rename setting --- dbms/src/Core/Settings.h | 2 +- dbms/src/Interpreters/AnalyzedJoin.cpp | 2 +- dbms/src/Interpreters/AnalyzedJoin.h | 4 ++-- dbms/src/Interpreters/MergeJoin.cpp | 6 +++--- dbms/src/Interpreters/MergeJoin.h | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index cf81b8bec79..5bf0761ab07 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -52,7 +52,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ M(SettingUInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ M(SettingUInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ - M(SettingUInt64, max_joined_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algo support it). 0 means unlimited.", 0) \ + M(SettingUInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algo support it). 0 means unlimited.", 0) \ M(SettingMaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ M(SettingMaxThreads, max_alter_threads, 0, "The maximum number of threads to execute the ALTER requests. By default, it is determined automatically.", 0) \ M(SettingUInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index bffb058d081..35365e4b1b1 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -23,7 +23,7 @@ AnalyzedJoin::AnalyzedJoin(const Settings & settings, const String & tmp_path_) : size_limits(SizeLimits{settings.max_rows_in_join, settings.max_bytes_in_join, settings.join_overflow_mode}) , default_max_bytes(settings.default_max_bytes_in_join) , join_use_nulls(settings.join_use_nulls) - , max_joined_block_size(settings.max_joined_block_size) + , max_joined_block_rows(settings.max_joined_block_size_rows) , partial_merge_join(settings.partial_merge_join) , partial_merge_join_optimizations(settings.partial_merge_join_optimizations) , partial_merge_join_rows_in_right_blocks(settings.partial_merge_join_rows_in_right_blocks) diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index f944d1278ee..4832f968695 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -40,7 +40,7 @@ class AnalyzedJoin const SizeLimits size_limits; const size_t default_max_bytes; const bool join_use_nulls; - const size_t max_joined_block_size = 0; + const size_t max_joined_block_rows = 0; const bool partial_merge_join = false; const bool partial_merge_join_optimizations = false; const size_t partial_merge_join_rows_in_right_blocks = 0; @@ -87,7 +87,7 @@ public: bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); } bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); } size_t defaultMaxBytes() const { return default_max_bytes; } - size_t maxJoinedBlockSize() const { return max_joined_block_size; } + size_t maxJoinedBlockRows() const { return max_joined_block_rows; } size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; } bool enablePartialMergeJoinOptimizations() const { return partial_merge_join_optimizations; } diff --git a/dbms/src/Interpreters/MergeJoin.cpp b/dbms/src/Interpreters/MergeJoin.cpp index ae35ce037a5..a62b9a9ec97 100644 --- a/dbms/src/Interpreters/MergeJoin.cpp +++ b/dbms/src/Interpreters/MergeJoin.cpp @@ -450,7 +450,7 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & ri , is_inner(isInner(table_join->kind())) , is_left(isLeft(table_join->kind())) , skip_not_intersected(table_join->enablePartialMergeJoinOptimizations()) - , max_joined_block_size(table_join->maxJoinedBlockSize()) + , max_joined_block_rows(table_join->maxJoinedBlockRows()) , max_rows_in_right_block(table_join->maxRowsInRightBlock()) { if (!isLeft(table_join->kind()) && !isInner(table_join->kind())) @@ -738,7 +738,7 @@ bool MergeJoin::leftJoin(MergeJoinCursor & left_cursor, const Block & left_block if constexpr (is_all) { - size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_size); + size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_rows); if (!max_rows) return false; @@ -792,7 +792,7 @@ bool MergeJoin::innerJoin(MergeJoinCursor & left_cursor, const Block & left_bloc if constexpr (is_all) { - size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_size); + size_t max_rows = maxRangeRows(left_columns.size(), max_joined_block_rows); if (!max_rows) return false; diff --git a/dbms/src/Interpreters/MergeJoin.h b/dbms/src/Interpreters/MergeJoin.h index f3a06e16f91..e20a42bb0d3 100644 --- a/dbms/src/Interpreters/MergeJoin.h +++ b/dbms/src/Interpreters/MergeJoin.h @@ -96,7 +96,7 @@ private: const bool is_inner; const bool is_left; const bool skip_not_intersected; - const size_t max_joined_block_size; + const size_t max_joined_block_rows; const size_t max_rows_in_right_block; void changeLeftColumns(Block & block, MutableColumns && columns); From 8346fb59864817bcba0d96b042f870facb1ff36a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 18 Jan 2020 21:05:54 +0300 Subject: [PATCH 062/312] Update .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 8147eb31799..206ae1e186e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,7 +46,7 @@ url = https://github.com/ClickHouse-Extras/protobuf.git [submodule "contrib/boost"] path = contrib/boost - url = https://github.com/oandrew/clickhouse-boost + url = https://github.com/ClickHouse-Extras/boost.git [submodule "contrib/base64"] path = contrib/base64 url = https://github.com/aklomp/base64.git From 997ea7ed53a7ee6a14647b97ea8ab66ed310343c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 22:29:53 +0300 Subject: [PATCH 063/312] Fixed error --- dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h | 2 +- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 4 ++-- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index 61fd28f2a70..47240db8b0d 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -309,7 +309,7 @@ protected: /// Uses a DFA based approach in order to better handle patterns without /// time assertions. /// - /// NOTE: This implementation relies on the assumption that the pattern are *small*. + /// NOTE: This implementation relies on the assumption that the pattern is *small*. /// /// This algorithm performs in O(mn) (with m the number of DFA states and N the number /// of events) with a memory consumption and memory allocations in O(m). It means that diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 6f2faaf09f0..59cdbcb8650 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -484,7 +484,7 @@ bool AvroRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) return false; } -#ifdef USE_POCO_JSON +#if USE_POCO_JSON class AvroConfluentRowInputFormat::SchemaRegistry { public: @@ -616,7 +616,7 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) return std::make_shared(sample, buf, params); }); -#ifdef USE_POCO_JSON +#if USE_POCO_JSON factory.registerInputFormatProcessor("AvroConfluent",[=]( ReadBuffer & buf, const Block & sample, diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h index 1941fb14300..7a9f9b239c4 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -1,5 +1,6 @@ #pragma once #include "config_formats.h" +#include "config_core.h" #if USE_AVRO #include @@ -46,7 +47,7 @@ private: AvroDeserializer deserializer; }; -#ifdef USE_POCO_JSON +#if USE_POCO_JSON class AvroConfluentRowInputFormat : public IRowInputFormat { public: From e6b407a961b1cdd81bf276ce806c8f34b246aa58 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 22:34:09 +0300 Subject: [PATCH 064/312] Added TODO --- .../Processors/Formats/Impl/AvroRowInputFormat.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 59cdbcb8650..0332cd4e354 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -498,18 +498,20 @@ public: { base_url = base_url_; } - catch (Poco::SyntaxException & e) + catch (const Poco::SyntaxException & e) { throw Exception("Invalid Schema Registry URL: " + e.displayText(), ErrorCodes::BAD_ARGUMENTS); } } - avro::ValidSchema getSchema(uint32_t id) + avro::ValidSchema getSchema(uint32_t id) const { try { try { + /// TODO Host checking to prevent SSRF + Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); Poco::Net::HTTPClientSession session(url.getHost(), url.getPort()); Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, url.getPathAndQuery()); @@ -518,16 +520,16 @@ public: auto & response_body = session.receiveResponse(response); if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) { - throw Exception("http code " + std::to_string(response.getStatus()), ErrorCodes::INCORRECT_DATA); + throw Exception("HTTP code " + std::to_string(response.getStatus()), ErrorCodes::INCORRECT_DATA); } Poco::JSON::Parser parser; auto json_body = parser.parse(response_body).extract(); auto schema = json_body->getValue("schema"); return avro::compileJsonSchemaFromString(schema); } - catch (const Exception & e) + catch (const Exception &) { - throw e; + throw; } catch (const Poco::Exception & e) { @@ -540,7 +542,7 @@ public: } catch (Exception & e) { - e.addMessage("while fetching schema id=" + std::to_string(id)); + e.addMessage("while fetching schema id = " + std::to_string(id)); throw; } } From 85f2e86e5bcde32f7c7e7aa71389f3277893ceaf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:10:11 +0300 Subject: [PATCH 065/312] Added performance test for Avro format --- dbms/tests/performance/parse_engine_file.xml | 1 + dbms/tests/performance/select_format.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/dbms/tests/performance/parse_engine_file.xml b/dbms/tests/performance/parse_engine_file.xml index 080acbd53f2..8a0054bdd7f 100644 --- a/dbms/tests/performance/parse_engine_file.xml +++ b/dbms/tests/performance/parse_engine_file.xml @@ -34,6 +34,7 @@ TSKV RowBinary Native + Avro diff --git a/dbms/tests/performance/select_format.xml b/dbms/tests/performance/select_format.xml index 621247fee1e..189b35a2700 100644 --- a/dbms/tests/performance/select_format.xml +++ b/dbms/tests/performance/select_format.xml @@ -44,6 +44,7 @@ XML ODBCDriver2 MySQLWire + Avro From f0b4fcee1f3eca214365a96b0aa685a58f1f996e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:10:32 +0300 Subject: [PATCH 066/312] Minor modifications --- dbms/src/IO/ReadHelpers.h | 18 ++++++++++++++++++ .../Formats/Impl/AvroRowInputFormat.cpp | 13 ++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index fc8e444330c..9cb26434930 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -746,6 +747,23 @@ inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf) inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); } +template +inline std::enable_if_t && (sizeof(T) <= 8), void> +readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. +{ + readPODBinary(x, buf); + + if constexpr (sizeof(x) == 1) + return; + else if constexpr (sizeof(x) == 2) + x = bswap_16(x); + else if constexpr (sizeof(x) == 4) + x = bswap_32(x); + else if constexpr (sizeof(x) == 8) + x = bswap_64(x); +} + + /// Generic methods to read value in text tab-separated format. template inline std::enable_if_t, void> diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 0332cd4e354..25cdc0e9abc 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -553,16 +553,16 @@ private: static uint32_t readConfluentSchemaId(ReadBuffer & in) { - Poco::Buffer buf(5); - in.readStrict(buf.begin(), buf.capacity()); - Poco::MemoryBinaryReader binary_reader(buf, Poco::BinaryReader::BIG_ENDIAN_BYTE_ORDER); - uint8_t magic; uint32_t schema_id; - binary_reader >> magic >> schema_id; + + readBinaryBigEndian(magic, in); + readBinaryBigEndian(schema_id, in); + if (magic != 0x00) { - throw Exception("Invalid magic byte", ErrorCodes::INCORRECT_DATA); + throw Exception("Invalid magic byte before AvroConfluent schema identifier." + " Must be zero byte, found " + std::to_string(int(magic)) + " instead", ErrorCodes::INCORRECT_DATA); } return schema_id; @@ -577,7 +577,6 @@ AvroConfluentRowInputFormat::AvroConfluentRowInputFormat( , decoder(avro::binaryDecoder()) { - (void)format_settings_; decoder->init(*input_stream); } From ac46498f60b2ff54a390e3f0615f7915eb4b4faf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:12:58 +0300 Subject: [PATCH 067/312] Minor modification --- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 4 ++-- dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 25cdc0e9abc..75f5634578e 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -608,7 +608,7 @@ AvroDeserializer & AvroConfluentRowInputFormat::getOrCreateDeserializer(SchemaId void registerInputFormatProcessorAvro(FormatFactory & factory) { - factory.registerInputFormatProcessor("Avro", [=]( + factory.registerInputFormatProcessor("Avro", []( ReadBuffer & buf, const Block & sample, const RowInputFormatParams & params, @@ -618,7 +618,7 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) }); #if USE_POCO_JSON - factory.registerInputFormatProcessor("AvroConfluent",[=]( + factory.registerInputFormatProcessor("AvroConfluent",[]( ReadBuffer & buf, const Block & sample, const RowInputFormatParams & params, diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 9f6233303ad..233d9284fc7 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -342,7 +342,7 @@ void AvroRowOutputFormat::writeSuffix() void registerOutputFormatProcessorAvro(FormatFactory & factory) { - factory.registerOutputFormatProcessor("Avro",[=]( + factory.registerOutputFormatProcessor("Avro", []( WriteBuffer & buf, const Block & sample, FormatFactory::WriteCallback callback, From 216b39a30de3a65104c9608712ba81b9f618b293 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:15:49 +0300 Subject: [PATCH 068/312] Minor changes for consistency --- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 7 +------ dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h | 4 ++-- dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 2 -- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 75f5634578e..8d96afc80dc 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -8,12 +8,9 @@ #include #include - #include - #include - #include #include #include @@ -27,14 +24,12 @@ #include #include - #include #include #include #include #include - #include #include #include @@ -414,7 +409,7 @@ AvroDeserializer::SkipFn AvroDeserializer::createSkipFn(avro::NodePtr root_node) } -AvroDeserializer::AvroDeserializer(const DB::ColumnsWithTypeAndName & columns, avro::ValidSchema schema) +AvroDeserializer::AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema) { auto schema_root = schema.root(); if (schema_root->type() != avro::AVRO_RECORD) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h index 7a9f9b239c4..353f611a36e 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -21,7 +21,7 @@ namespace DB class AvroDeserializer { public: - AvroDeserializer(const DB::ColumnsWithTypeAndName & columns, avro::ValidSchema schema); + AvroDeserializer(const ColumnsWithTypeAndName & columns, avro::ValidSchema schema); void deserializeRow(MutableColumns & columns, avro::Decoder & decoder); private: @@ -56,7 +56,7 @@ public: String getName() const override { return "AvroConfluentRowInputFormat"; } private: - const DB::ColumnsWithTypeAndName header_columns; + const ColumnsWithTypeAndName header_columns; class SchemaRegistry; std::unique_ptr schema_registry; diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 233d9284fc7..55fc9d08ca9 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -8,7 +8,6 @@ #include #include - #include #include @@ -19,7 +18,6 @@ #include #include - #include #include #include From 457c50c7d71bbf393480d4a5f0fd6c5c2f768c60 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:19:10 +0300 Subject: [PATCH 069/312] Code cleanups around --- dbms/src/Common/RemoteHostFilter.cpp | 5 +++-- dbms/src/Common/RemoteHostFilter.h | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dbms/src/Common/RemoteHostFilter.cpp b/dbms/src/Common/RemoteHostFilter.cpp index 16aaac35dbe..4c4aa3bca81 100644 --- a/dbms/src/Common/RemoteHostFilter.cpp +++ b/dbms/src/Common/RemoteHostFilter.cpp @@ -1,12 +1,13 @@ #include -#include #include -#include #include +#include +#include #include #include #include + namespace DB { namespace ErrorCodes diff --git a/dbms/src/Common/RemoteHostFilter.h b/dbms/src/Common/RemoteHostFilter.h index 86743891051..48d9b2bda7c 100644 --- a/dbms/src/Common/RemoteHostFilter.h +++ b/dbms/src/Common/RemoteHostFilter.h @@ -1,17 +1,19 @@ #pragma once +#include #include #include -#include -#include +namespace Poco { class URI; } +namespace Poco { namespace Util { class AbstractConfiguration; } } + namespace DB { class RemoteHostFilter { /** - * This class checks if url is allowed. + * This class checks if URL is allowed. * If primary_hosts and regexp_hosts are empty all urls are allowed. */ public: @@ -25,6 +27,7 @@ private: std::unordered_set primary_hosts; /// Allowed primary () URL from config.xml std::vector regexp_hosts; /// Allowed regexp () URL from config.xml - bool checkForDirectEntry(const std::string & str) const; /// Checks if the primary_hosts and regexp_hosts contain str. If primary_hosts and regexp_hosts are empty return true. + /// Checks if the primary_hosts and regexp_hosts contain str. If primary_hosts and regexp_hosts are empty return true. + bool checkForDirectEntry(const std::string & str) const; }; } From 3c7df7ca315c5cb52a7bffe9934c41988b44bbb0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Jan 2020 23:42:50 +0300 Subject: [PATCH 070/312] Minor modifications --- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 2 +- dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 8d96afc80dc..795d69c7505 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -438,7 +438,7 @@ AvroDeserializer::AvroDeserializer(const ColumnsWithTypeAndName & columns, avro: catch (Exception & e) { e.addMessage("column " + column.name); - e.rethrow(); + throw; } field_mapping[field_index] = i; } diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index 55fc9d08ca9..f303fea2b28 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -271,7 +271,7 @@ AvroSerializer::AvroSerializer(const ColumnsWithTypeAndName & columns) catch (Exception & e) { e.addMessage("column " + column.name); - e.rethrow(); + throw; } } schema.setSchema(record_schema); From d732f854b73e79c9f23a3ad33448423316c86f92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 00:42:51 +0300 Subject: [PATCH 071/312] Added timeouts for schema request --- .../Formats/Impl/AvroRowInputFormat.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 795d69c7505..a64ad0cd705 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -5,8 +5,10 @@ #include #include + #include #include +#include #include #include @@ -508,15 +510,23 @@ public: /// TODO Host checking to prevent SSRF Poco::URI url(base_url, "/schemas/ids/" + std::to_string(id)); - Poco::Net::HTTPClientSession session(url.getHost(), url.getPort()); + + /// One second for connect/send/receive. Just in case. + ConnectionTimeouts timeouts({1, 0}, {1, 0}, {1, 0}); + Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, url.getPathAndQuery()); - session.sendRequest(request); + + auto session = makePooledHTTPSession(url, timeouts, 1); + session->sendRequest(request); + Poco::Net::HTTPResponse response; - auto & response_body = session.receiveResponse(response); + auto & response_body = session->receiveResponse(response); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) { throw Exception("HTTP code " + std::to_string(response.getStatus()), ErrorCodes::INCORRECT_DATA); } + Poco::JSON::Parser parser; auto json_body = parser.parse(response_body).extract(); auto schema = json_body->getValue("schema"); From ae185a24d201d777bcfda0bf1b7288941f44570a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 00:45:16 +0300 Subject: [PATCH 072/312] Disable AvroConfluent --- dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index a64ad0cd705..13797a9ca56 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -623,6 +623,12 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) }); #if USE_POCO_JSON + + /// AvroConfluent format is disabled for the following reasons: + /// 1. There is no test for it. + /// 2. RemoteHostFilter is not used to prevent CSRF attacks. + +#if 0 factory.registerInputFormatProcessor("AvroConfluent",[]( ReadBuffer & buf, const Block & sample, @@ -633,6 +639,8 @@ void registerInputFormatProcessorAvro(FormatFactory & factory) }); #endif +#endif + } } From 4fdfb02b2f87814d3669f5fb478a202c90a55592 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 03:43:59 +0300 Subject: [PATCH 073/312] Fixed build --- dbms/src/IO/ReadHelpers.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/src/IO/ReadHelpers.h b/dbms/src/IO/ReadHelpers.h index 9cb26434930..2e6e51a7835 100644 --- a/dbms/src/IO/ReadHelpers.h +++ b/dbms/src/IO/ReadHelpers.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -756,11 +755,11 @@ readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian archi if constexpr (sizeof(x) == 1) return; else if constexpr (sizeof(x) == 2) - x = bswap_16(x); + x = __builtin_bswap16(x); else if constexpr (sizeof(x) == 4) - x = bswap_32(x); + x = __builtin_bswap32(x); else if constexpr (sizeof(x) == 8) - x = bswap_64(x); + x = __builtin_bswap64(x); } From b5ff5341d5b8154c294315b2a8341088e050bc24 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 04:22:27 +0300 Subject: [PATCH 074/312] Support more types for output --- .../Formats/Impl/AvroRowOutputFormat.cpp | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index f303fea2b28..b0375c7e2ae 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -83,15 +83,41 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF switch (data_type->getTypeId()) { case TypeIndex::UInt8: - return {avro::BoolSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { - encoder.encodeBool(assert_cast(column).getElement(row_num)); + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int8: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::UInt16: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::Int16: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); + }}; + case TypeIndex::UInt32: [[fallthrough]]; + case TypeIndex::DateTime: + return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeInt(assert_cast(column).getElement(row_num)); }}; case TypeIndex::Int32: return {avro::IntSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { encoder.encodeInt(assert_cast(column).getElement(row_num)); }}; + case TypeIndex::UInt64: + return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + { + encoder.encodeLong(assert_cast(column).getElement(row_num)); + }}; case TypeIndex::Int64: return {avro::LongSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { @@ -136,7 +162,7 @@ AvroSerializer::SchemaWithSerializeFn AvroSerializer::createSchemaWithSerializeF }}; } case TypeIndex::String: - return {avro::StringSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) + return {avro::BytesSchema(), [](const IColumn & column, size_t row_num, avro::Encoder & encoder) { const StringRef & s = assert_cast(column).getDataAt(row_num); encoder.encodeBytes(reinterpret_cast(s.data), s.size); From 40e35c36e7849ca2bbd9036e75f5c466cb18833b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 20:02:29 +0300 Subject: [PATCH 075/312] Minor modifications --- .../Formats/Impl/AvroRowInputFormat.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 13797a9ca56..649ed777c4f 100644 --- a/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/dbms/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -64,12 +64,9 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_TYPE_OF_FIELD; extern const int BAD_ARGUMENTS; extern const int THERE_IS_NO_COLUMN; - extern const int LOGICAL_ERROR; extern const int INCORRECT_DATA; - extern const int CANNOT_READ_ALL_DATA; extern const int ILLEGAL_COLUMN; extern const int TYPE_MISMATCH; } @@ -114,18 +111,9 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node WhichDataType target(target_type); switch (root_node->type()) { - case avro::AVRO_STRING: - if (target.isString()) - { - return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable - { - decoder.decodeString(tmp); - column.insertData(tmp.c_str(), tmp.length()); - }; - } - break; + case avro::AVRO_STRING: [[fallthrough]]; case avro::AVRO_BYTES: - if (target.isString()) + if (target.isString() || target.isFixedString()) { return [tmp = std::string()](IColumn & column, avro::Decoder & decoder) mutable { From 05048bf0c1869b959db60d0d243fbaa654471355 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Jan 2020 20:08:15 +0300 Subject: [PATCH 076/312] Fixed "unbundled" build --- cmake/find/replxx.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/find/replxx.cmake b/cmake/find/replxx.cmake index 13df104515e..3a0e5917b04 100644 --- a/cmake/find/replxx.cmake +++ b/cmake/find/replxx.cmake @@ -1,4 +1,4 @@ -option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) +option (ENABLE_REPLXX "Enable replxx support" ${NOT_UNBUNDLED}) if (ENABLE_REPLXX) option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED}) From 0669eff031befb2db096f97282bb717d3bfc59d4 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 20 Jan 2020 17:17:55 +0300 Subject: [PATCH 077/312] version with processors --- .../ExpressionBlockInputStream.cpp | 2 +- .../DataStreams/ExpressionBlockInputStream.h | 4 +- .../Interpreters/InterpreterSelectQuery.cpp | 5 +- .../InflatingExpressionTransform.cpp | 102 ++++++++++++++++++ .../Transforms/InflatingExpressionTransform.h | 37 +++++++ ...all_join_with_block_continuation.reference | 2 +- ...62_pm_all_join_with_block_continuation.sql | 6 +- 7 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp create mode 100644 dbms/src/Processors/Transforms/InflatingExpressionTransform.h diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index d41d21915e1..5a4fe4ecf1d 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -44,7 +44,7 @@ Block ExpressionBlockInputStream::readImpl() return res; } -Block SplittingExpressionBlockInputStream::readImpl() +Block InflatingExpressionBlockInputStream::readImpl() { if (!initialized) { diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.h b/dbms/src/DataStreams/ExpressionBlockInputStream.h index b1a1f9674fd..62141a060af 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.h @@ -35,10 +35,10 @@ private: }; /// ExpressionBlockInputStream that could generate many out blocks for single input block. -class SplittingExpressionBlockInputStream : public ExpressionBlockInputStream +class InflatingExpressionBlockInputStream : public ExpressionBlockInputStream { public: - SplittingExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) + InflatingExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) : ExpressionBlockInputStream(input, expression_) {} diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index a29267913c8..16a6ebc2b11 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -1104,7 +1105,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type) { bool on_totals = type == QueryPipeline::StreamType::Totals; - return std::make_shared(header, expressions.before_join, on_totals, default_totals); + return std::make_shared(header, expressions.before_join, on_totals, default_totals); }); } else @@ -1112,7 +1113,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS header_before_join = pipeline.firstStream()->getHeader(); /// Applies to all sources except stream_with_non_joined_data. for (auto & stream : pipeline.streams) - stream = std::make_shared(stream, expressions.before_join); + stream = std::make_shared(stream, expressions.before_join); if (isMergeJoin(expressions.before_join->getTableJoinAlgo()) && settings.partial_merge_join_optimizations) { diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp new file mode 100644 index 00000000000..e13a6bfa236 --- /dev/null +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.cpp @@ -0,0 +1,102 @@ +#include +#include + +namespace DB +{ + +static Block transformHeader(Block header, const ExpressionActionsPtr & expression) +{ + expression->execute(header, true); + return header; +} + +InflatingExpressionTransform::InflatingExpressionTransform(Block input_header, ExpressionActionsPtr expression_, + bool on_totals_, bool default_totals_) + : ISimpleTransform(input_header, transformHeader(input_header, expression_), on_totals_) + , expression(std::move(expression_)) + , on_totals(on_totals_) + , default_totals(default_totals_) +{} + +void InflatingExpressionTransform::work() +{ + if (current_data.exception) + return; + + try + { + transform(current_data.chunk); + } + catch (DB::Exception &) + { + current_data.exception = std::current_exception(); + transformed = true; + has_input = false; + return; + } + + /// The only change from ISimpleTransform::work() + if (!not_processed) + has_input = false; + + if (!skip_empty_chunks || current_data.chunk) + transformed = true; + + if (transformed && !current_data.chunk) + /// Support invariant that chunks must have the same number of columns as header. + current_data.chunk = Chunk(getOutputPort().getHeader().cloneEmpty().getColumns(), 0); +} + +void InflatingExpressionTransform::transform(Chunk & chunk) +{ + if (!initialized) + { + initialized = true; + + if (expression->resultIsAlwaysEmpty()) + { + stopReading(); + chunk = Chunk(getOutputPort().getHeader().getColumns(), 0); + return; + } + } + + Block block; + if (on_totals) + { + if (default_totals && !expression->hasTotalsInJoin()) + return; + block = readExecOnTotals(chunk); + } + else + block = readExec(chunk); + + auto num_rows = block.rows(); + chunk.setColumns(block.getColumns(), num_rows); +} + +Block InflatingExpressionTransform::readExec(Chunk & chunk) +{ + Block res; + if (likely(!not_processed)) + { + res = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + if (res) + expression->execute(res, not_processed, action_number); + } + else + { + res = std::move(not_processed->block); + expression->execute(res, not_processed, action_number); + } + return res; +} + +Block InflatingExpressionTransform::readExecOnTotals(Chunk & chunk) +{ + Block block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); + expression->executeOnTotals(block); + return block; +} + +} diff --git a/dbms/src/Processors/Transforms/InflatingExpressionTransform.h b/dbms/src/Processors/Transforms/InflatingExpressionTransform.h new file mode 100644 index 00000000000..ef114ea4b9b --- /dev/null +++ b/dbms/src/Processors/Transforms/InflatingExpressionTransform.h @@ -0,0 +1,37 @@ +#pragma once +#include + + +namespace DB +{ + +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + +class InflatingExpressionTransform : public ISimpleTransform +{ +public: + InflatingExpressionTransform(Block input_header, ExpressionActionsPtr expression_, + bool on_totals_ = false, bool default_totals_ = false); + + String getName() const override { return "InflatingExpressionTransform"; } + void work() override; + +protected: + void transform(Chunk & chunk) override; + +private: + ExpressionActionsPtr expression; + bool on_totals; + bool default_totals; + bool initialized = false; + + Chunk src_chunk; + ExtraBlockPtr not_processed; + size_t action_number = 0; + + Block readExec(Chunk & chunk); + Block readExecOnTotals(Chunk & chunk); +}; + +} diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference index b376600bdaf..b2220afe01d 100644 --- a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.reference @@ -3,7 +3,7 @@ defaults 10000000 10000000 10 errors -max_joined_block_size = 2000 +max_joined_block_size_rows = 2000 10000000 10000000 10000 10000000 10 diff --git a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql index 6c2754fc6cb..947bc64ece2 100644 --- a/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql +++ b/dbms/tests/queries/0_stateless/01062_pm_all_join_with_block_continuation.sql @@ -19,7 +19,7 @@ SELECT count(1), uniqExact(n) FROM ( USING k); SELECT 'errors'; -SET max_joined_block_size = 0; +SET max_joined_block_size_rows = 0; SELECT count(1) FROM ( SELECT materialize(1) as k, n FROM numbers(10) @@ -31,8 +31,8 @@ SELECT count(1) FROM ( JOIN (SELECT materialize(1) AS k, number n FROM numbers(10000)) j USING k); -- { serverError 241 } -SELECT 'max_joined_block_size = 2000'; -SET max_joined_block_size = 2000; +SELECT 'max_joined_block_size_rows = 2000'; +SET max_joined_block_size_rows = 2000; SELECT count(1) FROM ( SELECT materialize(1) as k, n FROM numbers(10) From b26c9a1f6777f50f67f940b1fc464459aa6577d9 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 20 Jan 2020 19:36:21 +0300 Subject: [PATCH 078/312] add test for pmj with consts and nullables --- ...4_pm_all_join_const_and_nullable.reference | 19 ++++ .../01064_pm_all_join_const_and_nullable.sql | 87 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference create mode 100644 dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql diff --git a/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference new file mode 100644 index 00000000000..917fac168c2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.reference @@ -0,0 +1,19 @@ +100000 1 +100000 1 +100000 1 +100000 1 +first nullable +100000 1 +100000 1 +100000 1 +100000 1 +second nullable +100000 1 +100000 1 +100000 1 +100000 1 +both nullable +100000 1 +100000 1 +100000 1 +100000 1 diff --git a/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql new file mode 100644 index 00000000000..631fda510b0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_pm_all_join_const_and_nullable.sql @@ -0,0 +1,87 @@ +SET partial_merge_join = 1; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'first nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT materialize(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT 1 AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'second nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(1) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT 1 as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT 'both nullable'; + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT materialize(toNullable(1)) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT materialize(toNullable(1)) AS k, number n FROM numbers(100000)) j +USING k); + +SELECT count(1), uniqExact(1) FROM ( +SELECT toNullable(1) as k FROM numbers(1) +JOIN (SELECT toNullable(1) AS k, number n FROM numbers(100000)) j +USING k); From 3f0d1e21c0a669a081b8c0960e1a2125dd5e1951 Mon Sep 17 00:00:00 2001 From: hcz Date: Tue, 21 Jan 2020 11:26:26 +0800 Subject: [PATCH 079/312] Remove a duplicated document entry --- docs/en/query_language/functions/rounding_functions.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/en/query_language/functions/rounding_functions.md b/docs/en/query_language/functions/rounding_functions.md index 2640472f955..ec365235381 100644 --- a/docs/en/query_language/functions/rounding_functions.md +++ b/docs/en/query_language/functions/rounding_functions.md @@ -161,8 +161,6 @@ roundBankers(10.755, 2) = 11,76 - [round](#rounding_functions-round) - - ## roundToExp2(num) Accepts a number. If the number is less than one, it returns 0. Otherwise, it rounds the number down to the nearest (whole non-negative) degree of two. @@ -180,9 +178,3 @@ Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rou Accept a number, round it down to an element in the specified array. If the value is less than the lowest bound, the lowest bound is returned. [Original article](https://clickhouse.yandex/docs/en/query_language/functions/rounding_functions/) - -## roundBankers(x\[, N\]) - -Rounds a value to a specified number of decimal places. - -The function returns the nearest number of the specified order. In case when given number has equal distance to surrounding numbers, the function always return the number having the nearest even digit (banker's rounding). From 414de57b999d0aeb31b2e7dc2c52745f6704fbec Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 21 Jan 2020 13:11:30 +0300 Subject: [PATCH 080/312] fix hash join with processors --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 16a6ebc2b11..86e706be76a 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1105,7 +1105,12 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType type) { bool on_totals = type == QueryPipeline::StreamType::Totals; - return std::make_shared(header, expressions.before_join, on_totals, default_totals); + std::shared_ptr ret; + if (settings.partial_merge_join) + ret = std::make_shared(header, expressions.before_join, on_totals, default_totals); + else + ret = std::make_shared(header, expressions.before_join, on_totals, default_totals); + return ret; }); } else From cc6d9e0bea725f34dc1b58b597ea617915d74728 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 21 Jan 2020 14:59:25 +0300 Subject: [PATCH 081/312] Try alternative documentation search backend --- docs/tools/build.py | 11 ++-------- .../assets/stylesheets/custom.css | 20 +++++++++++++++++++ docs/tools/mkdocs-material-theme/base.html | 16 ++++++++++++++- .../partials/header.html | 6 ++---- .../partials/search.html | 2 +- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 0e855ce9f1e..d6762a708d2 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -66,7 +66,7 @@ def build_for_lang(lang, args): 'logo': 'images/logo.svg', 'favicon': 'assets/images/favicon.ico', 'include_search_page': False, - 'search_index_only': True, + 'search_index_only': False, 'static_templates': ['404.html'], 'extra': { 'now': int(time.mktime(datetime.datetime.now().timetuple())) # TODO better way to avoid caching @@ -113,15 +113,8 @@ def build_for_lang(lang, args): } } ], - plugins=[{ - 'search': { - 'lang': ['en', 'ru'] if lang == 'ru' else ['en'] - } - }], + plugins=[], extra={ - 'search': { - 'language': 'en,ru' if lang == 'ru' else 'en' - }, 'stable_releases': args.stable_releases, 'version_prefix': args.version_prefix } diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index e452e88ad66..21d6ab80b38 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -209,3 +209,23 @@ h1, h2, h3, .md-logo { content: "" !important; } } + +.algolia-autocomplete .algolia-docsearch-suggestion--subcategory-column { + display: none !important; +} + +.algolia-autocomplete .algolia-docsearch-suggestion--content { + float: none !important; + width: 100% !important; +} + +.algolia-autocomplete .algolia-docsearch-suggestion--content:before { + content: none !important; +} + +.algolia-autocomplete .ds-dropdown-menu { + width: 100% !important; + max-height: 512px; + overflow-x: hidden; + overflow-y: auto; +} diff --git a/docs/tools/mkdocs-material-theme/base.html b/docs/tools/mkdocs-material-theme/base.html index 8b04617c721..850d3647dda 100644 --- a/docs/tools/mkdocs-material-theme/base.html +++ b/docs/tools/mkdocs-material-theme/base.html @@ -3,7 +3,7 @@ {% set palette = config.theme.palette %} {% set font = config.theme.font %} - +
{% block site_meta %} @@ -49,6 +49,7 @@ {% endif %} {% endblock %} {% block styles %} + {% if palette.primary or palette.accent %} @@ -269,6 +270,19 @@ ) }); + + + @@ -2141,6 +2447,7 @@ Results for Xeon 2176G are from Sergey Golod.
Results for Azure DS3v2 are from Boris Granveaud.
Results for AWS are from Wolf Kreuzerkrieg.
Results for Huawei Taishan are from Peng Gao in sina.com.
+Results for Selectel and AMD EPYC 7402P are from Andrey Dudin.
Xeon Gold 6230 server is using 4 x SAMSUNG datacenter class SSD in RAID-10.
Results for Yandex Managed ClickHouse for "cold cache" are biased and should not be compared, because cache was not flushed for every next query.
From 4d13f020cd2b8ece16e82f810d1d1be29cdf62ed Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jan 2020 16:36:37 +0300 Subject: [PATCH 211/312] Removed obsolete setting from test --- .../queries/0_stateless/01010_pmj_right_table_memory_limits.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql b/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql index b28b32ff787..fc0af53c5ae 100644 --- a/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql +++ b/dbms/tests/queries/0_stateless/01010_pmj_right_table_memory_limits.sql @@ -35,7 +35,6 @@ ORDER BY n SETTINGS max_bytes_in_join = 10000000; SET partial_merge_join_optimizations = 1; -SET partial_merge_join_rows_in_left_blocks = 100000; SELECT number * 200000 as n, j FROM numbers(5) LEFT JOIN ( From d364447824fdc31213187674a6b00c475640353b Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 26 Jan 2020 17:34:16 +0300 Subject: [PATCH 212/312] Better image for woboq --- docker/test/codebrowser/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 3ed3b250f0c..f9d239ef8ef 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -37,11 +37,12 @@ ENV SOURCE_DIRECTORY=/repo_folder ENV BUILD_DIRECTORY=/build ENV HTML_RESULT_DIRECTORY=$BUILD_DIRECTORY/html_report ENV SHA=nosha +ENV DATA="data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-8 -DCMAKE_C_COMPILER=/usr/bin/clang-8 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON && \ mkdir -p $HTML_RESULT_DIRECTORY && \ - $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA && \ + $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ - $CODEINDEX $HTML_RESULT_DIRECTORY -d "data" && \ + $CODEINDEX $HTML_RESULT_DIRECTORY -d $DATA && \ mv $HTML_RESULT_DIRECTORY /test_output From 5d522612662a79abbbd64a29e5839240b0252f70 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jan 2020 18:25:58 +0300 Subject: [PATCH 213/312] Remove useless files --- dbms/src/Compression/CompressionCodecWriteBuffer.h | 0 dbms/src/DataStreams/tests/filter_stream_hitlog.cpp | 0 dbms/src/DataStreams/tests/native_streams.cpp | 0 dbms/src/DataStreams/tests/sorting_stream.cpp | 0 dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp | 0 dbms/src/Storages/tests/hit_log.cpp | 0 6 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dbms/src/Compression/CompressionCodecWriteBuffer.h delete mode 100644 dbms/src/DataStreams/tests/filter_stream_hitlog.cpp delete mode 100644 dbms/src/DataStreams/tests/native_streams.cpp delete mode 100644 dbms/src/DataStreams/tests/sorting_stream.cpp delete mode 100644 dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp delete mode 100644 dbms/src/Storages/tests/hit_log.cpp diff --git a/dbms/src/Compression/CompressionCodecWriteBuffer.h b/dbms/src/Compression/CompressionCodecWriteBuffer.h deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp b/dbms/src/DataTypes/tests/gtest_DecimalUtils.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/src/Storages/tests/hit_log.cpp deleted file mode 100644 index e69de29bb2d..00000000000 From c2a7021e37f22a6d58840ff0e94f2066ea9924b8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 26 Jan 2020 19:14:13 +0300 Subject: [PATCH 214/312] Add ability to change user/group for clickhouse-server docker image This will allow to create files owned by subuid/subgid if docker is running with userns. --- docker/server/README.md | 9 +++++++-- docker/server/entrypoint.sh | 21 ++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docker/server/README.md b/docker/server/README.md index b1225e8a27a..b36bbe40dc7 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -28,7 +28,7 @@ Container exposes 8123 port for [HTTP interface](https://clickhouse.yandex/docs/ ClickHouse configuration represented with a file "config.xml" ([documentation](https://clickhouse.yandex/docs/en/operations/configuration_files/)) -### start server instance with custom configuration +### Start server instance with custom configuration ```bash $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v /path/to/your/config.xml:/etc/clickhouse-server/config.xml yandex/clickhouse-server ``` @@ -36,10 +36,15 @@ $ docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 -v / ### Start server as custom user ``` # $(pwd)/data/clickhouse should exist and be owned by current user -$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/data/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server +$ docker run --rm --user ${UID}:${GID} --name some-clickhouse-server --ulimit nofile=262144:262144 -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server ``` When you use the image with mounting local directories inside you probably would like to not mess your directory tree with files owner and permissions. Then you could use `--user` argument. In this case, you should mount every necessary directory (`/var/lib/clickhouse` and `/var/log/clickhouse-server`) inside the container. Otherwise, image will complain and not start. +### Start server from root (useful in case of userns enabled) +``` +$ docker run --rm -e CLICKHOUSE_UID=0 -e CLICKHOUSE_GID=0 --name clickhouse-server-userns -v "$(pwd)/logs/clickhouse:/var/log/clickhouse-server" -v "$(pwd)/data/clickhouse:/var/lib/clickhouse" yandex/clickhouse-server +``` + ## How to extend this image If you would like to do additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb` it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. diff --git a/docker/server/entrypoint.sh b/docker/server/entrypoint.sh index 03bacb7ac88..fbab44de2b7 100644 --- a/docker/server/entrypoint.sh +++ b/docker/server/entrypoint.sh @@ -1,17 +1,28 @@ #!/bin/bash -# set some vars -CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" +DO_CHOWN=1 +if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then + DO_CHOWN=0 +fi + +CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" +CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + +# support --user if [ x"$UID" == x0 ]; then - USER="$(id -u clickhouse)" - GROUP="$(id -g clickhouse)" + USER=$CLICKHOUSE_UID + GROUP=$CLICKHOUSE_GID gosu="gosu $USER:$GROUP" else USER="$(id -u)" GROUP="$(id -g)" gosu="" + DO_CHOWN=0 fi +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" + # port is needed to check if clickhouse-server is ready for connections HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" @@ -41,7 +52,7 @@ do exit 1 fi - if [ x"$UID" == x0 ] && [ "$CLICKHOUSE_DO_NOT_CHOWN" != "1" ]; then + if [ "$DO_CHOWN" = "1" ]; then # ensure proper directories permissions chown -R "$USER:$GROUP" "$dir" elif [ "$(stat -c %u "$dir")" != "$USER" ]; then From 87ee9ac09916817f48f97c17b07d18c7add751cb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jan 2020 21:43:44 +0300 Subject: [PATCH 215/312] Fixed ugly error in Client --- dbms/programs/client/Client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 6c5f9e5fb30..c41a26a6c63 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -513,6 +513,7 @@ private: if (input.empty()) break; + has_vertical_output_suffix = false; if (input.ends_with("\\G")) { input.resize(input.size() - 2); From 77cefbbd427b25b20f838ac415dea7b05f2d1ca1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 12:14:01 +0300 Subject: [PATCH 216/312] Do not dump pipeline in exception message. --- .../Processors/Executors/PipelineExecutor.cpp | 88 ++++++++++--------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 04031319873..c0af990bf84 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -11,9 +11,6 @@ #include #include -#if !defined(__APPLE__) && !defined(__FreeBSD__) -#include -#endif namespace DB { @@ -143,8 +140,7 @@ static void executeJob(IProcessor * processor) catch (Exception & exception) { if (checkCanAddAdditionalInfoToException(exception)) - exception.addMessage("While executing " + processor->getName() + " (" - + toString(reinterpret_cast(processor)) + ") "); + exception.addMessage("While executing " + processor->getName()); throw; } } @@ -265,7 +261,9 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue std::vector updated_direct_edges; { - /// Stopwatch watch; +#ifndef N_DEBUG + Stopwatch watch; +#endif std::unique_lock lock(std::move(node_lock)); @@ -279,7 +277,9 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue return false; } - /// node.execution_state->preparation_time_ns += watch.elapsed(); +#ifndef N_DEBUG + node.execution_state->preparation_time_ns += watch.elapsed(); +#endif node.updated_input_ports.clear(); node.updated_output_ports.clear(); @@ -466,9 +466,9 @@ void PipelineExecutor::execute(size_t num_threads) } catch (Exception & exception) { - if (checkCanAddAdditionalInfoToException(exception)) - exception.addMessage("\nCurrent state:\n" + dumpPipeline()); - +#ifndef N_DEBUG + LOG_TRACE(log, "Exception while executing query. Current state:\n" << dumpPipeline()); +#endif throw; } @@ -486,28 +486,15 @@ void PipelineExecutor::execute(size_t num_threads) void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads) { -#if !defined(__APPLE__) && !defined(__FreeBSD__) - /// Specify CPU core for thread if can. - /// It may reduce the number of context swithches. - /* - if (num_threads > 1) - { - cpu_set_t cpu_set; - CPU_ZERO(&cpu_set); - CPU_SET(thread_num, &cpu_set); +#ifndef N_DEBUG + UInt64 total_time_ns = 0; + UInt64 execution_time_ns = 0; + UInt64 processing_time_ns = 0; + UInt64 wait_time_ns = 0; - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set) == -1) - LOG_TRACE(log, "Cannot set affinity for thread " << num_threads); - } - */ + Stopwatch total_time_watch; #endif -// UInt64 total_time_ns = 0; -// UInt64 execution_time_ns = 0; -// UInt64 processing_time_ns = 0; -// UInt64 wait_time_ns = 0; - -// Stopwatch total_time_watch; ExecutionState * state = nullptr; auto prepare_processor = [&](UInt64 pid, Queue & queue) @@ -585,9 +572,15 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads addJob(state); { - // Stopwatch execution_time_watch; +#ifndef N_DEBUG + Stopwatch execution_time_watch; +#endif + state->job(); - // execution_time_ns += execution_time_watch.elapsed(); + +#ifndef N_DEBUG + execution_time_ns += execution_time_watch.elapsed(); +#endif } if (state->exception) @@ -596,7 +589,9 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads if (finished) break; - // Stopwatch processing_time_watch; +#ifndef N_DEBUG + Stopwatch processing_time_watch; +#endif /// Try to execute neighbour processor. { @@ -648,19 +643,22 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads doExpandPipeline(task, false); } - // processing_time_ns += processing_time_watch.elapsed(); +#ifndef N_DEBUG + processing_time_ns += processing_time_watch.elapsed(); +#endif } } -// total_time_ns = total_time_watch.elapsed(); -// wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; -/* +#ifndef N_DEBUG + total_time_ns = total_time_watch.elapsed(); + wait_time_ns = total_time_ns - execution_time_ns - processing_time_ns; + LOG_TRACE(log, "Thread finished." << " Total time: " << (total_time_ns / 1e9) << " sec." << " Execution time: " << (execution_time_ns / 1e9) << " sec." << " Processing time: " << (processing_time_ns / 1e9) << " sec." << " Wait time: " << (wait_time_ns / 1e9) << "sec."); -*/ +#endif } void PipelineExecutor::executeImpl(size_t num_threads) @@ -762,10 +760,18 @@ String PipelineExecutor::dumpPipeline() const for (auto & node : graph) { if (node.execution_state) - node.processor->setDescription( - "(" + std::to_string(node.execution_state->num_executed_jobs) + " jobs, execution time: " - + std::to_string(node.execution_state->execution_time_ns / 1e9) + " sec., preparation time: " - + std::to_string(node.execution_state->preparation_time_ns / 1e9) + " sec.)"); + { + WriteBufferFromOwnString buffer; + buffer << "(" << node.execution_state->num_executed_jobs << " jobs"; + +#ifndef N_DEBUG + buffer << ", execution time: " << node.execution_state->execution_time_ns / 1e9 << " sec."; + buffer << ", preparation time: " << node.execution_state->preparation_time_ns / 1e9 << " sec."; +#endif + + buffer << ")"; + node.processor->setDescription(buffer.str()); + } } std::vector statuses; From 0734fd7f7d50c2a95ce4be46540b0c8a8c0ade04 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Mon, 27 Jan 2020 12:19:32 +0300 Subject: [PATCH 217/312] Update index.md (#8849) --- docs/ru/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/index.md b/docs/ru/index.md index fbb80bfb8a9..15ad4c5853c 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -125,7 +125,7 @@ SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIM 2. Кодогенерация. Для запроса генерируется код, в котором подставлены все косвенные вызовы. -В "обычных" БД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения latency при выполнении SQL запросов. (Для сравнения - в аналитических СУБД, требуется оптимизация throughput, а не latency). +В "обычных" БД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения latency при выполнении SQL запросов. Для сравнения, в аналитических СУБД требуется оптимизация throughput, а не latency. Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации. From 2c025a8bd1009bb7c1a3bda3db99f55ba3a6cba9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 13:05:57 +0300 Subject: [PATCH 218/312] Add progress. --- dbms/src/Storages/System/StorageSystemNumbers.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 2634838b6aa..8ff8db99629 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -37,6 +37,8 @@ protected: next += step; + progress({column->size(), column->byteSize()}); + return { Columns {std::move(column)}, block_size }; } @@ -93,6 +95,8 @@ protected: while (pos < end) *pos++ = curr++; + progress({column->size(), column->byteSize()}); + return { Columns {std::move(column)}, block_size }; } From 3d0d31f62a94e6a41c8a63d9c68ce122419d129f Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 27 Jan 2020 13:19:14 +0300 Subject: [PATCH 219/312] Fixed `StorageInput::StorageInput` a little bit. --- dbms/src/Storages/StorageInput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageInput.cpp b/dbms/src/Storages/StorageInput.cpp index 4f41ba20dfa..68b9dfe8810 100644 --- a/dbms/src/Storages/StorageInput.cpp +++ b/dbms/src/Storages/StorageInput.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes } StorageInput::StorageInput(const String & table_name_, const ColumnsDescription & columns_) - : IStorage({"", table_name_}, columns_) + : IStorage({"", table_name_}) { setColumns(columns_); } From 0886bfb9bc3a481561e041428fbefc316c957925 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 27 Jan 2020 13:57:31 +0300 Subject: [PATCH 220/312] fix test --- .../0_stateless/01049_join_lc_crach.reference | 6 ------ .../0_stateless/01049_join_lc_crach.sql | 15 ------------- .../01049_join_low_card_crash.reference | 12 +++++++++++ .../0_stateless/01049_join_low_card_crash.sql | 21 +++++++++++++++++++ 4 files changed, 33 insertions(+), 21 deletions(-) delete mode 100644 dbms/tests/queries/0_stateless/01049_join_lc_crach.reference delete mode 100644 dbms/tests/queries/0_stateless/01049_join_lc_crach.sql create mode 100644 dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference create mode 100644 dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql diff --git a/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference b/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference deleted file mode 100644 index e7e860b7ce8..00000000000 --- a/dbms/tests/queries/0_stateless/01049_join_lc_crach.reference +++ /dev/null @@ -1,6 +0,0 @@ -a 1 -b 0 -a 1 2 -b 0 3 -a 1 a 2 - 0 b 3 diff --git a/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql b/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql deleted file mode 100644 index f66b2610d85..00000000000 --- a/dbms/tests/queries/0_stateless/01049_join_lc_crach.sql +++ /dev/null @@ -1,15 +0,0 @@ -DROP TABLE IF EXISTS Alpha; -DROP TABLE IF EXISTS Beta; - -CREATE TABLE Alpha (foo String, bar UInt64) ENGINE = Memory; -CREATE TABLE Beta (foo LowCardinality(String), baz UInt64) ENGINE = Memory; - -INSERT INTO Alpha VALUES ('a', 1); -INSERT INTO Beta VALUES ('a', 2), ('b', 3); - -SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo); -SELECT * FROM Alpha FULL JOIN Beta USING (foo); -SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo; - -DROP TABLE Alpha; -DROP TABLE Beta; diff --git a/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference new file mode 100644 index 00000000000..648a5de0a87 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.reference @@ -0,0 +1,12 @@ +a 1 +b 0 +a 1 2 +b 0 3 + 0 b 3 +a 1 a 2 +a 1 +b \N +a 1 2 +b \N 3 +a 1 a 2 +\N \N b 3 diff --git a/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql new file mode 100644 index 00000000000..99bf09c42e1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01049_join_low_card_crash.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS Alpha; +DROP TABLE IF EXISTS Beta; + +CREATE TABLE Alpha (foo String, bar UInt64) ENGINE = Memory; +CREATE TABLE Beta (foo LowCardinality(String), baz UInt64) ENGINE = Memory; + +INSERT INTO Alpha VALUES ('a', 1); +INSERT INTO Beta VALUES ('a', 2), ('b', 3); + +SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; + +SET join_use_nulls = 1; + +SELECT * FROM Alpha FULL JOIN (SELECT 'b' as foo) USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta USING (foo) ORDER BY foo; +SELECT * FROM Alpha FULL JOIN Beta ON Alpha.foo = Beta.foo ORDER BY foo; + +DROP TABLE Alpha; +DROP TABLE Beta; From a9c09b88c3fe6947040a2b188d662ef1bd157eb7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 13:59:54 +0300 Subject: [PATCH 221/312] Update ISource --- dbms/src/Processors/ISource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Processors/ISource.cpp b/dbms/src/Processors/ISource.cpp index d40f0e32fb7..d68c3345073 100644 --- a/dbms/src/Processors/ISource.cpp +++ b/dbms/src/Processors/ISource.cpp @@ -46,7 +46,7 @@ void ISource::work() try { current_chunk.chunk = generate(); - if (!current_chunk.chunk) + if (!current_chunk.chunk || isCancelled()) finished = true; else has_input = true; From 507692fb1c8b37074d11a20c1dd6a84b5322e47a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 27 Jan 2020 15:35:56 +0300 Subject: [PATCH 222/312] [wip] performance comparison --- docker/test/performance-comparison/compare.sh | 12 ++++++------ docker/test/performance-comparison/entrypoint.sh | 7 +++++++ docker/test/performance-comparison/eqmed.sql | 8 ++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 4b4501892e7..4d69bd47f26 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -27,15 +27,15 @@ function download # might have the same version on left and right if ! [ "$la" = "$ra" ] then - wget -q -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && tar -C left --strip-components=1 -zxvf "$la" & - wget -q -nd -c "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/performance/performance.tgz" -O "$ra" && tar -C right --strip-components=1 -zxvf "$ra" & + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && tar -C left --strip-components=1 -zxvf "$la" & + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$right_pr/$right_sha/performance/performance.tgz" -O "$ra" && tar -C right --strip-components=1 -zxvf "$ra" & else - wget -q -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && { tar -C left --strip-components=1 -zxvf "$la" & tar -C right --strip-components=1 -zxvf "$ra" & } & + wget -nv -nd -c "https://clickhouse-builds.s3.yandex.net/$left_pr/$left_sha/performance/performance.tgz" -O "$la" && { tar -C left --strip-components=1 -zxvf "$la" & tar -C right --strip-components=1 -zxvf "$ra" & } & fi - cd db0 && wget -q -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" && tar -xvf hits_10m_single.tar & - cd db0 && wget -q -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar" && tar -xvf hits_100m_single.tar & - cd db0 && wget -q -nd -c "https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar" && tar -xvf hits_v1.tar & + cd db0 && wget -nv -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" && tar -xvf hits_10m_single.tar & + cd db0 && wget -nv -nd -c "https://s3.mds.yandex.net/clickhouse-private-datasets/hits_100m_single/partitions/hits_100m_single.tar" && tar -xvf hits_100m_single.tar & + cd db0 && wget -nv -nd -c "https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_v1.tar" && tar -xvf hits_v1.tar & wait } diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 3a4d33326af..3f86599b547 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -15,7 +15,14 @@ echo Reference tag is $ref_tag # We use annotated tags which have their own shas, so we have to further # dereference the tag to get the commit it points to, hence the '~0' thing. ref_sha=$(cd ch && git rev-parse $ref_tag~0) + +# Show what we're testing echo Reference SHA is $ref_sha +(cd ch && git log -1 --decorate $ref_sha) ||: +echo +echo SHA to test is $SHA_TO_TEST +(cd ch && git log -1 --decorate $SHA_TO_TEST) ||: +echo # Set python output encoding so that we can print queries with Russian letters. export PYTHONIOENCODING=utf-8 diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql index cdc7cbec85f..d7265533208 100644 --- a/docker/test/performance-comparison/eqmed.sql +++ b/docker/test/performance-comparison/eqmed.sql @@ -1,10 +1,10 @@ -- input is table(query text, run UInt32, version int, time float) select -- abs(diff_percent) > rd_quantiles_percent[3] fail, - floor(original_medians_array.time_by_version[1], 4) left, - floor(original_medians_array.time_by_version[2], 4) right, - floor((right - left) / left, 3) diff_percent, - arrayMap(x -> floor(x / left, 3), rd.rd_quantiles) rd_quantiles_percent, + floor(original_medians_array.time_by_version[1], 4) l, + floor(original_medians_array.time_by_version[2], 4) r, + floor((r - l) / l, 3) diff_percent, + arrayMap(x -> floor(x / l, 3), rd.rd_quantiles) rd_quantiles_percent, query from ( From 205c0f7fbbebe3ff7e2a0fa58f0eef987a51aa7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Jan 2020 15:43:28 +0300 Subject: [PATCH 223/312] Added comment --- dbms/src/Dictionaries/DictionaryFactory.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Dictionaries/DictionaryFactory.h b/dbms/src/Dictionaries/DictionaryFactory.h index b0d0c246ef5..3d3f793e2b2 100644 --- a/dbms/src/Dictionaries/DictionaryFactory.h +++ b/dbms/src/Dictionaries/DictionaryFactory.h @@ -23,6 +23,8 @@ namespace DB class Context; +/** Create dictionary according to its layout. + */ class DictionaryFactory : private boost::noncopyable { public: From ef268d7016244b2057e6f5f1036e598843fa65a7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 16:00:18 +0300 Subject: [PATCH 224/312] Fix TablesBlockSource. --- dbms/src/Storages/System/StorageSystemTables.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 456031bb02c..294f9f8aa77 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -189,6 +189,7 @@ protected: } UInt64 num_rows = res_columns.at(0)->size(); + done = true; return Chunk(std::move(res_columns), num_rows); } From 419f2c2c2b47a59420897d62b6909aecf0c909d2 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Mon, 27 Jan 2020 16:04:01 +0300 Subject: [PATCH 225/312] Fix segmentation fault. --- dbms/src/Storages/StorageFactory.cpp | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index ae1ea8d1819..106eaec67cc 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -143,20 +143,23 @@ StoragePtr StorageFactory::get( } }; - if (storage_def->settings) - checkFeature( - "SETTINGS clause", - [](StorageFeatures features) { return features.supports_settings; }); + if (storage_def) + { + if (storage_def->settings) + checkFeature( + "SETTINGS clause", + [](StorageFeatures features) { return features.supports_settings; }); - if (storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by) - checkFeature( - "PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses", - [](StorageFeatures features) { return features.supports_sort_order; }); + if (storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by) + checkFeature( + "PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses", + [](StorageFeatures features) { return features.supports_sort_order; }); - if (storage_def->ttl_table || !columns.getColumnTTLs().empty()) - checkFeature( - "TTL clause", - [](StorageFeatures features) { return features.supports_ttl; }); + if (storage_def->ttl_table || !columns.getColumnTTLs().empty()) + checkFeature( + "TTL clause", + [](StorageFeatures features) { return features.supports_ttl; }); + } if (query.columns_list && query.columns_list->indices && !query.columns_list->indices->children.empty()) checkFeature( From 675d0f549fde4bf90297f9ba48553565f0348787 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Mon, 27 Jan 2020 17:14:49 +0300 Subject: [PATCH 226/312] Add replication and deduplication properties. --- .../Storages/MergeTree/registerStorageMergeTree.cpp | 3 +++ dbms/src/Storages/StorageFactory.h | 4 ++++ .../Storages/System/StorageSystemTableEngines.cpp | 6 +++++- docs/en/operations/system_tables.md | 13 ++++++++----- docs/ru/operations/system_tables.md | 13 ++++++++----- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 5ea55de7ab5..0c4359b0f65 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -684,6 +684,9 @@ void registerStorageMergeTree(StorageFactory & factory) factory.registerStorage("GraphiteMergeTree", create, features); factory.registerStorage("VersionedCollapsingMergeTree", create, features); + features.supports_replication = true; + features.supports_deduplication = true; + factory.registerStorage("ReplicatedMergeTree", create, features); factory.registerStorage("ReplicatedCollapsingMergeTree", create, features); factory.registerStorage("ReplicatedReplacingMergeTree", create, features); diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h index 024d177ab1f..e64d8647dd8 100644 --- a/dbms/src/Storages/StorageFactory.h +++ b/dbms/src/Storages/StorageFactory.h @@ -52,6 +52,8 @@ public: bool supports_skipping_indices = false; bool supports_sort_order = false; bool supports_ttl = false; + bool supports_replication = false; + bool supports_deduplication = false; }; using CreatorFn = std::function; @@ -79,6 +81,8 @@ public: .supports_skipping_indices = false, .supports_sort_order = false, .supports_ttl = false, + .supports_replication = false, + .supports_deduplication = false, }); const Storages & getAllStorages() const diff --git a/dbms/src/Storages/System/StorageSystemTableEngines.cpp b/dbms/src/Storages/System/StorageSystemTableEngines.cpp index 1998c95be52..e63923f69b6 100644 --- a/dbms/src/Storages/System/StorageSystemTableEngines.cpp +++ b/dbms/src/Storages/System/StorageSystemTableEngines.cpp @@ -12,7 +12,9 @@ NamesAndTypesList StorageSystemTableEngines::getNamesAndTypes() {"supports_settings", std::make_shared()}, {"supports_skipping_indices", std::make_shared()}, {"supports_sort_order", std::make_shared()}, - {"supports_ttl", std::make_shared()}}; + {"supports_ttl", std::make_shared()}, + {"supports_replication", std::make_shared()}, + {"supports_deduplication", std::make_shared()}}; } void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const @@ -24,6 +26,8 @@ void StorageSystemTableEngines::fillData(MutableColumns & res_columns, const Con res_columns[2]->insert(pair.second.features.supports_skipping_indices); res_columns[3]->insert(pair.second.features.supports_sort_order); res_columns[4]->insert(pair.second.features.supports_ttl); + res_columns[5]->insert(pair.second.features.supports_replication); + res_columns[6]->insert(pair.second.features.supports_deduplication); } } diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 484cc910764..5907f432f8b 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -755,20 +755,23 @@ This table contains the following columns (the column type is shown in brackets) - `supports_skipping_indices` (UInt8) — Flag that indicates if table engine supports [skipping indices](table_engines/mergetree/#table_engine-mergetree-data_skipping-indexes). - `supports_ttl` (UInt8) — Flag that indicates if table engine supports [TTL](table_engines/mergetree/#table_engine-mergetree-ttl). - `supports_sort_order` (UInt8) — Flag that indicates if table engine supports clauses `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` and `SAMPLE_BY`. +- `supports_replication` (UInt8) — Flag that indicates if table engine supports [data replication](table_engines/replication/). +- `supports_duduplication` (UInt8) — Flag that indicates if table engine supports data deduplication. Example: ```sql SELECT * FROM system.table_engines -WHERE name in ('Kafka', 'MergeTree') +WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') ``` ```text -┌─name──────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ -└───────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┘ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ ``` **See also** diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 9e7d80b4e6b..086e079358b 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -706,20 +706,23 @@ WHERE changed - `supports_skipping_indices` (UInt8) — флаг, показывающий поддержку [индексов пропуска данных](table_engines/mergetree/#table_engine-mergetree-data_skipping-indexes). - `supports_ttl` (UInt8) — флаг, показывающий поддержку [TTL](table_engines/mergetree/#table_engine-mergetree-ttl). - `supports_sort_order` (UInt8) — флаг, показывающий поддержку секций `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` и `SAMPLE_BY`. +- `supports_replication` (UInt8) — флаг, показвыающий поддержку [репликации](table_engines/replication/). +- `supports_duduplication` (UInt8) — флаг, показывающий наличие в движке дедупликации данных. Пример: ```sql SELECT * FROM system.table_engines -WHERE name in ('Kafka', 'MergeTree') +WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') ``` ```text -┌─name──────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┐ -│ Kafka │ 1 │ 0 │ 0 │ 0 │ -│ MergeTree │ 1 │ 1 │ 1 │ 1 │ -└───────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┘ +┌─name──────────────────────────┬─supports_settings─┬─supports_skipping_indices─┬─supports_sort_order─┬─supports_ttl─┬─supports_replication─┬─supports_deduplication─┐ +│ Kafka │ 1 │ 0 │ 0 │ 0 │ 0 │ 0 │ +│ MergeTree │ 1 │ 1 │ 1 │ 1 │ 0 │ 0 │ +│ ReplicatedCollapsingMergeTree │ 1 │ 1 │ 1 │ 1 │ 1 │ 1 │ +└───────────────────────────────┴───────────────────┴───────────────────────────┴─────────────────────┴──────────────┴──────────────────────┴────────────────────────┘ ``` **Смотрите также** From af7e66f0dac54ec4a8878bb6219c63c19cfaf0c2 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Sun, 26 Jan 2020 01:21:12 +0300 Subject: [PATCH 227/312] Make NULLs comparable to everything. --- dbms/src/Common/FieldVisitors.h | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/dbms/src/Common/FieldVisitors.h b/dbms/src/Common/FieldVisitors.h index e3fa487477a..4f9e34f4969 100644 --- a/dbms/src/Common/FieldVisitors.h +++ b/dbms/src/Common/FieldVisitors.h @@ -184,7 +184,7 @@ template <> constexpr bool isDecimalField>() { return t class FieldVisitorAccurateEquals : public StaticVisitor { public: - bool operator() (const UInt64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const UInt64 &, const Null &) const { return false; } bool operator() (const UInt64 & l, const UInt64 & r) const { return l == r; } bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } @@ -194,7 +194,7 @@ public: bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const Int64 &, const Null &) const { return false; } bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const Int64 & l, const Int64 & r) const { return l == r; } @@ -204,7 +204,7 @@ public: bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const Float64 &, const Null &) const { return false; } bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::equalsOp(l, r); } bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const Float64 & l, const Int64 & r) const { return accurate::equalsOp(l, r); } @@ -227,6 +227,8 @@ public: return l == r; if constexpr (std::is_same_v) return stringToUUID(l) == r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -237,6 +239,8 @@ public: return l == r; if constexpr (std::is_same_v) return l == stringToUUID(r); + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -245,6 +249,8 @@ public: { if constexpr (std::is_same_v) return l == r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -253,6 +259,8 @@ public: { if constexpr (std::is_same_v) return l == r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -263,6 +271,8 @@ public: return l == r; if constexpr (std::is_same_v || std::is_same_v) return l == DecimalField(r, 0); + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -289,11 +299,10 @@ private: } }; - class FieldVisitorAccurateLess : public StaticVisitor { public: - bool operator() (const UInt64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const UInt64 &, const Null &) const { return false; } bool operator() (const UInt64 & l, const UInt64 & r) const { return l < r; } bool operator() (const UInt64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const UInt64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } @@ -303,7 +312,7 @@ public: bool operator() (const UInt64 & l, const Tuple & r) const { return cantCompare(l, r); } bool operator() (const UInt64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - bool operator() (const Int64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const Int64 &, const Null &) const { return false; } bool operator() (const Int64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } bool operator() (const Int64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const Int64 & l, const Int64 & r) const { return l < r; } @@ -313,7 +322,7 @@ public: bool operator() (const Int64 & l, const Tuple & r) const { return cantCompare(l, r); } bool operator() (const Int64 & l, const AggregateFunctionStateData & r) const { return cantCompare(l, r); } - bool operator() (const Float64 & l, const Null & r) const { return cantCompare(l, r); } + bool operator() (const Float64 &, const Null &) const { return false; } bool operator() (const Float64 & l, const UInt64 & r) const { return accurate::lessOp(l, r); } bool operator() (const Float64 & l, const UInt128 & r) const { return cantCompare(l, r); } bool operator() (const Float64 & l, const Int64 & r) const { return accurate::lessOp(l, r); } @@ -336,6 +345,8 @@ public: return l < r; if constexpr (std::is_same_v) return stringToUUID(l) < r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -346,6 +357,8 @@ public: return l < r; if constexpr (std::is_same_v) return l < stringToUUID(r); + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -354,6 +367,8 @@ public: { if constexpr (std::is_same_v) return l < r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -362,6 +377,8 @@ public: { if constexpr (std::is_same_v) return l < r; + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } @@ -372,6 +389,8 @@ public: return l < r; else if constexpr (std::is_same_v || std::is_same_v) return l < DecimalField(r, 0); + if constexpr (std::is_same_v) + return false; return cantCompare(l, r); } From 73ee23c1d8288ebd64964cfb5d4b9307ac3a31b8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 17:31:40 +0300 Subject: [PATCH 228/312] Add cancell to SourceFromInputStream. --- dbms/src/Processors/IProcessor.h | 9 ++++++++- dbms/src/Processors/Sources/SourceFromInputStream.h | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dbms/src/Processors/IProcessor.h b/dbms/src/Processors/IProcessor.h index 5296f36de87..811da7e0925 100644 --- a/dbms/src/Processors/IProcessor.h +++ b/dbms/src/Processors/IProcessor.h @@ -222,7 +222,11 @@ public: /// In case if query was cancelled executor will wait till all processors finish their jobs. /// Generally, there is no reason to check this flag. However, it may be reasonable for long operations (e.g. i/o). bool isCancelled() const { return is_cancelled; } - void cancel() { is_cancelled = true; } + void cancel() + { + onCancell(); + is_cancelled = true; + } virtual ~IProcessor() = default; @@ -275,6 +279,9 @@ public: void enableQuota() { has_quota = true; } bool hasQuota() const { return has_quota; } +protected: + virtual void onCancell() {} + private: std::atomic is_cancelled{false}; diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.h b/dbms/src/Processors/Sources/SourceFromInputStream.h index 8e750a33faf..b5704fc521f 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.h +++ b/dbms/src/Processors/Sources/SourceFromInputStream.h @@ -30,6 +30,9 @@ public: void setProgressCallback(const ProgressCallback & callback) final { stream->setProgressCallback(callback); } void addTotalRowsApprox(size_t value) final { stream->addTotalRowsApprox(value); } +protected: + void onCancel() override { stream->cancel(false); } + private: bool has_aggregate_functions = false; bool force_add_aggregating_info; From 902b977d9de694c7b0416bb10c67ef7fb09568db Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 17:39:14 +0300 Subject: [PATCH 229/312] Add cancell to SourceFromInputStream. --- dbms/src/Processors/IProcessor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Processors/IProcessor.h b/dbms/src/Processors/IProcessor.h index 811da7e0925..87081b89199 100644 --- a/dbms/src/Processors/IProcessor.h +++ b/dbms/src/Processors/IProcessor.h @@ -224,7 +224,7 @@ public: bool isCancelled() const { return is_cancelled; } void cancel() { - onCancell(); + onCancel(); is_cancelled = true; } @@ -280,7 +280,7 @@ public: bool hasQuota() const { return has_quota; } protected: - virtual void onCancell() {} + virtual void onCancel() {} private: std::atomic is_cancelled{false}; From 605031a36920c677beeabd4e1f566ab7a0431adb Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 18:57:25 +0300 Subject: [PATCH 230/312] Better --- CHANGELOG.md | 480 ++++++++++++++++++++++++--------------------------- 1 file changed, 221 insertions(+), 259 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4cd66878e4..4eaa2127324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,336 +2,298 @@ ### Backward Incompatible Change * Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Changed return type of the function `greatCircleDistance` to Float32 because now the result of calculation is Float32. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Now it's expected that query parameters are represented in "escaped" format. For example, to pass string `ab` you have to write `a\tb` or `a\b` and respectively, `a%5Ctb` or `a%5C%09b` in URL. This is needed to add the possibility to pass NULL as `\N`. This fixes [#7488](https://github.com/ClickHouse/ClickHouse/issues/7488). [#8517](https://github.com/ClickHouse/ClickHouse/pull/8517) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add new ANY JOIN logic for StorageJoin consistent with JOIN operation. Backward incompatible change: to upgrade you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or remake these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) +* Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) ### New Feature -* Added information about paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Implement ON CLUSTER feature for SYSTEM RELOAD DICTIONARY [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) -* Implement multiple inheritances for user's profiles [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) +* Added information about part paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) +* Now user's profile in `users.xml` can inherit multiple profiles. [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) * Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576) [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* DateTime64 datatype with configurable sub-second precision. ... [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) -* Add ability to execute `CREATE DICTIONARY` queries with `ON CLUSTER`. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) -* Clickhouse 7262 [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) -* Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -* Speed up parsing of data files in CSV, TSV and JSONEachRow format by doing it in parallel. This is enabled by default and controlled by the `max_threads_for_parallel_parsing` setting. ... [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Add function `bankerRound` which performs banker's rounding [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) +* Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) +* Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) +* Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) +* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. This is enabled by default and controlled by the `max_threads_for_parallel_parsing` setting. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Add function `bankerRound` which performs banker's rounding. [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) * Support more languages in embedded dictionary for region names: 'ru', 'en', 'ua', 'uk', 'by', 'kz', 'tr', 'de', 'uz', 'lv', 'lt', 'et', 'pt', 'he', 'vi'. [#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* New consistent ANY JOIN logic: ```t1 ANY LEFT JOIN t2``` equals ```t2 ANY RIGHT JOIN t1```. Adds new SEMI and ANTY JOIN. Old ```ANY INNER JOIN``` logic is allowed as ```SEMI LEFT JOIN```. There's a ragne of versons (19.14 - 19.17) that has no default `ANY INNER JOIN` and `ANY RIGHT JOIN`. Older has old one, newer has new one. ... [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -* Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by `StorageDistributedDirectoryMonitor`. Those files are created while asynchronous insert into `Distributed` table. Examples: `select * from file('/path/to/file.bin', 'Distributed')` or `create table t (dummy UInt32) engine = File('Distributed', '/path/to/file.bin')`. ... [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Reset column argument for runningAccumulate [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -* ALTER MODIFY QUERY [#7533](https://github.com/ClickHouse/ClickHouse/pull/7533) ([nvartolomei](https://github.com/nvartolomei)) -* Prometheus endpoint[#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) -* User can add in `` from `config.xml` hosts, which other users can go to. ... [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) -* Added function `greatCircleAngle`; changed Earth radius to be consistent with H3 library. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Created JSONCompactEachRow and JSONCompactEachRowWithNamesAndTypes formats for input and output. [#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) -* Added feature to read and write gzip files based on engine parameters and path (URI) extension. Supported options are ... [#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) +* Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add new `SEMI` and `ANTY JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) +* Add ability to alter materialized views with `ALTER MODIFY QUERY `. [#7533](https://github.com/ClickHouse/ClickHouse/pull/7533) ([nvartolomei](https://github.com/nvartolomei)) +* Add ability to use ClickHouse as Prometheus endpoint. [#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) +* Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) +* Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Changed Earth radius to be consistent with H3 library. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `JSONCompactEachRow` and `JSONCompactEachRowWithNamesAndTypes` formats for input and output. [#7841](https://github.com/ClickHouse/ClickHouse/pull/7841) ([Mikhail Korotov](https://github.com/millb)) +* Added feature for file-related table engines and table functions (`File`, `S3`, `URL`, `HDFS`) which allows to read and write `gzip` files based on additional engine parameter or file extension. [#7840](https://github.com/ClickHouse/ClickHouse/pull/7840) ([Andrey Bodrov](https://github.com/apbodrov)) * Added the `randomASCII(length)` function, generating a string with a random set of [ASCII](https://en.wikipedia.org/wiki/ASCII#Printable_characters) printable characters. [#8401](https://github.com/ClickHouse/ClickHouse/pull/8401) ([BayoNet](https://github.com/BayoNet)) -* Added function JSONExtractArrayRaw [#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) -* Add arrayZip function for combine multiple array type columns [#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) -* Move parts between storage volumes according to configured TTL expressions. [#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) -* new aggregate function avgWeighted [#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) -* Returning back functionality which was deleted during refactoring [#6553](https://github.com/ClickHouse/ClickHouse/issues/6553) in [#7780](https://github.com/ClickHouse/ClickHouse/issues/7780) ... [#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Support of several `h3` function in addition to `geoToH3`: ... [#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) -* Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [#8156](https://github.com/ClickHouse/ClickHouse/issues/8156) ... [#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Filesystem abstraction layer - supporting changes for running ClickHouse over S3 / HDFS. [#7946](https://github.com/ClickHouse/ClickHouse/pull/7946) ([Alexander Burmak](https://github.com/Alex-Burmak)) -* Add `groupBit*` functions for the `SimpleAggregationFunction` type [#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) +* Added function `JSONExtractArrayRaw` which returns an array on unparsed json array elements from `JSON` string. [#8081](https://github.com/ClickHouse/ClickHouse/pull/8081) ([Oleg Matrokhin](https://github.com/errx)) +* Add `arrayZip` function which allows to combine multiple arrays of equal lengths into one array of tuples. [#8149](https://github.com/ClickHouse/ClickHouse/pull/8149) ([Winter Zhang](https://github.com/zhang2014)) +* Add ability to move data between disks according to configured `TTL`-expressions for `*MergeTree` table engines family. [#8140](https://github.com/ClickHouse/ClickHouse/pull/8140) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Added new aggregate function `avgWeighted` which allows to calculate weighted average. [#7898](https://github.com/ClickHouse/ClickHouse/pull/7898) ([Andrey Bodrov](https://github.com/apbodrov)) +* Now parallel parsing is enabled by default for `TSV`, `TSKV`, `CSV` and `JSONEachRow` formats. [#7894](https://github.com/ClickHouse/ClickHouse/pull/7894) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Add several geo functions from `H3` library: `h3GetResolution`, `h3EdgeAngle`, `h3EdgeLength`, `h3IsValid` and `h3kRing`. [#8034](https://github.com/ClickHouse/ClickHouse/pull/8034) ([Konstantin Malanchev](https://github.com/hombit)) +* Added support for brotli (`br`) compression in file-related storages and table functions. This fixes [#8156](https://github.com/ClickHouse/ClickHouse/issues/8156). [#8526](https://github.com/ClickHouse/ClickHouse/pull/8526) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add `groupBit*` functions for the `SimpleAggregationFunction` type. [#8485](https://github.com/ClickHouse/ClickHouse/pull/8485) ([Guillaume Tassery](https://github.com/YiuRULE)) ### Bug Fix -* Fixes [#7868](https://github.com/ClickHouse/ClickHouse/issues/7868) [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) -* Now dictionaries support string as expression and fix for default string values. [#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) -* Insert select from mysql(...) table function now should work ... [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) -* Fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653) [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) -* Now an exception will be thrown in case of using WITH TIES alongside LIMIT BY. And also you can use TOP with LIMIT BY. ... [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* This pr reverts https://github.com/ClickHouse/ClickHouse/pull/8028 and tries to interpose all malloc symbols correctly. ... [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) -* Fix bug in check function of *MergeTree engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) -* Fix enum conversion in native format for backward compatibility. [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) -* Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832) [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) +* Fix rename of tables with `Distributed` engine. Fixes issue [#7868](https://github.com/ClickHouse/ClickHouse/issues/7868). [#8306](https://github.com/ClickHouse/ClickHouse/pull/8306) ([tavplubix](https://github.com/tavplubix)) +* Now dictionaries support `EXPRESSION` for attributes in arbitrary string in non-ClickHouse SQL dialect. [#8098](https://github.com/ClickHouse/ClickHouse/pull/8098) ([alesapin](https://github.com/alesapin)) +* Fix broken `INSERT SELECT FROM mysql(...)` query. This fixes [#8070](https://github.com/ClickHouse/ClickHouse/issues/8070) and [#7960](https://github.com/ClickHouse/ClickHouse/issues/7960). [#8234](https://github.com/ClickHouse/ClickHouse/pull/8234) ([tavplubix](https://github.com/tavplubix)) +* Fix error "Mismatch column sizes" when inserting default `Tuple` from `JSONEachRow`. This fixes [#5653](https://github.com/ClickHouse/ClickHouse/issues/5653). [#8606](https://github.com/ClickHouse/ClickHouse/pull/8606) ([tavplubix](https://github.com/tavplubix)) +* Now an exception will be thrown in case of using `WITH TIES` alongside `LIMIT BY`. Also add ability to use `TOP` with `LIMIT BY`. This fixes [#7472](https://github.com/ClickHouse/ClickHouse/issues/7472). [#7637](https://github.com/ClickHouse/ClickHouse/pull/7637) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fix unintendent dependency from fresh glibc version in `clickhouse-odbc-bridge` binary. [#8046](https://github.com/ClickHouse/ClickHouse/pull/8046) ([Amos Bird](https://github.com/amosbird)) +* Fix bug in check function of `*MergeTree` engines family. Now it doesn't fail in case when we have equal amount of rows in last granule and last mark (non-final). [#8047](https://github.com/ClickHouse/ClickHouse/pull/8047) ([alesapin](https://github.com/alesapin)) +* Fix insert into `Enum*` columns after `ALTER` query, when underlying numeric type is equal to table specified type. This fixes [#7836](https://github.com/ClickHouse/ClickHouse/issues/7836). [#7908](https://github.com/ClickHouse/ClickHouse/pull/7908) ([Anton Popov](https://github.com/CurtizJ)) +* Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) * Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -* Fix checks if a client host is allowed in case it's the localhost. This PR fixes the solution provided in https://github.com/ClickHouse/ClickHouse/pull/8241. [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) -* if the args type is string , the template should be true [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) -* Fix Buffer memory overcommit ... [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) -* Fixed potential bug in functions that can take NULL as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in https://github.com/ClickHouse/ClickHouse/pull/8241. [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) +* Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) +* Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed metrics in `BackgroundProcessingPool` (they were interfering with metrics of processing pool for moves and could throw exceptions `invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size` during merges and mutations if there is a storage policy set for a table). [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix function IN inside WHERE statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) -* An exception is thrown if the integral value is not parsed completely. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) -* case: ```select count(1) from distributed_table``` ... [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) -* I fixed two bugs in the bloom_filter index. ... [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) +* Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) +* Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) +* Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) +* Now bloom filter can handle zero length arrays and doesn't perform redundant calculations. [#8242](https://github.com/ClickHouse/ClickHouse/pull/8242) ([achimbab](https://github.com/achimbab)) * Fixed checking if a client host is allowed by matching the client host to `host_regexp` specified in `users.xml`. [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241) ([Vitaly Baranov](https://github.com/vitlibar)) -* Relax ambiguous column check that leads to false positives in multiple JOIN ON section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) -* Fixed possible server crash (`std::terminate`) when the server cannot send or write data in JSON or XML format with values of String data type (that require UTF-8 validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix race in `StorageDistributedDirectoryMonitor`. [#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Do not allow to merge data moving it against storage policy volume order [#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fixed Native format for Kafka, resolves [#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [#8003](https://github.com/ClickHouse/ClickHouse/issues/8003) ... [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) -* Fixed a bug with making set from subquery in right part of IN section. ... [#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Relax ambiguous column check that leads to false positives in multiple `JOIN ON` section. [#8385](https://github.com/ClickHouse/ClickHouse/pull/8385) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed possible server crash (`std::terminate`) when the server cannot send or write data in `JSON` or `XML` format with values of `String` data type (that require `UTF-8` validation) or when compressing result data with Brotli algorithm or in some other rare cases. This fixes [#7603](https://github.com/ClickHouse/ClickHouse/issues/7603) [#8384](https://github.com/ClickHouse/ClickHouse/pull/8384) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix race condition in `StorageDistributedDirectoryMonitor` found by CI. This fixes [#8364](https://github.com/ClickHouse/ClickHouse/issues/8364). [#8383](https://github.com/ClickHouse/ClickHouse/pull/8383) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now background merges in `*MergeTree` table engines family preserve storage policy volume order more accurately. [#8549](https://github.com/ClickHouse/ClickHouse/pull/8549) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now table engine `Kafka` works properly with `Native` format. This fixes [#6731](https://github.com/ClickHouse/ClickHouse/issues/6731) [#7337](https://github.com/ClickHouse/ClickHouse/issues/7337) [#8003](https://github.com/ClickHouse/ClickHouse/issues/8003). [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed formats with headers (like `CSVWithNames`) which were throwing exception about EOF for table engine `Kafka`. [#8016](https://github.com/ClickHouse/ClickHouse/pull/8016) ([filimonov](https://github.com/filimonov)) +* Fixed a bug with making set from subquery in right part of `IN` section. This fixes [#5767](https://github.com/ClickHouse/ClickHouse/issues/5767) and [#2542](https://github.com/ClickHouse/ClickHouse/issues/2542). [#7755](https://github.com/ClickHouse/ClickHouse/pull/7755) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Fix possible crash while reading from storage `File`. [#7756](https://github.com/ClickHouse/ClickHouse/pull/7756) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Parquet list reading fix [#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) -* Fix error `Not found column` for distributed queries with prewhere condition dependent on sampling key if `max_parallel_replicas > 1`. [#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix error `Not found column` if query used `prewhere` dependent on table's alias and the result set was empty because of PK condition. [#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed return type for functions `rand` and `randConstant` in case of nullable argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* disabled predicate optimize for `WITH FILL` expression [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) -* Fixed incorrect count() result for SummingMergeTree while FINAL section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Fixed reading of the files in `Parquet` format containing columns of type `list`. [#8334](https://github.com/ClickHouse/ClickHouse/pull/8334) ([maxulan](https://github.com/maxulan)) +* Fix error `Not found column` for distributed queries with `PREWHERE` condition dependent on sampling key if `max_parallel_replicas > 1`. [#7913](https://github.com/ClickHouse/ClickHouse/pull/7913) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix error `Not found column` if query used `PREWHERE` dependent on table's alias and the result set was empty because of primary key condition. [#7911](https://github.com/ClickHouse/ClickHouse/pull/7911) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Disabled predicate push-down for `WITH FILL` expression. This fixes [#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) +* Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666) [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix some predicate optimizer wrong results [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) -* Avoid SIGSEGV on CREATE TABLE .. AS dictionary [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) -* 1. rename "error" to "err" in .g4 file to avoid symbol conflict ... [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) -* Fix bug that leads to crashes in JOINs with Engine Join tables. This fixes [#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [#8100](https://github.com/ClickHouse/ClickHouse/issues/8100) [#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) -* Do not reload *all* dictionaries on CREATE DATABASE (This ignores any lifetime, while dictionaries can be quite big). [#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) +* Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) +* Fix crash in `CREATE TABLE .. AS dictionary` query. [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) +* Several improvements ClickHouse grammar in `.g4` file. [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) +* Fix bug that leads to crashes in `JOIN`s with tables with engine `Join`. This fixes [#7556](https://github.com/ClickHouse/ClickHouse/issues/7556) [#8254](https://github.com/ClickHouse/ClickHouse/issues/8254) [#7915](https://github.com/ClickHouse/ClickHouse/issues/7915) [#8100](https://github.com/ClickHouse/ClickHouse/issues/8100). [#8298](https://github.com/ClickHouse/ClickHouse/pull/8298) ([Artem Zuikov](https://github.com/4ertus2)) +* Fix redundant dictionaries reload on `CREATE DATABASE`. [#7916](https://github.com/ClickHouse/ClickHouse/pull/7916) ([Azat Khuzhin](https://github.com/azat)) * Limit maximum number of streams for read from `StorageFile` and `StorageHDFS`. Fixes https://github.com/ClickHouse/ClickHouse/issues/7650. [#7981](https://github.com/ClickHouse/ClickHouse/pull/7981) ([alesapin](https://github.com/alesapin)) * Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) -* Continuation of [#8522](https://github.com/ClickHouse/ClickHouse/issues/8522). [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix tests with processors. [#8376](https://github.com/ClickHouse/ClickHouse/pull/8376) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fixed type check in toDateTime64 [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -* Do not crash on LEFT or FULL JOIN with Join engine and unsupported join_use_nulls settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) -* Fix `DROP DICTIONARY IF EXISTS db.dict`, now it doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) +* Now server do not crash on `LEFT` or `FULL JOIN` wittable and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) +* Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) -* Ensure network be up before starting clickhouse-server. Fix [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) -* Timeouts were not correctly handled if secure connection is used. Queries may hang indefinitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix clickhouse-copier cleaning-tainting contention between concurrent workers [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* Fixed the bug that mutations be skipped for some attached parts due to their data_version are larger than the table mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) -* Remove mutation number from part_name by default. [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) -* Ignore redundant copies of parts after move and restart. Progress on [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). ... [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix crash in FULL JOIN with LowCardinality in JOIN key [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) -* Omitting frame pointer somehow breaks clang release build. ... [#8151](https://github.com/ClickHouse/ClickHouse/pull/8151) ([Amos Bird](https://github.com/amosbird)) -* Fix unintendent dependency from `GLIBC@2.16` in `clickhouse-odbc-bridge`. [#8028](https://github.com/ClickHouse/ClickHouse/pull/8028) ([alesapin](https://github.com/alesapin)) -* Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. ... [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) -* Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239) [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix unknown column check added in 19.17 for not ordinary columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) -* Fixed segfault when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* We have free functions implemented for factories such as https://github.com/yandex/ClickHouse/blob/master/dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp#L318 [#8281](https://github.com/ClickHouse/ClickHouse/pull/8281) ([Amos Bird](https://github.com/amosbird)) -* Fix the case where PODArray.insert works with iterator after invalidation. [#7791](https://github.com/ClickHouse/ClickHouse/pull/7791) ([DimasKovas](https://github.com/DimasKovas)) -* Fix the "Sizes of columns doesn't match" error that might appear when using aggregate function columns. ... [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) -* Fix bug where user with empty allow_databases got access to all databases (and same for allow_dictionaries). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) -* Fix a typo that might have caused excessive memory copying. [#7797](https://github.com/ClickHouse/ClickHouse/pull/7797) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Avoid null dereference after "Unknown packet X from server" [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) -* Fix `ORDER BY` in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) -* Check if qualified column present in the table. [#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) -* FIxed behavior with ALTER MOVE ran immediately after merge finish moves superpart of specified. Fixes [#8103](https://github.com/ClickHouse/ClickHouse/issues/8103) [#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix possible server crash while using `UNION` with different number of columns. Fixes [#7279](https://github.com/ClickHouse/ClickHouse/issues/7279) [#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) +* Fix timeouts handling for secure connections, so queries doesn't hang indefenitely. This fixes [#8126](https://github.com/ClickHouse/ClickHouse/issues/8126). [#8128](https://github.com/ClickHouse/ClickHouse/pull/8128) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `clickhouse-copier`'s redundant contention between concurrent workers. [#7816](https://github.com/ClickHouse/ClickHouse/pull/7816) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* Now mutations doesn't skip attached parts, even if their mutation version were larger than current mutation version. [#7812](https://github.com/ClickHouse/ClickHouse/pull/7812) ([Zhichang Yu](https://github.com/yuzhichang)) [#8250](https://github.com/ClickHouse/ClickHouse/pull/8250) ([alesapin](https://github.com/alesapin)) +* Ignore redundant copies of `*MergeTree` data parts after move to another disk and server restart. [#7810](https://github.com/ClickHouse/ClickHouse/pull/7810) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash in `FULL JOIN` with `LowCardinality` in `JOIN` key. [#8252](https://github.com/ClickHouse/ClickHouse/pull/8252) ([Artem Zuikov](https://github.com/4ertus2)) +* Forbidden to use column name more than once in insert query like `INSERT INTO tbl (x, y, x)`. This fixes [#5465](https://github.com/ClickHouse/ClickHouse/issues/5465), [#7681](https://github.com/ClickHouse/ClickHouse/issues/7681). [#7685](https://github.com/ClickHouse/ClickHouse/pull/7685) ([alesapin](https://github.com/alesapin)) +* Added fallback for detection the number of physical CPU cores for unknown CPUs (using the number of logical CPU cores). This fixes [#5239](https://github.com/ClickHouse/ClickHouse/issues/5239). [#7726](https://github.com/ClickHouse/ClickHouse/pull/7726) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `There's no column` error for materialized and alias columns. [#8210](https://github.com/ClickHouse/ClickHouse/pull/8210) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed sever crash when `EXISTS` query was used without `TABLE` or `DICTIONARY` qualifier. Just like `EXISTS t`. This fixes [#8172](https://github.com/ClickHouse/ClickHouse/issues/8172). This bug was introduced in version 19.17. [#8213](https://github.com/ClickHouse/ClickHouse/pull/8213) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix rare bug with error `"Sizes of columns doesn't match"` that might appear when using `SimpleAggregateFunction` column. [#7790](https://github.com/ClickHouse/ClickHouse/pull/7790) ([Boris Granveaud](https://github.com/bgranvea)) +* Fix bug where user with empty `allow_databases` got access to all databases (and same for `allow_dictionaries`). [#7793](https://github.com/ClickHouse/ClickHouse/pull/7793) ([DeifyTheGod](https://github.com/DeifyTheGod)) +* Fix client crash when server already disconnected from client. [#8071](https://github.com/ClickHouse/ClickHouse/pull/8071) ([Azat Khuzhin](https://github.com/azat)) +* Fix `ORDER BY` behaviour in case of sorting by primary key prefix and non primary key suffix. [#7759](https://github.com/ClickHouse/ClickHouse/pull/7759) ([Anton Popov](https://github.com/CurtizJ)) +* Check if qualified column present in the table. This fixes [#6836](https://github.com/ClickHouse/ClickHouse/issues/6836). [#7758](https://github.com/ClickHouse/ClickHouse/pull/7758) ([Artem Zuikov](https://github.com/4ertus2)) +* Fixed behavior with `ALTER MOVE` ran immediately after merge finish moves superpart of specified. Fixes [#8103](https://github.com/ClickHouse/ClickHouse/issues/8103). [#8104](https://github.com/ClickHouse/ClickHouse/pull/8104) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix possible server crash while using `UNION` with different number of columns. Fixes [#7279](https://github.com/ClickHouse/ClickHouse/issues/7279). [#7929](https://github.com/ClickHouse/ClickHouse/pull/7929) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix size of result substring for function `substr` with negative size. [#8589](https://github.com/ClickHouse/ClickHouse/pull/8589) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Do not execute part mutation in `MergeTree` if there are not enough free threads in background pool. ... [#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) -* Fix a minor typo on formatting UNION ALL AST [#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) -* Fixed incorrect bloom filter results under negative. This fixes [#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) +* Now server does not execute part mutation in `MergeTree` if there are not enough free threads in background pool. [#8588](https://github.com/ClickHouse/ClickHouse/pull/8588) ([tavplubix](https://github.com/tavplubix)) +* Fix a minor typo on formatting `UNION ALL` AST. [#7999](https://github.com/ClickHouse/ClickHouse/pull/7999) ([litao91](https://github.com/litao91)) +* Fixed incorrect bloom filter results for negative numbers. This fixes [#8317](https://github.com/ClickHouse/ClickHouse/issues/8317). [#8566](https://github.com/ClickHouse/ClickHouse/pull/8566) ([Winter Zhang](https://github.com/zhang2014)) * Fixed potential buffer overflow in decompress. Malicious user can pass fabricated compressed data that will cause read after buffer. This issue was found by Eldar Zaitov from Yandex information security team. [#8404](https://github.com/ClickHouse/ClickHouse/pull/8404) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix incorrect result because of integers overflow in `arrayIntersect`. [#7777](https://github.com/ClickHouse/ClickHouse/pull/7777) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* check for active replicas when waiting for commands [#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) -* Fixed ALTER TTL for replicated tables. [#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Read temporary tables on failure to avoid interpreting that bits as query [#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) -* fix bitmapAnd error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) -* Refine the definition of ZXid according to the ZooKeeper Programmer's Guide. ... [#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) -* odbc table function now respects external_table_functions_use_nulls [#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) -* macOS Clang builds [#8148](https://github.com/ClickHouse/ClickHouse/pull/8148) ([Matt Keranen](https://github.com/kmatt)) -* Fixed bug that lead to a data race in DB::BlockStreamProfileInfo::calculateRowsBeforeLimit() [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) -* SYSTEM RELOAD DICTIONARY reloads a dictionary completely [#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) +* Now `OPTIMIZE TABLE` query will not wait for offline replicas to perform the operation. [#8314](https://github.com/ClickHouse/ClickHouse/pull/8314) ([javi santana](https://github.com/javisantana)) +* Fixed `ALTER TTL` parser for `Replicated*MergeTree` tables. [#8318](https://github.com/ClickHouse/ClickHouse/pull/8318) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix communication between server and client, so server read temporary tables info after query failure. [#8084](https://github.com/ClickHouse/ClickHouse/pull/8084) ([Azat Khuzhin](https://github.com/azat)) +* Fix `bitmapAnd` function error when intersecting an aggregated bitmap and a scalar bitmap. [#8082](https://github.com/ClickHouse/ClickHouse/pull/8082) ([Yue Huang](https://github.com/moon03432)) +* Refine the definition of `ZXid` according to the ZooKeeper Programmer's Guide which fixes bug in `clickhouse-cluster-copier`. [#8088](https://github.com/ClickHouse/ClickHouse/pull/8088) ([Ding Xiang Fei](https://github.com/dingxiangfei2009)) +* `odbc` table function now respects `external_table_functions_use_nulls` setting. [#7506](https://github.com/ClickHouse/ClickHouse/pull/7506) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug that lead to a rare data race. [#8143](https://github.com/ClickHouse/ClickHouse/pull/8143) ([Alexander Kazakov](https://github.com/Akazz)) +* Now `SYSTEM RELOAD DICTIONARY` reloads a dictionary completely, ignoring `update_field`. This fixes [#7440](https://github.com/ClickHouse/ClickHouse/issues/7440). [#8037](https://github.com/ClickHouse/ClickHouse/pull/8037) ([Vitaly Baranov](https://github.com/vitlibar)) * Add ability to check if dictionary exists in create query. [#8032](https://github.com/ClickHouse/ClickHouse/pull/8032) ([alesapin](https://github.com/alesapin)) -* Fixes [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817) [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) -* Fixed errors with space reservation introduced in [#7558](https://github.com/ClickHouse/ClickHouse/issues/7558) and [#7602](https://github.com/ClickHouse/ClickHouse/issues/7602) [#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Fix merge/OPTIMIIZE of SimpleAggregateFunction(LowCardinality) ... [#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) -* Restore support of all ICU locales, add the ability to apply collations for constant expressions and add language name to system.collations table. [#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) -* Restore support of all ICU locales, add the ability to apply collations for constant expressions and add language name to `system.collations` table. [#8050](https://github.com/ClickHouse/ClickHouse/pull/8050) ([alesapin](https://github.com/alesapin)) +* Fix `Float*` parsing in `Values` format. This fixes [#7817](https://github.com/ClickHouse/ClickHouse/issues/7817). [#7870](https://github.com/ClickHouse/ClickHouse/pull/7870) ([tavplubix](https://github.com/tavplubix)) +* Fix crash when we cannot reserve space in some background operations of `*MergeTree` table engines family. [#7873](https://github.com/ClickHouse/ClickHouse/pull/7873) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix crash of merge operation when table contains `SimpleAggregateFunction(LowCardinality)` column. This fixes [#8515](https://github.com/ClickHouse/ClickHouse/issues/8515). [#8522](https://github.com/ClickHouse/ClickHouse/pull/8522) ([Azat Khuzhin](https://github.com/azat)) +* Restore support of all ICU locales and add the ability to apply collations for constant expressions. Also add language name to `system.collations` table. [#8051](https://github.com/ClickHouse/ClickHouse/pull/8051) ([alesapin](https://github.com/alesapin)) * Fix bug when external dictionaries with zero minimal lifetime (`LIFETIME(MIN 0 MAX N)`, `LIFETIME(N)`) don't update in background. [#7983](https://github.com/ClickHouse/ClickHouse/pull/7983) ([alesapin](https://github.com/alesapin)) -* Fix dict with clickhouse source and subquery [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Fix incorrect parsing of file extension in URL. This fixes [#8157](https://github.com/ClickHouse/ClickHouse/issues/8157) [#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) +* Fix crash when external dictionary with ClickHouse source has subquery in query. [#8351](https://github.com/ClickHouse/ClickHouse/pull/8351) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix incorrect parsing of file extension in table with engine `URL`. This fixes [#8157](https://github.com/ClickHouse/ClickHouse/issues/8157). [#8419](https://github.com/ClickHouse/ClickHouse/pull/8419) ([Andrey Bodrov](https://github.com/apbodrov)) * Fix `CHECK TABLE` query for `*MergeTree` tables without key. Fixes [#7543](https://github.com/ClickHouse/ClickHouse/issues/7543). [#7979](https://github.com/ClickHouse/ClickHouse/pull/7979) ([alesapin](https://github.com/alesapin)) -* Fixed conversion of Float64 to MySQL type, and added test for all supported types. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) -* If table was not completely dropped because of server crash, try to restore and load it [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) -* Fixed segfault in table function file while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) -* Fix rare deadlock which can happen when trace_log is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) -* Add ability to work with different types besides Date in RangeHashed external dictionary created from DDL query. Fixes https://github.com/ClickHouse/ClickHouse/issues/7899. [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) -* Fixes crash when now64() is called with result of another function. ... [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) -* Fixed bug with detecting client ip when connection to mysql wire protocol ... [#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) -* Fix [#7708](https://github.com/ClickHouse/ClickHouse/issues/7708) empty array handling in `arraySplit` [#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) -* Fixed the issue when pid file of another running clickhouse-server may be deleted. [#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) +* Fixed conversion of `Float64` to MySQL type. [#8079](https://github.com/ClickHouse/ClickHouse/pull/8079) ([Yuriy Baranov](https://github.com/yurriy)) +* Now if table was not completely dropped because of server crash, server will try to restore and load it. [#8176](https://github.com/ClickHouse/ClickHouse/pull/8176) ([tavplubix](https://github.com/tavplubix)) +* Fixed crash in table function `file` while inserting into file that doesn't exist. Now in this case file would be created and then insert would be processed. [#8177](https://github.com/ClickHouse/ClickHouse/pull/8177) ([Olga Khvostikova](https://github.com/stavrolia)) +* Fix rare deadlock which can happen when `trace_log` is in enabled. [#7838](https://github.com/ClickHouse/ClickHouse/pull/7838) ([filimonov](https://github.com/filimonov)) +* Add ability to work with different types besides `Date` in `RangeHashed` external dictionary created from DDL query. Fixes [7899](https://github.com/ClickHouse/ClickHouse/issues/7899). [#8275](https://github.com/ClickHouse/ClickHouse/pull/8275) ([alesapin](https://github.com/alesapin)) +* Fixes crash when `now64()` is called with result of another function. [#8270](https://github.com/ClickHouse/ClickHouse/pull/8270) ([Vasily Nemkov](https://github.com/Enmk)) +* Fixed bug with detecting client IP for connections through mysql wire protocol. [#7743](https://github.com/ClickHouse/ClickHouse/pull/7743) ([Dmitry Muzyka](https://github.com/dmitriy-myz)) +* Fix empty array handling in `arraySplit` function. This fixes [#7708](https://github.com/ClickHouse/ClickHouse/issues/7708). [#7747](https://github.com/ClickHouse/ClickHouse/pull/7747) ([hcz](https://github.com/hczhcz)) +* Fixed the issue when `pid-file` of another running `clickhouse-server` may be deleted. [#8487](https://github.com/ClickHouse/ClickHouse/pull/8487) ([Weiqing Xu](https://github.com/weiqxu)) +* Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) +* Fixed error in function `arrayReduce` that may lead to "double free" and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) ### Improvement -* Forward all aws sdk log messages into CH log system to help debug scenarios with s3 [#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Improved logging when working with `S3` table engine. [#8251](https://github.com/ClickHouse/ClickHouse/pull/8251) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) * Printed help message when no arguments are passed when calling `clickhouse-local`. This fixes [#5335](https://github.com/ClickHouse/ClickHouse/issues/5335). [#8230](https://github.com/ClickHouse/ClickHouse/pull/8230) ([Andrey Nagorny](https://github.com/Melancholic)) -* Add setting `mutations_sync` which allows you to wait `ALTER UPDATE/DELETE` queries synchronously. [#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) -* Allow relative `user_files_path` (in the way similar to `format_schema_path`) [#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) -* Add exception for illegal types for conversion functions with -OrZero postfix [#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) +* Add setting `mutations_sync` which allows to wait `ALTER UPDATE/DELETE` queries synchronously. [#8237](https://github.com/ClickHouse/ClickHouse/pull/8237) ([alesapin](https://github.com/alesapin)) +* Allow to set up relative `user_files_path` in `config.xml` (in the way similar to `format_schema_path`). [#7632](https://github.com/ClickHouse/ClickHouse/pull/7632) ([hcz](https://github.com/hczhcz)) +* Add exception for illegal types for conversion functions with `-OrZero` postfix. [#7880](https://github.com/ClickHouse/ClickHouse/pull/7880) ([Andrey Konyaev](https://github.com/akonyaev90)) * Simplify format of the header of data sending to a shard in a distributed query. [#8044](https://github.com/ClickHouse/ClickHouse/pull/8044) ([Vitaly Baranov](https://github.com/vitlibar)) -* Updates to the Live View storage engine. Refactoring of getNewBlocks() and writeIntoLiveView() methods. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) -* Add additional checks for external dictionaries created from DDL-queries. Now it's not possible to create dictionaries with `Executable` source with DDL. Files for dictionaries from `File` source have to be located in `user_files` directory. Remote hosts for HTTP dictionaries now checked with `remote_url_allow_hosts` section from config. Shared libraries for corresponding dictionaries' sources have to be placed in `dictionaries_lib` folder. ... [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) -* Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186) [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* mutations might use joinGet without specifying default db. However if the argument isn't treated as special, it won't get currentdb appended thus breaks the background mutation job. Before this pr, we have to specify the database name explicitly, now we can use table identifier, https://github.com/ClickHouse/ClickHouse/issues/7699 ... [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) -* Allow using MV with subqueries above Kafka tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) -* Added separated pool for background moves. ... [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Synchronous system reload dictionary [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -* Remove `Context` from formats. [#8388](https://github.com/ClickHouse/ClickHouse/pull/8388) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) +* Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) +* Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) +* Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) +* `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) * Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360) [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Support new syntax for row-level security filters: `…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779) [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) -* thirdparty libraries such as libhdfs calls libgcc `backtrace` directly, which might lead to ABI mismatch issue when libunwind is enabled and running on old systems [#7956](https://github.com/ClickHouse/ClickHouse/pull/7956) ([Amos Bird](https://github.com/amosbird)) -* Created cityHash for decimal and UUID types. ... [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) +* Support new syntax for row-level security filters: `…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) +* Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) * Removed fixed index granularity (it was 1024) from system logs because it's obsolete after implementation of adaptive granularity. [#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Enabled MySQL compatibility server when ClickHouse is compiled without SSL. [#7852](https://github.com/ClickHouse/ClickHouse/pull/7852) ([Yuriy Baranov](https://github.com/yurriy)) -* Added Comprehension to Improve Performance [#7857](https://github.com/ClickHouse/ClickHouse/pull/7857) ([Metehan Çetinkaya](https://github.com/MeteHanC)) -* Avoid interpreting distributed batches with corrupted extras (query, query settings, revision), by checksumming them. [#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) -* Support `DROP DATABASE` & `DETACH TABLE` & `DROP TABLE` & `ATTACH TABLE` for MySQL Database Engine. [#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) -* Authentication in S3 table function and storage [#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Use SIGRTMIN instead of SIGPROF for user debugging purposes [#7823](https://github.com/ClickHouse/ClickHouse/pull/7823) ([Ivan](https://github.com/abyss7)) +* Now server checksums distributed batches, which gives more verbose errors in case of corrupted data in batch. [#7914](https://github.com/ClickHouse/ClickHouse/pull/7914) ([Azat Khuzhin](https://github.com/azat)) +* Support `DROP DATABASE`, `DETACH TABLE`, `DROP TABLE` and `ATTACH TABLE` for `MySQL` database engine. [#8202](https://github.com/ClickHouse/ClickHouse/pull/8202) ([Winter Zhang](https://github.com/zhang2014)) +* Add authentication in S3 table function and table engine. [#7623](https://github.com/ClickHouse/ClickHouse/pull/7623) ([Vladimir Chebotarev](https://github.com/excitoon)) * Added check for extra parts of `MergeTree` at different disks, in order to not allow to miss data parts at undefined disks. [#8118](https://github.com/ClickHouse/ClickHouse/pull/8118) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Enable SSL support for Mac client and server [#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) -* Support adding Clickhouse as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html) [#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) -* performance optimizations in processors pipeline. [#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* clickhouse-client now only enable `bracketed-paste` when multiquery is on and multiline is off. ... [#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) -* Support array(Decimal) in ```if``` function. [#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) -* Support Decimals in ```arrayDifference() arrayCumSum() arrayCumSumNegative()``` [#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) -* Added lifetime to system.dictionaries [#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) -* Improved check for existing parts on different disks. Addresses [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Integration with AWS SDK for S3 interactions. This is needed to have all S3 features out of the box. Native AWS S3 client can be easily re-used in S3 VFS development in the future. [#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) -* Added support for subqueries in Live View tables. ... [#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) -* Check for using `Date` or `DateTime` column from TTL expressions was removed. [#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Disk info was added to `system.detached_parts`. [#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) -* Move setMax(Table|Partition)SizeToDrop call into config updater to change these settings without a restart. [#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) +* Enable SSL support for Mac client and server. [#8297](https://github.com/ClickHouse/ClickHouse/pull/8297) ([Ivan](https://github.com/abyss7)) +* Now ClickHouse can work as MySQL federated server (see https://dev.mysql.com/doc/refman/5.7/en/federated-create-server.html). [#7717](https://github.com/ClickHouse/ClickHouse/pull/7717) ([Maxim Fedotov](https://github.com/MaxFedotov)) +* `clickhouse-client` now only enable `bracketed-paste` when multiquery is on and multiline is off. This fixes (#7757)[https://github.com/ClickHouse/ClickHouse/issues/7757]. [#7761](https://github.com/ClickHouse/ClickHouse/pull/7761) ([Amos Bird](https://github.com/amosbird)) +* Support `Array(Decimal)` in `if` function. [#7721](https://github.com/ClickHouse/ClickHouse/pull/7721) ([Artem Zuikov](https://github.com/4ertus2)) +* Support Decimals in `arrayDifference`, `arrayCumSum` and `arrayCumSumNegative` functions. [#7724](https://github.com/ClickHouse/ClickHouse/pull/7724) ([Artem Zuikov](https://github.com/4ertus2)) +* Added `lifetime` column to `system.dictionaries` table. [#6820](https://github.com/ClickHouse/ClickHouse/issues/6820) [#7727](https://github.com/ClickHouse/ClickHouse/pull/7727) ([kekekekule](https://github.com/kekekekule)) +* Improved check for existing parts on different disks for `*MergeTree` table engines. Addresses [#7660](https://github.com/ClickHouse/ClickHouse/issues/7660). [#8440](https://github.com/ClickHouse/ClickHouse/pull/8440) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Integration with `AWS SDK` for `S3` interactions which allows to use all S3 features out of the box. [#8011](https://github.com/ClickHouse/ClickHouse/pull/8011) ([Pavel Kovalenko](https://github.com/Jokser)) +* Added support for subqueries in `Live View` tables. [#7792](https://github.com/ClickHouse/ClickHouse/pull/7792) ([vzakaznikov](https://github.com/vzakaznikov)) +* Check for using `Date` or `DateTime` column from `TTL` expressions was removed. [#7920](https://github.com/ClickHouse/ClickHouse/pull/7920) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Information about disk was added to `system.detached_parts` table. [#7833](https://github.com/ClickHouse/ClickHouse/pull/7833) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Now settings `max_(table|partition)_size_to_drop` can be changed without a restart. [#7779](https://github.com/ClickHouse/ClickHouse/pull/7779) ([Grigory Pervakov](https://github.com/GrigoryPervakov)) * Slightly better usability of error messages. Ask user not to remove the lines below `Stack trace:`. [#7897](https://github.com/ClickHouse/ClickHouse/pull/7897) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better reading messages from Kafka in various formats after [#7935](https://github.com/ClickHouse/ClickHouse/issues/7935) [#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) -* Compatibility with MySQL clients which don't support sha2_password auth plugin [#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) -* Support more column types in MySQL compatibility server [#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) -* Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying MergeTree tables. [#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) -* old kernels (< 3.17) doens't have SYS_getrandom. Always use POSIX implementation to have better compatibility ... [#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) -* Added check for valid destination in a move TTL rule [#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) -* DirectoryMonitor: mark batch as broken on UNKNOWN_CODEC/CANNOT_DECOMPRESS errors [#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) -* Add column with array of parts name which mutations must process in future. [#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) -* Continuation of [#8545](https://github.com/ClickHouse/ClickHouse/issues/8545) [#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Add forgotten rows number check to chunk constructor. [#8005](https://github.com/ClickHouse/ClickHouse/pull/8005) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Better reading messages from `Kafka` engine in various formats after [#7935](https://github.com/ClickHouse/ClickHouse/issues/7935). [#8035](https://github.com/ClickHouse/ClickHouse/pull/8035) ([Ivan](https://github.com/abyss7)) +* Better compatibility with MySQL clients which don't support `sha2_password` auth plugin. [#8036](https://github.com/ClickHouse/ClickHouse/pull/8036) ([Yuriy Baranov](https://github.com/yurriy)) +* Support more column types in MySQL compatibility server. [#7975](https://github.com/ClickHouse/ClickHouse/pull/7975) ([Yuriy Baranov](https://github.com/yurriy)) +* Implement `ORDER BY` optimization for `Merge`, `Buffer` and `Materilized View` storages with underlying `MergeTree` tables. [#8130](https://github.com/ClickHouse/ClickHouse/pull/8130) ([Anton Popov](https://github.com/CurtizJ)) +* Now we always use POSIX implementation of `getrandom` to have better compatibility with old kernels (< 3.17). [#7940](https://github.com/ClickHouse/ClickHouse/pull/7940) ([Amos Bird](https://github.com/amosbird)) +* Better check for valid destination in a move TTL rule. [#8410](https://github.com/ClickHouse/ClickHouse/pull/8410) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Better checks for broken insert batches for `Distributed` table engine. [#7933](https://github.com/ClickHouse/ClickHouse/pull/7933) ([Azat Khuzhin](https://github.com/azat)) +* Add column with array of parts name which mutations must process in future to `system.mutations` table. [#8179](https://github.com/ClickHouse/ClickHouse/pull/8179) ([alesapin](https://github.com/alesapin)) +* Parallel merge sort optimization for processors. [#8552](https://github.com/ClickHouse/ClickHouse/pull/8552) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * The settings `mark_cache_min_lifetime` is now obsolete and does nothing. In previous versions, mark cache can grow in memory larger than `mark_cache_size` to accomodate data within `mark_cache_min_lifetime` seconds. That was leading to confusion and higher memory usage than expected, that is especially bad on memory constrained systems. If you will see performance degradation after installing this release, you should increase the `mark_cache_size`. [#8484](https://github.com/ClickHouse/ClickHouse/pull/8484) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Preparation to use `tid` everywhere. This is needed for [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477). [#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) ### Performance Improvement -* Non-blocking updates of expired keys in cache dictionaries (with with permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* omit frame pointer globally to spare one more register. Let's see how perf test goes. [#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) -* Speedup and test correctness of current function. Add perf test. ... [#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) -* Performance degradation is not yet in release, do not include in changelog. [#8462](https://github.com/ClickHouse/ClickHouse/pull/8462) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Performance degradation is not yet in release. Do not include in changelog. [#8463](https://github.com/ClickHouse/ClickHouse/pull/8463) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Performance optimizations in processors pipeline. [#7988](https://github.com/ClickHouse/ClickHouse/pull/7988) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Non-blocking updates of expired keys in cache dictionaries (with permission to read old ones). [#8303](https://github.com/ClickHouse/ClickHouse/pull/8303) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Compile ClickHouse without `-fno-omit-frame-pointer` globally to spare one more register. [#8097](https://github.com/ClickHouse/ClickHouse/pull/8097) ([Amos Bird](https://github.com/amosbird)) +* Speedup `greatCircleDistance` function and add performance tests for it. [#7307](https://github.com/ClickHouse/ClickHouse/pull/7307) ([Olga Khvostikova](https://github.com/stavrolia)) * Improved performance of function `roundDown`. [#8465](https://github.com/ClickHouse/ClickHouse/pull/8465) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improved performance of max(), min(), argMin(), argMax() for DateTime64 ... [#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of `max`, `min`, `argMin`, `argMax` for `DateTime64` data type. [#8199](https://github.com/ClickHouse/ClickHouse/pull/8199) ([Vasily Nemkov](https://github.com/Enmk)) * Improved performance of sorting without a limit or with big limit and external sorting. [#8545](https://github.com/ClickHouse/ClickHouse/pull/8545) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Improved performance of formatting floating point numbers up to 6 times. [#8542](https://github.com/ClickHouse/ClickHouse/pull/8542) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* gcc generates suboptimal instructions when dealing with `select ... group by column % 128` ... [#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) -* Optimized ORDER BY and merging with single column key. [#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Better implementation of addBatchArray (one more free register) . ... [#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) -* Now prewhere should be at least as efficient as where. As a result, any viable predicates can be moved into prewhere without fancy heuristic ... [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) -* Improve the way `round` and `roundBankers` handling negative numbers [#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) -* Fixed potential performance issue in "Context::getTable" method. [#8538](https://github.com/ClickHouse/ClickHouse/pull/8538) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Improved decoding performance of DoubleDelta and Gorilla codecs by (TODO: edit after performance tests are done) roughly 30-40%. ... [#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) -* Improved performance of base64 related functions. [#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Improved performance of `modulo` function. [#7750](https://github.com/ClickHouse/ClickHouse/pull/7750) ([Amos Bird](https://github.com/amosbird)) +* Optimized `ORDER BY` and merging with single column key. [#8335](https://github.com/ClickHouse/ClickHouse/pull/8335) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Better implementation for `arrayReduce`, `-Array` and `-State` combinators. [#7710](https://github.com/ClickHouse/ClickHouse/pull/7710) ([Amos Bird](https://github.com/amosbird)) +* Now `PREWHERE` should be optimized to be at least as efficient as `WHERE`. [#7769](https://github.com/ClickHouse/ClickHouse/pull/7769) ([Amos Bird](https://github.com/amosbird)) +* Improve the way `round` and `roundBankers` handling negative numbers. [#8229](https://github.com/ClickHouse/ClickHouse/pull/8229) ([hcz](https://github.com/hczhcz)) +* Improved decoding performance of `DoubleDelta` and `Gorilla` codecs by roughly 30-40%. This fixes [#7082](https://github.com/ClickHouse/ClickHouse/issues/7082). [#8019](https://github.com/ClickHouse/ClickHouse/pull/8019) ([Vasily Nemkov](https://github.com/Enmk)) +* Improved performance of `base64` related functions. [#8444](https://github.com/ClickHouse/ClickHouse/pull/8444) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Added a function `geoDistance`. It is similar to `greatCircleDistance` but uses approximation to WGS-84 ellipsoid model. The performance of both functions are near the same. [#8086](https://github.com/ClickHouse/ClickHouse/pull/8086) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Faster mix/max aggregtion functions for Decimals [#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) -* vectorize processing arrayReduce similar to Aggregator addBatch. Might also serve as an infrastructure for https://github.com/ClickHouse/ClickHouse/pull/7550 ... [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) -* If chains are now optimized as multiIf. [#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) -* Fix performance regression introduced in 19.15. [#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) -* Removed "pie" code generation that gcc from Debian packages occasionally brings by default. [#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Faster `min` and `max` aggregation functions for `Decimal` data type. [#8144](https://github.com/ClickHouse/ClickHouse/pull/8144) ([Artem Zuikov](https://github.com/4ertus2)) +* Vectorize processing `arrayReduce`. [#7608](https://github.com/ClickHouse/ClickHouse/pull/7608) ([Amos Bird](https://github.com/amosbird)) +* `if` chains are now optimized as `multiIf`. [#8355](https://github.com/ClickHouse/ClickHouse/pull/8355) ([kamalov-ruslan](https://github.com/kamalov-ruslan)) +* Fix performance regression of `Kafka` table engine introduced in 19.15. This fixes [#7261](https://github.com/ClickHouse/ClickHouse/issues/7261). [#7935](https://github.com/ClickHouse/ClickHouse/pull/7935) ([filimonov](https://github.com/filimonov)) +* Removed "pie" code generation that `gcc` from Debian packages occasionally brings by default. [#8483](https://github.com/ClickHouse/ClickHouse/pull/8483) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Parallel parsing data formats [#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) +* Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`). [#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) ### Build/Testing/Packaging Improvement -* Build fixes (arm, minimal) [#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) -* Add coverage file flush for clickhouse-server when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) +* Build fixes for `ARM` and in minimal mode. [#8304](https://github.com/ClickHouse/ClickHouse/pull/8304) ([proller](https://github.com/proller)) +* Add coverage file flush for `clickhouse-server` when std::atexit is not called. Also slightly improved logging in stateless tests with coverage. [#8267](https://github.com/ClickHouse/ClickHouse/pull/8267) ([alesapin](https://github.com/alesapin)) * Update LLVM library in contrib. Avoid using LLVM from OS packages. [#8258](https://github.com/ClickHouse/ClickHouse/pull/8258) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* No changes to previous release. [#8623](https://github.com/ClickHouse/ClickHouse/pull/8623) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Attempt to make curl build fully quiet. ... [#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) ([Pavel Kovalenko](https://github.com/Jokser)) -* Fix some MemorySanitizer warnings. [#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Use add_warning() and no_warning() macros in CMakeLists.txt [#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) -* For better S3 integration testing Minio S3 Compatible object storage is used (https://min.io/). ... [#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) ([Pavel Kovalenko](https://github.com/Jokser)) -* Imported libc headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Remove -fPIC from some libraries. [#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Clean CMakeLists for curl a little. See https://github.com/ClickHouse/ClickHouse/pull/8011#issuecomment-569478910 [#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Silent warnings in CapNProto library. [#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* to perf test https://github.com/ClickHouse/ClickHouse/pull/7662 [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) -* ClickHouse will build on AArch64 even if MADV_FREE is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027) [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) -* Update zlib-ng to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765) [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374) [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Make Field methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Make bundled `curl` build fully quiet. [#8232](https://github.com/ClickHouse/ClickHouse/pull/8232) [#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) +* Fix some `MemorySanitizer` warnings. [#8235](https://github.com/ClickHouse/ClickHouse/pull/8235) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Use `add_warning` and `no_warning` macros in `CMakeLists.txt`. [#8604](https://github.com/ClickHouse/ClickHouse/pull/8604) ([Ivan](https://github.com/abyss7)) +* Add support of Minio S3 Compatible object (https://min.io/) for better integration tests. [#7863](https://github.com/ClickHouse/ClickHouse/pull/7863) [#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) +* Imported `libc` headers to contrib. It allows to make builds more consistent across various systems (only for `x86_64-linux-gnu`). [#5773](https://github.com/ClickHouse/ClickHouse/pull/5773) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Remove `-fPIC` from some libraries. [#8464](https://github.com/ClickHouse/ClickHouse/pull/8464) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Clean `CMakeLists.txt` for curl. See https://github.com/ClickHouse/ClickHouse/pull/8011#issuecomment-569478910 [#8459](https://github.com/ClickHouse/ClickHouse/pull/8459) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Silent warnings in `CapNProto` library. [#8220](https://github.com/ClickHouse/ClickHouse/pull/8220) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Add performance tests for short string optimized hash tables. [#7679](https://github.com/ClickHouse/ClickHouse/pull/7679) ([Amos Bird](https://github.com/amosbird)) +* Now ClickHouse will build on `AArch64` even if `MADV_FREE` is not available. This fixes [#8027](https://github.com/ClickHouse/ClickHouse/issues/8027). [#8243](https://github.com/ClickHouse/ClickHouse/pull/8243) ([Amos Bird](https://github.com/amosbird)) +* Update `zlib-ng` to fix memory sanitizer problems. [#7182](https://github.com/ClickHouse/ClickHouse/pull/7182) [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Enable internal MySQL library on non-Linux system, because usage of OS packages is very fragile and usually doesn't work at all. This fixes [#5765](https://github.com/ClickHouse/ClickHouse/issues/5765). [#8426](https://github.com/ClickHouse/ClickHouse/pull/8426) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed build on some systems after enabling `libc++`. This supersedes [#8374](https://github.com/ClickHouse/ClickHouse/issues/8374). [#8380](https://github.com/ClickHouse/ClickHouse/pull/8380) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Make `Field` methods more type-safe to find more errors. [#7386](https://github.com/ClickHouse/ClickHouse/pull/7386) [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Added missing files to the `libc-headers` submodule. [#8507](https://github.com/ClickHouse/ClickHouse/pull/8507) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix wrong JSON quoting in performance test output. [#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Make Field methods more type-safe to find more errors. [#8209](https://github.com/ClickHouse/ClickHouse/pull/8209) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Update zlib-ng to fix memory sanitizer problems. [#8206](https://github.com/ClickHouse/ClickHouse/pull/8206) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Disable "Performing test" messages during curl build process. ... [#8203](https://github.com/ClickHouse/ClickHouse/pull/8203) ([Pavel Kovalenko](https://github.com/Jokser)) +* Fix wrong `JSON` quoting in performance test output. [#8497](https://github.com/ClickHouse/ClickHouse/pull/8497) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Now stack trace is displayed for `std::exception` and `Poco::Exception`. In previous versions it was available only for `DB::Exception`. This improves diagnostics. [#8501](https://github.com/ClickHouse/ClickHouse/pull/8501) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* drop tables after tests are finished. [#8502](https://github.com/ClickHouse/ClickHouse/pull/8502) ([Amos Bird](https://github.com/amosbird)) -* glibc2.30 removes the faulty ifunc compat redirects of clock_gettime and clock_nanosleep. Linking to librt won't help anymore. [#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) +* Porting `clock_gettime` and `clock_nanosleep` for fresh glibc versions. [#8054](https://github.com/ClickHouse/ClickHouse/pull/8054) ([Amos Bird](https://github.com/amosbird)) * Enable `part_log` in example config for developers. [#8609](https://github.com/ClickHouse/ClickHouse/pull/8609) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix async nature of reload in 01036_no_superfluous_dict_reload_on_create_database* [#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) -* Fixed codec performance tests ... [#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) -* Documentation for `.tgz` build and ability to work from any directory. [#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) ([alesapin](https://github.com/alesapin)) -* Removed old ZSTD test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [#8618](https://github.com/ClickHouse/ClickHouse/issues/8618) [#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix async nature of reload in `01036_no_superfluous_dict_reload_on_create_database*`. [#8111](https://github.com/ClickHouse/ClickHouse/pull/8111) ([Azat Khuzhin](https://github.com/azat)) +* Fixed codec performance tests. [#8615](https://github.com/ClickHouse/ClickHouse/pull/8615) ([Vasily Nemkov](https://github.com/Enmk)) +* Add install scripts for `.tgz` build and documentation for them. [#8612](https://github.com/ClickHouse/ClickHouse/pull/8612) [#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) +* Removed old `ZSTD` test (it was created in year 2016 to reproduce the bug that pre 1.0 version of ZSTD has had). This fixes [#8618](https://github.com/ClickHouse/ClickHouse/issues/8618). [#8619](https://github.com/ClickHouse/ClickHouse/pull/8619) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed build on Mac OS Catalina. [#8600](https://github.com/ClickHouse/ClickHouse/pull/8600) ([meo](https://github.com/meob)) -* Increased number of rows in codec performance tests to make results noticeable ... [#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) -* In debug builds, treat LOGICAL_ERROR exceptions as assertion failures, so that they are easier to notice. [#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) -* Make formats-related perf test more deterministic. [#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Update lz4 to fix a MemorySanitizer failure. [#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Increased number of rows in codec performance tests to make results noticeable. [#8574](https://github.com/ClickHouse/ClickHouse/pull/8574) ([Vasily Nemkov](https://github.com/Enmk)) +* In debug builds, treat `LOGICAL_ERROR` exceptions as assertion failures, so that they are easier to notice. [#8475](https://github.com/ClickHouse/ClickHouse/pull/8475) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Make formats-related performance test more deterministic. [#8477](https://github.com/ClickHouse/ClickHouse/pull/8477) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Update `lz4` to fix a MemorySanitizer failure. [#8181](https://github.com/ClickHouse/ClickHouse/pull/8181) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Suppress a known MemorySanitizer false positive in exception handling. [#8182](https://github.com/ClickHouse/ClickHouse/pull/8182) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Update `gcc` and `g++` to version 9 in `build/docker/build.sh` [#7766](https://github.com/ClickHouse/ClickHouse/pull/7766) ([TLightSky](https://github.com/tlightsky)) -* Add install scripts for .tgz packages. [#8591](https://github.com/ClickHouse/ClickHouse/pull/8591) ([alesapin](https://github.com/alesapin)) -* example perf test case which prewhere is worse than where. ... [#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) +* Add performance test case to test that `PREWHERE` is worse than `WHERE`. [#7768](https://github.com/ClickHouse/ClickHouse/pull/7768) ([Amos Bird](https://github.com/amosbird)) * Progress towards fixing one flacky test. [#8621](https://github.com/ClickHouse/ClickHouse/pull/8621) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Avoid MSan report for data from libunwind. [#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Do not include in changelog. [#7706](https://github.com/ClickHouse/ClickHouse/issues/7706) [#7722](https://github.com/ClickHouse/ClickHouse/pull/7722) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Avoid MemorySanitizer report for data from `libunwind`. [#8539](https://github.com/ClickHouse/ClickHouse/pull/8539) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Updated `libc++` to the latest version. [#8324](https://github.com/ClickHouse/ClickHouse/pull/8324) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Build ICU library from sources. This fixes [#6460](https://github.com/ClickHouse/ClickHouse/issues/6460). [#8219](https://github.com/ClickHouse/ClickHouse/pull/8219) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Switched from `libressl` to `openssl`. ClickHouse should support TLS 1.3 and SNI after this change. This fixes [#8171](https://github.com/ClickHouse/ClickHouse/issues/8171). [#8218](https://github.com/ClickHouse/ClickHouse/pull/8218) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fixed UBSan report when using `chacha20_poly1305` from SSL (happens on connect to https://yandex.ru/). [#8214](https://github.com/ClickHouse/ClickHouse/pull/8214) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Debian: Fix mode of default password file [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) -* Improved expression for getting clickhouse-server pid in clickhouse-test [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) -* Updated contrib/googletest to v1.10.0 ... [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) -* Fixed TSan report in base64 library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fix 00600_replace_running_query for processors. ... [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Remove support for `tcmalloc` to make CMakeLists simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix mode of default password file for `.deb` linux distros. [#8075](https://github.com/ClickHouse/ClickHouse/pull/8075) ([proller](https://github.com/proller)) +* Improved expression for getting `clickhouse-server` PID in `clickhouse-test`. [#8063](https://github.com/ClickHouse/ClickHouse/pull/8063) ([Alexander Kazakov](https://github.com/Akazz)) +* Updated contrib/googletest to v1.10.0. [#8587](https://github.com/ClickHouse/ClickHouse/pull/8587) ([Alexander Burmak](https://github.com/Alex-Burmak)) +* Fixed ThreadSaninitizer report in `base64` library. Also updated this library to the latest version, but it doesn't matter. This fixes [#8397](https://github.com/ClickHouse/ClickHouse/issues/8397). [#8403](https://github.com/ClickHouse/ClickHouse/pull/8403) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fix `00600_replace_running_query` for processors. [#8272](https://github.com/ClickHouse/ClickHouse/pull/8272) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Remove support for `tcmalloc` to make `CMakeLists.txt` simpler. [#8310](https://github.com/ClickHouse/ClickHouse/pull/8310) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Release gcc builds now use `libc++` instead of `libstdc++`. Recently `libc++` was used only with clang. This will improve consistency of build configurations and portability. [#8311](https://github.com/ClickHouse/ClickHouse/pull/8311) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Enable ICU library for build with Memory Sanitizer. [#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Suppress warnings from CapNProto library. [#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Removed special cases of code for tcmalloc, because it's no longer supported. [#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Enable ICU library for build with MemorySanitizer. [#8222](https://github.com/ClickHouse/ClickHouse/pull/8222) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Suppress warnings from `CapNProto` library. [#8224](https://github.com/ClickHouse/ClickHouse/pull/8224) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Removed special cases of code for `tcmalloc`, because it's no longer supported. [#8225](https://github.com/ClickHouse/ClickHouse/pull/8225) ([alexey-milovidov](https://github.com/alexey-milovidov)) * In CI coverage task, kill the server gracefully to allow it to save the coverage report. This fixes incomplete coverage reports we've been seeing lately. [#8142](https://github.com/ClickHouse/ClickHouse/pull/8142) ([alesapin](https://github.com/alesapin)) -* Performance tests for all codecs against Float64 and UInt64 values (where applicable). ... [#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) -* Python module `minio` need to be added to integration tests runner docker image to have ability to run intergration tests for https://github.com/ClickHouse/ClickHouse/pull/7863 ... [#7875](https://github.com/ClickHouse/ClickHouse/pull/7875) ([Pavel Kovalenko](https://github.com/Jokser)) -* In preparation of string optimized IN. [#7739](https://github.com/ClickHouse/ClickHouse/pull/7739) ([Amos Bird](https://github.com/amosbird)) -* termcap is very much deprecated and lead to various problems (f.g. missing "up" cap and echoing `^J` instead of multi line) . Favor terminfo or bundled ncurses. [#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) +* Performance tests for all codecs against `Float64` and `UInt64` values. [#8349](https://github.com/ClickHouse/ClickHouse/pull/8349) ([Vasily Nemkov](https://github.com/Enmk)) +* `termcap` is very much deprecated and lead to various problems (f.g. missing "up" cap and echoing `^J` instead of multi line) . Favor `terminfo` or bundled `ncurses`. [#7737](https://github.com/ClickHouse/ClickHouse/pull/7737) ([Amos Bird](https://github.com/amosbird)) * Fix `test_storage_s3` integration test. [#7734](https://github.com/ClickHouse/ClickHouse/pull/7734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* support `StorageFile(, null) ` to deser block into given format file without actually write to disk. It's used to perf test format parsers. ... [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) +* Support `StorageFile(, null) ` to insert block into given format file without actually write to disk. This is required for performance tests. [#8455](https://github.com/ClickHouse/ClickHouse/pull/8455) ([Amos Bird](https://github.com/amosbird)) * Added argument `--print-time` to functional tests which prints execution time per test. [#8001](https://github.com/ClickHouse/ClickHouse/pull/8001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Added asserts to KeyCondition while evaluating RPN. This will fix warning from gcc-9. [#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added asserts to `KeyCondition` while evaluating RPN. This will fix warning from gcc-9. [#8279](https://github.com/ClickHouse/ClickHouse/pull/8279) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Dump cmake options in CI builds. [#8273](https://github.com/ClickHouse/ClickHouse/pull/8273) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Don't generate debug info for some fat libraries. [#8271](https://github.com/ClickHouse/ClickHouse/pull/8271) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Make log_to_console.xml always log to stderr, regardless of is it interactive or not. [#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Make `log_to_console.xml` always log to stderr, regardless of is it interactive or not. [#8395](https://github.com/ClickHouse/ClickHouse/pull/8395) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Removed some unused features from `clickhouse-performance-test` tool. [#8555](https://github.com/ClickHouse/ClickHouse/pull/8555) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Now we will also search for `lld-X` with corresponding `clang-X` version. [#8092](https://github.com/ClickHouse/ClickHouse/pull/8092) ([alesapin](https://github.com/alesapin)) -* No changes to prev. release. [#8480](https://github.com/ClickHouse/ClickHouse/pull/8480) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Add tests to https://github.com/ClickHouse/ClickHouse/pull/7769 . One of them is from @CurtizJ in https://github.com/ClickHouse/ClickHouse/issues/8474 [#8486](https://github.com/ClickHouse/ClickHouse/pull/8486) ([Amos Bird](https://github.com/amosbird)) - -### Other -* Enable optimized parser of `Values` with expressions by default (`input_format_values_deduce_templates_of_expressions=1`) ... [#8231](https://github.com/ClickHouse/ClickHouse/pull/8231) ([tavplubix](https://github.com/tavplubix)) -* Fix test with processors. Tiny performance fixes. [#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Update contrib/protobuf [#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) -* In preparation of switching to c++20 as a new year celebration. Let's see how CI goes. "May the C++ force be with ClickHouse." [#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) -* Addition to [#5773](https://github.com/ClickHouse/ClickHouse/issues/5773) [#8321](https://github.com/ClickHouse/ClickHouse/pull/8321) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Preparation to use `tid` everywhere. This is needed for [#7477](https://github.com/ClickHouse/ClickHouse/issues/7477) [#8276](https://github.com/ClickHouse/ClickHouse/pull/8276) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### Bugfix -* Fix dictionary reload if it has `invalidate_query`, which stopped updates and some exception on previous update tries. [#8029](https://github.com/ClickHouse/ClickHouse/pull/8029) ([alesapin](https://github.com/alesapin)) - -### Bugfix; -* Fixed error in function `arrayReduce` that may lead to "double free" and error in aggregate function combinator `Resample` that may lead to memory leak. Added aggregate function `aggThrow`. This function can be used for testing purposes. [#8446](https://github.com/ClickHouse/ClickHouse/pull/8446) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### Build Improvement -* Arrow flatc cleanup [#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) +* Parquet build improvement. [#8421](https://github.com/ClickHouse/ClickHouse/pull/8421) ([maxulan](https://github.com/maxulan)) * More GCC warnings [#8221](https://github.com/ClickHouse/ClickHouse/pull/8221) ([kreuzerkrieg](https://github.com/kreuzerkrieg)) - -### Build/Testing/Packaging Improvement; * Package for Arch Linux now allows to run ClickHouse server, and not only client. [#8534](https://github.com/ClickHouse/ClickHouse/pull/8534) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Fix test with processors. Tiny performance fixes. [#7672](https://github.com/ClickHouse/ClickHouse/pull/7672) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Update contrib/protobuf. [#8256](https://github.com/ClickHouse/ClickHouse/pull/8256) ([Matwey V. Kornilov](https://github.com/matwey)) +* In preparation of switching to c++20 as a new year celebration. "May the C++ force be with ClickHouse." [#8447](https://github.com/ClickHouse/ClickHouse/pull/8447) ([Amos Bird](https://github.com/amosbird)) ### Experimental Feature * Added experimental setting `min_bytes_to_use_mmap_io`. It allows to read big files without copying data from kernel to userspace. The setting is disabled by default. Recommended threshold is about 64 MB, because mmap/munmap is slow. [#8520](https://github.com/ClickHouse/ClickHouse/pull/8520) ([alexey-milovidov](https://github.com/alexey-milovidov)) - -### New feature -* Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota()`, `currentQuotaKey()`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) +* Reworked quotas as a part of access control system. Added new table `system.quotas`, new functions `currentQuota`, `currentQuotaKey`, new SQL syntax `CREATE QUOTA`, `ALTER QUOTA`, `DROP QUOTA`, `SHOW QUOTA`. [#7257](https://github.com/ClickHouse/ClickHouse/pull/7257) ([Vitaly Baranov](https://github.com/vitlibar)) * Allow skipping unknown settings with warnings instead of throwing exceptions. [#7653](https://github.com/ClickHouse/ClickHouse/pull/7653) ([Vitaly Baranov](https://github.com/vitlibar)) -* Parallel parsing data formats [#6553](https://github.com/ClickHouse/ClickHouse/pull/6553) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) * Reworked row policies as a part of access control system. Added new table `system.row_policies`, new function `currentRowPolicies()`, new SQL syntax `CREATE POLICY`, `ALTER POLICY`, `DROP POLICY`, `SHOW CREATE POLICY`, `SHOW POLICIES`. [#7808](https://github.com/ClickHouse/ClickHouse/pull/7808) ([Vitaly Baranov](https://github.com/vitlibar)) ### Security Fix -* Fixed the possibility of reading directories structure in StorageFile. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536) [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov)) ## ClickHouse release v19.17 From 5e45f9ec986d71c093cd41866a3443f7eaabc869 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:00:09 +0300 Subject: [PATCH 231/312] Less strange --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eaa2127324..d60ef93a8f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ * Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) * Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) * Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) -* Fixed metrics in `BackgroundProcessingPool` (they were interfering with metrics of processing pool for moves and could throw exceptions `invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size` during merges and mutations if there is a storage policy set for a table). [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) +* Better metrics calculations in thread pool for background processes for `MergeTree` table engines. [#8194](https://github.com/ClickHouse/ClickHouse/pull/8194) ([Vladimir Chebotarev](https://github.com/excitoon)) * Fix function `IN` inside `WHERE` statement when row-level table filter is present. Fixes [#6687](https://github.com/ClickHouse/ClickHouse/issues/6687) [#8357](https://github.com/ClickHouse/ClickHouse/pull/8357) ([Ivan](https://github.com/abyss7)) * Now an exception is thrown if the integral value is not parsed completely for settings values. [#7678](https://github.com/ClickHouse/ClickHouse/pull/7678) ([Mikhail Korotov](https://github.com/millb)) * Fix exception when aggregate function is used in query to distributed table with more than two local shards. [#8164](https://github.com/ClickHouse/ClickHouse/pull/8164) ([小路](https://github.com/nicelulu)) From d71a09c336ee5558ccca97a9e5601f28fc2b2fb6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:00:57 +0300 Subject: [PATCH 232/312] Better --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d60ef93a8f6..bb0fd2a18b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,7 @@ * Allowed non-constant negative "size" argument for function `substring`. It was not allowed by mistake. This fixes [#4832](https://github.com/ClickHouse/ClickHouse/issues/4832). [#7703](https://github.com/ClickHouse/ClickHouse/pull/7703) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Fix parsing bug when wrong number of arguments passed to `(O|J)DBC` table engine. [#7709](https://github.com/ClickHouse/ClickHouse/pull/7709) ([alesapin](https://github.com/alesapin)) * Using command name of the running clickhouse process when sending logs to syslog. In previous versions, empty string was used instead of command name. [#8460](https://github.com/ClickHouse/ClickHouse/pull/8460) ([Michael Nacharov](https://github.com/mnach)) -* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in https://github.com/ClickHouse/ClickHouse/pull/8241. [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) +* Fix check of allowed hosts for `localhost`. This PR fixes the solution provided in [#8241](https://github.com/ClickHouse/ClickHouse/pull/8241). [#8342](https://github.com/ClickHouse/ClickHouse/pull/8342) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix rare crash in `argMin` and `argMax` functions for long string arguments, when result is used in `runningAccumulate` function. This fixes [#8325](https://github.com/ClickHouse/ClickHouse/issues/8325) [#8341](https://github.com/ClickHouse/ClickHouse/pull/8341) ([dinosaur](https://github.com/769344359)) * Fix memory overcommit for tables with `Buffer` engine. [#8345](https://github.com/ClickHouse/ClickHouse/pull/8345) ([Azat Khuzhin](https://github.com/azat)) * Fixed potential bug in functions that can take `NULL` as one of the arguments and return non-NULL. [#8196](https://github.com/ClickHouse/ClickHouse/pull/8196) ([alexey-milovidov](https://github.com/alexey-milovidov)) From 32b98dbf26d31a0511a79a6c395339bf9086ad38 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:02:39 +0300 Subject: [PATCH 233/312] Add dot --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0fd2a18b9..ce77aec0ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ * Add ability to execute `SYSTEM RELOAD DICTIONARY` query in `ON CLUSTER` mode. [#8288](https://github.com/ClickHouse/ClickHouse/pull/8288) ([Guillaume Tassery](https://github.com/YiuRULE)) * Add ability to execute `CREATE DICTIONARY` queries in `ON CLUSTER` mode. [#8163](https://github.com/ClickHouse/ClickHouse/pull/8163) ([alesapin](https://github.com/alesapin)) * Now user's profile in `users.xml` can inherit multiple profiles. [#8343](https://github.com/ClickHouse/ClickHouse/pull/8343) ([Mikhail f. Shiryaev](https://github.com/Felixoid)) -* Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576) [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Added `system.stack_trace` table that allows to look at stack traces of all server threads. This is useful for developers to introspect server state. This fixes [#7576](https://github.com/ClickHouse/ClickHouse/issues/7576). [#8344](https://github.com/ClickHouse/ClickHouse/pull/8344) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) * Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) * Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) From 6a656e8ee81931b5f036c3e7c922561c283f34bd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 19:03:55 +0300 Subject: [PATCH 234/312] Add cancell to SourceFromInputStream. --- dbms/src/Processors/IProcessor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Processors/IProcessor.h b/dbms/src/Processors/IProcessor.h index 87081b89199..3c5db7d5fb9 100644 --- a/dbms/src/Processors/IProcessor.h +++ b/dbms/src/Processors/IProcessor.h @@ -224,8 +224,8 @@ public: bool isCancelled() const { return is_cancelled; } void cancel() { - onCancel(); is_cancelled = true; + onCancel(); } virtual ~IProcessor() = default; From 4734a89644c8d5092f05c30924279fc229c266a0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:04:17 +0300 Subject: [PATCH 235/312] Better --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce77aec0ed1..4a8b33d207a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,7 +78,7 @@ * Fixed return type for functions `rand` and `randConstant` in case of `Nullable` argument. Now functions always return `UInt32` and never `Nullable(UInt32)`. [#8204](https://github.com/ClickHouse/ClickHouse/pull/8204) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Disabled predicate push-down for `WITH FILL` expression. This fixes [#7784](https://github.com/ClickHouse/ClickHouse/issues/7784). [#7789](https://github.com/ClickHouse/ClickHouse/pull/7789) ([Winter Zhang](https://github.com/zhang2014)) * Fixed incorrect `count()` result for `SummingMergeTree` when `FINAL` section is used. [#3280](https://github.com/ClickHouse/ClickHouse/issues/3280) [#7786](https://github.com/ClickHouse/ClickHouse/pull/7786) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)) -* Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666) [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) +* Fix possible incorrect result for constant functions from remote servers. It happened for queries with functions like `version()`, `uptime()`, etc. which returns different constant values for different servers. This fixes [#7666](https://github.com/ClickHouse/ClickHouse/issues/7666). [#7689](https://github.com/ClickHouse/ClickHouse/pull/7689) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fix complicated bug in push-down predicate optimization which leads to wrong results. This fixes a lot of issues on push-down predicate optimization. [#8503](https://github.com/ClickHouse/ClickHouse/pull/8503) ([Winter Zhang](https://github.com/zhang2014)) * Fix crash in `CREATE TABLE .. AS dictionary` query. [#8508](https://github.com/ClickHouse/ClickHouse/pull/8508) ([Azat Khuzhin](https://github.com/azat)) * Several improvements ClickHouse grammar in `.g4` file. [#8294](https://github.com/ClickHouse/ClickHouse/pull/8294) ([taiyang-li](https://github.com/taiyang-li)) From 5c9f3d942d42c8110fd33df5c8e742892d7d0342 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:05:15 +0300 Subject: [PATCH 236/312] Better --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a8b33d207a..4eb666a5fde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -157,7 +157,7 @@ * Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) * Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) * `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) -* Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360) [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) +* Stack traces now display physical addresses (offsets in object file) instead of virtual memory addresses (where the object file was loaded). That allows the use of `addr2line` when binary is position independent and ASLR is active. This fixes [#8360](https://github.com/ClickHouse/ClickHouse/issues/8360). [#8387](https://github.com/ClickHouse/ClickHouse/pull/8387) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Support new syntax for row-level security filters: `…
`. Fixes [#5779](https://github.com/ClickHouse/ClickHouse/issues/5779). [#8381](https://github.com/ClickHouse/ClickHouse/pull/8381) ([Ivan](https://github.com/abyss7)) * Now `cityHash` function can work with `Decimal` and `UUID` types. Fixes [#5184](https://github.com/ClickHouse/ClickHouse/issues/5184). [#7693](https://github.com/ClickHouse/ClickHouse/pull/7693) ([Mikhail Korotov](https://github.com/millb)) * Removed fixed index granularity (it was 1024) from system logs because it's obsolete after implementation of adaptive granularity. [#7698](https://github.com/ClickHouse/ClickHouse/pull/7698) ([alexey-milovidov](https://github.com/alexey-milovidov)) From 45741eb03ab12d93643da6cdb55777f43b19029e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 19:16:06 +0300 Subject: [PATCH 237/312] Add cancell to SourceFromInputStream. --- dbms/src/Processors/Sources/SourceFromInputStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.cpp b/dbms/src/Processors/Sources/SourceFromInputStream.cpp index 08eb0254097..06206dbb667 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.cpp +++ b/dbms/src/Processors/Sources/SourceFromInputStream.cpp @@ -35,7 +35,7 @@ IProcessor::Status SourceFromInputStream::prepare() is_generating_finished = true; /// Read postfix and get totals if needed. - if (!is_stream_finished) + if (!is_stream_finished && !isCancelled()) return Status::Ready; if (has_totals_port) @@ -109,7 +109,7 @@ Chunk SourceFromInputStream::generate() } auto block = stream->read(); - if (!block) + if (!block && !isCancelled()) { stream->readSuffix(); From 3377f17967a05e9e4afce914b28cf44ebcda1e44 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 27 Jan 2020 19:23:04 +0300 Subject: [PATCH 238/312] Fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb666a5fde..d72e4b12de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,7 @@ * Fix bug in `ALTER ... MODIFY ... CODEC` query, when user specify both default expression and codec. Fixes [8593](https://github.com/ClickHouse/ClickHouse/issues/8593). [#8614](https://github.com/ClickHouse/ClickHouse/pull/8614) ([alesapin](https://github.com/alesapin)) * Fix error in background merge of columns with `SimpleAggregateFunction(LowCardinality)` type. [#8613](https://github.com/ClickHouse/ClickHouse/pull/8613) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Fixed type check in function `toDateTime64`. [#8375](https://github.com/ClickHouse/ClickHouse/pull/8375) ([Vasily Nemkov](https://github.com/Enmk)) -* Now server do not crash on `LEFT` or `FULL JOIN` wittable and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) +* Now server do not crash on `LEFT` or `FULL JOIN` with and Join engine and unsupported `join_use_nulls` settings. [#8479](https://github.com/ClickHouse/ClickHouse/pull/8479) ([Artem Zuikov](https://github.com/4ertus2)) * Now `DROP DICTIONARY IF EXISTS db.dict` query doesn't throw exception if `db` doesn't exist. [#8185](https://github.com/ClickHouse/ClickHouse/pull/8185) ([Vitaly Baranov](https://github.com/vitlibar)) * Fix possible crashes in table functions (`file`, `mysql`, `remote`) caused by usage of reference to removed `IStorage` object. Fix incorrect parsing of columns specified at insertion into table function. [#7762](https://github.com/ClickHouse/ClickHouse/pull/7762) ([tavplubix](https://github.com/tavplubix)) * Ensure network be up before starting `clickhouse-server`. This fixes [#7507](https://github.com/ClickHouse/ClickHouse/issues/7507). [#8570](https://github.com/ClickHouse/ClickHouse/pull/8570) ([Zhichang Yu](https://github.com/yuzhichang)) From 0106040a2f2c056626ed8a0eada45c43dd4844ea Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Mon, 27 Jan 2020 19:51:48 +0300 Subject: [PATCH 239/312] In clickhouse-test: Updated check for hung queries --- dbms/tests/clickhouse-test | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index a216db2d633..aeb60286c80 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -99,7 +99,7 @@ def get_processlist(client_cmd): def get_stacktraces(server_pid): - cmd = "gdb -q -ex 'set pagination off' -ex 'backtrace' -ex 'thread apply all backtrace' -ex 'detach' -ex 'quit' --pid {} 2>/dev/null".format(server_pid) + cmd = "gdb -batch -ex 'thread apply all backtrace' -p {} 2>/dev/null".format(server_pid) try: return subprocess.check_output(cmd, shell=True) except Exception as ex: @@ -107,12 +107,11 @@ def get_stacktraces(server_pid): def get_server_pid(server_tcp_port): - cmd = "lsof -i tcp:{port} | fgrep 'TCP *:{port} (LISTEN)'".format(port=server_tcp_port) + cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | gawk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) try: output = subprocess.check_output(cmd, shell=True) if output: - columns = output.strip().split(' ') - return int(columns[1]) + return int(output[1:]) else: return None # server dead except Exception as ex: @@ -451,17 +450,26 @@ def main(args): total_tests_run += tests_n if args.hung_check: - processlist = get_processlist(args.client_with_database) - if processlist: - server_pid = get_server_pid(os.getenv("CLICKHOUSE_PORT_TCP", '9000')) - print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) - print(processlist) - if server_pid: + clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') + server_pid = get_server_pid(clickhouse_tcp_port) + if server_pid: + print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) + processlist = get_processlist(args.client_with_database) + if processlist: + print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) + print(processlist) + print("\nStacktraces of all threads:") print(get_stacktraces(server_pid)) - exit_code = 1 + exit_code = 1 + else: + print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) else: - print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) + print( + colored( + "\nUnable to locate ClickHouse server process listening at TCP port {}. " + "It must have crashed or exited prematurely!".format(clickhouse_tcp_port), + args, "red", attrs=["bold"])) if total_tests_run == 0: print("No tests were run.") From 43455d79b4765749b444a856e5bdf43514efb0e3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 19:58:25 +0300 Subject: [PATCH 240/312] Add time limit to SourceWithProgress. --- dbms/src/DataStreams/ExecutionSpeedLimits.cpp | 27 ++++++++++++++ dbms/src/DataStreams/ExecutionSpeedLimits.h | 3 ++ dbms/src/DataStreams/IBlockInputStream.cpp | 29 +-------------- .../Processors/Sources/SourceWithProgress.cpp | 9 +++++ .../Processors/Sources/SourceWithProgress.h | 2 ++ .../Transforms/LimitsCheckingTransform.cpp | 35 +------------------ .../Transforms/LimitsCheckingTransform.h | 3 -- 7 files changed, 43 insertions(+), 65 deletions(-) diff --git a/dbms/src/DataStreams/ExecutionSpeedLimits.cpp b/dbms/src/DataStreams/ExecutionSpeedLimits.cpp index 532c693bd47..ca79138655a 100644 --- a/dbms/src/DataStreams/ExecutionSpeedLimits.cpp +++ b/dbms/src/DataStreams/ExecutionSpeedLimits.cpp @@ -17,6 +17,8 @@ namespace DB namespace ErrorCodes { extern const int TOO_SLOW; + extern const int LOGICAL_ERROR; + extern const int TIMEOUT_EXCEEDED; } static void limitProgressingSpeed(size_t total_progress_size, size_t max_speed_in_seconds, UInt64 total_elapsed_microseconds) @@ -88,4 +90,29 @@ void ExecutionSpeedLimits::throttle( } } +static bool handleOverflowMode(OverflowMode mode, const String & message, int code) +{ + switch (mode) + { + case OverflowMode::THROW: + throw Exception(message, code); + case OverflowMode::BREAK: + return false; + default: + throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR); + } +} + +bool ExecutionSpeedLimits::checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode) +{ + if (max_execution_time != 0 + && elapsed_ns > static_cast(max_execution_time.totalMicroseconds()) * 1000) + return handleOverflowMode(overflow_mode, + "Timeout exceeded: elapsed " + toString(static_cast(elapsed_ns) / 1000000000ULL) + + " seconds, maximum: " + toString(max_execution_time.totalMicroseconds() / 1000000.0), + ErrorCodes::TIMEOUT_EXCEEDED); + + return true; +} + } diff --git a/dbms/src/DataStreams/ExecutionSpeedLimits.h b/dbms/src/DataStreams/ExecutionSpeedLimits.h index a067fc86000..539a5b0108b 100644 --- a/dbms/src/DataStreams/ExecutionSpeedLimits.h +++ b/dbms/src/DataStreams/ExecutionSpeedLimits.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -23,6 +24,8 @@ public: /// Pause execution in case if speed limits were exceeded. void throttle(size_t read_rows, size_t read_bytes, size_t total_rows_to_read, UInt64 total_elapsed_microseconds); + + bool checkTimeLimit(UInt64 elapsed_ns, OverflowMode overflow_mode); }; } diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index df81f26f665..db25c082626 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -54,7 +54,7 @@ Block IBlockInputStream::read() if (isCancelledOrThrowIfKilled()) return res; - if (!checkTimeLimit()) + if (!limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode)) limit_exceeded_need_break = true; if (!limit_exceeded_need_break) @@ -203,33 +203,6 @@ void IBlockInputStream::updateExtremes(Block & block) } -static bool handleOverflowMode(OverflowMode mode, const String & message, int code) -{ - switch (mode) - { - case OverflowMode::THROW: - throw Exception(message, code); - case OverflowMode::BREAK: - return false; - default: - throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR); - } -} - - -bool IBlockInputStream::checkTimeLimit() -{ - if (limits.speed_limits.max_execution_time != 0 - && info.total_stopwatch.elapsed() > static_cast(limits.speed_limits.max_execution_time.totalMicroseconds()) * 1000) - return handleOverflowMode(limits.timeout_overflow_mode, - "Timeout exceeded: elapsed " + toString(info.total_stopwatch.elapsedSeconds()) - + " seconds, maximum: " + toString(limits.speed_limits.max_execution_time.totalMicroseconds() / 1000000.0), - ErrorCodes::TIMEOUT_EXCEEDED); - - return true; -} - - void IBlockInputStream::checkQuota(Block & block) { switch (limits.mode) diff --git a/dbms/src/Processors/Sources/SourceWithProgress.cpp b/dbms/src/Processors/Sources/SourceWithProgress.cpp index fac2a53ea54..e7964ef81f3 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.cpp +++ b/dbms/src/Processors/Sources/SourceWithProgress.cpp @@ -10,6 +10,15 @@ namespace ErrorCodes { extern const int TOO_MANY_ROWS; extern const int TOO_MANY_BYTES; + extern const int TIMEOUT_EXCEEDED; +} + +void SourceWithProgress::work() +{ + if (!limits.speed_limits.checkTimeLimit(total_stopwatch.elapsed(), limits.timeout_overflow_mode)) + cancel(); + else + ISourceWithProgress::work(); } /// Aggregated copy-paste from IBlockInputStream::progressImpl. diff --git a/dbms/src/Processors/Sources/SourceWithProgress.h b/dbms/src/Processors/Sources/SourceWithProgress.h index 59e8c6afa20..abc96cbbe2f 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.h +++ b/dbms/src/Processors/Sources/SourceWithProgress.h @@ -58,6 +58,8 @@ protected: /// Call this method to provide information about progress. void progress(const Progress & value); + void work() override; + private: LocalLimits limits; std::shared_ptr quota; diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp index 1f621439048..57641a54b70 100644 --- a/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.cpp @@ -17,20 +17,6 @@ namespace ErrorCodes } -static bool handleOverflowMode(OverflowMode mode, const String & message, int code) -{ - switch (mode) - { - case OverflowMode::THROW: - throw Exception(message, code); - case OverflowMode::BREAK: - return false; - default: - throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR); - } -} - - void ProcessorProfileInfo::update(const Chunk & block) { ++blocks; @@ -44,13 +30,6 @@ LimitsCheckingTransform::LimitsCheckingTransform(const Block & header_, LocalLim { } -//LimitsCheckingTransform::LimitsCheckingTransform(const Block & header, LocalLimits limits, QueryStatus * process_list_elem) -// : ISimpleTransform(header, header, false) -// , limits(std::move(limits)) -// , process_list_elem(process_list_elem) -//{ -//} - void LimitsCheckingTransform::transform(Chunk & chunk) { if (!info.started) @@ -59,7 +38,7 @@ void LimitsCheckingTransform::transform(Chunk & chunk) info.started = true; } - if (!checkTimeLimit()) + if (!limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode)) { stopReading(); return; @@ -78,18 +57,6 @@ void LimitsCheckingTransform::transform(Chunk & chunk) } } -bool LimitsCheckingTransform::checkTimeLimit() -{ - if (limits.speed_limits.max_execution_time != 0 - && info.total_stopwatch.elapsed() > static_cast(limits.speed_limits.max_execution_time.totalMicroseconds()) * 1000) - return handleOverflowMode(limits.timeout_overflow_mode, - "Timeout exceeded: elapsed " + toString(info.total_stopwatch.elapsedSeconds()) - + " seconds, maximum: " + toString(limits.speed_limits.max_execution_time.totalMicroseconds() / 1000000.0), - ErrorCodes::TIMEOUT_EXCEEDED); - - return true; -} - void LimitsCheckingTransform::checkQuota(Chunk & chunk) { switch (limits.mode) diff --git a/dbms/src/Processors/Transforms/LimitsCheckingTransform.h b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h index 9410301030a..8746563ac78 100644 --- a/dbms/src/Processors/Transforms/LimitsCheckingTransform.h +++ b/dbms/src/Processors/Transforms/LimitsCheckingTransform.h @@ -29,10 +29,7 @@ public: using LocalLimits = IBlockInputStream::LocalLimits; using LimitsMode = IBlockInputStream::LimitsMode; - /// LIMITS_CURRENT LimitsCheckingTransform(const Block & header_, LocalLimits limits_); - /// LIMITS_TOTAL - /// LimitsCheckingTransform(const Block & header, LocalLimits limits, QueryStatus * process_list_elem); String getName() const override { return "LimitsCheckingTransform"; } From a1f05d3a56cffde9822304f76431504947aa2940 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Jan 2020 20:06:32 +0300 Subject: [PATCH 241/312] Fixed the issue with too many open files when using StorageFile with globs --- dbms/src/Storages/StorageFile.cpp | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index d3d15a72c54..4bfe502d556 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -228,7 +228,6 @@ public: } storage->table_fd_was_used = true; - read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(storage->table_fd), compression_method); } else { @@ -236,11 +235,7 @@ public: file_path = std::make_optional(file_path_); with_file_column = need_file; with_path_column = need_path; - read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(file_path.value()), compression_method); } - - if (!reader) - reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); } String getName() const override @@ -250,7 +245,20 @@ public: Block readImpl() override { + /// Open file lazily on first read. This is needed to avoid too many open files from different streams. + if (!reader) + { + read_buf = wrapReadBufferWithCompressionMethod(storage->use_table_fd + ? std::make_unique(storage->table_fd) + : std::make_unique(file_path.value()), + compression_method); + + reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); + } + auto res = reader->read(); + + /// Enrich with virtual columns. if (res && file_path) { if (with_path_column) @@ -264,6 +272,14 @@ public: std::make_shared(), "_file"}); } } + + /// Close file prematurally if stream was ended. + if (!res) + { + reader.reset(); + read_buf.reset(); + } + return res; } @@ -315,13 +331,13 @@ BlockInputStreams StorageFile::read( const ColumnsDescription & columns_ = getColumns(); auto column_defaults = columns_.getDefaults(); BlockInputStreams blocks_input; + if (use_table_fd) /// need to call ctr BlockInputStream paths = {""}; /// when use fd, paths are empty else - { if (paths.size() == 1 && !Poco::File(paths[0]).exists()) throw Exception("File " + paths[0] + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); - } + blocks_input.reserve(paths.size()); bool need_path_column = false; bool need_file_column = false; From 733df795a394fc009153cd15cd9c804be1665db6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 20:12:55 +0300 Subject: [PATCH 242/312] Fix IBlockInputStream --- dbms/src/DataStreams/IBlockInputStream.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index db25c082626..c894bcdf381 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -54,7 +54,7 @@ Block IBlockInputStream::read() if (isCancelledOrThrowIfKilled()) return res; - if (!limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode)) + if (!checkTimeLimit()) limit_exceeded_need_break = true; if (!limit_exceeded_need_break) @@ -203,6 +203,12 @@ void IBlockInputStream::updateExtremes(Block & block) } +bool IBlockInputStream::checkTimeLimit() +{ + return limits.speed_limits.checkTimeLimit(info.total_stopwatch.elapsed(), limits.timeout_overflow_mode); +} + + void IBlockInputStream::checkQuota(Block & block) { switch (limits.mode) From c6c3c4347b403a9dd70beef88a10efbb9d2f1567 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Mon, 27 Jan 2020 20:47:05 +0300 Subject: [PATCH 243/312] Keep stderr when collecting stacktraces --- dbms/tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index aeb60286c80..fc02d28a8fe 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -99,7 +99,7 @@ def get_processlist(client_cmd): def get_stacktraces(server_pid): - cmd = "gdb -batch -ex 'thread apply all backtrace' -p {} 2>/dev/null".format(server_pid) + cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: return subprocess.check_output(cmd, shell=True) except Exception as ex: From 774a98b4061c71fff99244879899eeecff25b8d5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 27 Jan 2020 21:03:30 +0300 Subject: [PATCH 244/312] Update CONTRIBUTING.md --- CONTRIBUTING.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 264d88f8553..e16957b759d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ If you want to contribute to documentation, read the [Contributing to ClickHouse ## Legal Info -In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: +In order for us (YANDEX LLC) to accept patches and other contributions from you, you may adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: 1) https://yandex.ru/legal/cla/?lang=en (in English) and 2) https://yandex.ru/legal/cla/?lang=ru (in Russian). @@ -37,4 +37,7 @@ Replace the bracketed text as follows: It is enough to provide us such notification once. -If you don't agree with the CLA, you still can open a pull request to provide your contributions. +As an alternative, you can provide DCO instead of CLA. You can find the text of DCO here: https://developercertificate.org/ +It is enough to read and copy it verbatim to your pull request. + +If you don't agree with the CLA and don't want to provide DCO, you still can open a pull request to provide your contributions. From 6b355aa273d90c511fc1c1f65d3167637f7456f1 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Mon, 27 Jan 2020 21:04:12 +0300 Subject: [PATCH 245/312] Moved around some logic of hung_check detector --- dbms/tests/clickhouse-test | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index fc02d28a8fe..acac89003ee 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -450,26 +450,27 @@ def main(args): total_tests_run += tests_n if args.hung_check: - clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') - server_pid = get_server_pid(clickhouse_tcp_port) - if server_pid: - print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) - processlist = get_processlist(args.client_with_database) - if processlist: - print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) - print(processlist) + processlist = get_processlist(args.client_with_database) + if processlist: + print(colored("\nFound hung queries in processlist:", args, "red", attrs=["bold"])) + print(processlist) - print("\nStacktraces of all threads:") + clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') + server_pid = get_server_pid(clickhouse_tcp_port) + if server_pid: + print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) + print("\nCollecting stacktraces from all running threads:") print(get_stacktraces(server_pid)) - exit_code = 1 else: - print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) + print( + colored( + "\nUnable to locate ClickHouse server process listening at TCP port {}. " + "It must have crashed or exited prematurely!".format(clickhouse_tcp_port), + args, "red", attrs=["bold"])) + + exit_code = 1 else: - print( - colored( - "\nUnable to locate ClickHouse server process listening at TCP port {}. " - "It must have crashed or exited prematurely!".format(clickhouse_tcp_port), - args, "red", attrs=["bold"])) + print(colored("\nNo queries hung.", args, "green", attrs=["bold"])) if total_tests_run == 0: print("No tests were run.") From f94eb7a0ff4fd5443509ba4cb13e2efd96aa8514 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 21:09:20 +0300 Subject: [PATCH 246/312] Update ISource. --- dbms/src/Processors/ISource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Processors/ISource.cpp b/dbms/src/Processors/ISource.cpp index d68c3345073..7c620a98a74 100644 --- a/dbms/src/Processors/ISource.cpp +++ b/dbms/src/Processors/ISource.cpp @@ -11,7 +11,7 @@ ISource::ISource(Block header) ISource::Status ISource::prepare() { - if (finished) + if (finished || isCancelled()) { output.finish(); return Status::Finished; From d34cf8a5bbc6777e70dc6718edc926b8020418e3 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 27 Jan 2020 21:12:27 +0300 Subject: [PATCH 247/312] Fix build. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index c0af990bf84..05d4398d3c9 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -464,7 +464,7 @@ void PipelineExecutor::execute(size_t num_threads) if (node.execution_state->exception) std::rethrow_exception(node.execution_state->exception); } - catch (Exception & exception) + catch (...) { #ifndef N_DEBUG LOG_TRACE(log, "Exception while executing query. Current state:\n" << dumpPipeline()); From 59ef3c803439d287862f707310a63ce8d6c3dbad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Jan 2020 21:13:02 +0300 Subject: [PATCH 248/312] Fixed the issue with too many open files when using StorageFile with globs --- dbms/src/Storages/StorageFile.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 4bfe502d556..7baa919b7d2 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -204,11 +204,12 @@ class StorageFileBlockInputStream : public IBlockInputStream { public: StorageFileBlockInputStream(std::shared_ptr storage_, - const Context & context, UInt64 max_block_size, + const Context & context_, UInt64 max_block_size_, std::string file_path_, bool need_path, bool need_file, - const CompressionMethod compression_method, + const CompressionMethod compression_method_, BlockInputStreamPtr prepared_reader = nullptr) - : storage(std::move(storage_)), reader(std::move(prepared_reader)) + : storage(std::move(storage_)), reader(std::move(prepared_reader)), + context(context_), max_block_size(max_block_size_), compression_method(compression_method_) { if (storage->use_table_fd) { @@ -315,6 +316,10 @@ private: std::unique_ptr read_buf; BlockInputStreamPtr reader; + const Context & context; /// TODO Untangle potential issues with context lifetime. + UInt64 max_block_size; + const CompressionMethod compression_method; + std::shared_lock shared_lock; std::unique_lock unique_lock; }; From 1ce32211cf8e94e40f804a179b429678abff87e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Jan 2020 21:25:54 +0300 Subject: [PATCH 249/312] Fixed the issue with too many open files when using StorageFile with globs --- dbms/src/Storages/StorageFile.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 7baa919b7d2..8cd3510917b 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -286,7 +286,8 @@ public: Block getHeader() const override { - auto res = reader->getHeader(); + auto res = storage->getSampleBlock(); + if (res && file_path) { if (with_path_column) @@ -294,6 +295,7 @@ public: if (with_file_column) res.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); } + return res; } From 390745cc8168adf23d21a735330338b4004c5d26 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 27 Jan 2020 21:26:58 +0300 Subject: [PATCH 250/312] Fixed the issue with too many open files when using StorageFile with globs --- dbms/src/Storages/StorageFile.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 8cd3510917b..5e7177b3e25 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -255,6 +255,7 @@ public: compression_method); reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); + reader->readPrefix(); } auto res = reader->read(); @@ -277,6 +278,7 @@ public: /// Close file prematurally if stream was ended. if (!res) { + reader->readSuffix(); reader.reset(); read_buf.reset(); } @@ -299,16 +301,6 @@ public: return res; } - void readPrefixImpl() override - { - reader->readPrefix(); - } - - void readSuffixImpl() override - { - reader->readSuffix(); - } - private: std::shared_ptr storage; std::optional file_path; From 2e9538f768826b5f7e8f8d24329539d120d0bef3 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Mon, 27 Jan 2020 21:28:27 +0300 Subject: [PATCH 251/312] Another fix for tests. Better handling of MV. --- dbms/src/Storages/StorageFactory.cpp | 96 +++++++++++++--------------- 1 file changed, 46 insertions(+), 50 deletions(-) diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 106eaec67cc..d61d2a04fdf 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -111,61 +111,57 @@ StoragePtr StorageFactory::get( "Direct creation of tables with ENGINE LiveView is not supported, use CREATE LIVE VIEW statement", ErrorCodes::INCORRECT_QUERY); } - } - } - auto it = storages.find(name); - if (it == storages.end()) - { - auto hints = getHints(name); - if (!hints.empty()) - throw Exception("Unknown table engine " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_STORAGE); - else - throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); - } - - - auto checkFeature = [&](String feature_description, FeatureMatcherFn feature_matcher_fn) - { - if (!feature_matcher_fn(it->second.features)) - { - String msg = "Engine " + name + " doesn't support " + feature_description + ". " - "Currently only the following engines have support for the feature: ["; - auto supporting_engines = getAllRegisteredNamesByFeatureMatcherFn(feature_matcher_fn); - for (size_t index = 0; index < supporting_engines.size(); ++index) + auto it = storages.find(name); + if (it == storages.end()) { - if (index) - msg += ", "; - msg += supporting_engines[index]; + auto hints = getHints(name); + if (!hints.empty()) + throw Exception("Unknown table engine " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_STORAGE); + else + throw Exception("Unknown table engine " + name, ErrorCodes::UNKNOWN_STORAGE); } - msg += "]"; - throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); + + auto checkFeature = [&](String feature_description, FeatureMatcherFn feature_matcher_fn) + { + if (!feature_matcher_fn(it->second.features)) + { + String msg = "Engine " + name + " doesn't support " + feature_description + ". " + "Currently only the following engines have support for the feature: ["; + auto supporting_engines = getAllRegisteredNamesByFeatureMatcherFn(feature_matcher_fn); + for (size_t index = 0; index < supporting_engines.size(); ++index) + { + if (index) + msg += ", "; + msg += supporting_engines[index]; + } + msg += "]"; + throw Exception(msg, ErrorCodes::BAD_ARGUMENTS); + } + }; + + if (storage_def->settings) + checkFeature( + "SETTINGS clause", + [](StorageFeatures features) { return features.supports_settings; }); + + if (storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by) + checkFeature( + "PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses", + [](StorageFeatures features) { return features.supports_sort_order; }); + + if (storage_def->ttl_table || !columns.getColumnTTLs().empty()) + checkFeature( + "TTL clause", + [](StorageFeatures features) { return features.supports_ttl; }); + + if (query.columns_list && query.columns_list->indices && !query.columns_list->indices->children.empty()) + checkFeature( + "skipping indices", + [](StorageFeatures features) { return features.supports_skipping_indices; }); } - }; - - if (storage_def) - { - if (storage_def->settings) - checkFeature( - "SETTINGS clause", - [](StorageFeatures features) { return features.supports_settings; }); - - if (storage_def->partition_by || storage_def->primary_key || storage_def->order_by || storage_def->sample_by) - checkFeature( - "PARTITION_BY, PRIMARY_KEY, ORDER_BY or SAMPLE_BY clauses", - [](StorageFeatures features) { return features.supports_sort_order; }); - - if (storage_def->ttl_table || !columns.getColumnTTLs().empty()) - checkFeature( - "TTL clause", - [](StorageFeatures features) { return features.supports_ttl; }); } - if (query.columns_list && query.columns_list->indices && !query.columns_list->indices->children.empty()) - checkFeature( - "skipping indices", - [](StorageFeatures features) { return features.supports_skipping_indices; }); - Arguments arguments { .engine_name = name, @@ -182,7 +178,7 @@ StoragePtr StorageFactory::get( .has_force_restore_data_flag = has_force_restore_data_flag }; - return it->second.creator_fn(arguments); + return storages.at(name).creator_fn(arguments); } StorageFactory & StorageFactory::instance() From b7c9075e53744b0eb744bf0b858b6ea67146aa09 Mon Sep 17 00:00:00 2001 From: achulkov2 Date: Mon, 27 Jan 2020 23:26:39 +0300 Subject: [PATCH 252/312] Stop flattening unnamed nested tuples --- dbms/src/Storages/ColumnsDescription.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index b3caeaa767b..8b491679138 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -208,6 +208,12 @@ void ColumnsDescription::flattenNested() continue; } + if (!type_tuple->haveExplicitNames()) + { + ++it; + continue; + } + ColumnDescription column = std::move(*it); it = columns.get<0>().erase(it); From d8f7383fd7ee869264eaa668b541721efc25c97e Mon Sep 17 00:00:00 2001 From: Andrei Chulkov Date: Tue, 28 Jan 2020 00:04:46 +0300 Subject: [PATCH 253/312] Add test --- .../01038_array_of_unnamed_tuples.reference | 4 ++++ .../01038_array_of_unnamed_tuples.sql | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference create mode 100644 dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql diff --git a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference new file mode 100644 index 00000000000..0c1eaef5ab3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.reference @@ -0,0 +1,4 @@ +[(1,2),(2,3),(3,4)] +[(1,2)] +[(1,2,3),(2,3,4)] +[(4,3,1)] diff --git a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql new file mode 100644 index 00000000000..e2ad9b54658 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql @@ -0,0 +1,16 @@ +SET send_logs_level = 'none'; + +DROP TABLE IF EXISTS array_of_tuples; + +CREATE TABLE array_of_tuples +( + first Array(Tuple(Float64, Float64)), + second Array(Tuple(UInt8, UInt16, UInt32)) +) ENGINE = Memory; + +INSERT INTO array_of_tuples values ([(1, 2), (2, 3), (3, 4)], array(tuple(1, 2, 3), tuple(2, 3, 4))), (array((1.0, 2.0)), [tuple(4, 3, 1)]); + +SELECT first from array_of_tuples; +SELECT second from array_of_tuples; + +DROP TABLE array_of_tuples; From 7395c0eadc8d2cca22341aa6bc2ce72675b56cde Mon Sep 17 00:00:00 2001 From: Andrei Chulkov Date: Tue, 28 Jan 2020 00:09:38 +0300 Subject: [PATCH 254/312] Rename columns to avoid ambiguity with sql --- .../queries/0_stateless/01038_array_of_unnamed_tuples.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql index e2ad9b54658..c9805599549 100644 --- a/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql +++ b/dbms/tests/queries/0_stateless/01038_array_of_unnamed_tuples.sql @@ -4,13 +4,13 @@ DROP TABLE IF EXISTS array_of_tuples; CREATE TABLE array_of_tuples ( - first Array(Tuple(Float64, Float64)), - second Array(Tuple(UInt8, UInt16, UInt32)) + f Array(Tuple(Float64, Float64)), + s Array(Tuple(UInt8, UInt16, UInt32)) ) ENGINE = Memory; INSERT INTO array_of_tuples values ([(1, 2), (2, 3), (3, 4)], array(tuple(1, 2, 3), tuple(2, 3, 4))), (array((1.0, 2.0)), [tuple(4, 3, 1)]); -SELECT first from array_of_tuples; -SELECT second from array_of_tuples; +SELECT f from array_of_tuples; +SELECT s from array_of_tuples; DROP TABLE array_of_tuples; From e53ad72edda58bae9d0da991b79843e73a606dc4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 28 Jan 2020 02:29:30 +0300 Subject: [PATCH 255/312] Add link to Woboq to docs --- docs/en/development/browse_code.md | 7 +++++++ docs/fa/development/browse_code.md | 1 + docs/ja/development/browse_code.md | 1 + docs/ru/development/browse_code.md | 7 +++++++ docs/toc_en.yml | 3 ++- docs/toc_fa.yml | 9 +++++---- docs/toc_ja.yml | 5 +++-- docs/toc_ru.yml | 5 +++-- docs/toc_zh.yml | 7 ++++--- docs/zh/development/browse_code.md | 1 + 10 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 docs/en/development/browse_code.md create mode 120000 docs/fa/development/browse_code.md create mode 120000 docs/ja/development/browse_code.md create mode 100644 docs/ru/development/browse_code.md create mode 120000 docs/zh/development/browse_code.md diff --git a/docs/en/development/browse_code.md b/docs/en/development/browse_code.md new file mode 100644 index 00000000000..76c52709941 --- /dev/null +++ b/docs/en/development/browse_code.md @@ -0,0 +1,7 @@ +# Browse ClickHouse Source Code + +You can use **Woboq** online code browser available [here](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). It provide code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. + +Also you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. + +If you're interested what IDE to use, we recommend CLion, QT Creator, VS Code and KDevelop (with caveats). You can use any favourite IDE. Vim and Emacs also count. diff --git a/docs/fa/development/browse_code.md b/docs/fa/development/browse_code.md new file mode 120000 index 00000000000..8c08c622129 --- /dev/null +++ b/docs/fa/development/browse_code.md @@ -0,0 +1 @@ +../../en/development/browse_code.md \ No newline at end of file diff --git a/docs/ja/development/browse_code.md b/docs/ja/development/browse_code.md new file mode 120000 index 00000000000..8c08c622129 --- /dev/null +++ b/docs/ja/development/browse_code.md @@ -0,0 +1 @@ +../../en/development/browse_code.md \ No newline at end of file diff --git a/docs/ru/development/browse_code.md b/docs/ru/development/browse_code.md new file mode 100644 index 00000000000..4593192bda9 --- /dev/null +++ b/docs/ru/development/browse_code.md @@ -0,0 +1,7 @@ +# Навигация по коду ClickHouse + +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходниками, семантическая подсветка, подсказки, индексацию и поиск. Слепок кода обновляется ежедневно. + +Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). + +Если вы интересуетесь, какую среду разработки выбрать для работы с ClickHouse, мы рекомендуем CLion, QT Creator, VSCode или KDevelop (с некоторыми предостережениями). Вы можете использовать свою любимую среду разработки, Vim и Emacs тоже считаются. diff --git a/docs/toc_en.yml b/docs/toc_en.yml index f5bbea39945..042b0332bbc 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -218,14 +218,15 @@ nav: - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index b13c8092889..9da4346dbbc 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -59,7 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Interval': 'data_types/special_data_types/interval.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domains': - 'Overview': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -167,7 +167,7 @@ nav: - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - - 'input': 'query_language/table_functions/input.md' + - 'input': 'query_language/table_functions/input.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': @@ -210,18 +210,19 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - 'F.A.Q.': - 'General Questions': 'faq/general.md' - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Write C++ code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index 31b384f97b5..786559124f5 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -214,17 +214,18 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - 'Development': - 'hidden': 'development/index.md' + - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Overview of ClickHouse Architecture': 'development/architecture.md' + - 'Browse ClickHouse Source Code': 'development/browse_code.md' - 'How to Build ClickHouse on Linux': 'development/build.md' - 'How to Build ClickHouse on Mac OS X': 'development/build_osx.md' - 'How to Build ClickHouse on Linux for Mac OS X': 'development/build_cross_osx.md' - 'How to Build ClickHouse on Linux for AARCH64 (ARM64)': 'development/build_cross_arm.md' - 'How to Write C++ Code': 'development/style.md' - 'How to Run ClickHouse Tests': 'development/tests.md' - - 'The Beginner ClickHouse Developer Instruction': 'development/developer_instruction.md' - 'Third-Party Libraries Used': 'development/contrib.md' - 'What''s New': diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index cd422773282..c2b7ab6961a 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -138,7 +138,7 @@ nav: - 'mysql': 'query_language/table_functions/mysql.md' - 'jdbc': 'query_language/table_functions/jdbc.md' - 'odbc': 'query_language/table_functions/odbc.md' - - 'hdfs': 'query_language/table_functions/hdfs.md' + - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' @@ -216,13 +216,14 @@ nav: - 'Разработка': - 'hidden': 'development/index.md' + - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - 'Обзор архитектуры ClickHouse': 'development/architecture.md' + - 'Навигация по коду ClickHouse': 'development/browse_code.md' - 'Как собрать ClickHouse на Linux': 'development/build.md' - 'Как собрать ClickHouse на Mac OS X': 'development/build_osx.md' - 'Как собрать ClickHouse на Linux для Mac OS X': 'development/build_cross_osx.md' - 'Как писать код на C++': 'development/style.md' - 'Как запустить тесты': 'development/tests.md' - - 'Инструкция для начинающего разработчика ClickHouse': 'development/developer_instruction.md' - 'Сторонние библиотеки': 'development/contrib.md' - 'Что нового': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index c07c3bb2711..8f33e5579d2 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -59,7 +59,7 @@ nav: - 'Expression': 'data_types/special_data_types/expression.md' - 'Set': 'data_types/special_data_types/set.md' - 'Nothing': 'data_types/special_data_types/nothing.md' - - 'Interval': 'data_types/special_data_types/interval.md' + - 'Interval': 'data_types/special_data_types/interval.md' - 'Domain类型': - '介绍': 'data_types/domains/overview.md' - 'IPv4': 'data_types/domains/ipv4.md' @@ -209,19 +209,20 @@ nav: - 'clickhouse-copier': 'operations/utils/clickhouse-copier.md' - 'clickhouse-local': 'operations/utils/clickhouse-local.md' - 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md' - + - '常见问题': - '一般的问题': 'faq/general.md' - '开发者指南': - 'hidden': 'development/index.md' + - '开发者指南': 'development/developer_instruction.md' - 'ClickHouse架构概述': 'development/architecture.md' + - 'ClickHouse Code Browser': 'development/browse_code.md' - '如何在Linux中编译ClickHouse': 'development/build.md' - '如何在Mac OS X中编译ClickHouse': 'development/build_osx.md' - '如何在Linux中编译Mac OS X ClickHouse': 'development/build_cross_osx.md' - '如何编写C++代码': 'development/style.md' - '如何运行ClickHouse测试': 'development/tests.md' - - '开发者指南': 'development/developer_instruction.md' - '使用的第三方库': 'development/contrib.md' - '新功能特性': diff --git a/docs/zh/development/browse_code.md b/docs/zh/development/browse_code.md new file mode 120000 index 00000000000..8c08c622129 --- /dev/null +++ b/docs/zh/development/browse_code.md @@ -0,0 +1 @@ +../../en/development/browse_code.md \ No newline at end of file From 8729f7c2b39672a1f89b18ab06a694332bf1f11b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 28 Jan 2020 02:30:43 +0300 Subject: [PATCH 256/312] Update browse_code.md --- docs/en/development/browse_code.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/browse_code.md b/docs/en/development/browse_code.md index 76c52709941..0a99f5128cb 100644 --- a/docs/en/development/browse_code.md +++ b/docs/en/development/browse_code.md @@ -1,6 +1,6 @@ # Browse ClickHouse Source Code -You can use **Woboq** online code browser available [here](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). It provide code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. +You can use **Woboq** online code browser available [here](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. Also you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. From 8c68ffde6e7bd2125d0411a89b27b299d6079d1c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 28 Jan 2020 02:31:37 +0300 Subject: [PATCH 257/312] Update browse_code.md --- docs/ru/development/browse_code.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/development/browse_code.md b/docs/ru/development/browse_code.md index 4593192bda9..47ee2d4b5a1 100644 --- a/docs/ru/development/browse_code.md +++ b/docs/ru/development/browse_code.md @@ -1,6 +1,6 @@ # Навигация по коду ClickHouse -Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходниками, семантическая подсветка, подсказки, индексацию и поиск. Слепок кода обновляется ежедневно. +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексацию и поиск. Слепок кода обновляется ежедневно. Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). From 9596c25c3649c55bbb37d253a45e625c54285bc6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 28 Jan 2020 02:31:49 +0300 Subject: [PATCH 258/312] Update browse_code.md --- docs/ru/development/browse_code.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/development/browse_code.md b/docs/ru/development/browse_code.md index 47ee2d4b5a1..e18ba21b889 100644 --- a/docs/ru/development/browse_code.md +++ b/docs/ru/development/browse_code.md @@ -1,6 +1,6 @@ # Навигация по коду ClickHouse -Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексацию и поиск. Слепок кода обновляется ежедневно. +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). From e8619ce99dfa6b6df7a80ec2882a42a0b683f094 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 11:50:29 +0300 Subject: [PATCH 259/312] Now we have branches in .tgz repository --- utils/release/push_packages | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/utils/release/push_packages b/utils/release/push_packages index 487e939c47e..1e384198ca8 100755 --- a/utils/release/push_packages +++ b/utils/release/push_packages @@ -160,12 +160,8 @@ def clear_old_incoming_packages(ssh_connection, user): for pkg in ('deb', 'rpm', 'tgz'): for release_type in ('stable', 'testing', 'prestable', 'lts'): try: - if pkg != 'tgz': - ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/{release_type}/*".format( - user=user, pkg=pkg, release_type=release_type)) - else: - ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/*".format( - user=user, pkg=pkg)) + ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/{release_type}/*".format( + user=user, pkg=pkg, release_type=release_type)) except Exception: logging.info("rm is not required") From 12e70d08f65834fab6b7bc72314707b192d5c8aa Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 11:58:11 +0300 Subject: [PATCH 260/312] Remove another if --- utils/release/push_packages | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/utils/release/push_packages b/utils/release/push_packages index 1e384198ca8..4a0548bb4ca 100755 --- a/utils/release/push_packages +++ b/utils/release/push_packages @@ -168,13 +168,8 @@ def clear_old_incoming_packages(ssh_connection, user): def _get_incoming_path(repo_url, user=None, pkg_type=None, release_type=None): if repo_url == 'repo.mirror.yandex.net': - if pkg_type != 'tgz': - return "/home/{user}/incoming/clickhouse/{pkg}/{release_type}".format( - user=user, pkg=pkg_type, release_type=release_type) - else: - return "/home/{user}/incoming/clickhouse/{pkg}".format( - user=user, pkg=pkg_type) - + return "/home/{user}/incoming/clickhouse/{pkg}/{release_type}".format( + user=user, pkg=pkg_type, release_type=release_type) else: return "/repo/{0}/mini-dinstall/incoming/".format(repo_url.split('.')[0]) From 7b6709a5cae04b1f41de8c018f769f41c8c1abae Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 12:06:32 +0300 Subject: [PATCH 261/312] Fix typo and remove setting --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d72e4b12de1..cdfaed101b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,12 @@ * Add `DateTime64` datatype with configurable sub-second precision. [#7170](https://github.com/ClickHouse/ClickHouse/pull/7170) ([Vasily Nemkov](https://github.com/Enmk)) * Add table function `clusterAllReplicas` which allows to query all the nodes in the cluster. [#8493](https://github.com/ClickHouse/ClickHouse/pull/8493) ([kiran sunkari](https://github.com/kiransunkari)) * Add aggregate function `categoricalInformationValue` which calculates the information value of a discrete feature. [#8117](https://github.com/ClickHouse/ClickHouse/pull/8117) ([hcz](https://github.com/hczhcz)) -* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. This is enabled by default and controlled by the `max_threads_for_parallel_parsing` setting. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) +* Speed up parsing of data files in `CSV`, `TSV` and `JSONEachRow` format by doing it in parallel. [#7780](https://github.com/ClickHouse/ClickHouse/pull/7780) ([Alexander Kuzmenkov](https://github.com/akuzm)) * Add function `bankerRound` which performs banker's rounding. [#8112](https://github.com/ClickHouse/ClickHouse/pull/8112) ([hcz](https://github.com/hczhcz)) * Support more languages in embedded dictionary for region names: 'ru', 'en', 'ua', 'uk', 'by', 'kz', 'tr', 'de', 'uz', 'lv', 'lt', 'et', 'pt', 'he', 'vi'. [#8189](https://github.com/ClickHouse/ClickHouse/pull/8189) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Improvements in consistency of `ANY JOIN` logic. Now `t1 ANY LEFT JOIN t2` equals `t2 ANY RIGHT JOIN t1`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) * Add setting `any_join_distinct_right_table_keys` which enables old behaviour for `ANY INNER JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) -* Add new `SEMI` and `ANTY JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) +* Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) * Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) * Add ability to alter materialized views with `ALTER MODIFY QUERY `. [#7533](https://github.com/ClickHouse/ClickHouse/pull/7533) ([nvartolomei](https://github.com/nvartolomei)) From c63b1c84c0471f7c742c0b2ceab27c90441882ef Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 28 Jan 2020 13:18:09 +0300 Subject: [PATCH 262/312] [docs] Improved retrieval of stable releases list --- docs/tools/github.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/tools/github.py b/docs/tools/github.py index d92dfe7435b..ecf30c6363c 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -12,14 +12,22 @@ import util def choose_latest_releases(): seen = collections.OrderedDict() - candidates = requests.get('https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100').json() + candidates = [] + for page in range(1, 10): + url = 'https://api.github.com/repos/ClickHouse/ClickHouse/tags?per_page=100&page=%d' % page + candidates += requests.get(url).json() + for tag in candidates: name = tag.get('name', '') - if ('v18' in name) or ('stable' not in name) or ('prestable' in name): + is_unstable = ('stable' not in name) and ('lts' not in name) + is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name) + if is_unstable or is_in_blacklist: continue major_version = '.'.join((name.split('.', 2))[:2]) if major_version not in seen: seen[major_version] = (name, tag.get('tarball_url'),) + if len(seen) > 10: + break return seen.items() From e27a3170bc61981f824fd02776d6be4942e7ecce Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 13:58:37 +0300 Subject: [PATCH 263/312] Less flaps in tests --- dbms/tests/integration/helpers/client.py | 6 +++- .../test.py | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/dbms/tests/integration/helpers/client.py b/dbms/tests/integration/helpers/client.py index fd59166b137..e986a9ef7c8 100644 --- a/dbms/tests/integration/helpers/client.py +++ b/dbms/tests/integration/helpers/client.py @@ -67,7 +67,11 @@ class CommandRequest: #print " ".join(command) - self.process = sp.Popen(command, stdin=stdin_file, stdout=self.stdout_file, stderr=self.stderr_file) + # we suppress stderror on client becase sometimes thread sanitizer + # can print some debug information there + env = {} + env["TSAN_OPTIONS"] = "verbosity=0" + self.process = sp.Popen(command, stdin=stdin_file, stdout=self.stdout_file, stderr=self.stderr_file, env=env) self.timer = None self.process_finished_before_timeout = True diff --git a/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py index 01f9b15b51f..784276707b4 100644 --- a/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py +++ b/dbms/tests/integration/test_dictionaries_all_layouts_and_sources/test.py @@ -5,6 +5,7 @@ from helpers.cluster import ClickHouseCluster from dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from external_sources import SourceMySQL, SourceClickHouse, SourceFile, SourceExecutableCache, SourceExecutableHashed from external_sources import SourceMongo, SourceHTTP, SourceHTTPS, SourceRedis +import math SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) dict_configs_path = os.path.join(SCRIPT_DIR, 'configs/dictionaries') @@ -160,10 +161,10 @@ def setup_module(module): for layout in LAYOUTS: for source in SOURCES: if source.compatible_with_layout(layout): - DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type])) + DICTIONARIES.append(get_dict(source, layout, FIELDS[layout.layout_type])) else: print "Source", source.name, "incompatible with layout", layout.name - + for layout in LAYOUTS: field_keys = list(filter(lambda x: x.is_key, FIELDS[layout.layout_type])) for source in SOURCES_KV: @@ -198,12 +199,23 @@ def started_cluster(): cluster.shutdown() -def test_simple_dictionaries(started_cluster): +def get_dictionaries(fold, total_folds, all_dicts): + chunk_len = int(math.ceil(len(all_dicts) / float(total_folds))) + if chunk_len * fold >= len(all_dicts): + return [] + return all_dicts[fold * chunk_len : (fold + 1) * chunk_len] + + +@pytest.mark.parametrize("fold", list(range(10))) +def test_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] values = VALUES["simple"] data = [Row(fields, vals) for vals in values] - simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"] + all_simple_dicts = [d for d in DICTIONARIES if d.structure.layout.layout_type == "simple"] + simple_dicts = get_dictionaries(fold, 10, all_simple_dicts) + + print "Length of dicts:", len(simple_dicts) for dct in simple_dicts: dct.load_data(data) @@ -295,12 +307,14 @@ def test_ranged_dictionaries(started_cluster): assert node.query(query) == str(answer) + '\n' -def test_key_value_simple_dictionaries(started_cluster): +@pytest.mark.parametrize("fold", list(range(10))) +def test_key_value_simple_dictionaries(started_cluster, fold): fields = FIELDS["simple"] values = VALUES["simple"] data = [Row(fields, vals) for vals in values] - simple_dicts = [d for d in DICTIONARIES_KV if d.structure.layout.layout_type == "simple"] + all_simple_dicts = [d for d in DICTIONARIES_KV if d.structure.layout.layout_type == "simple"] + simple_dicts = get_dictionaries(fold, 10, all_simple_dicts) for dct in simple_dicts: queries_with_answers = [] @@ -309,7 +323,7 @@ def test_key_value_simple_dictionaries(started_cluster): local_fields = dct.get_fields() local_values = [row.get_value_by_name(field.name) for field in local_fields if row.has_field(field.name)] local_data.append(Row(local_fields, local_values)) - + dct.load_data(local_data) node.query("system reload dictionary {}".format(dct.name)) @@ -368,7 +382,7 @@ def test_key_value_complex_dictionaries(started_cluster): local_fields = dct.get_fields() local_values = [row.get_value_by_name(field.name) for field in local_fields if row.has_field(field.name)] local_data.append(Row(local_fields, local_values)) - + dct.load_data(local_data) node.query("system reload dictionary {}".format(dct.name)) From a19c8b3d37630392be0ee4397d0195e941ef8fdd Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 28 Jan 2020 14:50:22 +0300 Subject: [PATCH 264/312] Update LimitTransform. --- dbms/src/Processors/LimitTransform.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Processors/LimitTransform.cpp b/dbms/src/Processors/LimitTransform.cpp index 266267d4e56..7bf61a2a9bc 100644 --- a/dbms/src/Processors/LimitTransform.cpp +++ b/dbms/src/Processors/LimitTransform.cpp @@ -76,7 +76,7 @@ LimitTransform::Status LimitTransform::prepare() if (!input.hasData()) return Status::NeedData; - current_chunk = input.pull(); + current_chunk = input.pull(true); has_block = true; auto rows = current_chunk.getNumRows(); @@ -95,6 +95,7 @@ LimitTransform::Status LimitTransform::prepare() } /// Now, we pulled from input, and it must be empty. + input.setNeeded(); return Status::NeedData; } @@ -114,6 +115,7 @@ LimitTransform::Status LimitTransform::prepare() } /// Now, we pulled from input, and it must be empty. + input.setNeeded(); return Status::NeedData; } From 06f5a274ec4af2db768faeed13c6d4b5dc3ff51f Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 28 Jan 2020 14:59:06 +0300 Subject: [PATCH 265/312] Fix initialization of QueryProfiler with regard to query context. --- dbms/src/Interpreters/ThreadStatusExt.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index e0124a0618c..a91abc29721 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -1,4 +1,5 @@ #include + #include #include #include @@ -7,11 +8,11 @@ #include #include -#if defined(__linux__) -#include -#include +#if defined(OS_LINUX) +# include -#include +# include +# include #endif @@ -37,10 +38,13 @@ void ThreadStatus::attachQueryContext(Context & query_context_) if (thread_group) { std::lock_guard lock(thread_group->mutex); + thread_group->query_context = query_context; if (!thread_group->global_context) thread_group->global_context = global_context; } + + initQueryProfiler(); } void CurrentThread::defaultThreadDeleter() @@ -77,6 +81,7 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) if (query_context) { query_id = query_context->getCurrentQueryId(); + initQueryProfiler(); #if defined(OS_LINUX) /// Set "nice" value if required. @@ -94,7 +99,6 @@ void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) } initPerformanceCounters(); - initQueryProfiler(); thread_state = ThreadState::AttachedToQuery; } From 27986b66587be2d88e38b143fef91ce86696069a Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Tue, 28 Jan 2020 15:01:45 +0300 Subject: [PATCH 266/312] Revert "Merge pull request #8831 from ClickHouse/revert-8820-issue-6964" This reverts commit a21d3712eb1dceea554db1877ba45500241363c0, reversing changes made to 667e9ed6d4f2cc6a6901187f3c82b9f4246aa920. --- dbms/src/Common/ThreadStatus.h | 3 + dbms/src/Interpreters/ThreadStatusExt.cpp | 68 +++++++++++------------ 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index 47c612c6ddf..30779b1610b 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -203,6 +203,9 @@ protected: /// Set to non-nullptr only if we have enough capabilities. std::unique_ptr taskstats_getter; + +private: + void setupState(const ThreadGroupStatusPtr & thread_group_); }; } diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp index df649c20ca6..d7f8829b016 100644 --- a/dbms/src/Interpreters/ThreadStatusExt.cpp +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -54,40 +54,11 @@ void CurrentThread::defaultThreadDeleter() current_thread->detachQuery(true, true); } -void ThreadStatus::initializeQuery() +void ThreadStatus::setupState(const ThreadGroupStatusPtr & thread_group_) { assertState({ThreadState::DetachedFromQuery}, __PRETTY_FUNCTION__); - thread_group = std::make_shared(); - - performance_counters.setParent(&thread_group->performance_counters); - memory_tracker.setParent(&thread_group->memory_tracker); - thread_group->memory_tracker.setDescription("(for query)"); - - thread_group->thread_numbers.emplace_back(thread_number); - thread_group->os_thread_ids.emplace_back(os_thread_id); - thread_group->master_thread_number = thread_number; - thread_group->master_thread_os_id = os_thread_id; - - initPerformanceCounters(); - thread_state = ThreadState::AttachedToQuery; -} - -void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached) -{ - if (thread_state == ThreadState::AttachedToQuery) - { - if (check_detached) - throw Exception("Can't attach query to the thread, it is already attached", ErrorCodes::LOGICAL_ERROR); - return; - } - - assertState({ThreadState::DetachedFromQuery}, __PRETTY_FUNCTION__); - - if (!thread_group_) - throw Exception("Attempt to attach to nullptr thread group", ErrorCodes::LOGICAL_ERROR); - - /// Attach current thread to thread group and copy useful information from it + /// Attach or init current thread to thread group and copy useful information from it thread_group = thread_group_; performance_counters.setParent(&thread_group->performance_counters); @@ -96,15 +67,15 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool { std::lock_guard lock(thread_group->mutex); + /// NOTE: thread may be attached multiple times if it is reused from a thread pool. + thread_group->thread_numbers.emplace_back(thread_number); + thread_group->os_thread_ids.emplace_back(os_thread_id); + logs_queue_ptr = thread_group->logs_queue_ptr; query_context = thread_group->query_context; if (!global_context) global_context = thread_group->global_context; - - /// NOTE: A thread may be attached multiple times if it is reused from a thread pool. - thread_group->thread_numbers.emplace_back(thread_number); - thread_group->os_thread_ids.emplace_back(os_thread_id); } if (query_context) @@ -112,7 +83,7 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool query_id = query_context->getCurrentQueryId(); initQueryProfiler(); -#if defined(__linux__) +#if defined(OS_LINUX) /// Set "nice" value if required. Int32 new_os_thread_priority = query_context->getSettingsRef().os_thread_priority; if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE)) @@ -132,6 +103,31 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool thread_state = ThreadState::AttachedToQuery; } +void ThreadStatus::initializeQuery() +{ + setupState(std::make_shared()); + + /// No need to lock on mutex here + thread_group->memory_tracker.setDescription("(for query)"); + thread_group->master_thread_number = thread_number; + thread_group->master_thread_os_id = os_thread_id; +} + +void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached) +{ + if (thread_state == ThreadState::AttachedToQuery) + { + if (check_detached) + throw Exception("Can't attach query to the thread, it is already attached", ErrorCodes::LOGICAL_ERROR); + return; + } + + if (!thread_group_) + throw Exception("Attempt to attach to nullptr thread group", ErrorCodes::LOGICAL_ERROR); + + setupState(thread_group_); +} + void ThreadStatus::finalizePerformanceCounters() { if (performance_counters_finalized) From ead37dc2947182dc9d96b0264eb56999089c86cd Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Tue, 28 Jan 2020 15:31:17 +0300 Subject: [PATCH 267/312] Fix comparison between decimal field and null. --- dbms/src/Common/FieldVisitors.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/FieldVisitors.h b/dbms/src/Common/FieldVisitors.h index 4f9e34f4969..17c092800fb 100644 --- a/dbms/src/Common/FieldVisitors.h +++ b/dbms/src/Common/FieldVisitors.h @@ -387,9 +387,9 @@ public: { if constexpr (isDecimalField()) return l < r; - else if constexpr (std::is_same_v || std::is_same_v) + if constexpr (std::is_same_v || std::is_same_v) return l < DecimalField(r, 0); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) return false; return cantCompare(l, r); } From 65356ea893e9d27b0411e2ab24026caff3304f4b Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Tue, 28 Jan 2020 15:33:25 +0300 Subject: [PATCH 268/312] And one more fix. --- dbms/src/Common/FieldVisitors.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/FieldVisitors.h b/dbms/src/Common/FieldVisitors.h index 17c092800fb..90f80974ab1 100644 --- a/dbms/src/Common/FieldVisitors.h +++ b/dbms/src/Common/FieldVisitors.h @@ -271,7 +271,7 @@ public: return l == r; if constexpr (std::is_same_v || std::is_same_v) return l == DecimalField(r, 0); - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) return false; return cantCompare(l, r); } From 5a8a367228371d21af41b204a8d8ed547633bc6a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 28 Jan 2020 15:59:34 +0300 Subject: [PATCH 269/312] Add processlist to PipelineExecutor. Throw exception if query was killed. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 9 ++++++++- dbms/src/Processors/Executors/PipelineExecutor.h | 6 +++++- dbms/src/Processors/QueryPipeline.cpp | 4 +++- dbms/src/Processors/QueryPipeline.h | 2 ++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 04031319873..0b79b37de27 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #if !defined(__APPLE__) && !defined(__FreeBSD__) #include @@ -33,12 +34,13 @@ static bool checkCanAddAdditionalInfoToException(const DB::Exception & exception && exception.code() != ErrorCodes::QUERY_WAS_CANCELLED; } -PipelineExecutor::PipelineExecutor(Processors & processors_) +PipelineExecutor::PipelineExecutor(Processors & processors_, QueryStatus * elem) : processors(processors_) , cancelled(false) , finished(false) , num_processing_executors(0) , expand_pipeline_task(nullptr) + , process_list_element(elem) { buildGraph(); } @@ -473,7 +475,12 @@ void PipelineExecutor::execute(size_t num_threads) } if (cancelled) + { + if (process_list_element && process_list_element->isKilled()) + throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); + return; + } bool all_processors_finished = true; for (auto & node : graph) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.h b/dbms/src/Processors/Executors/PipelineExecutor.h index e8abc5788e3..3ca5c670e7b 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.h +++ b/dbms/src/Processors/Executors/PipelineExecutor.h @@ -13,6 +13,7 @@ namespace DB { +class QueryStatus; /// Executes query pipeline. class PipelineExecutor @@ -24,7 +25,7 @@ public: /// During pipeline execution new processors can appear. They will be added to existing set. /// /// Explicit graph representation is built in constructor. Throws if graph is not correct. - explicit PipelineExecutor(Processors & processors_); + explicit PipelineExecutor(Processors & processors_, QueryStatus * elem = nullptr); /// Execute pipeline in multiple threads. Must be called once. /// In case of exception during execution throws any occurred. @@ -242,6 +243,9 @@ private: using ProcessorsMap = std::unordered_map; ProcessorsMap processors_map; + /// Now it's used to check if query was killed. + QueryStatus * process_list_element = nullptr; + /// Graph related methods. bool addEdges(UInt64 node); void buildGraph(); diff --git a/dbms/src/Processors/QueryPipeline.cpp b/dbms/src/Processors/QueryPipeline.cpp index ed62e2a1e58..7cccd83fee4 100644 --- a/dbms/src/Processors/QueryPipeline.cpp +++ b/dbms/src/Processors/QueryPipeline.cpp @@ -523,6 +523,8 @@ void QueryPipeline::setProgressCallback(const ProgressCallback & callback) void QueryPipeline::setProcessListElement(QueryStatus * elem) { + process_list_element = elem; + for (auto & processor : processors) { if (auto * source = dynamic_cast(processor.get())) @@ -630,7 +632,7 @@ PipelineExecutorPtr QueryPipeline::execute() if (!output_format) throw Exception("Cannot execute pipeline because it doesn't have output.", ErrorCodes::LOGICAL_ERROR); - return std::make_shared(processors); + return std::make_shared(processors, process_list_element); } } diff --git a/dbms/src/Processors/QueryPipeline.h b/dbms/src/Processors/QueryPipeline.h index 85fc7d04513..27e29264ea6 100644 --- a/dbms/src/Processors/QueryPipeline.h +++ b/dbms/src/Processors/QueryPipeline.h @@ -120,6 +120,8 @@ private: size_t max_threads = 0; + QueryStatus * process_list_element = nullptr; + void checkInitialized(); void checkSource(const ProcessorPtr & source, bool can_have_totals); From fea33f8b60da51e995e8719a7f1e25ce870ee3a6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 28 Jan 2020 16:16:02 +0300 Subject: [PATCH 270/312] Add processlist to PipelineExecutor. Throw exception if query was killed. --- dbms/src/Processors/Executors/PipelineExecutor.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbms/src/Processors/Executors/PipelineExecutor.cpp b/dbms/src/Processors/Executors/PipelineExecutor.cpp index 0b79b37de27..3ca796258c2 100644 --- a/dbms/src/Processors/Executors/PipelineExecutor.cpp +++ b/dbms/src/Processors/Executors/PipelineExecutor.cpp @@ -474,13 +474,11 @@ void PipelineExecutor::execute(size_t num_threads) throw; } - if (cancelled) - { - if (process_list_element && process_list_element->isKilled()) - throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); + if (process_list_element && process_list_element->isKilled()) + throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); + if (cancelled) return; - } bool all_processors_finished = true; for (auto & node : graph) From 4bdfd4ddcd2b2baca0e0d401c622c74f72068d35 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 16:40:23 +0300 Subject: [PATCH 271/312] Remove not merged feature --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdfaed101b1..b147eeeed5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,6 @@ * Add new `SEMI` and `ANTI JOIN`. Old `ANY INNER JOIN` behaviour now available as `SEMI LEFT JOIN`. [#7665](https://github.com/ClickHouse/ClickHouse/pull/7665) ([Artem Zuikov](https://github.com/4ertus2)) * Added `Distributed` format for `File` engine and `file` table function which allows to read from `.bin` files generated by asynchronous inserts into `Distributed` table. [#8535](https://github.com/ClickHouse/ClickHouse/pull/8535) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) * Add optional reset column argument for `runningAccumulate` which allows to reset aggregation results for each new key value. [#8326](https://github.com/ClickHouse/ClickHouse/pull/8326) ([Sergey Kononenko](https://github.com/kononencheg)) -* Add ability to alter materialized views with `ALTER MODIFY QUERY `. [#7533](https://github.com/ClickHouse/ClickHouse/pull/7533) ([nvartolomei](https://github.com/nvartolomei)) * Add ability to use ClickHouse as Prometheus endpoint. [#7900](https://github.com/ClickHouse/ClickHouse/pull/7900) ([vdimir](https://github.com/Vdimir)) * Add section `` in `config.xml` which restricts allowed hosts for remote table engines and table functions `URL`, `S3`, `HDFS`. [#7154](https://github.com/ClickHouse/ClickHouse/pull/7154) ([Mikhail Korotov](https://github.com/millb)) * Added function `greatCircleAngle` which calculates the distance on a sphere in degrees. [#8105](https://github.com/ClickHouse/ClickHouse/pull/8105) ([alexey-milovidov](https://github.com/alexey-milovidov)) From 55b81c97e6b13c4ac5f53da5eec7fb4ff5b875de Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei <6884440+dingxiangfei2009@users.noreply.github.com> Date: Tue, 28 Jan 2020 21:47:47 +0800 Subject: [PATCH 272/312] Set signal before rethrowing exception (#8736) * set signal before rethrowing exception * Add comment --- dbms/src/Common/ThreadPool.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbms/src/Common/ThreadPool.h b/dbms/src/Common/ThreadPool.h index 1b3071f732c..662d34afadd 100644 --- a/dbms/src/Common/ThreadPool.h +++ b/dbms/src/Common/ThreadPool.h @@ -151,10 +151,18 @@ public: func = std::forward(func), args = std::make_tuple(std::forward(args)...)] { + try { + /// Thread status holds raw pointer on query context, thus it always must be destroyed + /// before sending signal that permits to join this thread. DB::ThreadStatus thread_status; std::apply(func, args); } + catch (...) + { + state->set(); + throw; + } state->set(); }); } From 37daed283047d7957f12307c890f301d8dfea914 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 28 Jan 2020 17:39:24 +0300 Subject: [PATCH 273/312] Reformat stack traces for readability. --- dbms/src/Common/StackTrace.cpp | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/dbms/src/Common/StackTrace.cpp b/dbms/src/Common/StackTrace.cpp index e43bc4c287e..785a97e4d74 100644 --- a/dbms/src/Common/StackTrace.cpp +++ b/dbms/src/Common/StackTrace.cpp @@ -265,7 +265,19 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); - out << i << ". " << physical_addr << " "; + out << i << ". "; + + if (object) + { + if (std::filesystem::exists(object->name)) + { + auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; + + DB::Dwarf::LocationInfo location; + if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) + out << location.file.toString() << ":" << location.line << ": "; + } + } auto symbol = symbol_index.findSymbol(virtual_addr); if (symbol) @@ -276,22 +288,8 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs else out << "?"; - out << " "; - - if (object) - { - if (std::filesystem::exists(object->name)) - { - auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; - - DB::Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) - out << location.file.toString() << ":" << location.line; - } - out << " in " << object->name; - } - else - out << "?"; + out << " @ " << physical_addr; + out << " in " << (object ? object->name : "?"); callback(out.str()); out.str({}); From 62fdc6c8790470b26f9a07eefe01459de9cf15f2 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 28 Jan 2020 18:13:12 +0300 Subject: [PATCH 274/312] [wip] performance comparison --- docker/test/performance-comparison/compare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 4d69bd47f26..ea7d9e74786 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -151,4 +151,4 @@ right/clickhouse local --file '*-report.tsv' -S "$result_structure" --query "sel right/clickhouse local --file '*-client-time.tsv' -S "query text, client float, server float" -q "select client, server, floor(client/server, 3) p, query from table where p > 1.01 order by p desc" > slow-on-client.tsv grep Exception:[^:] *-err.log > run-errors.log -./report.py > report.html +$script_dir/report.py > report.html From 67dc8f24227304a07e43beed577a40e278b1ca26 Mon Sep 17 00:00:00 2001 From: Sergei Shtykov Date: Tue, 28 Jan 2020 18:27:44 +0300 Subject: [PATCH 275/312] Links fixed. --- docs/en/operations/server_settings/settings.md | 12 ++++++------ docs/en/operations/system_tables.md | 4 ++-- docs/en/operations/table_engines/distributed.md | 2 +- docs/en/operations/table_engines/join.md | 2 +- docs/en/operations/table_engines/mergetree.md | 2 +- docs/ru/operations/system_tables.md | 4 ++-- docs/ru/operations/table_engines/join.md | 2 +- docs/ru/operations/table_engines/mergetree.md | 2 +- docs/zh/operations/table_engines/join.md | 11 +++++++++++ docs/zh/operations/table_engines/kafka.md | 2 ++ docs/zh/operations/table_engines/mergetree.md | 4 +++- 11 files changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 550a84350d9..7766be91fd5 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -368,7 +368,7 @@ For more information, see the section "[Creating replicated tables](../../operat ## mark_cache_size {#server-mark-cache-size} -Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../../operations/table_engines/mergetree.md) family. +Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../table_engines/mergetree.md) family. The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120. @@ -420,7 +420,7 @@ We recommend using this option in Mac OS X, since the `getrlimit()` function ret Restriction on deleting tables. -If the size of a [MergeTree](../../operations/table_engines/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. +If the size of a [MergeTree](../table_engines/mergetree.md) table exceeds `max_table_size_to_drop` (in bytes), you can't delete it using a DROP query. If you still need to delete the table without restarting the ClickHouse server, create the `/flags/force_drop_table` file and run the DROP query. @@ -437,7 +437,7 @@ The value 0 means that you can delete all tables without any restrictions. ## merge_tree {#server_settings-merge_tree} -Fine tuning for tables in the [MergeTree](../../operations/table_engines/mergetree.md). +Fine tuning for tables in the [MergeTree](../table_engines/mergetree.md). For more information, see the MergeTreeSettings.h header file. @@ -512,7 +512,7 @@ Keys for server/client settings: ## part_log {#server_settings-part-log} -Logging events that are associated with [MergeTree](../../operations/table_engines/mergetree.md). For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. +Logging events that are associated with [MergeTree](../table_engines/mergetree.md). For instance, adding or merging data. You can use the log to simulate merge algorithms and compare their characteristics. You can visualize the merge process. Queries are logged in the [system.part_log](../system_tables.md#system_tables-part-log) table, not in a separate file. You can configure the name of this table in the `table` parameter (see below). @@ -739,7 +739,7 @@ Path to temporary data for processing large queries. ## tmp_policy {#server-settings-tmp_policy} -Policy from [`storage_configuration`](mergetree.md#table_engine-mergetree-multiple-volumes) to store temporary files. +Policy from [`storage_configuration`](../table_engines/mergetree.md#table_engine-mergetree-multiple-volumes) to store temporary files. If not set [`tmp_path`](#server-settings-tmp_path) is used, otherwise it is ignored. !!! note @@ -750,7 +750,7 @@ If not set [`tmp_path`](#server-settings-tmp_path) is used, otherwise it is igno ## uncompressed_cache_size {#server-settings-uncompressed_cache_size} -Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../../operations/table_engines/mergetree.md). +Cache size (in bytes) for uncompressed data used by table engines from the [MergeTree](../table_engines/mergetree.md). There is one shared cache for the server. Memory is allocated on demand. The cache is used if the option [use_uncompressed_cache](../settings/settings.md#setting-use_uncompressed_cache) is enabled. diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md index 563bd3864bd..a2fc6c44d06 100644 --- a/docs/en/operations/system_tables.md +++ b/docs/en/operations/system_tables.md @@ -805,9 +805,9 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') **See also** -- MergeTree family [query clauses](table_engines/mergetree/#sektsii-zaprosa) +- MergeTree family [query clauses](table_engines/mergetree.md#mergetree-query-clauses) - Kafka [settings](table_engines/kafka.md#table_engine-kafka-creating-a-table) -- Join [settings](table_engines/join/#limitations-and-settings) +- Join [settings](table_engines/join.md#join-limitations-and-settings) ## system.tables diff --git a/docs/en/operations/table_engines/distributed.md b/docs/en/operations/table_engines/distributed.md index 1e0145dda15..e26e83dbf5e 100644 --- a/docs/en/operations/table_engines/distributed.md +++ b/docs/en/operations/table_engines/distributed.md @@ -14,7 +14,7 @@ The Distributed engine accepts parameters: See also: - `insert_distributed_sync` setting - - [MergeTree](../mergetree.md#table_engine-mergetree-multiple-volumes) for the examples + - [MergeTree](mergetree.md#table_engine-mergetree-multiple-volumes) for the examples Example: diff --git a/docs/en/operations/table_engines/join.md b/docs/en/operations/table_engines/join.md index 4182ac936f8..385a64747c2 100644 --- a/docs/en/operations/table_engines/join.md +++ b/docs/en/operations/table_engines/join.md @@ -77,7 +77,7 @@ You cannot perform a `SELECT` query directly from the table. Instead, use one of - Place the table to the right side in a `JOIN` clause. - Call the [joinGet](../../query_language/functions/other_functions.md#other_functions-joinget) function, which lets you extract data from the table the same way as from a dictionary. -### Limitations and Settings +### Limitations and Settings {#join-limitations-and-settings} When creating a table, the following settings are applied: diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md index e9c9ec26c53..d2cf9e43857 100644 --- a/docs/en/operations/table_engines/mergetree.md +++ b/docs/en/operations/table_engines/mergetree.md @@ -50,7 +50,7 @@ For a description of parameters, see the [CREATE query description](../../query_ !!!note "Note" `INDEX` is an experimental feature, see [Data Skipping Indexes](#table_engine-mergetree-data_skipping-indexes). -### Query Clauses +### Query Clauses {#mergetree-query-clauses} - `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md index 086e079358b..c0a1de519bf 100644 --- a/docs/ru/operations/system_tables.md +++ b/docs/ru/operations/system_tables.md @@ -727,9 +727,9 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') **Смотрите также** -- [Секции движка](table_engines/mergetree/#sektsii-zaprosa) семейства MergeTree +- [Секции движка](table_engines/mergetree/#mergetree-query-clauses) семейства MergeTree - [Настройки](table_engines/kafka.md#table_engine-kafka-creating-a-table) Kafka -- [Настройки](table_engines/join/#limitations-and-settings) Join +- [Настройки](table_engines/join/#join-limitations-and-settings) Join ## system.tables diff --git a/docs/ru/operations/table_engines/join.md b/docs/ru/operations/table_engines/join.md index 75676bdfa0a..f10f2dd60b0 100644 --- a/docs/ru/operations/table_engines/join.md +++ b/docs/ru/operations/table_engines/join.md @@ -79,7 +79,7 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) - Используйте таблицу как правую в секции `JOIN`. - Используйте функцию [joinGet](../../query_language/functions/other_functions.md#other_functions-joinget), которая позволяет извлекать данные из таблицы таким же образом как из словаря. -### Ограничения и настройки +### Ограничения и настройки {#join-limitations-and-settings} При создании таблицы, применяются следующие параметры : diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md index 058e46eed99..f1815290064 100644 --- a/docs/ru/operations/table_engines/mergetree.md +++ b/docs/ru/operations/table_engines/mergetree.md @@ -49,7 +49,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] !!!note "Note" `INDEX` — экспериментальная возможность, смотрите [Индексы пропуска данных](#table_engine-mergetree-data_skipping-indexes). -### Секции запроса +### Секции запроса {#mergetree-query-clauses} - `ENGINE` — имя и параметры движка. `ENGINE = MergeTree()`. `MergeTree` не имеет параметров. diff --git a/docs/zh/operations/table_engines/join.md b/docs/zh/operations/table_engines/join.md index 9fabb7004af..d037ddbed9a 100644 --- a/docs/zh/operations/table_engines/join.md +++ b/docs/zh/operations/table_engines/join.md @@ -15,5 +15,16 @@ Join(ANY|ALL, LEFT|INNER, k1[, k2, ...]) 跟 Set 引擎类似，Join 引擎把数据存储在磁盘中。 +### Limitations and Settings {#join-limitations-and-settings} + +When creating a table, the following settings are applied: + +- join_use_nulls +- max_rows_in_join +- max_bytes_in_join +- join_overflow_mode +- join_any_take_last_row + +The `Join`-engine tables can't be used in `GLOBAL JOIN` operations. [来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/join/) diff --git a/docs/zh/operations/table_engines/kafka.md b/docs/zh/operations/table_engines/kafka.md index 22e001f2c1c..cda367793bb 100644 --- a/docs/zh/operations/table_engines/kafka.md +++ b/docs/zh/operations/table_engines/kafka.md @@ -8,6 +8,8 @@ Kafka 特性： - 容错存储机制。 - 处理流数据。 + + 老版格式： ``` diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md index fc7b4967571..2beeca1a9cb 100644 --- a/docs/zh/operations/table_engines/mergetree.md +++ b/docs/zh/operations/table_engines/mergetree.md @@ -46,6 +46,8 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] 请求参数的描述，参考 [请求描述](../../query_language/create.md) 。 + + **子句** - `ENGINE` - 引擎名和参数。 `ENGINE = MergeTree()`. `MergeTree` 引擎没有参数。 @@ -270,7 +272,7 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### 索引的可用类型 +#### 索引的可用类型 {#table_engine-mergetree-data_skipping-indexes} * `minmax` 存储指定表达式的极值（如果表达式是 `tuple` ，则存储 `tuple` 中每个元素的极值），这些信息用于跳过数据块，类似主键。 From 196bb81a6eb9abcc0fa19eef1338b880caccd1bf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 28 Jan 2020 19:59:28 +0300 Subject: [PATCH 276/312] Update TreeExecutor. --- .../Executors/TreeExecutorBlockInputStream.cpp | 9 +++++++++ .../Processors/Executors/TreeExecutorBlockInputStream.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp index 0522e7a5323..ee482d62f27 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace DB @@ -152,6 +153,12 @@ void TreeExecutorBlockInputStream::execute() case IProcessor::Status::Ready: { node->work(); + + /// This is handled inside PipelineExecutor now, + /// and doesn't checked by processors as in IInputStream before. + if (process_list_element && process_list_element->isKilled()) + throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED); + break; } case IProcessor::Status::Async: @@ -188,6 +195,8 @@ void TreeExecutorBlockInputStream::setProgressCallback(const ProgressCallback & void TreeExecutorBlockInputStream::setProcessListElement(QueryStatus * elem) { + process_list_element = elem; + for (auto & source : sources_with_progress) source->setProcessListElement(elem); } diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h index 176fbd06af8..5ed6436c660 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h @@ -46,6 +46,8 @@ private: /// Remember sources that support progress. std::vector sources_with_progress; + QueryStatus * process_list_element = nullptr; + void init(); /// Execute tree step-by-step until root returns next chunk or execution is finished. void execute(); From 3a5de8b8746ac7e60c22d35a3f4b4d1cb60e7f6c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 28 Jan 2020 19:59:32 +0300 Subject: [PATCH 277/312] Update script for CHYT --- dbms/benchmark/clickhouse/benchmark-chyt.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/benchmark/clickhouse/benchmark-chyt.sh b/dbms/benchmark/clickhouse/benchmark-chyt.sh index efc790a029a..23f22024c74 100755 --- a/dbms/benchmark/clickhouse/benchmark-chyt.sh +++ b/dbms/benchmark/clickhouse/benchmark-chyt.sh @@ -9,7 +9,10 @@ cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do echo -n "[" for i in $(seq 1 $TRIES); do while true; do - RES=$(command time -f %e -o /dev/stdout curl -sS --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query?default_format=Null&database=*$YT_CLIQUE_ID" --data-binary @- <<< "$query" 2>/dev/null) && break; + RES=$(command time -f %e -o /dev/stdout curl -sS --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query?default_format=Null&max_memory_usage=100000000000&max_memory_usage_for_all_queries=100000000000&max_concurrent_queries_for_user=100&database=*$YT_CLIQUE_ID" --data-binary @- <<< "$query" 2>/dev/null); + if [[ $? == 0 ]]; then + [[ $RES =~ 'fail|exception' ]] || break; + fi done [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" From cebeae005504365b0aa486287d1a18658fe88034 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Tue, 28 Jan 2020 20:21:57 +0300 Subject: [PATCH 278/312] Fixed awk invokation in hung queries check --- dbms/tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index acac89003ee..7d3d65defc6 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -107,7 +107,7 @@ def get_stacktraces(server_pid): def get_server_pid(server_tcp_port): - cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | gawk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) + cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) try: output = subprocess.check_output(cmd, shell=True) if output: From 7970067ecf49b263035b14f5747dc61c24004b42 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 28 Jan 2020 21:42:33 +0300 Subject: [PATCH 279/312] [wip] performance comparison --- docker/test/performance-comparison/report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 64461ba0587..c116f5d16f0 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -87,15 +87,15 @@ params['header'] = "ClickHouse Performance Comparison" params['test_part'] = (table_template.format_map( collections.defaultdict(str, caption = 'Changes in performance', - header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']), + header = table_header(['Old, s', 'New, s', 'Relative difference (new - old)/old', 'Randomization distribution quantiles [5%, 50%, 95%]', 'Query']), rows = tsv_rows('changed-perf.tsv'))) + table_template.format( caption = 'Slow on client', - header = table_header(['Client', 'Server', 'Ratio', 'Query']), + header = table_header(['Client time, s', 'Server time, s', 'Ratio', 'Query']), rows = tsv_rows('slow-on-client.tsv')) + table_template.format( caption = 'Unstable', - header = table_header(['Left', 'Right', 'Diff', 'RD', 'Query']), + header = table_header(['Old, s', 'New, s', 'Relative difference (new - old)/old', 'Randomization distribution quantiles [5%, 50%, 95%]', 'Query']), rows = tsv_rows('unstable.tsv')) + table_template.format( caption = 'Run errors', From 2ff8f7de1ae441e4663bf5798a3ffa7d95532fd7 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 28 Jan 2020 21:51:44 +0300 Subject: [PATCH 280/312] Try alternative website deploy destination --- docs/tools/release.sh | 12 ++++++++++++ website/index.html | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 0627b98f3e1..d8f378771fd 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -3,6 +3,7 @@ set -ex BASE_DIR=$(dirname $(readlink -f $0)) BUILD_DIR="${BASE_DIR}/../build" +PUBLISH_DIR="${BASE_DIR}/../publish" IMAGE="clickhouse/website" if [[ -z "$1" ]] then @@ -17,6 +18,17 @@ if [[ -z "$1" ]] then source "${BASE_DIR}/venv/bin/activate" python "${BASE_DIR}/build.py" "--enable-stable-releases" + set +x + rm -rf "${PUBLISH_DIR}" || true + git clone git@github.com:ClickHouse/clickhouse.github.io.git "${PUBLISH_DIR}" + cd "${PUBLISH_DIR}" + git rm -rf * + git commit -a -m "wipe old release" + cp -R "${BUILD_DIR}"/* . + git add * + git commit -a -m "add new release at $(date)" + git push origin master + set -x cd "${BUILD_DIR}" docker build -t "${FULL_NAME}" "${BUILD_DIR}" docker tag "${FULL_NAME}" "${REMOTE_NAME}" diff --git a/website/index.html b/website/index.html index fa9abdda140..2e4298e7e2d 100644 --- a/website/index.html +++ b/website/index.html @@ -567,7 +567,7 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh }); var hostParts = window.location.host.split('.'); - if (hostParts.length > 2 && hostParts[0] != 'test') { + if (hostParts.length > 2 && hostParts[0] != 'test' && hostParts[1] != 'github') { window.location.host = hostParts[0] + '.' + hostParts[1]; } From 3144e81cb56be1eb57a6f148139e3db519b4b7fd Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 28 Jan 2020 21:54:25 +0300 Subject: [PATCH 281/312] fix test prefer_localhost_replica --- .../test_insert_into_distributed/test.py | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/dbms/tests/integration/test_insert_into_distributed/test.py b/dbms/tests/integration/test_insert_into_distributed/test.py index 8d656cef3ea..b9767e8b694 100644 --- a/dbms/tests/integration/test_insert_into_distributed/test.py +++ b/dbms/tests/integration/test_insert_into_distributed/test.py @@ -162,35 +162,48 @@ def test_inserts_local(started_cluster): assert instance.query("SELECT count(*) FROM local").strip() == '1' def test_prefer_localhost_replica(started_cluster): - test_query = "SELECT * FROM distributed ORDER BY id;" + test_query = "SELECT * FROM distributed ORDER BY id" + node1.query("INSERT INTO distributed VALUES (toDate('2017-06-17'), 11)") node2.query("INSERT INTO distributed VALUES (toDate('2017-06-17'), 22)") time.sleep(1.0) + expected_distributed = '''\ 2017-06-17\t11 2017-06-17\t22 ''' - assert TSV(node1.query(test_query)) == TSV(expected_distributed) - assert TSV(node2.query(test_query)) == TSV(expected_distributed) - with PartitionManager() as pm: - pm.partition_instances(node1, node2, action='REJECT --reject-with tcp-reset') - node1.query("INSERT INTO replicated VALUES (toDate('2017-06-17'), 33)") - node2.query("INSERT INTO replicated VALUES (toDate('2017-06-17'), 44)") - time.sleep(1.0) + expected_from_node2 = '''\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t44 ''' - # Query is sent to node2, as it local and prefer_localhost_replica=1 - assert TSV(node2.query(test_query)) == TSV(expected_from_node2) + expected_from_node1 = '''\ 2017-06-17\t11 2017-06-17\t22 2017-06-17\t33 ''' + + assert TSV(node1.query(test_query)) == TSV(expected_distributed) + assert TSV(node2.query(test_query)) == TSV(expected_distributed) + + # Make replicas inconsistent by disabling merges and fetches + # for possibility of determining to which replica the query was send + node1.query("SYSTEM STOP MERGES") + node1.query("SYSTEM STOP FETCHES") + node2.query("SYSTEM STOP MERGES") + node2.query("SYSTEM STOP FETCHES") + + node1.query("INSERT INTO replicated VALUES (toDate('2017-06-17'), 33)") + node2.query("INSERT INTO replicated VALUES (toDate('2017-06-17'), 44)") + time.sleep(1.0) + + # Query is sent to node2, as it local and prefer_localhost_replica=1 + assert TSV(node2.query(test_query)) == TSV(expected_from_node2) + # Now query is sent to node1, as it higher in order - assert TSV(node2.query("SET load_balancing='in_order'; SET prefer_localhost_replica=0;" + test_query)) == TSV(expected_from_node1) + assert TSV(node2.query(test_query + " SETTINGS load_balancing='in_order', prefer_localhost_replica=0")) == TSV(expected_from_node1) def test_inserts_low_cardinality(started_cluster): instance = shard1 From 569e1ff6662046113d30eee6cb2d5d993113766b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 28 Jan 2020 22:17:30 +0300 Subject: [PATCH 282/312] Update testing script --- dbms/benchmark/clickhouse/benchmark-chyt.sh | 4 ++-- dbms/benchmark/clickhouse/queries.sql | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbms/benchmark/clickhouse/benchmark-chyt.sh b/dbms/benchmark/clickhouse/benchmark-chyt.sh index 23f22024c74..778ce4f86ce 100755 --- a/dbms/benchmark/clickhouse/benchmark-chyt.sh +++ b/dbms/benchmark/clickhouse/benchmark-chyt.sh @@ -9,9 +9,9 @@ cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do echo -n "[" for i in $(seq 1 $TRIES); do while true; do - RES=$(command time -f %e -o /dev/stdout curl -sS --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query?default_format=Null&max_memory_usage=100000000000&max_memory_usage_for_all_queries=100000000000&max_concurrent_queries_for_user=100&database=*$YT_CLIQUE_ID" --data-binary @- <<< "$query" 2>/dev/null); + RES=$(command time -f %e -o /dev/stdout curl -sS -G --data-urlencode "query=$query" --data "default_format=Null&max_memory_usage=100000000000&max_memory_usage_for_all_queries=100000000000&max_concurrent_queries_for_user=100&database=*$YT_CLIQUE_ID" --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query" 2>/dev/null); if [[ $? == 0 ]]; then - [[ $RES =~ 'fail|exception' ]] || break; + [[ $RES =~ 'fail|Exception' ]] || break; fi done diff --git a/dbms/benchmark/clickhouse/queries.sql b/dbms/benchmark/clickhouse/queries.sql index 726beebb064..89c4616c642 100644 --- a/dbms/benchmark/clickhouse/queries.sql +++ b/dbms/benchmark/clickhouse/queries.sql @@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM SELECT URL, count() AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10; SELECT 1, URL, count() AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10; SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM {table} GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10; -SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; -SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10; -SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; -SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; -SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; -SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; -SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; +SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000; +SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000; +SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100; +SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000; +SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute; From 3aeea41369c864fe12b1b5b114561982d2c8e258 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 28 Jan 2020 22:25:43 +0300 Subject: [PATCH 283/312] Add system.merge_tree_settings into clickhouse-client suggest --- dbms/programs/client/Suggest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/programs/client/Suggest.cpp b/dbms/programs/client/Suggest.cpp index 65487e07a93..e3e2b8b5b63 100644 --- a/dbms/programs/client/Suggest.cpp +++ b/dbms/programs/client/Suggest.cpp @@ -65,6 +65,8 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo " UNION ALL " "SELECT name FROM system.data_type_families" " UNION ALL " + "SELECT name FROM system.merge_tree_settings" + " UNION ALL " "SELECT name FROM system.settings" " UNION ALL " "SELECT cluster FROM system.clusters" From e05c253b5a373fda7137d0385515664b2aa9ffac Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 28 Jan 2020 22:31:35 +0300 Subject: [PATCH 284/312] Update release.sh --- docs/tools/release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index d8f378771fd..9220e9406a3 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -25,6 +25,8 @@ then git rm -rf * git commit -a -m "wipe old release" cp -R "${BUILD_DIR}"/* . + echo -n "clickhouse.tech" > CNAME + cp "${BASE_DIR}/../../LICENSE" . git add * git commit -a -m "add new release at $(date)" git push origin master From fbf04cb63043feaface294bad3818122579159e0 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 28 Jan 2020 22:47:58 +0300 Subject: [PATCH 285/312] Update release.sh --- docs/tools/release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 9220e9406a3..3865e4237ee 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -26,6 +26,7 @@ then git commit -a -m "wipe old release" cp -R "${BUILD_DIR}"/* . echo -n "clickhouse.tech" > CNAME + echo -n "" > README.md cp "${BASE_DIR}/../../LICENSE" . git add * git commit -a -m "add new release at $(date)" From 57ac725410386eddbc29f5ba8a8a899bfb44a58d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 28 Jan 2020 22:49:48 +0300 Subject: [PATCH 286/312] Update index.html --- website/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/index.html b/website/index.html index 2e4298e7e2d..f376e55b84e 100644 --- a/website/index.html +++ b/website/index.html @@ -32,7 +32,7 @@ Quick Start Performance Documentation - Blog + Blog Contacts From b650f1563abd19894c70f2e68c5bf5e29dfb4819 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 28 Jan 2020 05:57:44 +0800 Subject: [PATCH 287/312] Fix `MakeSet` in Group By --- dbms/src/Interpreters/ActionsVisitor.cpp | 2 +- dbms/src/Interpreters/ActionsVisitor.h | 4 +++- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 19 +++++++++++++++---- dbms/src/Interpreters/ExpressionAnalyzer.h | 6 ++++++ .../01069_set_in_group_by.reference | 1 + .../0_stateless/01069_set_in_group_by.sql | 9 +++++++++ 6 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01069_set_in_group_by.reference create mode 100644 dbms/tests/queries/0_stateless/01069_set_in_group_by.sql diff --git a/dbms/src/Interpreters/ActionsVisitor.cpp b/dbms/src/Interpreters/ActionsVisitor.cpp index 9fe6e218746..4bef41f3c3e 100644 --- a/dbms/src/Interpreters/ActionsVisitor.cpp +++ b/dbms/src/Interpreters/ActionsVisitor.cpp @@ -330,7 +330,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// Let's find the type of the first argument (then getActionsImpl will be called again and will not affect anything). visit(node.arguments->children.at(0), data); - if ((prepared_set = makeSet(node, data, data.no_subqueries))) + if (!data.no_makeset && (prepared_set = makeSet(node, data, data.no_subqueries))) { /// Transform tuple or subquery into a set. } diff --git a/dbms/src/Interpreters/ActionsVisitor.h b/dbms/src/Interpreters/ActionsVisitor.h index f48ccc1d88e..b850c615655 100644 --- a/dbms/src/Interpreters/ActionsVisitor.h +++ b/dbms/src/Interpreters/ActionsVisitor.h @@ -72,6 +72,7 @@ public: PreparedSets & prepared_sets; SubqueriesForSets & subqueries_for_sets; bool no_subqueries; + bool no_makeset; bool only_consts; bool no_storage_or_local; size_t visit_depth; @@ -80,7 +81,7 @@ public: Data(const Context & context_, SizeLimits set_size_limit_, size_t subquery_depth_, const NamesAndTypesList & source_columns_, const ExpressionActionsPtr & actions, PreparedSets & prepared_sets_, SubqueriesForSets & subqueries_for_sets_, - bool no_subqueries_, bool only_consts_, bool no_storage_or_local_) + bool no_subqueries_, bool no_makeset_, bool only_consts_, bool no_storage_or_local_) : context(context_), set_size_limit(set_size_limit_), subquery_depth(subquery_depth_), @@ -88,6 +89,7 @@ public: prepared_sets(prepared_sets_), subqueries_for_sets(subqueries_for_sets_), no_subqueries(no_subqueries_), + no_makeset(no_makeset_), only_consts(only_consts_), no_storage_or_local(no_storage_or_local_), visit_depth(0), diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 950edba96f2..c6c6f08f815 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -122,7 +122,7 @@ void ExpressionAnalyzer::analyzeAggregation() ASTPtr array_join_expression_list = select_query->array_join_expression_list(is_array_join_left); if (array_join_expression_list) { - getRootActions(array_join_expression_list, true, temp_actions); + getRootActionsNoMakeSet(array_join_expression_list, true, temp_actions, false); addMultipleArrayJoinAction(temp_actions, is_array_join_left); array_join_columns.clear(); @@ -134,7 +134,7 @@ void ExpressionAnalyzer::analyzeAggregation() const ASTTablesInSelectQueryElement * join = select_query->join(); if (join) { - getRootActions(analyzedJoin().leftKeysList(), true, temp_actions); + getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), true, temp_actions, false); addJoinAction(temp_actions); } } @@ -155,7 +155,7 @@ void ExpressionAnalyzer::analyzeAggregation() for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i) { ssize_t size = group_asts.size(); - getRootActions(group_asts[i], true, temp_actions); + getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); const auto & column_name = group_asts[i]->getColumnName(); const auto & block = temp_actions->getSampleBlock(); @@ -340,7 +340,18 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_subqueries, LogAST log; ActionsVisitor::Data visitor_data(context, settings.size_limits_for_set, subquery_depth, sourceColumns(), actions, prepared_sets, subqueries_for_sets, - no_subqueries, only_consts, !isRemoteStorage()); + no_subqueries, false, only_consts, !isRemoteStorage()); + ActionsVisitor(visitor_data, log.stream()).visit(ast); + visitor_data.updateActions(actions); +} + + +void ExpressionAnalyzer::getRootActionsNoMakeSet(const ASTPtr & ast, bool no_subqueries, ExpressionActionsPtr & actions, bool only_consts) +{ + LogAST log; + ActionsVisitor::Data visitor_data(context, settings.size_limits_for_set, subquery_depth, + sourceColumns(), actions, prepared_sets, subqueries_for_sets, + no_subqueries, true, only_consts, !isRemoteStorage()); ActionsVisitor(visitor_data, log.stream()).visit(ast); visitor_data.updateActions(actions); } diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index 2de2d2bfd18..0b077901c66 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -134,6 +134,12 @@ protected: void getRootActions(const ASTPtr & ast, bool no_subqueries, ExpressionActionsPtr & actions, bool only_consts = false); + /** Similar to getRootActions but do not make sets when analyzing IN functions. It's used in + * analyzeAggregation which happens earlier than analyzing PREWHERE and WHERE. If we did, the + * prepared sets would not be applicable for MergeTree index optimization. + */ + void getRootActionsNoMakeSet(const ASTPtr & ast, bool no_subqueries, ExpressionActionsPtr & actions, bool only_consts = false); + /** Add aggregation keys to aggregation_keys, aggregate functions to aggregate_descriptions, * Create a set of columns aggregated_columns resulting after the aggregation, if any, * or after all the actions that are normally performed before aggregation. diff --git a/dbms/tests/queries/0_stateless/01069_set_in_group_by.reference b/dbms/tests/queries/0_stateless/01069_set_in_group_by.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_set_in_group_by.reference @@ -0,0 +1 @@ +0 diff --git a/dbms/tests/queries/0_stateless/01069_set_in_group_by.sql b/dbms/tests/queries/0_stateless/01069_set_in_group_by.sql new file mode 100644 index 00000000000..5884fd0e134 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01069_set_in_group_by.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS testmt; + +CREATE TABLE testmt (`CounterID` UInt64, `value` String) ENGINE = MergeTree() ORDER BY CounterID; + +INSERT INTO testmt VALUES (1, '1'), (2, '2'); + +SELECT arrayJoin([CounterID NOT IN (2)]) AS counter FROM testmt WHERE CounterID IN (2) GROUP BY counter; + +DROP TABLE testmt; From 2d87ee822c473819cee643f8917e1a339b4aee9b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jan 2020 11:47:16 +0300 Subject: [PATCH 288/312] Add missed lines in changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b147eeeed5b..7a8a2c93722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## ClickHouse release v20.1 +### ClickHouse release v20.1.2.4, 2020-01-22 + ### Backward Incompatible Change * Make the setting `merge_tree_uniform_read_distribution` obsolete. The server still recognizes this setting but it has no effect. [#8308](https://github.com/ClickHouse/ClickHouse/pull/8308) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Changed return type of the function `greatCircleDistance` to `Float32` because now the result of calculation is `Float32`. [#7993](https://github.com/ClickHouse/ClickHouse/pull/7993) ([alexey-milovidov](https://github.com/alexey-milovidov)) @@ -7,6 +9,7 @@ * Enable `use_minimalistic_part_header_in_zookeeper` setting for `ReplicatedMergeTree` by default. This will significantly reduce amount of data stored in ZooKeeper. This setting is supported since version 19.1 and we already use it in production in multiple services without any issues for more than half a year. Disable this setting if you have a chance to downgrade to versions older than 19.1. [#6850](https://github.com/ClickHouse/ClickHouse/pull/6850) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Data skipping indices are production ready and enabled by default. The settings `allow_experimental_data_skipping_indices`, `allow_experimental_cross_to_join_conversion` and `allow_experimental_multiple_joins_emulation` are now obsolete and do nothing. [#7974](https://github.com/ClickHouse/ClickHouse/pull/7974) ([alexey-milovidov](https://github.com/alexey-milovidov)) * Add new `ANY JOIN` logic for `StorageJoin` consistent with `JOIN` operation. To upgrade without changes in behaviour you need add `SETTINGS any_join_distinct_right_table_keys = 1` to Engine Join tables metadata or recreate these tables after upgrade. [#8400](https://github.com/ClickHouse/ClickHouse/pull/8400) ([Artem Zuikov](https://github.com/4ertus2)) +* Require server to be restarted to apply the changes in logging configuration. This is a temporary workaround to avoid the bug where the server logs to a deleted log file (see [#8696](https://github.com/ClickHouse/ClickHouse/issues/8696)). [#8707](https://github.com/ClickHouse/ClickHouse/pull/8707) ([Alexander Kuzmenkov](https://github.com/akuzm)) ### New Feature * Added information about part paths to `system.merges`. [#8043](https://github.com/ClickHouse/ClickHouse/pull/8043) ([Vladimir Chebotarev](https://github.com/excitoon)) From a8e7f864e347835081e252f8f01f97512d06c5c3 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 12:42:06 +0300 Subject: [PATCH 289/312] Update release.sh --- docs/tools/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 3865e4237ee..d23463503d5 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -18,7 +18,7 @@ if [[ -z "$1" ]] then source "${BASE_DIR}/venv/bin/activate" python "${BASE_DIR}/build.py" "--enable-stable-releases" - set +x + set +e rm -rf "${PUBLISH_DIR}" || true git clone git@github.com:ClickHouse/clickhouse.github.io.git "${PUBLISH_DIR}" cd "${PUBLISH_DIR}" @@ -31,7 +31,7 @@ then git add * git commit -a -m "add new release at $(date)" git push origin master - set -x + set -e cd "${BUILD_DIR}" docker build -t "${FULL_NAME}" "${BUILD_DIR}" docker tag "${FULL_NAME}" "${REMOTE_NAME}" From 3038278a14ec0aa0daca975e78b76647c3ed2d8a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 12:43:34 +0300 Subject: [PATCH 290/312] Update roadmap.md --- docs/en/roadmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md index f8cbf1c17c0..1628f046c6f 100644 --- a/docs/en/roadmap.md +++ b/docs/en/roadmap.md @@ -2,8 +2,8 @@ ## Q1 2020 -- Resource pools for more precise distribution of cluster capacity between users -- Fine-grained authorization +- Role-based access control - Integration with external authentication services +- Resource pools for more precise distribution of cluster capacity between users [Original article](https://clickhouse.yandex/docs/en/roadmap/) From 74d1eb90dfda4be5bceb621fd8a6fa4696d5eb6c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 29 Jan 2020 13:26:08 +0300 Subject: [PATCH 291/312] Automatically call progress for SourceWithProgress. --- dbms/src/Processors/Sources/SourceWithProgress.cpp | 9 +++++++++ dbms/src/Processors/Sources/SourceWithProgress.h | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/dbms/src/Processors/Sources/SourceWithProgress.cpp b/dbms/src/Processors/Sources/SourceWithProgress.cpp index e7964ef81f3..6b62123933c 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.cpp +++ b/dbms/src/Processors/Sources/SourceWithProgress.cpp @@ -18,13 +18,22 @@ void SourceWithProgress::work() if (!limits.speed_limits.checkTimeLimit(total_stopwatch.elapsed(), limits.timeout_overflow_mode)) cancel(); else + { + was_progress_called = false; + ISourceWithProgress::work(); + + if (!was_progress_called && has_input) + progress({ current_chunk.chunk.getNumRows(), current_chunk.chunk.bytes() }); + } } /// Aggregated copy-paste from IBlockInputStream::progressImpl. /// Most of this must be done in PipelineExecutor outside. Now it's done for compatibility with IBlockInputStream. void SourceWithProgress::progress(const Progress & value) { + was_progress_called = true; + if (total_rows_approx != 0) { Progress total_rows_progress = {0, 0, total_rows_approx}; diff --git a/dbms/src/Processors/Sources/SourceWithProgress.h b/dbms/src/Processors/Sources/SourceWithProgress.h index abc96cbbe2f..8a508c59acf 100644 --- a/dbms/src/Processors/Sources/SourceWithProgress.h +++ b/dbms/src/Processors/Sources/SourceWithProgress.h @@ -72,6 +72,10 @@ private: Stopwatch total_stopwatch {CLOCK_MONOTONIC_COARSE}; /// Time with waiting time. /// According to total_stopwatch in microseconds. UInt64 last_profile_events_update_time = 0; + + /// This flag checks if progress() was manually called at generate() call. + /// If not, it will be called for chunk after generate() was finished. + bool was_progress_called = false; }; } From 7940cab38103cccf70f3d575e39122cb0adfca4d Mon Sep 17 00:00:00 2001 From: Ilya Mazaev Date: Wed, 29 Jan 2020 15:59:48 +0300 Subject: [PATCH 292/312] Update dockerfile for server fix issue#6608 --- docker/server/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 84b4ed0a2b6..fb41984f224 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -33,6 +33,7 @@ RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +ENV TZ UTC RUN mkdir /docker-entrypoint-initdb.d From 8e7fdfc3f0c4b85e4f080bd8f3155c21480c74ba Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 16:34:12 +0300 Subject: [PATCH 293/312] Try to add some parallelism to docs build --- docs/tools/build.py | 5 +++-- docs/tools/github.py | 32 ++++++++++++++++++++------------ docs/tools/util.py | 12 ++++++++++++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index d6762a708d2..47a69ff2279 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -11,7 +11,6 @@ import subprocess import sys import time -import markdown.extensions import markdown.util from mkdocs import config @@ -212,8 +211,10 @@ def build_redirects(args): def build_docs(args): + tasks = [] for lang in args.lang.split(','): - build_for_lang(lang, args) + tasks.append((lang, args,)) + util.run_function_in_parallel(build_for_lang, tasks, threads=True) def build(args): diff --git a/docs/tools/github.py b/docs/tools/github.py index ecf30c6363c..e407eff9cca 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -3,6 +3,7 @@ import copy import io import logging import os +import sys import tarfile import requests @@ -18,17 +19,22 @@ def choose_latest_releases(): candidates += requests.get(url).json() for tag in candidates: - name = tag.get('name', '') - is_unstable = ('stable' not in name) and ('lts' not in name) - is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name) - if is_unstable or is_in_blacklist: - continue - major_version = '.'.join((name.split('.', 2))[:2]) - if major_version not in seen: - seen[major_version] = (name, tag.get('tarball_url'),) - if len(seen) > 10: - break - + if isinstance(tag, dict): + name = tag.get('name', '') + is_unstable = ('stable' not in name) and ('lts' not in name) + is_in_blacklist = ('v18' in name) or ('prestable' in name) or ('v1.1' in name) + if is_unstable or is_in_blacklist: + continue + major_version = '.'.join((name.split('.', 2))[:2]) + if major_version not in seen: + seen[major_version] = (name, tag.get('tarball_url'),) + if len(seen) > 10: + break + else: + logging.fatal('Unexpected GitHub response: %s', str(candidates)) + sys.exit(1) + + logging.info('Found stable releases: %s', str(seen.keys())) return seen.items() @@ -47,7 +53,9 @@ def process_release(args, callback, release): def build_releases(args, callback): + tasks = [] for release in args.stable_releases: - process_release(args, callback, release) + tasks.append((args, callback, release,)) + util.run_function_in_parallel(process_release, tasks) diff --git a/docs/tools/util.py b/docs/tools/util.py index 27960231e8c..aad7364594d 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -1,7 +1,9 @@ import contextlib +import multiprocessing import os import shutil import tempfile +import threading @contextlib.contextmanager @@ -20,3 +22,13 @@ def autoremoved_file(path): yield handle finally: os.unlink(path) + + +def run_function_in_parallel(func, args_list, threads=False): + processes = [] + for task in args_list: + cls = threading.Thread if threads else multiprocessing.Process + processes.append(cls(target=func, args=task)) + processes[-1].start() + for process in processes: + process.join() From 666b18dca468d412f2574c97cac07d447f6a9e6a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 16:36:13 +0300 Subject: [PATCH 294/312] add git config --- docs/tools/release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index d23463503d5..0fa6bd70909 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -22,6 +22,8 @@ then rm -rf "${PUBLISH_DIR}" || true git clone git@github.com:ClickHouse/clickhouse.github.io.git "${PUBLISH_DIR}" cd "${PUBLISH_DIR}" + git config user.email "robot-clickhouse@yandex-team.ru" + git config user.name "robot-clickhouse" git rm -rf * git commit -a -m "wipe old release" cp -R "${BUILD_DIR}"/* . From 975f042dc0a61f4073ba68f37c234f0053e5cd43 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 29 Jan 2020 17:15:53 +0300 Subject: [PATCH 295/312] Report BAD_ARGUMENTS for a wrong user-supplied enum value. Before we reported LOGICAL_ERROR that is incorrect. --- dbms/src/DataTypes/DataTypeEnum.h | 6 +++--- dbms/tests/queries/0_stateless/00453_cast_enum.sql | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index 2e5ba74b3e6..2cb677984df 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -13,7 +13,7 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } @@ -70,7 +70,7 @@ public: { const auto it = value_to_name_map.find(value); if (it == std::end(value_to_name_map)) - throw Exception{"Unexpected value " + toString(value) + " for type " + getName(), ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Unexpected value " + toString(value) + " for type " + getName(), ErrorCodes::BAD_ARGUMENTS}; return it->second; } @@ -79,7 +79,7 @@ public: { const auto it = name_to_value_map.find(field_name); if (!it) - throw Exception{"Unknown element '" + field_name.toString() + "' for type " + getName(), ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Unknown element '" + field_name.toString() + "' for type " + getName(), ErrorCodes::BAD_ARGUMENTS}; return it->getMapped(); } diff --git a/dbms/tests/queries/0_stateless/00453_cast_enum.sql b/dbms/tests/queries/0_stateless/00453_cast_enum.sql index 3770a21f27e..70b7e2b1d6d 100644 --- a/dbms/tests/queries/0_stateless/00453_cast_enum.sql +++ b/dbms/tests/queries/0_stateless/00453_cast_enum.sql @@ -11,4 +11,6 @@ INSERT INTO cast_enums SELECT 2 AS type, toDate('2017-01-01') AS date, number AS SELECT type, date, id FROM cast_enums ORDER BY type, id; +INSERT INTO cast_enums VALUES ('wrong_value', '2017-01-02', 7); -- { clientError 36 } + DROP TABLE IF EXISTS cast_enums; From 162a2cb2f4885f8f3140191176685a26c2d1f06a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 29 Jan 2020 17:32:38 +0300 Subject: [PATCH 296/312] update SourceFromSingleChunk --- dbms/src/Processors/Sources/SourceFromSingleChunk.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Processors/Sources/SourceFromSingleChunk.h b/dbms/src/Processors/Sources/SourceFromSingleChunk.h index 4384597e28b..d304bdbab93 100644 --- a/dbms/src/Processors/Sources/SourceFromSingleChunk.h +++ b/dbms/src/Processors/Sources/SourceFromSingleChunk.h @@ -1,14 +1,14 @@ #pragma once -#include +#include namespace DB { -class SourceFromSingleChunk : public ISource +class SourceFromSingleChunk : public SourceWithProgress { public: - explicit SourceFromSingleChunk(Block header, Chunk chunk_) : ISource(std::move(header)), chunk(std::move(chunk_)) {} + explicit SourceFromSingleChunk(Block header, Chunk chunk_) : SourceWithProgress(std::move(header)), chunk(std::move(chunk_)) {} String getName() const override { return "SourceFromSingleChunk"; } protected: From 9848a50785988477ff83b5518344b88d775d9591 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 17:56:52 +0300 Subject: [PATCH 297/312] Use clickhouse-test.github.io repo as well --- docs/tools/release.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 0fa6bd70909..345af0d244e 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -5,6 +5,9 @@ BASE_DIR=$(dirname $(readlink -f $0)) BUILD_DIR="${BASE_DIR}/../build" PUBLISH_DIR="${BASE_DIR}/../publish" IMAGE="clickhouse/website" +GIT_TEST_URI="git@github.com:ClickHouse/clickhouse-test.github.io.git" +GIT_PROD_URI="git@github.com:ClickHouse/clickhouse.github.io.git" + if [[ -z "$1" ]] then TAG=$(head -c 8 /dev/urandom | xxd -p) @@ -20,14 +23,14 @@ then python "${BASE_DIR}/build.py" "--enable-stable-releases" set +e rm -rf "${PUBLISH_DIR}" || true - git clone git@github.com:ClickHouse/clickhouse.github.io.git "${PUBLISH_DIR}" + git clone "${GIT_TEST_URI}" "${PUBLISH_DIR}" cd "${PUBLISH_DIR}" git config user.email "robot-clickhouse@yandex-team.ru" git config user.name "robot-clickhouse" git rm -rf * git commit -a -m "wipe old release" cp -R "${BUILD_DIR}"/* . - echo -n "clickhouse.tech" > CNAME + echo -n "test.clickhouse.tech" > CNAME echo -n "" > README.md cp "${BASE_DIR}/../../LICENSE" . git add * @@ -39,6 +42,24 @@ then docker tag "${FULL_NAME}" "${REMOTE_NAME}" DOCKER_HASH=$(docker push "${REMOTE_NAME}" | tail -1 | awk '{print $3;}') docker rmi "${FULL_NAME}" +else + set +e + rm -rf "${BUILD_DIR}" || true + rm -rf "${PUBLISH_DIR}" || true + git clone "${GIT_TEST_URI}" "${BUILD_DIR}" + git clone "${GIT_PROD_URI}" "${PUBLISH_DIR}" + cd "${PUBLISH_DIR}" + git config user.email "robot-clickhouse@yandex-team.ru" + git config user.name "robot-clickhouse" + git rm -rf * + git commit -a -m "wipe old release" + rm -rf "${BUILD_DIR}/.git" + cp -R "${BUILD_DIR}"/* . + echo -n "clickhouse.tech" > CNAME + git add * + git commit -a -m "add new release at $(date)" + git push origin master + set -e fi QLOUD_ENDPOINT="https://platform.yandex-team.ru/api/v1" From b9cd915a0804e86d9e788307f0d3116289706d0d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 18:57:49 +0300 Subject: [PATCH 298/312] Generate client-side redirects for docs --- docs/tools/build.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/tools/build.py b/docs/tools/build.py index 47a69ff2279..4837a298395 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -196,12 +196,42 @@ def build_single_page_version(lang, args, cfg): shutil.copytree(test_dir, args.save_raw_single_page) +def build_redirect_html(args, from_path, to_path): + for lang in args.lang.split(','): + out_path = os.path.join(args.docs_output_dir, lang, from_path.replace('.md', '/index.html')) + out_dir = os.path.dirname(out_path) + try: + os.makedirs(out_dir) + except OSError: + pass + version_prefix = args.version_prefix + '/' if args.version_prefix else '/' + to_url = '/docs%s%s/%s' % (version_prefix, lang, to_path.replace('.md', '/')) + to_url = to_url.strip() + with open(out_path, 'w') as f: + f.write(''' + + + + + + Page Redirection + + + If you are not redirected automatically, follow this link. + +''' % (to_url, to_url, to_url)) + + def build_redirects(args): lang_re_fragment = args.lang.replace(',', '|') rewrites = [] + with open(os.path.join(args.docs_dir, 'redirects.txt'), 'r') as f: for line in f: from_path, to_path = line.split(' ', 1) + build_redirect_html(args, from_path, to_path) from_path = '^/docs/(' + lang_re_fragment + ')/' + from_path.replace('.md', '/?') + '$' to_path = '/docs/$1/' + to_path.replace('.md', '/') rewrites.append(' '.join(['rewrite', from_path, to_path, 'permanent;'])) From 8aa6a2146147a1fbc6ee7996859b3f14aa9c02b8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 19:03:20 +0300 Subject: [PATCH 299/312] Update github.py --- docs/tools/github.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tools/github.py b/docs/tools/github.py index e407eff9cca..f6878e1a0ca 100644 --- a/docs/tools/github.py +++ b/docs/tools/github.py @@ -55,7 +55,6 @@ def process_release(args, callback, release): def build_releases(args, callback): tasks = [] for release in args.stable_releases: - tasks.append((args, callback, release,)) - util.run_function_in_parallel(process_release, tasks) + process_release(args, callback, release) From efedfcdfd5bce1fae3921b08ca40145b02d3e5f1 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 21:01:51 +0300 Subject: [PATCH 300/312] fix redirects for stable releases --- docs/tools/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index 4837a298395..4733333d467 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -245,6 +245,7 @@ def build_docs(args): for lang in args.lang.split(','): tasks.append((lang, args,)) util.run_function_in_parallel(build_for_lang, tasks, threads=True) + build_redirects(args) def build(args): @@ -259,8 +260,6 @@ def build(args): from github import build_releases build_releases(args, build_docs) - build_redirects(args) - if not args.skip_website: minify_website(args) From 699c429e30f4ac178048a8660c6c7f34a3ac108a Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 21:05:47 +0300 Subject: [PATCH 301/312] Update release.sh --- docs/tools/release.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 345af0d244e..0d1c254857e 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -21,7 +21,6 @@ if [[ -z "$1" ]] then source "${BASE_DIR}/venv/bin/activate" python "${BASE_DIR}/build.py" "--enable-stable-releases" - set +e rm -rf "${PUBLISH_DIR}" || true git clone "${GIT_TEST_URI}" "${PUBLISH_DIR}" cd "${PUBLISH_DIR}" @@ -36,14 +35,12 @@ then git add * git commit -a -m "add new release at $(date)" git push origin master - set -e cd "${BUILD_DIR}" docker build -t "${FULL_NAME}" "${BUILD_DIR}" docker tag "${FULL_NAME}" "${REMOTE_NAME}" DOCKER_HASH=$(docker push "${REMOTE_NAME}" | tail -1 | awk '{print $3;}') docker rmi "${FULL_NAME}" else - set +e rm -rf "${BUILD_DIR}" || true rm -rf "${PUBLISH_DIR}" || true git clone "${GIT_TEST_URI}" "${BUILD_DIR}" @@ -59,7 +56,6 @@ else git add * git commit -a -m "add new release at $(date)" git push origin master - set -e fi QLOUD_ENDPOINT="https://platform.yandex-team.ru/api/v1" From e04504ecf63506b802cd7ec852e2164f7d9da927 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 29 Jan 2020 22:45:37 +0300 Subject: [PATCH 302/312] Update index.html --- website/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/index.html b/website/index.html index f376e55b84e..9dd8261f294 100644 --- a/website/index.html +++ b/website/index.html @@ -398,7 +398,7 @@

System requirements: Linux, x86_64 with SSE 4.2.

-

Install packages for Ubuntu/Debian or CentOS/RedHat or Other Linux:

+

Install packages for Ubuntu/Debian, CentOS/RedHat or other Linux:


From 2ecb0a150e03f9fc8441697d95f41cd54eb9335c Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Wed, 29 Jan 2020 23:20:14 +0300
Subject: [PATCH 303/312] Update index.html

---
 website/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/index.html b/website/index.html
index 9dd8261f294..e384097a4f1 100644
--- a/website/index.html
+++ b/website/index.html
@@ -455,7 +455,7 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh
             Yandex Managed Service for ClickHouse.
     

 
-    
After you got connected to your ClickHouse server, you can proceed to tutorial or full
+    
After you got connected to your ClickHouse server, you can proceed to tutorial or full
         documentation.

 
     
Contacts


From 223704ee9ca1f774069d4506897a647c284cfef4 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Wed, 29 Jan 2020 23:27:36 +0300
Subject: [PATCH 304/312] Add tutorial redirect

---
 docs/tools/build.py | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/docs/tools/build.py b/docs/tools/build.py
index 4733333d467..581658a0162 100755
--- a/docs/tools/build.py
+++ b/docs/tools/build.py
@@ -196,19 +196,9 @@ def build_single_page_version(lang, args, cfg):
                         shutil.copytree(test_dir, args.save_raw_single_page)
 
 
-def build_redirect_html(args, from_path, to_path):
-    for lang in args.lang.split(','):
-        out_path = os.path.join(args.docs_output_dir, lang, from_path.replace('.md', '/index.html'))
-        out_dir = os.path.dirname(out_path)
-        try:
-            os.makedirs(out_dir)
-        except OSError:
-            pass
-        version_prefix = args.version_prefix + '/' if args.version_prefix else '/'
-        to_url = '/docs%s%s/%s' % (version_prefix, lang, to_path.replace('.md', '/'))
-        to_url = to_url.strip()
-        with open(out_path, 'w') as f:
-            f.write('''
+def write_redirect_html(out_path, to_url):
+    with open(out_path, 'w') as f:
+        f.write('''
 
     
         
@@ -224,6 +214,20 @@ def build_redirect_html(args, from_path, to_path):
 ''' % (to_url, to_url, to_url))
 
 
+def build_redirect_html(args, from_path, to_path):
+    for lang in args.lang.split(','):
+        out_path = os.path.join(args.docs_output_dir, lang, from_path.replace('.md', '/index.html'))
+        out_dir = os.path.dirname(out_path)
+        try:
+            os.makedirs(out_dir)
+        except OSError:
+            pass
+        version_prefix = args.version_prefix + '/' if args.version_prefix else '/'
+        to_url = '/docs%s%s/%s' % (version_prefix, lang, to_path.replace('.md', '/'))
+        to_url = to_url.strip()
+        write_redirect_html(out_path, to_url)
+
+
 def build_redirects(args):
     lang_re_fragment = args.lang.replace(',', '|')
     rewrites = []
@@ -263,6 +267,11 @@ def build(args):
     if not args.skip_website:
         minify_website(args)
 
+    write_redirect_html(
+        os.path.join(args.output_dir, 'tutorial.html'),
+        '/docs/en/getting_started/tutorial/'
+    )
+
 
 if __name__ == '__main__':
     os.chdir(os.path.join(os.path.dirname(__file__), '..'))

From a20ebf5341c716db20f62bc2aea63292f386f074 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 12:44:57 +0300
Subject: [PATCH 305/312] Update README.md

---
 docs/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/README.md b/docs/README.md
index 91883444459..2fe6c75463a 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,6 @@
 # Contributing to ClickHouse Documentation
 
-## Why You Need to Document ClickHouse
+## Why Do You Need to Document ClickHouse
 
 The main reason is that ClickHouse is an open source project, and if you don't write the docs, nobody does. "Incomplete or Confusing Documentation" is the top complaint about open source software by the results of a [Github Open Source Survey](http://opensourcesurvey.org/2017/) of 2017. Documentation is highly valued but often overlooked. One of the most important contributions someone can make to an open source repository is a documentation update.
 

From ec5c513faa459f4aa678dec8d3ac9e09e6e1566e Mon Sep 17 00:00:00 2001
From: Ivan <5627721+abyss7@users.noreply.github.com>
Date: Thu, 30 Jan 2020 12:51:04 +0300
Subject: [PATCH 306/312] Implement separate CMake build file for curl library
 (#8905)

---
 contrib/CMakeLists.txt                        |    2 +-
 contrib/aws-s3-cmake/CMakeLists.txt           |    2 +-
 .../curl-cmake/CMake/CurlSymbolHiding.cmake   |   61 -
 contrib/curl-cmake/CMake/CurlTests.c          |  617 -------
 contrib/curl-cmake/CMake/Macros.cmake         |   84 -
 contrib/curl-cmake/CMake/OtherTests.cmake     |  260 ---
 contrib/curl-cmake/CMake/README.md            |   28 -
 contrib/curl-cmake/CMakeLists.patch           | 1509 -----------------
 contrib/curl-cmake/CMakeLists.txt             |  753 ++------
 contrib/curl-cmake/curl_config.h              |   38 +
 10 files changed, 188 insertions(+), 3166 deletions(-)
 delete mode 100644 contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
 delete mode 100644 contrib/curl-cmake/CMake/CurlTests.c
 delete mode 100644 contrib/curl-cmake/CMake/Macros.cmake
 delete mode 100644 contrib/curl-cmake/CMake/OtherTests.cmake
 delete mode 100644 contrib/curl-cmake/CMake/README.md
 delete mode 100644 contrib/curl-cmake/CMakeLists.patch
 create mode 100644 contrib/curl-cmake/curl_config.h

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 7c9db5bb06f..999d1e96c9a 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -306,7 +306,7 @@ if (USE_INTERNAL_AWS_S3_LIBRARY)
     # The library is large - avoid bloat.
     target_compile_options (aws_s3 PRIVATE -g0)
     target_compile_options (aws_s3_checksums PRIVATE -g0)
-    target_compile_options (libcurl PRIVATE -g0)
+    target_compile_options (curl PRIVATE -g0)
 endif ()
 
 if (USE_BASE64)
diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt
index ef9107e0d4f..6ed6434dcfd 100644
--- a/contrib/aws-s3-cmake/CMakeLists.txt
+++ b/contrib/aws-s3-cmake/CMakeLists.txt
@@ -102,4 +102,4 @@ if (OPENSSL_FOUND)
     target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES})
 endif()
 
-target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl)
+target_link_libraries(aws_s3 PRIVATE aws_s3_checksums curl)
diff --git a/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake b/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
deleted file mode 100644
index 224755a314c..00000000000
--- a/contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
+++ /dev/null
@@ -1,61 +0,0 @@
-include(CheckCSourceCompiles)
-
-option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON)
-mark_as_advanced(CURL_HIDDEN_SYMBOLS)
-
-if(CURL_HIDDEN_SYMBOLS)
-  set(SUPPORTS_SYMBOL_HIDING FALSE)
-
-  if(CMAKE_C_COMPILER_ID MATCHES "Clang")
-    set(SUPPORTS_SYMBOL_HIDING TRUE)
-    set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
-    set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
-  elseif(CMAKE_COMPILER_IS_GNUCC)
-    if(NOT CMAKE_VERSION VERSION_LESS 2.8.10)
-      set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION})
-    else()
-      execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
-                      OUTPUT_VARIABLE GCC_VERSION)
-    endif()
-    if(NOT GCC_VERSION VERSION_LESS 3.4)
-      # note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact
-      set(SUPPORTS_SYMBOL_HIDING TRUE)
-      set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
-      set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
-    endif()
-  elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0)
-    set(SUPPORTS_SYMBOL_HIDING TRUE)
-    set(_SYMBOL_EXTERN "__global")
-    set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden")
-  elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
-    # note: this should probably just check for version 9.1.045 but I'm not 100% sure
-    #       so let's do it the same way autotools do.
-    set(SUPPORTS_SYMBOL_HIDING TRUE)
-    set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
-    set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
-    check_c_source_compiles("#include 
-        int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug)
-    if(NOT _no_bug)
-      set(SUPPORTS_SYMBOL_HIDING FALSE)
-      set(_SYMBOL_EXTERN "")
-      set(_CFLAG_SYMBOLS_HIDE "")
-    endif()
-  elseif(MSVC)
-    set(SUPPORTS_SYMBOL_HIDING TRUE)
-  endif()
-
-  set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING})
-elseif(MSVC)
-  if(NOT CMAKE_VERSION VERSION_LESS 3.7)
-    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken
-    set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
-  else()
-    message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.")
-    set(HIDES_CURL_PRIVATE_SYMBOLS TRUE)
-  endif()
-else()
-  set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
-endif()
-
-set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE})
-set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN})
diff --git a/contrib/curl-cmake/CMake/CurlTests.c b/contrib/curl-cmake/CMake/CurlTests.c
deleted file mode 100644
index 3ef35f02535..00000000000
--- a/contrib/curl-cmake/CMake/CurlTests.c
+++ /dev/null
@@ -1,617 +0,0 @@
-/***************************************************************************
- *                                  _   _ ____  _
- *  Project                     ___| | | |  _ \| |
- *                             / __| | | | |_) | |
- *                            | (__| |_| |  _ <| |___
- *                             \___|\___/|_| \_\_____|
- *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, , et al.
- *
- * This software is licensed as described in the file COPYING, which
- * you should have received as part of this distribution. The terms
- * are also available at https://curl.haxx.se/docs/copyright.html.
- *
- * You may opt to use, copy, modify, merge, publish, distribute and/or sell
- * copies of the Software, and permit persons to whom the Software is
- * furnished to do so, under the terms of the COPYING file.
- *
- * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
- * KIND, either express or implied.
- *
- ***************************************************************************/
-#ifdef TIME_WITH_SYS_TIME
-/* Time with sys/time test */
-
-#include 
-#include 
-#include 
-
-int
-main ()
-{
-if ((struct tm *) 0)
-return 0;
-  ;
-  return 0;
-}
-
-#endif
-
-#ifdef HAVE_FCNTL_O_NONBLOCK
-
-/* headers for FCNTL_O_NONBLOCK test */
-#include 
-#include 
-#include 
-/* */
-#if defined(sun) || defined(__sun__) || \
-    defined(__SUNPRO_C) || defined(__SUNPRO_CC)
-# if defined(__SVR4) || defined(__srv4__)
-#  define PLATFORM_SOLARIS
-# else
-#  define PLATFORM_SUNOS4
-# endif
-#endif
-#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41)
-# define PLATFORM_AIX_V3
-#endif
-/* */
-#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__)
-#error "O_NONBLOCK does not work on this platform"
-#endif
-
-int
-main ()
-{
-      /* O_NONBLOCK source test */
-      int flags = 0;
-      if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK))
-          return 1;
-      return 0;
-}
-#endif
-
-/* tests for gethostbyaddr_r or gethostbyname_r */
-#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
-    defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
-    defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
-    defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \
-    defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
-    defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
-#   define _REENTRANT
-    /* no idea whether _REENTRANT is always set, just invent a new flag */
-#   define TEST_GETHOSTBYFOO_REENTRANT
-#endif
-#if defined(HAVE_GETHOSTBYADDR_R_5) || \
-    defined(HAVE_GETHOSTBYADDR_R_7) || \
-    defined(HAVE_GETHOSTBYADDR_R_8) || \
-    defined(HAVE_GETHOSTBYNAME_R_3) || \
-    defined(HAVE_GETHOSTBYNAME_R_5) || \
-    defined(HAVE_GETHOSTBYNAME_R_6) || \
-    defined(TEST_GETHOSTBYFOO_REENTRANT)
-#include 
-#include 
-int main(void)
-{
-  char *address = "example.com";
-  int length = 0;
-  int type = 0;
-  struct hostent h;
-  int rc = 0;
-#if defined(HAVE_GETHOSTBYADDR_R_5) || \
-    defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
-    \
-    defined(HAVE_GETHOSTBYNAME_R_3) || \
-    defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
-  struct hostent_data hdata;
-#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
-      defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
-      defined(HAVE_GETHOSTBYADDR_R_8) || \
-      defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
-      \
-      defined(HAVE_GETHOSTBYNAME_R_5) || \
-      defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
-      defined(HAVE_GETHOSTBYNAME_R_6) || \
-      defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
-  char buffer[8192];
-  int h_errnop;
-  struct hostent *hp;
-#endif
-
-#ifndef gethostbyaddr_r
-  (void)gethostbyaddr_r;
-#endif
-
-#if   defined(HAVE_GETHOSTBYADDR_R_5) || \
-      defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT)
-  rc = gethostbyaddr_r(address, length, type, &h, &hdata);
-  (void)rc;
-#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
-      defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT)
-  hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop);
-  (void)hp;
-#elif defined(HAVE_GETHOSTBYADDR_R_8) || \
-      defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT)
-  rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop);
-  (void)rc;
-#endif
-
-#if   defined(HAVE_GETHOSTBYNAME_R_3) || \
-      defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
-  rc = gethostbyname_r(address, &h, &hdata);
-#elif defined(HAVE_GETHOSTBYNAME_R_5) || \
-      defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT)
-  rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop);
-  (void)hp; /* not used for test */
-#elif defined(HAVE_GETHOSTBYNAME_R_6) || \
-      defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
-  rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop);
-#endif
-
-  (void)length;
-  (void)type;
-  (void)rc;
-  return 0;
-}
-#endif
-
-#ifdef HAVE_SOCKLEN_T
-#ifdef _WIN32
-#include 
-#else
-#include 
-#include 
-#endif
-int
-main ()
-{
-if ((socklen_t *) 0)
-  return 0;
-if (sizeof (socklen_t))
-  return 0;
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_IN_ADDR_T
-#include 
-#include 
-#include 
-
-int
-main ()
-{
-if ((in_addr_t *) 0)
-  return 0;
-if (sizeof (in_addr_t))
-  return 0;
-  ;
-  return 0;
-}
-#endif
-
-#ifdef HAVE_BOOL_T
-#ifdef HAVE_SYS_TYPES_H
-#include 
-#endif
-#ifdef HAVE_STDBOOL_H
-#include 
-#endif
-int
-main ()
-{
-if (sizeof (bool *) )
-  return 0;
-  ;
-  return 0;
-}
-#endif
-
-#ifdef STDC_HEADERS
-#include 
-#include 
-#include 
-#include 
-int main() { return 0; }
-#endif
-#ifdef RETSIGTYPE_TEST
-#include 
-#include 
-#ifdef signal
-# undef signal
-#endif
-#ifdef __cplusplus
-extern "C" void (*signal (int, void (*)(int)))(int);
-#else
-void (*signal ()) ();
-#endif
-
-int
-main ()
-{
-  return 0;
-}
-#endif
-#ifdef HAVE_INET_NTOA_R_DECL
-#include 
-
-typedef void (*func_type)();
-
-int main()
-{
-#ifndef inet_ntoa_r
-  func_type func;
-  func = (func_type)inet_ntoa_r;
-  (void)func;
-#endif
-  return 0;
-}
-#endif
-#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT
-#define _REENTRANT
-#include 
-
-typedef void (*func_type)();
-
-int main()
-{
-#ifndef inet_ntoa_r
-  func_type func;
-  func = (func_type)&inet_ntoa_r;
-  (void)func;
-#endif
-  return 0;
-}
-#endif
-#ifdef HAVE_GETADDRINFO
-#include 
-#include 
-#include 
-
-int main(void) {
-    struct addrinfo hints, *ai;
-    int error;
-
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = AF_UNSPEC;
-    hints.ai_socktype = SOCK_STREAM;
-#ifndef getaddrinfo
-    (void)getaddrinfo;
-#endif
-    error = getaddrinfo("127.0.0.1", "8080", &hints, &ai);
-    if (error) {
-        return 1;
-    }
-    return 0;
-}
-#endif
-#ifdef HAVE_FILE_OFFSET_BITS
-#ifdef _FILE_OFFSET_BITS
-#undef _FILE_OFFSET_BITS
-#endif
-#define _FILE_OFFSET_BITS 64
-#include 
- /* Check that off_t can represent 2**63 - 1 correctly.
-    We can't simply define LARGE_OFF_T to be 9223372036854775807,
-    since some C++ compilers masquerading as C compilers
-    incorrectly reject 9223372036854775807.  */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
-  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
-                       && LARGE_OFF_T % 2147483647 == 1)
-                      ? 1 : -1];
-int main () { ; return 0; }
-#endif
-#ifdef HAVE_IOCTLSOCKET
-/* includes start */
-#ifdef HAVE_WINDOWS_H
-#  ifndef WIN32_LEAN_AND_MEAN
-#    define WIN32_LEAN_AND_MEAN
-#  endif
-#  include 
-#  ifdef HAVE_WINSOCK2_H
-#    include 
-#  else
-#    ifdef HAVE_WINSOCK_H
-#      include 
-#    endif
-#  endif
-#endif
-
-int
-main ()
-{
-
-/* ioctlsocket source code */
- int socket;
- unsigned long flags = ioctlsocket(socket, FIONBIO, &flags);
-
-  ;
-  return 0;
-}
-
-#endif
-#ifdef HAVE_IOCTLSOCKET_CAMEL
-/* includes start */
-#ifdef HAVE_WINDOWS_H
-#  ifndef WIN32_LEAN_AND_MEAN
-#    define WIN32_LEAN_AND_MEAN
-#  endif
-#  include 
-#  ifdef HAVE_WINSOCK2_H
-#    include 
-#  else
-#    ifdef HAVE_WINSOCK_H
-#      include 
-#    endif
-#  endif
-#endif
-
-int
-main ()
-{
-
-/* IoctlSocket source code */
-    if(0 != IoctlSocket(0, 0, 0))
-      return 1;
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO
-/* includes start */
-#ifdef HAVE_WINDOWS_H
-#  ifndef WIN32_LEAN_AND_MEAN
-#    define WIN32_LEAN_AND_MEAN
-#  endif
-#  include 
-#  ifdef HAVE_WINSOCK2_H
-#    include 
-#  else
-#    ifdef HAVE_WINSOCK_H
-#      include 
-#    endif
-#  endif
-#endif
-
-int
-main ()
-{
-
-/* IoctlSocket source code */
-        long flags = 0;
-        if(0 != ioctlsocket(0, FIONBIO, &flags))
-          return 1;
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_IOCTLSOCKET_FIONBIO
-/* includes start */
-#ifdef HAVE_WINDOWS_H
-#  ifndef WIN32_LEAN_AND_MEAN
-#    define WIN32_LEAN_AND_MEAN
-#  endif
-#  include 
-#  ifdef HAVE_WINSOCK2_H
-#    include 
-#  else
-#    ifdef HAVE_WINSOCK_H
-#      include 
-#    endif
-#  endif
-#endif
-
-int
-main ()
-{
-
-        int flags = 0;
-        if(0 != ioctlsocket(0, FIONBIO, &flags))
-          return 1;
-
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_IOCTL_FIONBIO
-/* headers for FIONBIO test */
-/* includes start */
-#ifdef HAVE_SYS_TYPES_H
-#  include 
-#endif
-#ifdef HAVE_UNISTD_H
-#  include 
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#  include 
-#endif
-#ifdef HAVE_SYS_IOCTL_H
-#  include 
-#endif
-#ifdef HAVE_STROPTS_H
-#  include 
-#endif
-
-int
-main ()
-{
-
-        int flags = 0;
-        if(0 != ioctl(0, FIONBIO, &flags))
-          return 1;
-
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_IOCTL_SIOCGIFADDR
-/* headers for FIONBIO test */
-/* includes start */
-#ifdef HAVE_SYS_TYPES_H
-#  include 
-#endif
-#ifdef HAVE_UNISTD_H
-#  include 
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#  include 
-#endif
-#ifdef HAVE_SYS_IOCTL_H
-#  include 
-#endif
-#ifdef HAVE_STROPTS_H
-#  include 
-#endif
-#include 
-
-int
-main ()
-{
-        struct ifreq ifr;
-        if(0 != ioctl(0, SIOCGIFADDR, &ifr))
-          return 1;
-
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK
-/* includes start */
-#ifdef HAVE_WINDOWS_H
-#  ifndef WIN32_LEAN_AND_MEAN
-#    define WIN32_LEAN_AND_MEAN
-#  endif
-#  include 
-#  ifdef HAVE_WINSOCK2_H
-#    include 
-#  else
-#    ifdef HAVE_WINSOCK_H
-#      include 
-#    endif
-#  endif
-#endif
-/* includes start */
-#ifdef HAVE_SYS_TYPES_H
-#  include 
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#  include 
-#endif
-/* includes end */
-
-int
-main ()
-{
-        if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0))
-          return 1;
-  ;
-  return 0;
-}
-#endif
-#ifdef HAVE_GLIBC_STRERROR_R
-#include 
-#include 
-
-void check(char c) {}
-
-int
-main () {
-  char buffer[1024];
-  /* This will not compile if strerror_r does not return a char* */
-  check(strerror_r(EACCES, buffer, sizeof(buffer))[0]);
-  return 0;
-}
-#endif
-#ifdef HAVE_POSIX_STRERROR_R
-#include 
-#include 
-
-/* float, because a pointer can't be implicitly cast to float */
-void check(float f) {}
-
-int
-main () {
-  char buffer[1024];
-  /* This will not compile if strerror_r does not return an int */
-  check(strerror_r(EACCES, buffer, sizeof(buffer)));
-  return 0;
-}
-#endif
-#ifdef HAVE_FSETXATTR_6
-#include  /* header from libc, not from libattr */
-int
-main() {
-  fsetxattr(0, 0, 0, 0, 0, 0);
-  return 0;
-}
-#endif
-#ifdef HAVE_FSETXATTR_5
-#include  /* header from libc, not from libattr */
-int
-main() {
-  fsetxattr(0, 0, 0, 0, 0);
-  return 0;
-}
-#endif
-#ifdef HAVE_CLOCK_GETTIME_MONOTONIC
-#include 
-int
-main() {
-  struct timespec ts = {0, 0};
-  clock_gettime(CLOCK_MONOTONIC, &ts);
-  return 0;
-}
-#endif
-#ifdef HAVE_BUILTIN_AVAILABLE
-int
-main() {
-  if(__builtin_available(macOS 10.12, *)) {}
-  return 0;
-}
-#endif
-#ifdef HAVE_VARIADIC_MACROS_C99
-#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__)
-#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__)
-
-int fun3(int arg1, int arg2, int arg3);
-int fun2(int arg1, int arg2);
-
-int fun3(int arg1, int arg2, int arg3) {
-  return arg1 + arg2 + arg3;
-}
-int fun2(int arg1, int arg2) {
-  return arg1 + arg2;
-}
-
-int
-main() {
-  int res3 = c99_vmacro3(1, 2, 3);
-  int res2 = c99_vmacro2(1, 2);
-  (void)res3;
-  (void)res2;
-  return 0;
-}
-#endif
-#ifdef HAVE_VARIADIC_MACROS_GCC
-#define gcc_vmacro3(first, args...) fun3(first, args)
-#define gcc_vmacro2(first, args...) fun2(first, args)
-
-int fun3(int arg1, int arg2, int arg3);
-int fun2(int arg1, int arg2);
-
-int fun3(int arg1, int arg2, int arg3) {
-  return arg1 + arg2 + arg3;
-}
-int fun2(int arg1, int arg2) {
-  return arg1 + arg2;
-}
-
-int
-main() {
-  int res3 = gcc_vmacro3(1, 2, 3);
-  int res2 = gcc_vmacro2(1, 2);
-  (void)res3;
-  (void)res2;
-  return 0;
-}
-#endif
diff --git a/contrib/curl-cmake/CMake/Macros.cmake b/contrib/curl-cmake/CMake/Macros.cmake
deleted file mode 100644
index 39a60b04df9..00000000000
--- a/contrib/curl-cmake/CMake/Macros.cmake
+++ /dev/null
@@ -1,84 +0,0 @@
-#File defines convenience macros for available feature testing
-
-# This macro checks if the symbol exists in the library and if it
-# does, it prepends library to the list.  It is intended to be called
-# multiple times with a sequence of possibly dependent libraries in
-# order of least-to-most-dependent.  Some libraries depend on others
-# to link correctly.
-macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE)
-  check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}"
-    ${VARIABLE})
-  if(${VARIABLE})
-    set(CURL_LIBS ${LIBRARY} ${CURL_LIBS})
-  endif()
-endmacro()
-
-# Check if header file exists and add it to the list.
-# This macro is intended to be called multiple times with a sequence of
-# possibly dependent header files.  Some headers depend on others to be
-# compiled correctly.
-macro(check_include_file_concat FILE VARIABLE)
-  check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE})
-  if(${VARIABLE})
-    set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE})
-    set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}")
-  endif()
-endmacro()
-
-# For other curl specific tests, use this macro.
-macro(curl_internal_test CURL_TEST)
-  if(NOT DEFINED "${CURL_TEST}")
-    set(MACRO_CHECK_FUNCTION_DEFINITIONS
-      "-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}")
-    if(CMAKE_REQUIRED_LIBRARIES)
-      set(CURL_TEST_ADD_LIBRARIES
-        "-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
-    endif()
-
-    try_compile(${CURL_TEST}
-      ${CMAKE_BINARY_DIR}
-      ${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c
-      CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
-      "${CURL_TEST_ADD_LIBRARIES}"
-      OUTPUT_VARIABLE OUTPUT)
-    if(${CURL_TEST})
-      set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}")
-      file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
-        "Performing Curl Test ${CURL_TEST} passed with the following output:\n"
-        "${OUTPUT}\n")
-    else()
-      set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}")
-      file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
-        "Performing Curl Test ${CURL_TEST} failed with the following output:\n"
-        "${OUTPUT}\n")
-    endif()
-  endif()
-endmacro()
-
-macro(curl_nroff_check)
-  find_program(NROFF NAMES gnroff nroff)
-  if(NROFF)
-    # Need a way to write to stdin, this will do
-    file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test")
-    # Tests for a valid nroff option to generate a manpage
-    foreach(_MANOPT "-man" "-mandoc")
-      execute_process(COMMAND "${NROFF}" ${_MANOPT}
-        OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT
-        INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt"
-        ERROR_QUIET)
-      # Save the option if it was valid
-      if(NROFF_MANOPT_OUTPUT)
-        set(NROFF_MANOPT ${_MANOPT})
-        set(NROFF_USEFUL ON)
-        break()
-      endif()
-    endforeach()
-    # No need for the temporary file
-    file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt")
-    if(NOT NROFF_USEFUL)
-      message(WARNING "Found no *nroff option to get plaintext from man pages")
-    endif()
-  else()
-    message(WARNING "Found no *nroff program")
-  endif()
-endmacro()
diff --git a/contrib/curl-cmake/CMake/OtherTests.cmake b/contrib/curl-cmake/CMake/OtherTests.cmake
deleted file mode 100644
index 5ad70a62fb2..00000000000
--- a/contrib/curl-cmake/CMake/OtherTests.cmake
+++ /dev/null
@@ -1,260 +0,0 @@
-include(CheckCSourceCompiles)
-# The begin of the sources (macros and includes)
-set(_source_epilogue "#undef inline")
-
-macro(add_header_include check header)
-  if(${check})
-    set(_source_epilogue "${_source_epilogue}\n#include <${header}>")
-  endif()
-endmacro()
-
-set(signature_call_conv)
-if(HAVE_WINDOWS_H)
-  add_header_include(HAVE_WINSOCK2_H "winsock2.h")
-  add_header_include(HAVE_WINDOWS_H "windows.h")
-  add_header_include(HAVE_WINSOCK_H "winsock.h")
-  set(_source_epilogue
-      "${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif")
-  set(signature_call_conv "PASCAL")
-  if(HAVE_LIBWS2_32)
-    set(CMAKE_REQUIRED_LIBRARIES ws2_32)
-  endif()
-else()
-  add_header_include(HAVE_SYS_TYPES_H "sys/types.h")
-  add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h")
-endif()
-
-set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
-
-check_c_source_compiles("${_source_epilogue}
-int main(void) {
-    recv(0, 0, 0, 0);
-    return 0;
-}" curl_cv_recv)
-if(curl_cv_recv)
-  if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown")
-    foreach(recv_retv "int" "ssize_t" )
-      foreach(recv_arg1 "SOCKET" "int" )
-        foreach(recv_arg2 "char *" "void *" )
-          foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int")
-            foreach(recv_arg4 "int" "unsigned int")
-              if(NOT curl_cv_func_recv_done)
-                unset(curl_cv_func_recv_test CACHE)
-                check_c_source_compiles("
-                  ${_source_epilogue}
-                  extern ${recv_retv} ${signature_call_conv}
-                  recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4});
-                  int main(void) {
-                    ${recv_arg1} s=0;
-                    ${recv_arg2} buf=0;
-                    ${recv_arg3} len=0;
-                    ${recv_arg4} flags=0;
-                    ${recv_retv} res = recv(s, buf, len, flags);
-                    (void) res;
-                    return 0;
-                  }"
-                  curl_cv_func_recv_test)
-                if(curl_cv_func_recv_test)
-                  set(curl_cv_func_recv_args
-                    "${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}")
-                  set(RECV_TYPE_ARG1 "${recv_arg1}")
-                  set(RECV_TYPE_ARG2 "${recv_arg2}")
-                  set(RECV_TYPE_ARG3 "${recv_arg3}")
-                  set(RECV_TYPE_ARG4 "${recv_arg4}")
-                  set(RECV_TYPE_RETV "${recv_retv}")
-                  set(HAVE_RECV 1)
-                  set(curl_cv_func_recv_done 1)
-                endif()
-              endif()
-            endforeach()
-          endforeach()
-        endforeach()
-      endforeach()
-    endforeach()
-  else()
-    string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}")
-    string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}")
-  endif()
-
-  if("${curl_cv_func_recv_args}" STREQUAL "unknown")
-    message(FATAL_ERROR "Cannot find proper types to use for recv args")
-  endif()
-else()
-  message(FATAL_ERROR "Unable to link function recv")
-endif()
-set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv")
-set(HAVE_RECV 1)
-
-check_c_source_compiles("${_source_epilogue}
-int main(void) {
-    send(0, 0, 0, 0);
-    return 0;
-}" curl_cv_send)
-if(curl_cv_send)
-  if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown")
-    foreach(send_retv "int" "ssize_t" )
-      foreach(send_arg1 "SOCKET" "int" "ssize_t" )
-        foreach(send_arg2 "const char *" "const void *" "void *" "char *")
-          foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int")
-            foreach(send_arg4 "int" "unsigned int")
-              if(NOT curl_cv_func_send_done)
-                unset(curl_cv_func_send_test CACHE)
-                check_c_source_compiles("
-                  ${_source_epilogue}
-                  extern ${send_retv} ${signature_call_conv}
-                  send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4});
-                  int main(void) {
-                    ${send_arg1} s=0;
-                    ${send_arg2} buf=0;
-                    ${send_arg3} len=0;
-                    ${send_arg4} flags=0;
-                    ${send_retv} res = send(s, buf, len, flags);
-                    (void) res;
-                    return 0;
-                  }"
-                  curl_cv_func_send_test)
-                if(curl_cv_func_send_test)
-                  string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}")
-                  string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}")
-                  set(curl_cv_func_send_args
-                    "${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}")
-                  set(SEND_TYPE_ARG1 "${send_arg1}")
-                  set(SEND_TYPE_ARG2 "${send_arg2}")
-                  set(SEND_TYPE_ARG3 "${send_arg3}")
-                  set(SEND_TYPE_ARG4 "${send_arg4}")
-                  set(SEND_TYPE_RETV "${send_retv}")
-                  set(HAVE_SEND 1)
-                  set(curl_cv_func_send_done 1)
-                endif()
-              endif()
-            endforeach()
-          endforeach()
-        endforeach()
-      endforeach()
-    endforeach()
-  else()
-    string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}")
-    string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}")
-    string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}")
-  endif()
-
-  if("${curl_cv_func_send_args}" STREQUAL "unknown")
-    message(FATAL_ERROR "Cannot find proper types to use for send args")
-  endif()
-  set(SEND_QUAL_ARG2 "const")
-else()
-  message(FATAL_ERROR "Unable to link function send")
-endif()
-set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send")
-set(HAVE_SEND 1)
-
-check_c_source_compiles("${_source_epilogue}
-  int main(void) {
-    int flag = MSG_NOSIGNAL;
-    (void)flag;
-    return 0;
-  }" HAVE_MSG_NOSIGNAL)
-
-if(NOT HAVE_WINDOWS_H)
-  add_header_include(HAVE_SYS_TIME_H "sys/time.h")
-  add_header_include(TIME_WITH_SYS_TIME "time.h")
-  add_header_include(HAVE_TIME_H "time.h")
-endif()
-check_c_source_compiles("${_source_epilogue}
-int main(void) {
-  struct timeval ts;
-  ts.tv_sec  = 0;
-  ts.tv_usec = 0;
-  (void)ts;
-  return 0;
-}" HAVE_STRUCT_TIMEVAL)
-
-set(HAVE_SIG_ATOMIC_T 1)
-set(CMAKE_REQUIRED_FLAGS)
-if(HAVE_SIGNAL_H)
-  set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H")
-  set(CMAKE_EXTRA_INCLUDE_FILES "signal.h")
-endif()
-check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T)
-if(HAVE_SIZEOF_SIG_ATOMIC_T)
-  check_c_source_compiles("
-    #ifdef HAVE_SIGNAL_H
-    #  include 
-    #endif
-    int main(void) {
-      static volatile sig_atomic_t dummy = 0;
-      (void)dummy;
-      return 0;
-    }" HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
-  if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
-    set(HAVE_SIG_ATOMIC_T_VOLATILE 1)
-  endif()
-endif()
-
-if(HAVE_WINDOWS_H)
-  set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h)
-else()
-  set(CMAKE_EXTRA_INCLUDE_FILES)
-  if(HAVE_SYS_SOCKET_H)
-    set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
-  endif()
-endif()
-
-check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE)
-if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE)
-  set(HAVE_STRUCT_SOCKADDR_STORAGE 1)
-endif()
-
-unset(CMAKE_TRY_COMPILE_TARGET_TYPE)
-
-if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
-  # if not cross-compilation...
-  include(CheckCSourceRuns)
-  set(CMAKE_REQUIRED_FLAGS "")
-  if(HAVE_SYS_POLL_H)
-    set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H")
-  elseif(HAVE_POLL_H)
-    set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H")
-  endif()
-  check_c_source_runs("
-    #include 
-    #include 
-
-    #ifdef HAVE_SYS_POLL_H
-    #  include 
-    #elif  HAVE_POLL_H
-    #  include 
-    #endif
-
-    int main(void)
-    {
-        if(0 != poll(0, 0, 10)) {
-          return 1; /* fail */
-        }
-        else {
-          /* detect the 10.12 poll() breakage */
-          struct timeval before, after;
-          int rc;
-          size_t us;
-
-          gettimeofday(&before, NULL);
-          rc = poll(NULL, 0, 500);
-          gettimeofday(&after, NULL);
-
-          us = (after.tv_sec - before.tv_sec) * 1000000 +
-            (after.tv_usec - before.tv_usec);
-
-          if(us < 400000) {
-            return 1;
-          }
-        }
-        return 0;
-    }" HAVE_POLL_FINE)
-endif()
-
diff --git a/contrib/curl-cmake/CMake/README.md b/contrib/curl-cmake/CMake/README.md
deleted file mode 100644
index a96fe6849ed..00000000000
--- a/contrib/curl-cmake/CMake/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
-#***************************************************************************
-#                                  _   _ ____  _
-#  Project                     ___| | | |  _ \| |
-#                             / __| | | | |_) | |
-#                            | (__| |_| |  _ <| |___
-#                             \___|\___/|_| \_\_____|
-#
-# Copyright (C) 1998 - 2019, Daniel Stenberg, , et al.
-#
-# This software is licensed as described in the file COPYING, which
-# you should have received as part of this distribution. The terms
-# are also available at https://curl.haxx.se/docs/copyright.html.
-#
-# You may opt to use, copy, modify, merge, publish, distribute and/or sell
-# copies of the Software, and permit persons to whom the Software is
-# furnished to do so, under the terms of the COPYING file.
-#
-# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
-# KIND, either express or implied.
-#
-###########################################################################
-
-CMake files under this directory were reused from project curl.
-Here are links to original source files:
-https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake
-https://github.com/curl/curl/blob/master/CMake/CurlTests,c
-https://github.com/curl/curl/blob/master/CMake/Macros.cmake
-https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake
diff --git a/contrib/curl-cmake/CMakeLists.patch b/contrib/curl-cmake/CMakeLists.patch
deleted file mode 100644
index 3b4ef3bf9ab..00000000000
--- a/contrib/curl-cmake/CMakeLists.patch
+++ /dev/null
@@ -1,1509 +0,0 @@
---- ../curl/CMakeLists.txt	2019-12-09 13:33:38.140856959 +0300
-+++ CMakeLists.txt	2019-12-12 19:50:48.001094425 +0300
-@@ -19,220 +19,100 @@
- # KIND, either express or implied.
- #
- ###########################################################################
--# curl/libcurl CMake script
--# by Tetetest and Sukender (Benoit Neil)
--
--# TODO:
--# The output .so file lacks the soname number which we currently have within the lib/Makefile.am file
--# Add full (4 or 5 libs) SSL support
--# Add INSTALL target (EXTRA_DIST variables in Makefile.am may be moved to Makefile.inc so that CMake/CPack is aware of what's to include).
--# Add CTests(?)
--# Check on all possible platforms
--# Test with as many configurations possible (With or without any option)
--# Create scripts that help keeping the CMake build system up to date (to reduce maintenance). According to Tetetest:
--#  - lists of headers that 'configure' checks for;
--#  - curl-specific tests (the ones that are in m4/curl-*.m4 files);
--#  - (most obvious thing:) curl version numbers.
--# Add documentation subproject
--#
--# To check:
--# (From Daniel Stenberg) The cmake build selected to run gcc with -fPIC on my box while the plain configure script did not.
--# (From Daniel Stenberg) The gcc command line use neither -g nor any -O options. As a developer, I also treasure our configure scripts's --enable-debug option that sets a long range of "picky" compiler options.
-+# This file is mostly copy-pasted from original curl CMakeLists.txt
- cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
-+
-+SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
-+SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
- set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
--include(Utilities)
- include(Macros)
- include(CMakeDependentOption)
- include(CheckCCompilerFlag)
- 
--project(CURL C)
--
--message(WARNING "the curl cmake build system is poorly maintained. Be aware")
--
- file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
- string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
--  CURL_VERSION ${CURL_VERSION_H_CONTENTS})
-+        CURL_VERSION ${CURL_VERSION_H_CONTENTS})
- string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
- string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
--  CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
-+        CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
- string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
- 
--
--# Setup package meta-data
--# SET(PACKAGE "curl")
--message(STATUS "curl version=[${CURL_VERSION}]")
--# SET(PACKAGE_TARNAME "curl")
--# SET(PACKAGE_NAME "curl")
--# SET(PACKAGE_VERSION "-")
--# SET(PACKAGE_STRING "curl-")
--# SET(PACKAGE_BUGREPORT "a suitable curl mailing list => https://curl.haxx.se/mail/")
-+message(STATUS "Use curl version=[${CURL_VERSION}]")
- set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
- set(OS "\"${CMAKE_SYSTEM_NAME}\"")
- 
--include_directories(${CURL_SOURCE_DIR}/include)
--
--option(CURL_WERROR "Turn compiler warnings into errors" OFF)
- option(PICKY_COMPILER "Enable picky compiler options" ON)
--option(BUILD_CURL_EXE "Set to ON to build curl executable." ON)
--option(BUILD_SHARED_LIBS "Build shared libraries" ON)
--option(ENABLE_ARES "Set to ON to enable c-ares support" OFF)
--if(WIN32)
--  option(CURL_STATIC_CRT "Set to ON to build libcurl with static CRT on Windows (/MT)." OFF)
--  option(ENABLE_INET_PTON "Set to OFF to prevent usage of inet_pton when building against modern SDKs while still requiring compatibility with older Windows versions, such as Windows XP, Windows Server 2003 etc." ON)
--endif()
--
--cmake_dependent_option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup"
--        ON "NOT ENABLE_ARES"
--        OFF)
--
--option(ENABLE_DEBUG "Set to ON to enable curl debug features" OFF)
--option(ENABLE_CURLDEBUG "Set to ON to build with TrackMemory feature enabled" OFF)
-+option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
- 
- if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
--  if(PICKY_COMPILER)
--    foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
--      # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
--      # test result in.
--      check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
--      if(OPT${_CCOPT})
--        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
--      endif()
--    endforeach()
--  endif()
--endif()
--
--if(ENABLE_DEBUG)
--  # DEBUGBUILD will be defined only for Debug builds
--  set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS $<$:DEBUGBUILD>)
--  set(ENABLE_CURLDEBUG ON)
--endif()
--
--if(ENABLE_CURLDEBUG)
--  set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS CURLDEBUG)
-+    if(PICKY_COMPILER)
-+        foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
-+            # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
-+            # test result in.
-+            check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
-+            if(OPT${_CCOPT})
-+                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
-+            endif()
-+        endforeach()
-+    endif()
- endif()
- 
- # For debug libs and exes, add "-d" postfix
- if(NOT DEFINED CMAKE_DEBUG_POSTFIX)
--  set(CMAKE_DEBUG_POSTFIX "-d")
-+    set(CMAKE_DEBUG_POSTFIX "-d")
- endif()
- 
- # initialize CURL_LIBS
- set(CURL_LIBS "")
- 
--if(ENABLE_ARES)
--  set(USE_ARES 1)
--  find_package(CARES REQUIRED)
--  list(APPEND CURL_LIBS ${CARES_LIBRARY})
--  set(CURL_LIBS ${CURL_LIBS} ${CARES_LIBRARY})
--endif()
--
- include(CurlSymbolHiding)
- 
--option(HTTP_ONLY "disables all protocols except HTTP (This overrides all CURL_DISABLE_* options)" OFF)
--mark_as_advanced(HTTP_ONLY)
--option(CURL_DISABLE_FTP "disables FTP" OFF)
--mark_as_advanced(CURL_DISABLE_FTP)
--option(CURL_DISABLE_LDAP "disables LDAP" OFF)
--mark_as_advanced(CURL_DISABLE_LDAP)
--option(CURL_DISABLE_TELNET "disables Telnet" OFF)
--mark_as_advanced(CURL_DISABLE_TELNET)
--option(CURL_DISABLE_DICT "disables DICT" OFF)
--mark_as_advanced(CURL_DISABLE_DICT)
--option(CURL_DISABLE_FILE "disables FILE" OFF)
--mark_as_advanced(CURL_DISABLE_FILE)
--option(CURL_DISABLE_TFTP "disables TFTP" OFF)
--mark_as_advanced(CURL_DISABLE_TFTP)
--option(CURL_DISABLE_HTTP "disables HTTP" OFF)
--mark_as_advanced(CURL_DISABLE_HTTP)
--
--option(CURL_DISABLE_LDAPS "to disable LDAPS" OFF)
--mark_as_advanced(CURL_DISABLE_LDAPS)
--
--option(CURL_DISABLE_RTSP "to disable RTSP" OFF)
--mark_as_advanced(CURL_DISABLE_RTSP)
--option(CURL_DISABLE_PROXY "to disable proxy" OFF)
--mark_as_advanced(CURL_DISABLE_PROXY)
--option(CURL_DISABLE_POP3 "to disable POP3" OFF)
--mark_as_advanced(CURL_DISABLE_POP3)
--option(CURL_DISABLE_IMAP "to disable IMAP" OFF)
--mark_as_advanced(CURL_DISABLE_IMAP)
--option(CURL_DISABLE_SMTP "to disable SMTP" OFF)
--mark_as_advanced(CURL_DISABLE_SMTP)
--option(CURL_DISABLE_GOPHER "to disable Gopher" OFF)
--mark_as_advanced(CURL_DISABLE_GOPHER)
--
--if(HTTP_ONLY)
--  set(CURL_DISABLE_FTP ON)
--  set(CURL_DISABLE_LDAP ON)
--  set(CURL_DISABLE_LDAPS ON)
--  set(CURL_DISABLE_TELNET ON)
--  set(CURL_DISABLE_DICT ON)
--  set(CURL_DISABLE_FILE ON)
--  set(CURL_DISABLE_TFTP ON)
--  set(CURL_DISABLE_RTSP ON)
--  set(CURL_DISABLE_POP3 ON)
--  set(CURL_DISABLE_IMAP ON)
--  set(CURL_DISABLE_SMTP ON)
--  set(CURL_DISABLE_GOPHER ON)
--endif()
-+# Http only
-+set(CURL_DISABLE_FTP ON)
-+set(CURL_DISABLE_LDAP ON)
-+set(CURL_DISABLE_LDAPS ON)
-+set(CURL_DISABLE_TELNET ON)
-+set(CURL_DISABLE_DICT ON)
-+set(CURL_DISABLE_FILE ON)
-+set(CURL_DISABLE_TFTP ON)
-+set(CURL_DISABLE_RTSP ON)
-+set(CURL_DISABLE_POP3 ON)
-+set(CURL_DISABLE_IMAP ON)
-+set(CURL_DISABLE_SMTP ON)
-+set(CURL_DISABLE_GOPHER ON)
- 
- option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
- mark_as_advanced(CURL_DISABLE_COOKIES)
- 
- option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
- mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
-+
- option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
- mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
-+
- option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
- mark_as_advanced(ENABLE_IPV6)
-+
- if(ENABLE_IPV6 AND NOT WIN32)
--  include(CheckStructHasMember)
--  check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
--                          HAVE_SOCKADDR_IN6_SIN6_ADDR)
--  check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
--                          HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
--  if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
--    message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
--    # Force the feature off as this name is used as guard macro...
--    set(ENABLE_IPV6 OFF
--        CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
--  endif()
-+    include(CheckStructHasMember)
-+    check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
-+            HAVE_SOCKADDR_IN6_SIN6_ADDR)
-+    check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
-+            HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
-+    if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
-+        message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
-+        # Force the feature off as this name is used as guard macro...
-+        set(ENABLE_IPV6 OFF
-+                CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
-+    endif()
- endif()
- 
- curl_nroff_check()
--find_package(Perl)
--
--cmake_dependent_option(ENABLE_MANUAL "to provide the built-in manual"
--    ON "NROFF_USEFUL;PERL_FOUND"
--    OFF)
--
--if(NOT PERL_FOUND)
--  message(STATUS "Perl not found, testing disabled.")
--  set(BUILD_TESTING OFF)
--endif()
--if(ENABLE_MANUAL)
--  set(USE_MANUAL ON)
--endif()
- 
- # We need ansi c-flags, especially on HP
- set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
- set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
- 
--if(CURL_STATIC_CRT)
--  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /MT")
--  set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /MTd")
--endif()
--
--# Disable warnings on Borland to avoid changing 3rd party code.
--if(BORLAND)
--  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w-")
--endif()
--
--# If we are on AIX, do the _ALL_SOURCE magic
--if(${CMAKE_SYSTEM_NAME} MATCHES AIX)
--  set(_ALL_SOURCE 1)
--endif()
--
- # Include all the necessary files for macros
- include(CheckFunctionExists)
- include(CheckIncludeFile)
-@@ -242,21 +122,15 @@
- include(CheckTypeSize)
- include(CheckCSourceCompiles)
- 
--# On windows preload settings
--if(WIN32)
--  set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_WINSOCKAPI_=")
--  include(${CMAKE_CURRENT_SOURCE_DIR}/CMake/Platforms/WindowsCache.cmake)
--endif()
--
- if(ENABLE_THREADED_RESOLVER)
--  find_package(Threads REQUIRED)
--  if(WIN32)
--    set(USE_THREADS_WIN32 ON)
--  else()
--    set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
--    set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
--  endif()
--  set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
-+    find_package(Threads REQUIRED)
-+    if(WIN32)
-+        set(USE_THREADS_WIN32 ON)
-+    else()
-+        set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
-+        set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
-+    endif()
-+    set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
- endif()
- 
- # Check for all needed libraries
-@@ -264,245 +138,24 @@
- check_library_exists_concat("socket" connect      HAVE_LIBSOCKET)
- check_library_exists("c" gethostbyname "" NOT_NEED_LIBNSL)
- 
--# Yellowtab Zeta needs different libraries than BeOS 5.
--if(BEOS)
--  set(NOT_NEED_LIBNSL 1)
--  check_library_exists_concat("bind" gethostbyname HAVE_LIBBIND)
--  check_library_exists_concat("bnetapi" closesocket HAVE_LIBBNETAPI)
--endif()
--
--if(NOT NOT_NEED_LIBNSL)
--  check_library_exists_concat("nsl"    gethostbyname  HAVE_LIBNSL)
--endif()
--
- check_function_exists(gethostname HAVE_GETHOSTNAME)
- 
--if(WIN32)
--  check_library_exists_concat("ws2_32" getch        HAVE_LIBWS2_32)
--  check_library_exists_concat("winmm"  getch        HAVE_LIBWINMM)
--  list(APPEND CURL_LIBS "advapi32")
--endif()
--
--# check SSL libraries
--# TODO support GNUTLS, NSS, POLARSSL, CYASSL
--
--if(APPLE)
--  option(CMAKE_USE_SECTRANSP "enable Apple OS native SSL/TLS" OFF)
--endif()
--if(WIN32)
--  option(CMAKE_USE_WINSSL "enable Windows native SSL/TLS" OFF)
--  cmake_dependent_option(CURL_WINDOWS_SSPI "Use windows libraries to allow NTLM authentication without openssl" ON
--    CMAKE_USE_WINSSL OFF)
--endif()
--option(CMAKE_USE_MBEDTLS "Enable mbedTLS for SSL/TLS" OFF)
--option(CMAKE_USE_BEARSSL "Enable BearSSL for SSL/TLS" OFF)
--
--set(openssl_default ON)
--if(WIN32 OR CMAKE_USE_SECTRANSP OR CMAKE_USE_WINSSL OR CMAKE_USE_MBEDTLS)
--  set(openssl_default OFF)
--endif()
--option(CMAKE_USE_OPENSSL "Use OpenSSL code. Experimental" ${openssl_default})
--
--count_true(enabled_ssl_options_count
--  CMAKE_USE_WINSSL
--  CMAKE_USE_SECTRANSP
--  CMAKE_USE_OPENSSL
--  CMAKE_USE_MBEDTLS
--  CMAKE_USE_BEARSSL
--)
--if(enabled_ssl_options_count GREATER "1")
--  set(CURL_WITH_MULTI_SSL ON)
--endif()
--
--if(CMAKE_USE_WINSSL)
--  set(SSL_ENABLED ON)
--  set(USE_SCHANNEL ON) # Windows native SSL/TLS support
--  set(USE_WINDOWS_SSPI ON) # CMAKE_USE_WINSSL implies CURL_WINDOWS_SSPI
--  list(APPEND CURL_LIBS "crypt32")
--endif()
--if(CURL_WINDOWS_SSPI)
--  set(USE_WINDOWS_SSPI ON)
--  set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -DSECURITY_WIN32")
--endif()
-+# From cmake/find/ssl.cmake
-+if (OPENSSL_FOUND)
-+    set(SSL_ENABLED ON)
-+    set(USE_OPENSSL ON)
- 
--if(CMAKE_USE_DARWINSSL)
--  message(FATAL_ERROR "The cmake option CMAKE_USE_DARWINSSL was renamed to CMAKE_USE_SECTRANSP.")
--endif()
--
--if(CMAKE_USE_SECTRANSP)
--  find_library(COREFOUNDATION_FRAMEWORK "CoreFoundation")
--  if(NOT COREFOUNDATION_FRAMEWORK)
--      message(FATAL_ERROR "CoreFoundation framework not found")
--  endif()
--
--  find_library(SECURITY_FRAMEWORK "Security")
--  if(NOT SECURITY_FRAMEWORK)
--     message(FATAL_ERROR "Security framework not found")
--  endif()
--
--  set(SSL_ENABLED ON)
--  set(USE_SECTRANSP ON)
--  list(APPEND CURL_LIBS "${COREFOUNDATION_FRAMEWORK}" "${SECURITY_FRAMEWORK}")
--endif()
--
--if(CMAKE_USE_OPENSSL)
--  find_package(OpenSSL REQUIRED)
--  set(SSL_ENABLED ON)
--  set(USE_OPENSSL ON)
--
--  # Depend on OpenSSL via imported targets if supported by the running
--  # version of CMake.  This allows our dependents to get our dependencies
--  # transitively.
--  if(NOT CMAKE_VERSION VERSION_LESS 3.4)
--    list(APPEND CURL_LIBS OpenSSL::SSL OpenSSL::Crypto)
--  else()
-     list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
--    include_directories(${OPENSSL_INCLUDE_DIR})
--  endif()
--
--  set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
--  check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
--  check_include_file("openssl/err.h"    HAVE_OPENSSL_ERR_H)
--  check_include_file("openssl/pem.h"    HAVE_OPENSSL_PEM_H)
--  check_include_file("openssl/rsa.h"    HAVE_OPENSSL_RSA_H)
--  check_include_file("openssl/ssl.h"    HAVE_OPENSSL_SSL_H)
--  check_include_file("openssl/x509.h"   HAVE_OPENSSL_X509_H)
--  check_include_file("openssl/rand.h"   HAVE_OPENSSL_RAND_H)
--  check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
--  check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
--  check_symbol_exists(RAND_egd    "${CURL_INCLUDES}" HAVE_RAND_EGD)
--endif()
--
--if(CMAKE_USE_MBEDTLS)
--  find_package(MbedTLS REQUIRED)
--  set(SSL_ENABLED ON)
--  set(USE_MBEDTLS ON)
--  list(APPEND CURL_LIBS ${MBEDTLS_LIBRARIES})
--  include_directories(${MBEDTLS_INCLUDE_DIRS})
--endif()
--
--if(CMAKE_USE_BEARSSL)
--  find_package(BearSSL REQUIRED)
--  set(SSL_ENABLED ON)
--  set(USE_BEARSSL ON)
--  list(APPEND CURL_LIBS ${BEARSSL_LIBRARY})
--  include_directories(${BEARSSL_INCLUDE_DIRS})
--endif()
--
--option(USE_NGHTTP2 "Use Nghttp2 library" OFF)
--if(USE_NGHTTP2)
--  find_package(NGHTTP2 REQUIRED)
--  include_directories(${NGHTTP2_INCLUDE_DIRS})
--  list(APPEND CURL_LIBS ${NGHTTP2_LIBRARIES})
--endif()
--
--if(NOT CURL_DISABLE_LDAP)
--  if(WIN32)
--    option(USE_WIN32_LDAP "Use Windows LDAP implementation" ON)
--    if(USE_WIN32_LDAP)
--      check_library_exists_concat("wldap32" cldap_open HAVE_WLDAP32)
--      if(NOT HAVE_WLDAP32)
--        set(USE_WIN32_LDAP OFF)
--      endif()
--    endif()
--  endif()
--
--  option(CMAKE_USE_OPENLDAP "Use OpenLDAP code." OFF)
--  mark_as_advanced(CMAKE_USE_OPENLDAP)
--  set(CMAKE_LDAP_LIB "ldap" CACHE STRING "Name or full path to ldap library")
--  set(CMAKE_LBER_LIB "lber" CACHE STRING "Name or full path to lber library")
--
--  if(CMAKE_USE_OPENLDAP AND USE_WIN32_LDAP)
--    message(FATAL_ERROR "Cannot use USE_WIN32_LDAP and CMAKE_USE_OPENLDAP at the same time")
--  endif()
--
--  # Now that we know, we're not using windows LDAP...
--  if(USE_WIN32_LDAP)
--    check_include_file_concat("winldap.h" HAVE_WINLDAP_H)
--    check_include_file_concat("winber.h"  HAVE_WINBER_H)
--  else()
--    # Check for LDAP
--    set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARIES})
--    check_library_exists_concat(${CMAKE_LDAP_LIB} ldap_init HAVE_LIBLDAP)
--    check_library_exists_concat(${CMAKE_LBER_LIB} ber_init HAVE_LIBLBER)
--
--    set(CMAKE_REQUIRED_INCLUDES_BAK ${CMAKE_REQUIRED_INCLUDES})
--    set(CMAKE_LDAP_INCLUDE_DIR "" CACHE STRING "Path to LDAP include directory")
--    if(CMAKE_LDAP_INCLUDE_DIR)
--      list(APPEND CMAKE_REQUIRED_INCLUDES ${CMAKE_LDAP_INCLUDE_DIR})
--    endif()
--    check_include_file_concat("ldap.h"           HAVE_LDAP_H)
--    check_include_file_concat("lber.h"           HAVE_LBER_H)
--
--    if(NOT HAVE_LDAP_H)
--      message(STATUS "LDAP_H not found CURL_DISABLE_LDAP set ON")
--      set(CURL_DISABLE_LDAP ON CACHE BOOL "" FORCE)
--      set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_BAK}) #LDAP includes won't be used
--    elseif(NOT HAVE_LIBLDAP)
--      message(STATUS "LDAP library '${CMAKE_LDAP_LIB}' not found CURL_DISABLE_LDAP set ON")
--      set(CURL_DISABLE_LDAP ON CACHE BOOL "" FORCE)
--      set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_BAK}) #LDAP includes won't be used
--    else()
--      if(CMAKE_USE_OPENLDAP)
--        set(USE_OPENLDAP ON)
--      endif()
--      if(CMAKE_LDAP_INCLUDE_DIR)
--        include_directories(${CMAKE_LDAP_INCLUDE_DIR})
--      endif()
--      set(NEED_LBER_H ON)
--      set(_HEADER_LIST)
--      if(HAVE_WINDOWS_H)
--        list(APPEND _HEADER_LIST "windows.h")
--      endif()
--      if(HAVE_SYS_TYPES_H)
--        list(APPEND _HEADER_LIST "sys/types.h")
--      endif()
--      list(APPEND _HEADER_LIST "ldap.h")
--
--      set(_SRC_STRING "")
--      foreach(_HEADER ${_HEADER_LIST})
--        set(_INCLUDE_STRING "${_INCLUDE_STRING}#include <${_HEADER}>\n")
--      endforeach()
--
--      set(_SRC_STRING
--        "
--        ${_INCLUDE_STRING}
--        int main(int argc, char ** argv)
--        {
--          BerValue *bvp = NULL;
--          BerElement *bep = ber_init(bvp);
--          ber_free(bep, 1);
--          return 0;
--        }"
--      )
--      set(CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -DLDAP_DEPRECATED=1")
--      list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_LDAP_LIB})
--      if(HAVE_LIBLBER)
--        list(APPEND CMAKE_REQUIRED_LIBRARIES ${CMAKE_LBER_LIB})
--      endif()
--      check_c_source_compiles("${_SRC_STRING}" NOT_NEED_LBER_H)
--      unset(CMAKE_REQUIRED_LIBRARIES)
--
--      if(NOT_NEED_LBER_H)
--        set(NEED_LBER_H OFF)
--      else()
--        set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -DNEED_LBER_H")
--      endif()
--    endif()
--  endif()
--endif()
--
--# No ldap, no ldaps.
--if(CURL_DISABLE_LDAP)
--  if(NOT CURL_DISABLE_LDAPS)
--    message(STATUS "LDAP needs to be enabled to support LDAPS")
--    set(CURL_DISABLE_LDAPS ON CACHE BOOL "" FORCE)
--  endif()
--endif()
--
--if(NOT CURL_DISABLE_LDAPS)
--  check_include_file_concat("ldap_ssl.h" HAVE_LDAP_SSL_H)
--  check_include_file_concat("ldapssl.h"  HAVE_LDAPSSL_H)
-+    check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
-+    check_include_file("openssl/err.h"    HAVE_OPENSSL_ERR_H)
-+    check_include_file("openssl/pem.h"    HAVE_OPENSSL_PEM_H)
-+    check_include_file("openssl/rsa.h"    HAVE_OPENSSL_RSA_H)
-+    check_include_file("openssl/ssl.h"    HAVE_OPENSSL_SSL_H)
-+    check_include_file("openssl/x509.h"   HAVE_OPENSSL_X509_H)
-+    check_include_file("openssl/rand.h"   HAVE_OPENSSL_RAND_H)
-+    check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
-+    check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
-+    check_symbol_exists(RAND_egd    "${CURL_INCLUDES}" HAVE_RAND_EGD)
- endif()
- 
- # Check for idn
-@@ -511,224 +164,30 @@
- # Check for symbol dlopen (same as HAVE_LIBDL)
- check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
- 
--option(CURL_ZLIB "Set to ON to enable building curl with zlib support." ON)
--set(HAVE_LIBZ OFF)
--set(HAVE_ZLIB_H OFF)
--set(USE_ZLIB OFF)
--if(CURL_ZLIB)
--  find_package(ZLIB QUIET)
--  if(ZLIB_FOUND)
-+# From /cmake/find/zlib.cmake
-+if (ZLIB_FOUND)
-     set(HAVE_ZLIB_H ON)
-     set(HAVE_LIBZ ON)
-     set(USE_ZLIB ON)
- 
--    # Depend on ZLIB via imported targets if supported by the running
--    # version of CMake.  This allows our dependents to get our dependencies
--    # transitively.
--    if(NOT CMAKE_VERSION VERSION_LESS 3.4)
--      list(APPEND CURL_LIBS ZLIB::ZLIB)
--    else()
--      list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
--      include_directories(${ZLIB_INCLUDE_DIRS})
--    endif()
--    list(APPEND CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIRS})
--  endif()
--endif()
--
--option(CURL_BROTLI "Set to ON to enable building curl with brotli support." OFF)
--set(HAVE_BROTLI OFF)
--if(CURL_BROTLI)
--  find_package(Brotli QUIET)
--  if(BROTLI_FOUND)
--    set(HAVE_BROTLI ON)
--    list(APPEND CURL_LIBS ${BROTLI_LIBRARIES})
--    include_directories(${BROTLI_INCLUDE_DIRS})
--    list(APPEND CMAKE_REQUIRED_INCLUDES ${BROTLI_INCLUDE_DIRS})
--  endif()
--endif()
--
--#libSSH2
--option(CMAKE_USE_LIBSSH2 "Use libSSH2" ON)
--mark_as_advanced(CMAKE_USE_LIBSSH2)
--set(USE_LIBSSH2 OFF)
--set(HAVE_LIBSSH2 OFF)
--set(HAVE_LIBSSH2_H OFF)
--
--if(CMAKE_USE_LIBSSH2)
--  find_package(LibSSH2)
--  if(LIBSSH2_FOUND)
--    list(APPEND CURL_LIBS ${LIBSSH2_LIBRARY})
--    set(CMAKE_REQUIRED_LIBRARIES ${LIBSSH2_LIBRARY})
--    list(APPEND CMAKE_REQUIRED_INCLUDES "${LIBSSH2_INCLUDE_DIR}")
--    include_directories("${LIBSSH2_INCLUDE_DIR}")
--    set(HAVE_LIBSSH2 ON)
--    set(USE_LIBSSH2 ON)
--
--    # find_package has already found the headers
--    set(HAVE_LIBSSH2_H ON)
--    set(CURL_INCLUDES ${CURL_INCLUDES} "${LIBSSH2_INCLUDE_DIR}/libssh2.h")
--    set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -DHAVE_LIBSSH2_H")
--
--    # now check for specific libssh2 symbols as they were added in different versions
--    set(CMAKE_EXTRA_INCLUDE_FILES "libssh2.h")
--    check_function_exists(libssh2_version           HAVE_LIBSSH2_VERSION)
--    check_function_exists(libssh2_init              HAVE_LIBSSH2_INIT)
--    check_function_exists(libssh2_exit              HAVE_LIBSSH2_EXIT)
--    check_function_exists(libssh2_scp_send64        HAVE_LIBSSH2_SCP_SEND64)
--    check_function_exists(libssh2_session_handshake HAVE_LIBSSH2_SESSION_HANDSHAKE)
--    set(CMAKE_EXTRA_INCLUDE_FILES "")
--    unset(CMAKE_REQUIRED_LIBRARIES)
--  endif()
--endif()
--
--option(CMAKE_USE_GSSAPI "Use GSSAPI implementation (right now only Heimdal is supported with CMake build)" OFF)
--mark_as_advanced(CMAKE_USE_GSSAPI)
--
--if(CMAKE_USE_GSSAPI)
--  find_package(GSS)
--
--  set(HAVE_GSSAPI ${GSS_FOUND})
--  if(GSS_FOUND)
--
--    message(STATUS "Found ${GSS_FLAVOUR} GSSAPI version: \"${GSS_VERSION}\"")
--
--    list(APPEND CMAKE_REQUIRED_INCLUDES ${GSS_INCLUDE_DIR})
--    check_include_file_concat("gssapi/gssapi.h"  HAVE_GSSAPI_GSSAPI_H)
--    check_include_file_concat("gssapi/gssapi_generic.h" HAVE_GSSAPI_GSSAPI_GENERIC_H)
--    check_include_file_concat("gssapi/gssapi_krb5.h" HAVE_GSSAPI_GSSAPI_KRB5_H)
--
--    if(GSS_FLAVOUR STREQUAL "Heimdal")
--      set(HAVE_GSSHEIMDAL ON)
--    else() # MIT
--      set(HAVE_GSSMIT ON)
--      set(_INCLUDE_LIST "")
--      if(HAVE_GSSAPI_GSSAPI_H)
--        list(APPEND _INCLUDE_LIST "gssapi/gssapi.h")
--      endif()
--      if(HAVE_GSSAPI_GSSAPI_GENERIC_H)
--        list(APPEND _INCLUDE_LIST "gssapi/gssapi_generic.h")
--      endif()
--      if(HAVE_GSSAPI_GSSAPI_KRB5_H)
--        list(APPEND _INCLUDE_LIST "gssapi/gssapi_krb5.h")
--      endif()
--
--      string(REPLACE ";" " " _COMPILER_FLAGS_STR "${GSS_COMPILER_FLAGS}")
--      string(REPLACE ";" " " _LINKER_FLAGS_STR "${GSS_LINKER_FLAGS}")
--
--      foreach(_dir ${GSS_LINK_DIRECTORIES})
--        set(_LINKER_FLAGS_STR "${_LINKER_FLAGS_STR} -L\"${_dir}\"")
--      endforeach()
--
--      set(CMAKE_REQUIRED_FLAGS "${_COMPILER_FLAGS_STR} ${_LINKER_FLAGS_STR}")
--      set(CMAKE_REQUIRED_LIBRARIES ${GSS_LIBRARIES})
--      check_symbol_exists("GSS_C_NT_HOSTBASED_SERVICE" ${_INCLUDE_LIST} HAVE_GSS_C_NT_HOSTBASED_SERVICE)
--      if(NOT HAVE_GSS_C_NT_HOSTBASED_SERVICE)
--        set(HAVE_OLD_GSSMIT ON)
--      endif()
--      unset(CMAKE_REQUIRED_LIBRARIES)
--
--    endif()
--
--    include_directories(${GSS_INCLUDE_DIR})
--    link_directories(${GSS_LINK_DIRECTORIES})
--    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GSS_COMPILER_FLAGS}")
--    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${GSS_LINKER_FLAGS}")
--    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GSS_LINKER_FLAGS}")
--    list(APPEND CURL_LIBS ${GSS_LIBRARIES})
--
--  else()
--    message(WARNING "GSSAPI support has been requested but no supporting libraries found. Skipping.")
--  endif()
-+    list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
- endif()
- 
- option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" ON)
- if(ENABLE_UNIX_SOCKETS)
--  include(CheckStructHasMember)
--  check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
-+    include(CheckStructHasMember)
-+    check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
- else()
--  unset(USE_UNIX_SOCKETS CACHE)
-+    unset(USE_UNIX_SOCKETS CACHE)
- endif()
- 
--#
- # CA handling
--#
--set(CURL_CA_BUNDLE "auto" CACHE STRING
--    "Path to the CA bundle. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.")
--set(CURL_CA_FALLBACK OFF CACHE BOOL
--    "Set ON to use built-in CA store of TLS backend. Defaults to OFF")
--set(CURL_CA_PATH "auto" CACHE STRING
--    "Location of default CA path. Set 'none' to disable or 'auto' for auto-detection. Defaults to 'auto'.")
--
--if("${CURL_CA_BUNDLE}" STREQUAL "")
--  message(FATAL_ERROR "Invalid value of CURL_CA_BUNDLE. Use 'none', 'auto' or file path.")
--elseif("${CURL_CA_BUNDLE}" STREQUAL "none")
--  unset(CURL_CA_BUNDLE CACHE)
--elseif("${CURL_CA_BUNDLE}" STREQUAL "auto")
--  unset(CURL_CA_BUNDLE CACHE)
--  set(CURL_CA_BUNDLE_AUTODETECT TRUE)
--else()
--  set(CURL_CA_BUNDLE_SET TRUE)
--endif()
--
--if("${CURL_CA_PATH}" STREQUAL "")
--  message(FATAL_ERROR "Invalid value of CURL_CA_PATH. Use 'none', 'auto' or directory path.")
--elseif("${CURL_CA_PATH}" STREQUAL "none")
--  unset(CURL_CA_PATH CACHE)
--elseif("${CURL_CA_PATH}" STREQUAL "auto")
--  unset(CURL_CA_PATH CACHE)
--  set(CURL_CA_PATH_AUTODETECT TRUE)
--else()
--  set(CURL_CA_PATH_SET TRUE)
--endif()
--
--if(CURL_CA_BUNDLE_SET AND CURL_CA_PATH_AUTODETECT)
--  # Skip autodetection of unset CA path because CA bundle is set explicitly
--elseif(CURL_CA_PATH_SET AND CURL_CA_BUNDLE_AUTODETECT)
--  # Skip autodetection of unset CA bundle because CA path is set explicitly
--elseif(CURL_CA_PATH_AUTODETECT OR CURL_CA_BUNDLE_AUTODETECT)
--  # first try autodetecting a CA bundle, then a CA path
--
--  if(CURL_CA_BUNDLE_AUTODETECT)
--    set(SEARCH_CA_BUNDLE_PATHS
--        /etc/ssl/certs/ca-certificates.crt
--        /etc/pki/tls/certs/ca-bundle.crt
--        /usr/share/ssl/certs/ca-bundle.crt
--        /usr/local/share/certs/ca-root-nss.crt
--        /etc/ssl/cert.pem)
--
--    foreach(SEARCH_CA_BUNDLE_PATH ${SEARCH_CA_BUNDLE_PATHS})
--      if(EXISTS "${SEARCH_CA_BUNDLE_PATH}")
--        message(STATUS "Found CA bundle: ${SEARCH_CA_BUNDLE_PATH}")
--        set(CURL_CA_BUNDLE "${SEARCH_CA_BUNDLE_PATH}")
--        set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
--        break()
--      endif()
--    endforeach()
--  endif()
--
--  if(CURL_CA_PATH_AUTODETECT AND (NOT CURL_CA_PATH_SET))
--    if(EXISTS "/etc/ssl/certs")
--      set(CURL_CA_PATH "/etc/ssl/certs")
--      set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
--    endif()
--  endif()
--endif()
--
--if(CURL_CA_PATH_SET AND NOT USE_OPENSSL AND NOT USE_MBEDTLS)
--  message(FATAL_ERROR
--          "CA path only supported by OpenSSL, GnuTLS or mbed TLS. "
--          "Set CURL_CA_PATH=none or enable one of those TLS backends.")
--endif()
--
--# Check for header files
--if(NOT UNIX)
--  check_include_file_concat("windows.h"      HAVE_WINDOWS_H)
--  check_include_file_concat("winsock.h"      HAVE_WINSOCK_H)
--  check_include_file_concat("ws2tcpip.h"     HAVE_WS2TCPIP_H)
--  check_include_file_concat("winsock2.h"     HAVE_WINSOCK2_H)
--  if(NOT CURL_WINDOWS_SSPI AND USE_OPENSSL)
--    set(CURL_LIBS ${CURL_LIBS} "crypt32")
--  endif()
-+# Explicitly set to most common case
-+if (OPENSSL_FOUND)
-+    set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
-+    set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
-+    set(CURL_CA_PATH "/etc/ssl/certs")
-+    set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
- endif()
- 
- check_include_file_concat("stdio.h"          HAVE_STDIO_H)
-@@ -810,18 +269,18 @@
- check_type_size("long double"  SIZEOF_LONG_DOUBLE)
- check_type_size("time_t"  SIZEOF_TIME_T)
- if(NOT HAVE_SIZEOF_SSIZE_T)
--  if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T)
--    set(ssize_t long)
--  endif()
--  if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T)
--    set(ssize_t __int64)
--  endif()
-+    if(SIZEOF_LONG EQUAL SIZEOF_SIZE_T)
-+        set(ssize_t long)
-+    endif()
-+    if(NOT ssize_t AND SIZEOF___INT64 EQUAL SIZEOF_SIZE_T)
-+        set(ssize_t __int64)
-+    endif()
- endif()
- # off_t is sized later, after the HAVE_FILE_OFFSET_BITS test
- 
- if(HAVE_SIZEOF_LONG_LONG)
--  set(HAVE_LONGLONG 1)
--  set(HAVE_LL 1)
-+    set(HAVE_LONGLONG 1)
-+    set(HAVE_LL 1)
- endif()
- 
- find_file(RANDOM_FILE urandom /dev)
-@@ -829,9 +288,9 @@
- 
- # Check for some functions that are used
- if(HAVE_LIBWS2_32)
--  set(CMAKE_REQUIRED_LIBRARIES ws2_32)
-+    set(CMAKE_REQUIRED_LIBRARIES ws2_32)
- elseif(HAVE_LIBSOCKET)
--  set(CMAKE_REQUIRED_LIBRARIES socket)
-+    set(CMAKE_REQUIRED_LIBRARIES socket)
- endif()
- 
- check_symbol_exists(basename      "${CURL_INCLUDES}" HAVE_BASENAME)
-@@ -849,7 +308,7 @@
- check_symbol_exists(strncmpi      "${CURL_INCLUDES}" HAVE_STRNCMPI)
- check_symbol_exists(alarm         "${CURL_INCLUDES}" HAVE_ALARM)
- if(NOT HAVE_STRNCMPI)
--  set(HAVE_STRCMPI)
-+    set(HAVE_STRCMPI)
- endif()
- check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
- check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
-@@ -879,7 +338,7 @@
- check_symbol_exists(signal        "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
- check_symbol_exists(SIGALRM       "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
- if(HAVE_SIGNAL_FUNC AND HAVE_SIGNAL_MACRO)
--  set(HAVE_SIGNAL 1)
-+    set(HAVE_SIGNAL 1)
- endif()
- check_symbol_exists(uname          "${CURL_INCLUDES}" HAVE_UNAME)
- check_symbol_exists(strtoll        "${CURL_INCLUDES}" HAVE_STRTOLL)
-@@ -906,89 +365,75 @@
- check_symbol_exists(setsockopt     "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
- check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
- 
--# symbol exists in win32, but function does not.
--if(WIN32)
--  if(ENABLE_INET_PTON)
--    check_function_exists(inet_pton HAVE_INET_PTON)
--    # _WIN32_WINNT_VISTA (0x0600)
--    add_definitions(-D_WIN32_WINNT=0x0600)
--  else()
--    # _WIN32_WINNT_WINXP (0x0501)
--    add_definitions(-D_WIN32_WINNT=0x0501)
--  endif()
--else()
--  check_function_exists(inet_pton HAVE_INET_PTON)
--endif()
--
- check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
- if(HAVE_FSETXATTR)
--  foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
--    curl_internal_test(${CURL_TEST})
--  endforeach()
-+    foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
-+        curl_internal_test(${CURL_TEST})
-+    endforeach()
- endif()
- 
- # sigaction and sigsetjmp are special. Use special mechanism for
- # detecting those, but only if previous attempt failed.
- if(HAVE_SIGNAL_H)
--  check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
-+    check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
- endif()
- 
- if(NOT HAVE_SIGSETJMP)
--  if(HAVE_SETJMP_H)
--    check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
--    if(HAVE_MACRO_SIGSETJMP)
--      set(HAVE_SIGSETJMP 1)
-+    if(HAVE_SETJMP_H)
-+        check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
-+        if(HAVE_MACRO_SIGSETJMP)
-+            set(HAVE_SIGSETJMP 1)
-+        endif()
-     endif()
--  endif()
- endif()
- 
- # If there is no stricmp(), do not allow LDAP to parse URLs
- if(NOT HAVE_STRICMP)
--  set(HAVE_LDAP_URL_PARSE 1)
-+    set(HAVE_LDAP_URL_PARSE 1)
- endif()
- 
- # Do curl specific tests
- foreach(CURL_TEST
--    HAVE_FCNTL_O_NONBLOCK
--    HAVE_IOCTLSOCKET
--    HAVE_IOCTLSOCKET_CAMEL
--    HAVE_IOCTLSOCKET_CAMEL_FIONBIO
--    HAVE_IOCTLSOCKET_FIONBIO
--    HAVE_IOCTL_FIONBIO
--    HAVE_IOCTL_SIOCGIFADDR
--    HAVE_SETSOCKOPT_SO_NONBLOCK
--    HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
--    TIME_WITH_SYS_TIME
--    HAVE_O_NONBLOCK
--    HAVE_GETHOSTBYADDR_R_5
--    HAVE_GETHOSTBYADDR_R_7
--    HAVE_GETHOSTBYADDR_R_8
--    HAVE_GETHOSTBYADDR_R_5_REENTRANT
--    HAVE_GETHOSTBYADDR_R_7_REENTRANT
--    HAVE_GETHOSTBYADDR_R_8_REENTRANT
--    HAVE_GETHOSTBYNAME_R_3
--    HAVE_GETHOSTBYNAME_R_5
--    HAVE_GETHOSTBYNAME_R_6
--    HAVE_GETHOSTBYNAME_R_3_REENTRANT
--    HAVE_GETHOSTBYNAME_R_5_REENTRANT
--    HAVE_GETHOSTBYNAME_R_6_REENTRANT
--    HAVE_IN_ADDR_T
--    HAVE_BOOL_T
--    STDC_HEADERS
--    RETSIGTYPE_TEST
--    HAVE_INET_NTOA_R_DECL
--    HAVE_INET_NTOA_R_DECL_REENTRANT
--    HAVE_GETADDRINFO
--    HAVE_FILE_OFFSET_BITS
--    HAVE_VARIADIC_MACROS_C99
--    HAVE_VARIADIC_MACROS_GCC
--    )
--  curl_internal_test(${CURL_TEST})
-+        HAVE_FCNTL_O_NONBLOCK
-+        HAVE_IOCTLSOCKET
-+        HAVE_IOCTLSOCKET_CAMEL
-+        HAVE_IOCTLSOCKET_CAMEL_FIONBIO
-+        HAVE_IOCTLSOCKET_FIONBIO
-+        HAVE_IOCTL_FIONBIO
-+        HAVE_IOCTL_SIOCGIFADDR
-+        HAVE_SETSOCKOPT_SO_NONBLOCK
-+        HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
-+        TIME_WITH_SYS_TIME
-+        HAVE_O_NONBLOCK
-+        HAVE_GETHOSTBYADDR_R_5
-+        HAVE_GETHOSTBYADDR_R_7
-+        HAVE_GETHOSTBYADDR_R_8
-+        HAVE_GETHOSTBYADDR_R_5_REENTRANT
-+        HAVE_GETHOSTBYADDR_R_7_REENTRANT
-+        HAVE_GETHOSTBYADDR_R_8_REENTRANT
-+        HAVE_GETHOSTBYNAME_R_3
-+        HAVE_GETHOSTBYNAME_R_5
-+        HAVE_GETHOSTBYNAME_R_6
-+        HAVE_GETHOSTBYNAME_R_3_REENTRANT
-+        HAVE_GETHOSTBYNAME_R_5_REENTRANT
-+        HAVE_GETHOSTBYNAME_R_6_REENTRANT
-+        HAVE_IN_ADDR_T
-+        HAVE_BOOL_T
-+        STDC_HEADERS
-+        RETSIGTYPE_TEST
-+        HAVE_INET_NTOA_R_DECL
-+        HAVE_INET_NTOA_R_DECL_REENTRANT
-+        HAVE_GETADDRINFO
-+        HAVE_FILE_OFFSET_BITS
-+        HAVE_VARIADIC_MACROS_C99
-+        HAVE_VARIADIC_MACROS_GCC
-+        )
-+    curl_internal_test(${CURL_TEST})
- endforeach()
- 
- if(HAVE_FILE_OFFSET_BITS)
--  set(_FILE_OFFSET_BITS 64)
--  set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
-+    set(_FILE_OFFSET_BITS 64)
-+    set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
- endif()
- check_type_size("off_t"  SIZEOF_OFF_T)
- 
-@@ -998,49 +443,47 @@
- check_type_size("curl_off_t"  SIZEOF_CURL_OFF_T)
- set(CMAKE_EXTRA_INCLUDE_FILES "")
- 
--set(CMAKE_REQUIRED_FLAGS)
--
- foreach(CURL_TEST
--    HAVE_GLIBC_STRERROR_R
--    HAVE_POSIX_STRERROR_R
--    )
--  curl_internal_test(${CURL_TEST})
-+        HAVE_GLIBC_STRERROR_R
-+        HAVE_POSIX_STRERROR_R
-+        )
-+    curl_internal_test(${CURL_TEST})
- endforeach()
- 
- # Check for reentrant
- foreach(CURL_TEST
--    HAVE_GETHOSTBYADDR_R_5
--    HAVE_GETHOSTBYADDR_R_7
--    HAVE_GETHOSTBYADDR_R_8
--    HAVE_GETHOSTBYNAME_R_3
--    HAVE_GETHOSTBYNAME_R_5
--    HAVE_GETHOSTBYNAME_R_6
--    HAVE_INET_NTOA_R_DECL_REENTRANT)
--  if(NOT ${CURL_TEST})
--    if(${CURL_TEST}_REENTRANT)
--      set(NEED_REENTRANT 1)
-+        HAVE_GETHOSTBYADDR_R_5
-+        HAVE_GETHOSTBYADDR_R_7
-+        HAVE_GETHOSTBYADDR_R_8
-+        HAVE_GETHOSTBYNAME_R_3
-+        HAVE_GETHOSTBYNAME_R_5
-+        HAVE_GETHOSTBYNAME_R_6
-+        HAVE_INET_NTOA_R_DECL_REENTRANT)
-+    if(NOT ${CURL_TEST})
-+        if(${CURL_TEST}_REENTRANT)
-+            set(NEED_REENTRANT 1)
-+        endif()
-     endif()
--  endif()
- endforeach()
- 
- if(NEED_REENTRANT)
--  foreach(CURL_TEST
--      HAVE_GETHOSTBYADDR_R_5
--      HAVE_GETHOSTBYADDR_R_7
--      HAVE_GETHOSTBYADDR_R_8
--      HAVE_GETHOSTBYNAME_R_3
--      HAVE_GETHOSTBYNAME_R_5
--      HAVE_GETHOSTBYNAME_R_6)
--    set(${CURL_TEST} 0)
--    if(${CURL_TEST}_REENTRANT)
--      set(${CURL_TEST} 1)
--    endif()
--  endforeach()
-+    foreach(CURL_TEST
-+            HAVE_GETHOSTBYADDR_R_5
-+            HAVE_GETHOSTBYADDR_R_7
-+            HAVE_GETHOSTBYADDR_R_8
-+            HAVE_GETHOSTBYNAME_R_3
-+            HAVE_GETHOSTBYNAME_R_5
-+            HAVE_GETHOSTBYNAME_R_6)
-+        set(${CURL_TEST} 0)
-+        if(${CURL_TEST}_REENTRANT)
-+            set(${CURL_TEST} 1)
-+        endif()
-+    endforeach()
- endif()
- 
- if(HAVE_INET_NTOA_R_DECL_REENTRANT)
--  set(HAVE_INET_NTOA_R_DECL 1)
--  set(NEED_REENTRANT 1)
-+    set(HAVE_INET_NTOA_R_DECL 1)
-+    set(NEED_REENTRANT 1)
- endif()
- 
- # Check clock_gettime(CLOCK_MONOTONIC, x) support
-@@ -1052,323 +495,132 @@
- # Some other minor tests
- 
- if(NOT HAVE_IN_ADDR_T)
--  set(in_addr_t "unsigned long")
--endif()
--
--# Fix libz / zlib.h
--
--if(NOT CURL_SPECIAL_LIBZ)
--  if(NOT HAVE_LIBZ)
--    set(HAVE_ZLIB_H 0)
--  endif()
--
--  if(NOT HAVE_ZLIB_H)
--    set(HAVE_LIBZ 0)
--  endif()
-+    set(in_addr_t "unsigned long")
- endif()
- 
- # Check for nonblocking
- set(HAVE_DISABLED_NONBLOCKING 1)
- if(HAVE_FIONBIO OR
--    HAVE_IOCTLSOCKET OR
--    HAVE_IOCTLSOCKET_CASE OR
--    HAVE_O_NONBLOCK)
--  set(HAVE_DISABLED_NONBLOCKING)
--endif()
--
--if(RETSIGTYPE_TEST)
--  set(RETSIGTYPE void)
--else()
--  set(RETSIGTYPE int)
--endif()
--
--if(CMAKE_COMPILER_IS_GNUCC AND APPLE)
--  include(CheckCCompilerFlag)
--  check_c_compiler_flag(-Wno-long-double HAVE_C_FLAG_Wno_long_double)
--  if(HAVE_C_FLAG_Wno_long_double)
--    # The Mac version of GCC warns about use of long double.  Disable it.
--    get_source_file_property(MPRINTF_COMPILE_FLAGS mprintf.c COMPILE_FLAGS)
--    if(MPRINTF_COMPILE_FLAGS)
--      set(MPRINTF_COMPILE_FLAGS "${MPRINTF_COMPILE_FLAGS} -Wno-long-double")
--    else()
--      set(MPRINTF_COMPILE_FLAGS "-Wno-long-double")
--    endif()
--    set_source_files_properties(mprintf.c PROPERTIES
--      COMPILE_FLAGS ${MPRINTF_COMPILE_FLAGS})
--  endif()
-+        HAVE_IOCTLSOCKET OR
-+        HAVE_IOCTLSOCKET_CASE OR
-+        HAVE_O_NONBLOCK)
-+    set(HAVE_DISABLED_NONBLOCKING)
- endif()
- 
--# TODO test which of these headers are required
--if(WIN32)
--  set(CURL_PULL_WS2TCPIP_H ${HAVE_WS2TCPIP_H})
--else()
--  set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
--  set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
--  set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
--endif()
-+set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
-+set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
-+set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
- set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
- set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
- 
- include(CMake/OtherTests.cmake)
- 
--add_definitions(-DHAVE_CONFIG_H)
-+SET(LIB_VAUTH_CFILES
-+        "${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
-+        "${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
-+        "${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
-+        "${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
-+
-+SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
-+
-+SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
-+        "${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
-+        "${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
-+        "${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
-+        "${CURL_LIBRARY_DIR}/vtls/bearssl.c")
-+
-+SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
-+        "${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
-+        "${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
-+        "${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
-+
-+SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
-+
-+SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
-+
-+SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
-+
-+SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
-+
-+SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
-+        "${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
-+        "${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
-+        "${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
-+        "${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
-+        "${CURL_LIBRARY_DIR}/fileinfo.c"  "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
-+        "${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
-+        "${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
-+        "${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
-+        "${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c"  "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
-+        "${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
-+        "${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
-+        "${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
-+        "${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
-+        "${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
-+        "${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
-+        "${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
-+        "${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
-+        "${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
-+        "${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
-+
-+SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
-+        "${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
-+        "${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
-+        "${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
-+        "${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h"  "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
-+        "${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
-+        "${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
-+        "${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
-+        "${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
-+        "${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
-+        "${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
-+        "${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
-+        "${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
-+        "${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
-+        "${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
-+        "${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
-+        "${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
-+        "${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
-+        "${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
-+        "${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
-+
-+SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
-+
-+SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
-+        ${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
-+SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
-+        ${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
-+
-+configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
-+        ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
-+
-+list(APPEND HHEADERS
-+        ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
-+        )
-+
-+add_library(libcurl ${HHEADERS} ${CSOURCES})
-+
-+if(NOT BUILD_SHARED_LIBS)
-+    set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
-+endif()
-+
-+if(HIDES_CURL_PRIVATE_SYMBOLS)
-+    set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
-+    set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
-+endif()
-+
-+if(OPENSSL_FOUND)
-+    target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
-+    message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
-+endif()
-+
-+if(ZLIB_FOUND)
-+    target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
-+    message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
-+endif()
-+
-+target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
-+target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
-+target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
- 
--# For Windows, all compilers used by CMake should support large files
--if(WIN32)
--  set(USE_WIN32_LARGE_FILES ON)
--
--  # Use the manifest embedded in the Windows Resource
--  set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -DCURL_EMBED_MANIFEST")
--endif()
--
--if(MSVC)
--  # Disable default manifest added by CMake
--  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
--
--  add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
--  if(CMAKE_C_FLAGS MATCHES "/W[0-4]")
--    string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
--  else()
--    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
--  endif()
--endif()
--
--if(CURL_WERROR)
--  if(MSVC_VERSION)
--    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX")
--  else()
--    # this assumes clang or gcc style options
--    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
--  endif()
--endif()
--
--# Ugly (but functional) way to include "Makefile.inc" by transforming it (= regenerate it).
--function(transform_makefile_inc INPUT_FILE OUTPUT_FILE)
--  file(READ ${INPUT_FILE} MAKEFILE_INC_TEXT)
--  string(REPLACE "$(top_srcdir)"   "\${CURL_SOURCE_DIR}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})
--  string(REPLACE "$(top_builddir)" "\${CURL_BINARY_DIR}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})
--
--  string(REGEX REPLACE "\\\\\n" "!π!α!" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})
--  string(REGEX REPLACE "([a-zA-Z_][a-zA-Z0-9_]*)[\t ]*=[\t ]*([^\n]*)" "SET(\\1 \\2)" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})
--  string(REPLACE "!π!α!" "\n" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})
--
--  string(REGEX REPLACE "\\$\\(([a-zA-Z_][a-zA-Z0-9_]*)\\)" "\${\\1}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})    # Replace $() with ${}
--  string(REGEX REPLACE "@([a-zA-Z_][a-zA-Z0-9_]*)@" "\${\\1}" MAKEFILE_INC_TEXT ${MAKEFILE_INC_TEXT})    # Replace @@ with ${}, even if that may not be read by CMake scripts.
--  file(WRITE ${OUTPUT_FILE} ${MAKEFILE_INC_TEXT})
--
--endfunction()
--
--include(GNUInstallDirs)
--
--set(CURL_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
--set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets")
--set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated")
--set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake")
--set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake")
--
--if(USE_MANUAL)
--  add_subdirectory(docs)
--endif()
--
--add_subdirectory(lib)
--
--if(BUILD_CURL_EXE)
--  add_subdirectory(src)
--endif()
--
--include(CTest)
--if(BUILD_TESTING)
--  add_subdirectory(tests)
--endif()
--
--# Helper to populate a list (_items) with a label when conditions (the remaining
--# args) are satisfied
--function(_add_if label)
--  # TODO need to disable policy CMP0054 (CMake 3.1) to allow this indirection
--  if(${ARGN})
--    set(_items ${_items} "${label}" PARENT_SCOPE)
--  endif()
--endfunction()
--
--# Clear list and try to detect available features
--set(_items)
--_add_if("SSL"           SSL_ENABLED)
--_add_if("IPv6"          ENABLE_IPV6)
--_add_if("unix-sockets"  USE_UNIX_SOCKETS)
--_add_if("libz"          HAVE_LIBZ)
--_add_if("AsynchDNS"     USE_ARES OR USE_THREADS_POSIX OR USE_THREADS_WIN32)
--_add_if("IDN"           HAVE_LIBIDN2)
--_add_if("Largefile"     (CURL_SIZEOF_CURL_OFF_T GREATER 4) AND
--                        ((SIZEOF_OFF_T GREATER 4) OR USE_WIN32_LARGE_FILES))
--# TODO SSP1 (WinSSL) check is missing
--_add_if("SSPI"          USE_WINDOWS_SSPI)
--_add_if("GSS-API"       HAVE_GSSAPI)
--# TODO SSP1 missing for SPNEGO
--_add_if("SPNEGO"        NOT CURL_DISABLE_CRYPTO_AUTH AND
--                        (HAVE_GSSAPI OR USE_WINDOWS_SSPI))
--_add_if("Kerberos"      NOT CURL_DISABLE_CRYPTO_AUTH AND
--                        (HAVE_GSSAPI OR USE_WINDOWS_SSPI))
--# NTLM support requires crypto function adaptions from various SSL libs
--# TODO alternative SSL libs tests for SSP1, GNUTLS, NSS
--if(NOT CURL_DISABLE_CRYPTO_AUTH AND (USE_OPENSSL OR USE_WINDOWS_SSPI OR USE_SECTRANSP OR USE_MBEDTLS))
--  _add_if("NTLM"        1)
--  # TODO missing option (autoconf: --enable-ntlm-wb)
--  _add_if("NTLM_WB"     NOT CURL_DISABLE_HTTP AND NTLM_WB_ENABLED)
--endif()
--# TODO missing option (--enable-tls-srp), depends on GNUTLS_SRP/OPENSSL_SRP
--_add_if("TLS-SRP"       USE_TLS_SRP)
--# TODO option --with-nghttp2 tests for nghttp2 lib and nghttp2/nghttp2.h header
--_add_if("HTTP2"         USE_NGHTTP2)
--string(REPLACE ";" " " SUPPORT_FEATURES "${_items}")
--message(STATUS "Enabled features: ${SUPPORT_FEATURES}")
--
--# Clear list and try to detect available protocols
--set(_items)
--_add_if("HTTP"          NOT CURL_DISABLE_HTTP)
--_add_if("HTTPS"         NOT CURL_DISABLE_HTTP AND SSL_ENABLED)
--_add_if("FTP"           NOT CURL_DISABLE_FTP)
--_add_if("FTPS"          NOT CURL_DISABLE_FTP AND SSL_ENABLED)
--_add_if("FILE"          NOT CURL_DISABLE_FILE)
--_add_if("TELNET"        NOT CURL_DISABLE_TELNET)
--_add_if("LDAP"          NOT CURL_DISABLE_LDAP)
--# CURL_DISABLE_LDAP implies CURL_DISABLE_LDAPS
--# TODO check HAVE_LDAP_SSL (in autoconf this is enabled with --enable-ldaps)
--_add_if("LDAPS"         NOT CURL_DISABLE_LDAPS AND
--                        ((USE_OPENLDAP AND SSL_ENABLED) OR
--                        (NOT USE_OPENLDAP AND HAVE_LDAP_SSL)))
--_add_if("DICT"          NOT CURL_DISABLE_DICT)
--_add_if("TFTP"          NOT CURL_DISABLE_TFTP)
--_add_if("GOPHER"        NOT CURL_DISABLE_GOPHER)
--_add_if("POP3"          NOT CURL_DISABLE_POP3)
--_add_if("POP3S"         NOT CURL_DISABLE_POP3 AND SSL_ENABLED)
--_add_if("IMAP"          NOT CURL_DISABLE_IMAP)
--_add_if("IMAPS"         NOT CURL_DISABLE_IMAP AND SSL_ENABLED)
--_add_if("SMTP"          NOT CURL_DISABLE_SMTP)
--_add_if("SMTPS"         NOT CURL_DISABLE_SMTP AND SSL_ENABLED)
--_add_if("SCP"           USE_LIBSSH2)
--_add_if("SFTP"          USE_LIBSSH2)
--_add_if("RTSP"          NOT CURL_DISABLE_RTSP)
--_add_if("RTMP"          USE_LIBRTMP)
--if(_items)
--  list(SORT _items)
--endif()
--string(REPLACE ";" " " SUPPORT_PROTOCOLS "${_items}")
--message(STATUS "Enabled protocols: ${SUPPORT_PROTOCOLS}")
--
--# Clear list and collect SSL backends
--set(_items)
--_add_if("WinSSL"           SSL_ENABLED AND USE_WINDOWS_SSPI)
--_add_if("OpenSSL"          SSL_ENABLED AND USE_OPENSSL)
--_add_if("Secure Transport" SSL_ENABLED AND USE_SECTRANSP)
--_add_if("mbedTLS"          SSL_ENABLED AND USE_MBEDTLS)
--_add_if("BearSSL"          SSL_ENABLED AND USE_BEARSSL)
--if(_items)
--  list(SORT _items)
--endif()
--string(REPLACE ";" " " SSL_BACKENDS "${_items}")
--message(STATUS "Enabled SSL backends: ${SSL_BACKENDS}")
--
--# curl-config needs the following options to be set.
--set(CC                      "${CMAKE_C_COMPILER}")
--# TODO probably put a -D... options here?
--set(CONFIGURE_OPTIONS       "")
--# TODO when to set "-DCURL_STATICLIB" for CPPFLAG_CURL_STATICLIB?
--set(CPPFLAG_CURL_STATICLIB  "")
--set(CURLVERSION             "${CURL_VERSION}")
--if(BUILD_SHARED_LIBS)
--  set(ENABLE_SHARED         "yes")
--  set(ENABLE_STATIC         "no")
--else()
--  set(ENABLE_SHARED         "no")
--  set(ENABLE_STATIC         "yes")
--endif()
--set(exec_prefix             "\${prefix}")
--set(includedir              "\${prefix}/include")
--set(LDFLAGS                 "${CMAKE_SHARED_LINKER_FLAGS}")
--set(LIBCURL_LIBS            "")
--set(libdir                  "${CMAKE_INSTALL_PREFIX}/lib")
--foreach(_lib ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} ${CURL_LIBS})
--  if(_lib MATCHES ".*/.*" OR _lib MATCHES "^-")
--    set(LIBCURL_LIBS          "${LIBCURL_LIBS} ${_lib}")
--  else()
--    set(LIBCURL_LIBS          "${LIBCURL_LIBS} -l${_lib}")
--  endif()
--endforeach()
--# "a" (Linux) or "lib" (Windows)
--string(REPLACE "." "" libext "${CMAKE_STATIC_LIBRARY_SUFFIX}")
--set(prefix                  "${CMAKE_INSTALL_PREFIX}")
--# Set this to "yes" to append all libraries on which -lcurl is dependent
--set(REQUIRE_LIB_DEPS        "no")
--# SUPPORT_FEATURES
--# SUPPORT_PROTOCOLS
--set(VERSIONNUM              "${CURL_VERSION_NUM}")
--
--# Finally generate a "curl-config" matching this config
--# Use:
--# * ENABLE_SHARED
--# * ENABLE_STATIC
--configure_file("${CURL_SOURCE_DIR}/curl-config.in"
--               "${CURL_BINARY_DIR}/curl-config" @ONLY)
--install(FILES "${CURL_BINARY_DIR}/curl-config"
--        DESTINATION ${CMAKE_INSTALL_BINDIR}
--        PERMISSIONS
--          OWNER_READ OWNER_WRITE OWNER_EXECUTE
--          GROUP_READ GROUP_EXECUTE
--          WORLD_READ WORLD_EXECUTE)
--
--# Finally generate a pkg-config file matching this config
--configure_file("${CURL_SOURCE_DIR}/libcurl.pc.in"
--               "${CURL_BINARY_DIR}/libcurl.pc" @ONLY)
--install(FILES "${CURL_BINARY_DIR}/libcurl.pc"
--        DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
--
--# install headers
--install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/curl"
--    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
--    FILES_MATCHING PATTERN "*.h")
--
--include(CMakePackageConfigHelpers)
--write_basic_package_version_file(
--    "${version_config}"
--    VERSION ${CURL_VERSION}
--    COMPATIBILITY SameMajorVersion
--)
--
--# Use:
--# * TARGETS_EXPORT_NAME
--# * PROJECT_NAME
--configure_package_config_file(CMake/curl-config.cmake.in
--        "${project_config}"
--        INSTALL_DESTINATION ${CURL_INSTALL_CMAKE_DIR}
--)
--
--install(
--        EXPORT "${TARGETS_EXPORT_NAME}"
--        NAMESPACE "${PROJECT_NAME}::"
--        DESTINATION ${CURL_INSTALL_CMAKE_DIR}
--)
--
--install(
--        FILES ${version_config} ${project_config}
--        DESTINATION ${CURL_INSTALL_CMAKE_DIR}
--)
--
--# Workaround for MSVS10 to avoid the Dialog Hell
--# FIXME: This could be removed with future version of CMake.
--if(MSVC_VERSION EQUAL 1600)
--  set(CURL_SLN_FILENAME "${CMAKE_CURRENT_BINARY_DIR}/CURL.sln")
--  if(EXISTS "${CURL_SLN_FILENAME}")
--    file(APPEND "${CURL_SLN_FILENAME}" "\n# This should be regenerated!\n")
--  endif()
--endif()
--
--if(NOT TARGET uninstall)
--  configure_file(
--      ${CMAKE_CURRENT_SOURCE_DIR}/CMake/cmake_uninstall.cmake.in
--      ${CMAKE_CURRENT_BINARY_DIR}/CMake/cmake_uninstall.cmake
--      IMMEDIATE @ONLY)
--
--  add_custom_target(uninstall
--      COMMAND ${CMAKE_COMMAND} -P
--      ${CMAKE_CURRENT_BINARY_DIR}/CMake/cmake_uninstall.cmake)
--endif()
-+target_link_libraries(libcurl ${CURL_LIBS})
diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt
index 53a255c7d8b..07a689adb6d 100644
--- a/contrib/curl-cmake/CMakeLists.txt
+++ b/contrib/curl-cmake/CMakeLists.txt
@@ -1,605 +1,148 @@
-#***************************************************************************
-#                                  _   _ ____  _
-#  Project                     ___| | | |  _ \| |
-#                             / __| | | | |_) | |
-#                            | (__| |_| |  _ <| |___
-#                             \___|\___/|_| \_\_____|
-#
-# Copyright (C) 1998 - 2019, Daniel Stenberg, , et al.
-#
-# This software is licensed as described in the file COPYING, which
-# you should have received as part of this distribution. The terms
-# are also available at https://curl.haxx.se/docs/copyright.html.
-#
-# You may opt to use, copy, modify, merge, publish, distribute and/or sell
-# copies of the Software, and permit persons to whom the Software is
-# furnished to do so, under the terms of the COPYING file.
-#
-# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
-# KIND, either express or implied.
-#
-###########################################################################
-
-# NOTE:
-# This file is shrinked and reworked version of original curl CMakeLists.txt
-# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt
-# If you need to update curl building you can find patch file in this directory
-# and apply it to fresh original CMakeLists.txt file.
-cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
-
-SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
-SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
-# Disable status messages when perform checks
-set(CMAKE_REQUIRED_QUIET TRUE)
-
-include(Macros)
-include(CMakeDependentOption)
-include(CheckCCompilerFlag)
-
-file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
-string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
-        CURL_VERSION ${CURL_VERSION_H_CONTENTS})
-string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
-string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
-        CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
-string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
-
-message(STATUS "Use curl version=[${CURL_VERSION}]")
-set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
-set(OS "\"${CMAKE_SYSTEM_NAME}\"")
-
-option(PICKY_COMPILER "Enable picky compiler options" ON)
-option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
-
-if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
-    if(PICKY_COMPILER)
-        foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
-            # surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
-            # test result in.
-            check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
-            if(OPT${_CCOPT})
-                set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
-            endif()
-        endforeach()
-    endif()
-endif()
-
-# initialize CURL_LIBS
-set(CURL_LIBS "")
-
-include(CurlSymbolHiding)
-
-# Http only
-set(CURL_DISABLE_FTP ON)
-set(CURL_DISABLE_LDAP ON)
-set(CURL_DISABLE_LDAPS ON)
-set(CURL_DISABLE_TELNET ON)
-set(CURL_DISABLE_DICT ON)
-set(CURL_DISABLE_FILE ON)
-set(CURL_DISABLE_TFTP ON)
-set(CURL_DISABLE_RTSP ON)
-set(CURL_DISABLE_POP3 ON)
-set(CURL_DISABLE_IMAP ON)
-set(CURL_DISABLE_SMTP ON)
-set(CURL_DISABLE_GOPHER ON)
-
-option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
-mark_as_advanced(CURL_DISABLE_COOKIES)
-
-option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
-mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
-
-option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
-mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
-
-option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
-mark_as_advanced(ENABLE_IPV6)
-
-if(ENABLE_IPV6 AND NOT WIN32)
-    include(CheckStructHasMember)
-    check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
-            HAVE_SOCKADDR_IN6_SIN6_ADDR)
-    check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
-            HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
-    if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
-        message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
-        # Force the feature off as this name is used as guard macro...
-        set(ENABLE_IPV6 OFF
-                CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
-    endif()
-endif()
-
-# We need ansi c-flags, especially on HP
-set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
-set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
-
-# Include all the necessary files for macros
-include(CheckFunctionExists)
-include(CheckIncludeFile)
-include(CheckIncludeFiles)
-include(CheckLibraryExists)
-include(CheckSymbolExists)
-include(CheckTypeSize)
-include(CheckCSourceCompiles)
-
-if(ENABLE_THREADED_RESOLVER)
-    find_package(Threads REQUIRED)
-    set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
-    set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
-    set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
-endif()
-
-# Check for all needed libraries
-
-# We don't want any plugin loading at runtime. It is harmful.
-#check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
-
-# This is unneeded.
-#check_library_exists_concat("socket" connect      HAVE_LIBSOCKET)
-
-set (NOT_NEED_LIBNSL 1)
-set (gethostname HAVE_GETHOSTNAME 1)
-
-# From cmake/find/ssl.cmake
-if (OPENSSL_FOUND)
-    set(SSL_ENABLED ON)
-    set(USE_OPENSSL ON)
-
-    list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
-    check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
-    check_include_file("openssl/err.h"    HAVE_OPENSSL_ERR_H)
-    check_include_file("openssl/pem.h"    HAVE_OPENSSL_PEM_H)
-    check_include_file("openssl/rsa.h"    HAVE_OPENSSL_RSA_H)
-    check_include_file("openssl/ssl.h"    HAVE_OPENSSL_SSL_H)
-    check_include_file("openssl/x509.h"   HAVE_OPENSSL_X509_H)
-    check_include_file("openssl/rand.h"   HAVE_OPENSSL_RAND_H)
-    check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
-    check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
-    check_symbol_exists(RAND_egd    "${CURL_INCLUDES}" HAVE_RAND_EGD)
-endif()
-
-# Check for idn
-# No, we don't need that.
-# check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
-
-# Check for symbol dlopen (same as HAVE_LIBDL)
-# We don't want any plugin loading at runtime. It is harmful.
-# check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
-
-# From /cmake/find/zlib.cmake
-if (ZLIB_FOUND)
-    set(HAVE_ZLIB_H ON)
-    set(HAVE_LIBZ ON)
-    set(USE_ZLIB ON)
-
-    list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
-endif()
-
-option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" OFF)
-if(ENABLE_UNIX_SOCKETS)
-    include(CheckStructHasMember)
-    check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
-else()
-    unset(USE_UNIX_SOCKETS CACHE)
-endif()
-
-# CA handling
-# Explicitly set to most common case
-if (OPENSSL_FOUND)
-    set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
-    set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
-    set(CURL_CA_PATH "/etc/ssl/certs")
-    set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
-endif()
-
-check_include_file_concat("stdio.h"          HAVE_STDIO_H)
-check_include_file_concat("inttypes.h"       HAVE_INTTYPES_H)
-check_include_file_concat("sys/filio.h"      HAVE_SYS_FILIO_H)
-check_include_file_concat("sys/ioctl.h"      HAVE_SYS_IOCTL_H)
-check_include_file_concat("sys/param.h"      HAVE_SYS_PARAM_H)
-check_include_file_concat("sys/poll.h"       HAVE_SYS_POLL_H)
-check_include_file_concat("sys/resource.h"   HAVE_SYS_RESOURCE_H)
-check_include_file_concat("sys/select.h"     HAVE_SYS_SELECT_H)
-check_include_file_concat("sys/socket.h"     HAVE_SYS_SOCKET_H)
-check_include_file_concat("sys/sockio.h"     HAVE_SYS_SOCKIO_H)
-check_include_file_concat("sys/stat.h"       HAVE_SYS_STAT_H)
-check_include_file_concat("sys/time.h"       HAVE_SYS_TIME_H)
-check_include_file_concat("sys/types.h"      HAVE_SYS_TYPES_H)
-check_include_file_concat("sys/uio.h"        HAVE_SYS_UIO_H)
-check_include_file_concat("sys/un.h"         HAVE_SYS_UN_H)
-check_include_file_concat("sys/utime.h"      HAVE_SYS_UTIME_H)
-check_include_file_concat("sys/xattr.h"      HAVE_SYS_XATTR_H)
-check_include_file_concat("alloca.h"         HAVE_ALLOCA_H)
-check_include_file_concat("arpa/inet.h"      HAVE_ARPA_INET_H)
-#check_include_file_concat("arpa/tftp.h"      HAVE_ARPA_TFTP_H)
-check_include_file_concat("assert.h"         HAVE_ASSERT_H)
-check_include_file_concat("crypto.h"         HAVE_CRYPTO_H)
-check_include_file_concat("des.h"            HAVE_DES_H)
-check_include_file_concat("err.h"            HAVE_ERR_H)
-check_include_file_concat("errno.h"          HAVE_ERRNO_H)
-check_include_file_concat("fcntl.h"          HAVE_FCNTL_H)
-#check_include_file_concat("idn2.h"           HAVE_IDN2_H)
-check_include_file_concat("ifaddrs.h"        HAVE_IFADDRS_H)
-check_include_file_concat("io.h"             HAVE_IO_H)
-check_include_file_concat("krb.h"            HAVE_KRB_H)
-check_include_file_concat("libgen.h"         HAVE_LIBGEN_H)
-check_include_file_concat("locale.h"         HAVE_LOCALE_H)
-check_include_file_concat("net/if.h"         HAVE_NET_IF_H)
-check_include_file_concat("netdb.h"          HAVE_NETDB_H)
-check_include_file_concat("netinet/in.h"     HAVE_NETINET_IN_H)
-check_include_file_concat("netinet/tcp.h"    HAVE_NETINET_TCP_H)
-
-check_include_file_concat("pem.h"            HAVE_PEM_H)
-check_include_file_concat("poll.h"           HAVE_POLL_H)
-check_include_file_concat("pwd.h"            HAVE_PWD_H)
-check_include_file_concat("rsa.h"            HAVE_RSA_H)
-check_include_file_concat("setjmp.h"         HAVE_SETJMP_H)
-check_include_file_concat("sgtty.h"          HAVE_SGTTY_H)
-check_include_file_concat("signal.h"         HAVE_SIGNAL_H)
-check_include_file_concat("ssl.h"            HAVE_SSL_H)
-check_include_file_concat("stdbool.h"        HAVE_STDBOOL_H)
-check_include_file_concat("stdint.h"         HAVE_STDINT_H)
-check_include_file_concat("stdio.h"          HAVE_STDIO_H)
-check_include_file_concat("stdlib.h"         HAVE_STDLIB_H)
-check_include_file_concat("string.h"         HAVE_STRING_H)
-check_include_file_concat("strings.h"        HAVE_STRINGS_H)
-check_include_file_concat("stropts.h"        HAVE_STROPTS_H)
-check_include_file_concat("termio.h"         HAVE_TERMIO_H)
-check_include_file_concat("termios.h"        HAVE_TERMIOS_H)
-check_include_file_concat("time.h"           HAVE_TIME_H)
-check_include_file_concat("unistd.h"         HAVE_UNISTD_H)
-check_include_file_concat("utime.h"          HAVE_UTIME_H)
-check_include_file_concat("x509.h"           HAVE_X509_H)
-
-check_include_file_concat("process.h"        HAVE_PROCESS_H)
-check_include_file_concat("stddef.h"         HAVE_STDDEF_H)
-#check_include_file_concat("dlfcn.h"          HAVE_DLFCN_H)
-check_include_file_concat("malloc.h"         HAVE_MALLOC_H)
-check_include_file_concat("memory.h"         HAVE_MEMORY_H)
-check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
-check_include_file_concat("stdint.h"        HAVE_STDINT_H)
-check_include_file_concat("sockio.h"        HAVE_SOCKIO_H)
-check_include_file_concat("sys/utsname.h"   HAVE_SYS_UTSNAME_H)
-
-check_type_size(size_t  SIZEOF_SIZE_T)
-check_type_size(ssize_t  SIZEOF_SSIZE_T)
-check_type_size("long long"  SIZEOF_LONG_LONG)
-check_type_size("long"  SIZEOF_LONG)
-check_type_size("short"  SIZEOF_SHORT)
-check_type_size("int"  SIZEOF_INT)
-check_type_size("__int64"  SIZEOF___INT64)
-check_type_size("long double"  SIZEOF_LONG_DOUBLE)
-check_type_size("time_t"  SIZEOF_TIME_T)
-
-set(HAVE_LONGLONG 1)
-set(HAVE_LL 1)
-
-set(RANDOM_FILE /dev/urandom)
-
-check_symbol_exists(basename      "${CURL_INCLUDES}" HAVE_BASENAME)
-check_symbol_exists(socket        "${CURL_INCLUDES}" HAVE_SOCKET)
-check_symbol_exists(select        "${CURL_INCLUDES}" HAVE_SELECT)
-check_symbol_exists(poll          "${CURL_INCLUDES}" HAVE_POLL)
-check_symbol_exists(strdup        "${CURL_INCLUDES}" HAVE_STRDUP)
-check_symbol_exists(strstr        "${CURL_INCLUDES}" HAVE_STRSTR)
-check_symbol_exists(strtok_r      "${CURL_INCLUDES}" HAVE_STRTOK_R)
-check_symbol_exists(strftime      "${CURL_INCLUDES}" HAVE_STRFTIME)
-check_symbol_exists(uname         "${CURL_INCLUDES}" HAVE_UNAME)
-check_symbol_exists(strcasecmp    "${CURL_INCLUDES}" HAVE_STRCASECMP)
-#check_symbol_exists(stricmp       "${CURL_INCLUDES}" HAVE_STRICMP)
-#check_symbol_exists(strcmpi       "${CURL_INCLUDES}" HAVE_STRCMPI)
-#check_symbol_exists(strncmpi      "${CURL_INCLUDES}" HAVE_STRNCMPI)
-check_symbol_exists(alarm         "${CURL_INCLUDES}" HAVE_ALARM)
-#check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
-check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
-check_symbol_exists(gettimeofday  "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
-check_symbol_exists(inet_addr     "${CURL_INCLUDES}" HAVE_INET_ADDR)
-#check_symbol_exists(inet_ntoa     "${CURL_INCLUDES}" HAVE_INET_NTOA)
-check_symbol_exists(inet_ntoa_r   "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
-check_symbol_exists(tcsetattr     "${CURL_INCLUDES}" HAVE_TCSETATTR)
-check_symbol_exists(tcgetattr     "${CURL_INCLUDES}" HAVE_TCGETATTR)
-check_symbol_exists(perror        "${CURL_INCLUDES}" HAVE_PERROR)
-check_symbol_exists(closesocket   "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
-check_symbol_exists(setvbuf       "${CURL_INCLUDES}" HAVE_SETVBUF)
-check_symbol_exists(sigsetjmp     "${CURL_INCLUDES}" HAVE_SIGSETJMP)
-check_symbol_exists(getpass_r     "${CURL_INCLUDES}" HAVE_GETPASS_R)
-#check_symbol_exists(strlcat       "${CURL_INCLUDES}" HAVE_STRLCAT)
-#check_symbol_exists(getpwuid      "${CURL_INCLUDES}" HAVE_GETPWUID)
-check_symbol_exists(getpwuid_r    "${CURL_INCLUDES}" HAVE_GETPWUID_R)
-check_symbol_exists(geteuid       "${CURL_INCLUDES}" HAVE_GETEUID)
-check_symbol_exists(usleep        "${CURL_INCLUDES}" HAVE_USLEEP)
-check_symbol_exists(utime         "${CURL_INCLUDES}" HAVE_UTIME)
-check_symbol_exists(gmtime_r      "${CURL_INCLUDES}" HAVE_GMTIME_R)
-check_symbol_exists(localtime_r   "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
-
-#check_symbol_exists(gethostbyname   "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
-check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
-
-check_symbol_exists(signal        "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
-check_symbol_exists(SIGALRM       "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
-set(HAVE_SIGNAL 1)
-check_symbol_exists(uname          "${CURL_INCLUDES}" HAVE_UNAME)
-check_symbol_exists(strtoll        "${CURL_INCLUDES}" HAVE_STRTOLL)
-#check_symbol_exists(_strtoi64      "${CURL_INCLUDES}" HAVE__STRTOI64)
-check_symbol_exists(strerror_r     "${CURL_INCLUDES}" HAVE_STRERROR_R)
-check_symbol_exists(siginterrupt   "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
-check_symbol_exists(perror         "${CURL_INCLUDES}" HAVE_PERROR)
-check_symbol_exists(fork           "${CURL_INCLUDES}" HAVE_FORK)
-check_symbol_exists(getaddrinfo    "${CURL_INCLUDES}" HAVE_GETADDRINFO)
-check_symbol_exists(freeaddrinfo   "${CURL_INCLUDES}" HAVE_FREEADDRINFO)
-check_symbol_exists(freeifaddrs    "${CURL_INCLUDES}" HAVE_FREEIFADDRS)
-check_symbol_exists(pipe           "${CURL_INCLUDES}" HAVE_PIPE)
-check_symbol_exists(ftruncate      "${CURL_INCLUDES}" HAVE_FTRUNCATE)
-check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME)
-check_symbol_exists(getpeername    "${CURL_INCLUDES}" HAVE_GETPEERNAME)
-check_symbol_exists(getsockname    "${CURL_INCLUDES}" HAVE_GETSOCKNAME)
-check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX)
-check_symbol_exists(getrlimit      "${CURL_INCLUDES}" HAVE_GETRLIMIT)
-check_symbol_exists(setlocale      "${CURL_INCLUDES}" HAVE_SETLOCALE)
-check_symbol_exists(setmode        "${CURL_INCLUDES}" HAVE_SETMODE)
-check_symbol_exists(setrlimit      "${CURL_INCLUDES}" HAVE_SETRLIMIT)
-check_symbol_exists(fcntl          "${CURL_INCLUDES}" HAVE_FCNTL)
-check_symbol_exists(ioctl          "${CURL_INCLUDES}" HAVE_IOCTL)
-check_symbol_exists(setsockopt     "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
-check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
-
-check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
-if(HAVE_FSETXATTR)
-    foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
-        curl_internal_test(${CURL_TEST})
-    endforeach()
-endif()
-
-# sigaction and sigsetjmp are special. Use special mechanism for
-# detecting those, but only if previous attempt failed.
-if(HAVE_SIGNAL_H)
-    check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
-endif()
-
-if(NOT HAVE_SIGSETJMP)
-    if(HAVE_SETJMP_H)
-        check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
-        if(HAVE_MACRO_SIGSETJMP)
-            set(HAVE_SIGSETJMP 1)
-        endif()
-    endif()
-endif()
-
-# If there is no stricmp(), do not allow LDAP to parse URLs
-if(NOT HAVE_STRICMP)
-    set(HAVE_LDAP_URL_PARSE 1)
-endif()
-
-# Do curl specific tests
-foreach(CURL_TEST
-        HAVE_FCNTL_O_NONBLOCK
-        HAVE_IOCTLSOCKET
-        HAVE_IOCTLSOCKET_CAMEL
-        HAVE_IOCTLSOCKET_CAMEL_FIONBIO
-        HAVE_IOCTLSOCKET_FIONBIO
-        HAVE_IOCTL_FIONBIO
-        HAVE_IOCTL_SIOCGIFADDR
-        HAVE_SETSOCKOPT_SO_NONBLOCK
-        HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
-        TIME_WITH_SYS_TIME
-        HAVE_O_NONBLOCK
-        HAVE_GETHOSTBYADDR_R_5
-        HAVE_GETHOSTBYADDR_R_7
-        HAVE_GETHOSTBYADDR_R_8
-        HAVE_GETHOSTBYADDR_R_5_REENTRANT
-        HAVE_GETHOSTBYADDR_R_7_REENTRANT
-        HAVE_GETHOSTBYADDR_R_8_REENTRANT
-        HAVE_GETHOSTBYNAME_R_3
-        HAVE_GETHOSTBYNAME_R_5
-        HAVE_GETHOSTBYNAME_R_6
-        HAVE_GETHOSTBYNAME_R_3_REENTRANT
-        HAVE_GETHOSTBYNAME_R_5_REENTRANT
-        HAVE_GETHOSTBYNAME_R_6_REENTRANT
-        HAVE_IN_ADDR_T
-        HAVE_BOOL_T
-        STDC_HEADERS
-        RETSIGTYPE_TEST
-        HAVE_INET_NTOA_R_DECL
-        HAVE_INET_NTOA_R_DECL_REENTRANT
-        HAVE_GETADDRINFO
-        HAVE_FILE_OFFSET_BITS
-        HAVE_VARIADIC_MACROS_C99
-        HAVE_VARIADIC_MACROS_GCC
-        )
-    curl_internal_test(${CURL_TEST})
-endforeach()
-
-if(HAVE_FILE_OFFSET_BITS)
-    set(_FILE_OFFSET_BITS 64)
-    set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
-endif()
-check_type_size("off_t"  SIZEOF_OFF_T)
-
-# include this header to get the type
-set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include")
-set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h")
-check_type_size("curl_off_t"  SIZEOF_CURL_OFF_T)
-set(CMAKE_EXTRA_INCLUDE_FILES "")
-
-foreach(CURL_TEST
-        HAVE_GLIBC_STRERROR_R
-        HAVE_POSIX_STRERROR_R
-        )
-    curl_internal_test(${CURL_TEST})
-endforeach()
-
-# Check for reentrant
-foreach(CURL_TEST
-        HAVE_GETHOSTBYADDR_R_5
-        HAVE_GETHOSTBYADDR_R_7
-        HAVE_GETHOSTBYADDR_R_8
-        HAVE_GETHOSTBYNAME_R_3
-        HAVE_GETHOSTBYNAME_R_5
-        HAVE_GETHOSTBYNAME_R_6
-        HAVE_INET_NTOA_R_DECL_REENTRANT)
-    if(NOT ${CURL_TEST})
-        if(${CURL_TEST}_REENTRANT)
-            set(NEED_REENTRANT 1)
-        endif()
-    endif()
-endforeach()
-
-if(NEED_REENTRANT)
-    foreach(CURL_TEST
-            HAVE_GETHOSTBYADDR_R_5
-            HAVE_GETHOSTBYADDR_R_7
-            HAVE_GETHOSTBYADDR_R_8
-            HAVE_GETHOSTBYNAME_R_3
-            HAVE_GETHOSTBYNAME_R_5
-            HAVE_GETHOSTBYNAME_R_6)
-        set(${CURL_TEST} 0)
-        if(${CURL_TEST}_REENTRANT)
-            set(${CURL_TEST} 1)
-        endif()
-    endforeach()
-endif()
-
-if(HAVE_INET_NTOA_R_DECL_REENTRANT)
-    set(HAVE_INET_NTOA_R_DECL 1)
-    set(NEED_REENTRANT 1)
-endif()
-
-# Check clock_gettime(CLOCK_MONOTONIC, x) support
-curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC)
-
-# Check compiler support of __builtin_available()
-curl_internal_test(HAVE_BUILTIN_AVAILABLE)
-
-# Some other minor tests
-
-if(NOT HAVE_IN_ADDR_T)
-    set(in_addr_t "unsigned long")
-endif()
-
-# Check for nonblocking
-set(HAVE_DISABLED_NONBLOCKING 1)
-if(HAVE_FIONBIO OR
-        HAVE_IOCTLSOCKET OR
-        HAVE_IOCTLSOCKET_CASE OR
-        HAVE_O_NONBLOCK)
-    set(HAVE_DISABLED_NONBLOCKING)
-endif()
-
-set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
-set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
-set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
-set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
-set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
-
-include(CMake/OtherTests.cmake)
-
-SET(LIB_VAUTH_CFILES
-        "${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
-        "${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
-        "${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
-        "${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
-
-SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
-
-SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
-        "${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
-        "${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
-        "${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
-        "${CURL_LIBRARY_DIR}/vtls/bearssl.c")
-
-SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
-        "${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
-        "${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
-        "${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
-
-SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
-
-SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
-
-SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
-
-SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
-
-SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
-        "${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
-        "${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
-        "${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
-        "${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
-        "${CURL_LIBRARY_DIR}/fileinfo.c"  "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
-        "${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
-        "${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
-        "${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
-        "${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c"  "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
-        "${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
-        "${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
-        "${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
-        "${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
-        "${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
-        "${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
-        "${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
-        "${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
-        "${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
-        "${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
-
-SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
-        "${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
-        "${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
-        "${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
-        "${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h"  "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
-        "${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
-        "${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
-        "${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
-        "${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
-        "${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
-        "${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
-        "${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
-        "${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
-        "${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
-        "${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
-        "${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
-        "${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
-        "${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
-        "${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
-        "${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
-
-SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
-
-SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
-        ${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
-SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
-        ${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
-
-configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
-        ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
-
-list(APPEND HHEADERS
-        ${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
-        )
-
-add_library(libcurl ${HHEADERS} ${CSOURCES})
-
-if(NOT BUILD_SHARED_LIBS)
-    set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
-endif()
-
-if(HIDES_CURL_PRIVATE_SYMBOLS)
-    set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
-    set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
-endif()
-
-if(OPENSSL_FOUND)
-    target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
-    message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
-endif()
-
-if(ZLIB_FOUND)
-    target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
-    message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
-endif()
-
-target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
-target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
-target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
-
-target_link_libraries(libcurl ${CURL_LIBS})
+set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
+
+set (SRCS
+    ${CURL_DIR}/lib/file.c
+    ${CURL_DIR}/lib/timeval.c
+    ${CURL_DIR}/lib/base64.c
+    ${CURL_DIR}/lib/hostip.c
+    ${CURL_DIR}/lib/progress.c
+    ${CURL_DIR}/lib/formdata.c
+    ${CURL_DIR}/lib/cookie.c
+    ${CURL_DIR}/lib/http.c
+    ${CURL_DIR}/lib/sendf.c
+    ${CURL_DIR}/lib/url.c
+    ${CURL_DIR}/lib/dict.c
+    ${CURL_DIR}/lib/if2ip.c
+    ${CURL_DIR}/lib/speedcheck.c
+    ${CURL_DIR}/lib/ldap.c
+    ${CURL_DIR}/lib/version.c
+    ${CURL_DIR}/lib/getenv.c
+    ${CURL_DIR}/lib/escape.c
+    ${CURL_DIR}/lib/mprintf.c
+    ${CURL_DIR}/lib/telnet.c
+    ${CURL_DIR}/lib/netrc.c
+    ${CURL_DIR}/lib/getinfo.c
+    ${CURL_DIR}/lib/transfer.c
+    ${CURL_DIR}/lib/strcase.c
+    ${CURL_DIR}/lib/easy.c
+    ${CURL_DIR}/lib/security.c
+    ${CURL_DIR}/lib/curl_fnmatch.c
+    ${CURL_DIR}/lib/fileinfo.c
+    ${CURL_DIR}/lib/wildcard.c
+    ${CURL_DIR}/lib/krb5.c
+    ${CURL_DIR}/lib/memdebug.c
+    ${CURL_DIR}/lib/http_chunks.c
+    ${CURL_DIR}/lib/strtok.c
+    ${CURL_DIR}/lib/connect.c
+    ${CURL_DIR}/lib/llist.c
+    ${CURL_DIR}/lib/hash.c
+    ${CURL_DIR}/lib/multi.c
+    ${CURL_DIR}/lib/content_encoding.c
+    ${CURL_DIR}/lib/share.c
+    ${CURL_DIR}/lib/http_digest.c
+    ${CURL_DIR}/lib/md4.c
+    ${CURL_DIR}/lib/md5.c
+    ${CURL_DIR}/lib/http_negotiate.c
+    ${CURL_DIR}/lib/inet_pton.c
+    ${CURL_DIR}/lib/strtoofft.c
+    ${CURL_DIR}/lib/strerror.c
+    ${CURL_DIR}/lib/amigaos.c
+    ${CURL_DIR}/lib/hostasyn.c
+    ${CURL_DIR}/lib/hostip4.c
+    ${CURL_DIR}/lib/hostip6.c
+    ${CURL_DIR}/lib/hostsyn.c
+    ${CURL_DIR}/lib/inet_ntop.c
+    ${CURL_DIR}/lib/parsedate.c
+    ${CURL_DIR}/lib/select.c
+    ${CURL_DIR}/lib/splay.c
+    ${CURL_DIR}/lib/strdup.c
+    ${CURL_DIR}/lib/socks.c
+    ${CURL_DIR}/lib/curl_addrinfo.c
+    ${CURL_DIR}/lib/socks_gssapi.c
+    ${CURL_DIR}/lib/socks_sspi.c
+    ${CURL_DIR}/lib/curl_sspi.c
+    ${CURL_DIR}/lib/slist.c
+    ${CURL_DIR}/lib/nonblock.c
+    ${CURL_DIR}/lib/curl_memrchr.c
+    ${CURL_DIR}/lib/imap.c
+    ${CURL_DIR}/lib/pop3.c
+    ${CURL_DIR}/lib/smtp.c
+    ${CURL_DIR}/lib/pingpong.c
+    ${CURL_DIR}/lib/rtsp.c
+    ${CURL_DIR}/lib/curl_threads.c
+    ${CURL_DIR}/lib/warnless.c
+    ${CURL_DIR}/lib/hmac.c
+    ${CURL_DIR}/lib/curl_rtmp.c
+    ${CURL_DIR}/lib/openldap.c
+    ${CURL_DIR}/lib/curl_gethostname.c
+    ${CURL_DIR}/lib/gopher.c
+    ${CURL_DIR}/lib/idn_win32.c
+    ${CURL_DIR}/lib/http_proxy.c
+    ${CURL_DIR}/lib/non-ascii.c
+    ${CURL_DIR}/lib/asyn-thread.c
+    ${CURL_DIR}/lib/curl_gssapi.c
+    ${CURL_DIR}/lib/http_ntlm.c
+    ${CURL_DIR}/lib/curl_ntlm_wb.c
+    ${CURL_DIR}/lib/curl_ntlm_core.c
+    ${CURL_DIR}/lib/curl_sasl.c
+    ${CURL_DIR}/lib/rand.c
+    ${CURL_DIR}/lib/curl_multibyte.c
+    ${CURL_DIR}/lib/hostcheck.c
+    ${CURL_DIR}/lib/conncache.c
+    ${CURL_DIR}/lib/dotdot.c
+    ${CURL_DIR}/lib/x509asn1.c
+    ${CURL_DIR}/lib/http2.c
+    ${CURL_DIR}/lib/smb.c
+    ${CURL_DIR}/lib/curl_endian.c
+    ${CURL_DIR}/lib/curl_des.c
+    ${CURL_DIR}/lib/system_win32.c
+    ${CURL_DIR}/lib/mime.c
+    ${CURL_DIR}/lib/sha256.c
+    ${CURL_DIR}/lib/setopt.c
+    ${CURL_DIR}/lib/curl_path.c
+    ${CURL_DIR}/lib/curl_ctype.c
+    ${CURL_DIR}/lib/curl_range.c
+    ${CURL_DIR}/lib/psl.c
+    ${CURL_DIR}/lib/doh.c
+    ${CURL_DIR}/lib/urlapi.c
+    ${CURL_DIR}/lib/curl_get_line.c
+    ${CURL_DIR}/lib/altsvc.c
+    ${CURL_DIR}/lib/socketpair.c
+    ${CURL_DIR}/lib/vauth/vauth.c
+    ${CURL_DIR}/lib/vauth/cleartext.c
+    ${CURL_DIR}/lib/vauth/cram.c
+    ${CURL_DIR}/lib/vauth/digest.c
+    ${CURL_DIR}/lib/vauth/digest_sspi.c
+    ${CURL_DIR}/lib/vauth/krb5_gssapi.c
+    ${CURL_DIR}/lib/vauth/krb5_sspi.c
+    ${CURL_DIR}/lib/vauth/ntlm.c
+    ${CURL_DIR}/lib/vauth/ntlm_sspi.c
+    ${CURL_DIR}/lib/vauth/oauth2.c
+    ${CURL_DIR}/lib/vauth/spnego_gssapi.c
+    ${CURL_DIR}/lib/vauth/spnego_sspi.c
+    ${CURL_DIR}/lib/vtls/openssl.c
+    ${CURL_DIR}/lib/vtls/gtls.c
+    ${CURL_DIR}/lib/vtls/vtls.c
+    ${CURL_DIR}/lib/vtls/nss.c
+    ${CURL_DIR}/lib/vtls/polarssl.c
+    ${CURL_DIR}/lib/vtls/polarssl_threadlock.c
+    ${CURL_DIR}/lib/vtls/wolfssl.c
+    ${CURL_DIR}/lib/vtls/schannel.c
+    ${CURL_DIR}/lib/vtls/schannel_verify.c
+    ${CURL_DIR}/lib/vtls/sectransp.c
+    ${CURL_DIR}/lib/vtls/gskit.c
+    ${CURL_DIR}/lib/vtls/mbedtls.c
+    ${CURL_DIR}/lib/vtls/mesalink.c
+    ${CURL_DIR}/lib/vtls/bearssl.c
+    ${CURL_DIR}/lib/vquic/ngtcp2.c
+    ${CURL_DIR}/lib/vquic/quiche.c
+    ${CURL_DIR}/lib/vssh/libssh2.c
+    ${CURL_DIR}/lib/vssh/libssh.c
+)
+
+add_library (curl ${SRCS})
+
+target_compile_definitions(curl PRIVATE HAVE_CONFIG_H BUILDING_LIBCURL CURL_HIDDEN_SYMBOLS libcurl_EXPORTS)
+target_include_directories(curl PUBLIC ${CURL_DIR}/include ${CURL_DIR}/lib .)
+
+target_compile_definitions(curl PRIVATE OS="${CMAKE_SYSTEM_NAME}")
diff --git a/contrib/curl-cmake/curl_config.h b/contrib/curl-cmake/curl_config.h
new file mode 100644
index 00000000000..c8f8e1b658f
--- /dev/null
+++ b/contrib/curl-cmake/curl_config.h
@@ -0,0 +1,38 @@
+#define CURL_DISABLE_FTP
+#define CURL_DISABLE_TFTP
+#define CURL_DISABLE_LDAP
+#define CURL_EXTERN_SYMBOL __attribute__ ((__visibility__ ("default")))
+
+#define SIZEOF_SHORT 2
+#define SIZEOF_INT   4
+#define SIZEOF_LONG  8
+#define SIZEOF_CURL_OFF_T 8
+#define SIZEOF_SIZE_T 8
+
+#define HAVE_FCNTL_O_NONBLOCK
+#define HAVE_LONGLONG
+#define HAVE_POLL_FINE
+#define HAVE_SOCKET
+#define HAVE_STRUCT_TIMEVAL
+
+#define HAVE_RECV
+#define RECV_TYPE_ARG1 int
+#define RECV_TYPE_ARG2 void*
+#define RECV_TYPE_ARG3 size_t
+#define RECV_TYPE_ARG4 int
+#define RECV_TYPE_RETV ssize_t
+
+#define HAVE_SEND
+#define SEND_TYPE_ARG1 int
+#define SEND_TYPE_ARG2 void*
+#define SEND_QUAL_ARG2 const
+#define SEND_TYPE_ARG3 size_t
+#define SEND_TYPE_ARG4 int
+#define SEND_TYPE_RETV ssize_t
+
+#define HAVE_ARPA_INET_H
+#define HAVE_ERRNO_H
+#define HAVE_FCNTL_H
+#define HAVE_NETDB_H
+#define HAVE_SYS_STAT_H
+#define HAVE_UNISTD_H

From de71bd6e85730683ba9ddc7f0d4da70784612960 Mon Sep 17 00:00:00 2001
From: alexey-milovidov 
Date: Thu, 30 Jan 2020 13:05:12 +0300
Subject: [PATCH 307/312] Update extended_roadmap.md

---
 docs/ru/extended_roadmap.md | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/docs/ru/extended_roadmap.md b/docs/ru/extended_roadmap.md
index 145779cfccb..c5e51290aac 100644
--- a/docs/ru/extended_roadmap.md
+++ b/docs/ru/extended_roadmap.md
@@ -231,7 +231,7 @@ Upd. Иван Блинков сделал эту задачу путём зам
 
 ### 4.1. Уменьшение числа потоков при распределённых запросах.
 
-[Никита Лапков](https://github.com/laplab), весна 2020. Upd. Есть прототип.
+[Никита Лапков](https://github.com/laplab), весна 2020. Upd. Есть прототип. Upd. Он не работает.
 
 ### 4.2. Спекулятивное выполнение запросов на нескольких репликах.
 
@@ -527,6 +527,7 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII
 
 Максим из YT сказал, что сделает это после нового года.
 Максим из YT сказал, что "мы планируем в январе добиться".
+Максим сейчас занимается собираемостью YT с новой версией ClickHouse.
 
 Нужно для CHYT и YQL.
 
@@ -536,7 +537,7 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII
 
 ### 7.27. Запуск автотестов в Аркадии.
 
-Требует 7.26.
+Требует 7.26. Коллеги начали делать, есть результат.
 
 ### 7.29. Опции clickhouse install, stop, start вместо postinst, init.d, systemd скриптов.
 
@@ -577,6 +578,7 @@ Upd. Сергей Штыков сделал функцию `randomPrintableASCII
 
 Контрибьюторы, у которых есть 5 померженных PR. Для их новых PR автотесты запускаются сразу.
 [Александр Сапин](https://github.com/alesapin). Может делегировать эту задачу кому угодно.
+Сейчас добавляем некоторых доверенных контрибьюторов в ручном режиме.
 
 ### 7.37. Разобраться с repo.yandex.ru.
 
@@ -644,15 +646,15 @@ Altinity.
 
 ### 8.15. Запись данных в CapNProto.
 
-### 8.16. Поддержка формата Avro.
+### 8.16. + Поддержка формата Avro.
 
-Andrew Onyshchuk. Есть pull request. Q1.
+Andrew Onyshchuk. Есть pull request. Q1. Сделано.
 
 Формат Apache Avro является компактным структурированным построчным бинарным форматом данных с внешней схемой. Этот формат часто используется совместно с Kafka и поддержка его в качестве одного из форматов ввода-вывода в ClickHouse является востребованной пользователями.
 
 ### 8.16.1. Поддержка формата JSONEachRow, засунутого в массив.
 
-Павел Круглов, ВШЭ и Яндекс.
+Павел Круглов, ВШЭ и Яндекс. Есть pull request.
 
 ### 8.16.2. Поддержка формата Thrift.
 
@@ -772,6 +774,7 @@ ClickHouse предоставляет возможность обратитьс
 ### 10.12. Layout direct для словарей.
 
 Артём Стрельцов, Николай Дегтеринский, Наталия Михненко, ВШЭ.
+Приступили к этой задаче.
 
 ### 10.13. Использование Join как generic layout для словарей.
 
@@ -900,7 +903,8 @@ zhang2014
 
 ### 14.5. Поддержка задания множества как массива в правой части секции IN.
 
-Василий Немков, Altinity, делал эту задачу, но временно приостановил работу над ней в пользу других задач.
+Василий Немков, Altinity, делал эту задачу, но забросил её в пользу других задач.
+В результате, сейчас доделывает Антон Попов.
 
 ### 14.6. Глобальный scope для WITH.
 
@@ -1119,6 +1123,8 @@ Hold. Полезно для заказчиков внутри Яндекса, н
 
 ### 19.6. Одновременный выбор кусков для слияния многими репликами, отказ от leader election в ZK.
 
+Обсуждается. Возможно, будет делать Александр Казаков.
+
 ### 19.7. Возможность записи данных при недоступности ZK и отказ от линейного порядка кусков в большинстве случаев.
 
 ### 19.8. Отказ от хранения в ZK множества кусков для каждой реплики отдельно.

From 18538f5c6579c2caf76fa7c736ac6e776a521133 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 13:34:55 +0300
Subject: [PATCH 308/312] Domain change in docs

---
 .gitignore                                    |  6 +-
 docs/README.md                                |  6 +-
 docs/en/data_types/array.md                   |  2 +-
 docs/en/data_types/boolean.md                 |  2 +-
 docs/en/data_types/date.md                    |  2 +-
 docs/en/data_types/datetime.md                |  2 +-
 docs/en/data_types/decimal.md                 |  2 +-
 docs/en/data_types/domains/ipv4.md            |  8 +--
 docs/en/data_types/domains/ipv6.md            |  8 +--
 docs/en/data_types/domains/overview.md        |  2 +-
 docs/en/data_types/enum.md                    |  2 +-
 docs/en/data_types/fixedstring.md             |  2 +-
 docs/en/data_types/float.md                   |  2 +-
 docs/en/data_types/index.md                   |  2 +-
 docs/en/data_types/int_uint.md                |  2 +-
 .../aggregatefunction.md                      |  2 +-
 .../nested_data_structures/index.md           |  2 +-
 .../nested_data_structures/nested.md          |  2 +-
 docs/en/data_types/nullable.md                |  2 +-
 .../special_data_types/expression.md          |  2 +-
 .../en/data_types/special_data_types/index.md |  2 +-
 .../data_types/special_data_types/nothing.md  |  2 +-
 docs/en/data_types/special_data_types/set.md  |  2 +-
 docs/en/data_types/string.md                  |  2 +-
 docs/en/data_types/tuple.md                   |  2 +-
 docs/en/data_types/uuid.md                    |  2 +-
 docs/en/database_engines/index.md             |  2 +-
 docs/en/database_engines/lazy.md              |  2 +-
 docs/en/database_engines/mysql.md             |  2 +-
 docs/en/development/architecture.md           |  2 +-
 docs/en/development/build.md                  |  2 +-
 docs/en/development/build_cross_osx.md        |  2 +-
 docs/en/development/build_osx.md              |  2 +-
 docs/en/development/developer_instruction.md  |  8 +--
 docs/en/development/index.md                  |  2 +-
 docs/en/development/style.md                  |  2 +-
 docs/en/development/tests.md                  |  4 +-
 docs/en/faq/general.md                        |  2 +-
 .../example_datasets/amplab_benchmark.md      |  2 +-
 .../example_datasets/criteo.md                |  2 +-
 .../example_datasets/nyc_taxi.md              |  2 +-
 .../example_datasets/ontime.md                |  2 +-
 .../example_datasets/star_schema.md           |  2 +-
 .../example_datasets/wikistat.md              |  2 +-
 docs/en/getting_started/index.md              |  2 +-
 docs/en/getting_started/install.md            |  4 +-
 docs/en/index.md                              |  2 +-
 docs/en/interfaces/cli.md                     |  2 +-
 docs/en/interfaces/cpp.md                     |  2 +-
 docs/en/interfaces/formats.md                 |  2 +-
 docs/en/interfaces/http.md                    |  2 +-
 docs/en/interfaces/index.md                   |  2 +-
 docs/en/interfaces/jdbc.md                    |  2 +-
 docs/en/interfaces/odbc.md                    |  2 +-
 docs/en/interfaces/tcp.md                     |  2 +-
 .../third-party/client_libraries.md           |  2 +-
 docs/en/interfaces/third-party/gui.md         |  2 +-
 .../en/interfaces/third-party/integrations.md |  2 +-
 docs/en/interfaces/third-party/proxy.md       |  2 +-
 docs/en/introduction/distinctive_features.md  |  2 +-
 .../features_considered_disadvantages.md      |  2 +-
 docs/en/introduction/history.md               |  2 +-
 docs/en/introduction/performance.md           |  6 +-
 docs/en/operations/access_rights.md           |  2 +-
 docs/en/operations/backup.md                  |  2 +-
 docs/en/operations/configuration_files.md     |  2 +-
 docs/en/operations/index.md                   |  2 +-
 docs/en/operations/performance_test.md        |  4 +-
 docs/en/operations/quotas.md                  |  2 +-
 docs/en/operations/server_settings/index.md   |  2 +-
 .../en/operations/server_settings/settings.md |  2 +-
 .../settings/constraints_on_settings.md       |  2 +-
 docs/en/operations/settings/index.md          |  2 +-
 .../settings/permissions_for_queries.md       |  2 +-
 .../operations/settings/query_complexity.md   |  2 +-
 docs/en/operations/settings/settings.md       |  2 +-
 .../operations/settings/settings_profiles.md  |  2 +-
 docs/en/operations/settings/settings_users.md |  2 +-
 docs/en/operations/system_tables.md           |  2 +-
 .../table_engines/aggregatingmergetree.md     |  2 +-
 docs/en/operations/table_engines/buffer.md    |  2 +-
 .../table_engines/collapsingmergetree.md      |  2 +-
 .../table_engines/custom_partitioning_key.md  |  2 +-
 .../en/operations/table_engines/dictionary.md |  2 +-
 .../operations/table_engines/distributed.md   |  2 +-
 .../operations/table_engines/external_data.md |  2 +-
 docs/en/operations/table_engines/file.md      |  2 +-
 .../table_engines/graphitemergetree.md        |  2 +-
 docs/en/operations/table_engines/hdfs.md      |  4 +-
 docs/en/operations/table_engines/index.md     |  2 +-
 docs/en/operations/table_engines/jdbc.md      |  2 +-
 docs/en/operations/table_engines/join.md      |  2 +-
 docs/en/operations/table_engines/kafka.md     |  2 +-
 docs/en/operations/table_engines/log.md       |  2 +-
 .../en/operations/table_engines/log_family.md |  2 +-
 .../table_engines/materializedview.md         |  2 +-
 docs/en/operations/table_engines/memory.md    |  2 +-
 docs/en/operations/table_engines/merge.md     |  2 +-
 docs/en/operations/table_engines/mergetree.md |  4 +-
 docs/en/operations/table_engines/mysql.md     |  2 +-
 docs/en/operations/table_engines/null.md      |  2 +-
 docs/en/operations/table_engines/odbc.md      |  2 +-
 .../table_engines/replacingmergetree.md       |  2 +-
 .../operations/table_engines/replication.md   |  2 +-
 docs/en/operations/table_engines/set.md       |  2 +-
 docs/en/operations/table_engines/stripelog.md |  2 +-
 .../table_engines/summingmergetree.md         |  2 +-
 docs/en/operations/table_engines/tinylog.md   |  2 +-
 docs/en/operations/table_engines/url.md       |  2 +-
 .../versionedcollapsingmergetree.md           |  2 +-
 docs/en/operations/table_engines/view.md      |  2 +-
 docs/en/operations/tips.md                    |  2 +-
 docs/en/operations/utils/clickhouse-copier.md |  2 +-
 docs/en/operations/utils/clickhouse-local.md  |  2 +-
 docs/en/operations/utils/index.md             |  2 +-
 .../agg_functions/combinators.md              |  2 +-
 docs/en/query_language/agg_functions/index.md |  2 +-
 .../agg_functions/parametric_functions.md     |  2 +-
 .../query_language/agg_functions/reference.md |  2 +-
 docs/en/query_language/alter.md               |  2 +-
 docs/en/query_language/create.md              |  2 +-
 .../en/query_language/dicts/external_dicts.md |  2 +-
 .../dicts/external_dicts_dict.md              |  2 +-
 .../dicts/external_dicts_dict_layout.md       |  2 +-
 .../dicts/external_dicts_dict_lifetime.md     |  2 +-
 .../dicts/external_dicts_dict_sources.md      |  2 +-
 .../dicts/external_dicts_dict_structure.md    |  2 +-
 docs/en/query_language/dicts/index.md         |  2 +-
 .../en/query_language/dicts/internal_dicts.md |  2 +-
 .../functions/arithmetic_functions.md         |  2 +-
 .../functions/array_functions.md              |  2 +-
 .../en/query_language/functions/array_join.md |  2 +-
 .../query_language/functions/bit_functions.md |  2 +-
 .../functions/bitmap_functions.md             |  2 +-
 .../functions/comparison_functions.md         |  2 +-
 .../functions/conditional_functions.md        |  2 +-
 .../functions/date_time_functions.md          |  2 +-
 .../functions/encoding_functions.md           |  2 +-
 .../functions/ext_dict_functions.md           |  2 +-
 .../functions/functions_for_nulls.md          |  2 +-
 docs/en/query_language/functions/geo.md       |  2 +-
 .../functions/hash_functions.md               |  2 +-
 .../functions/higher_order_functions.md       |  2 +-
 .../query_language/functions/in_functions.md  |  2 +-
 docs/en/query_language/functions/index.md     |  2 +-
 .../functions/ip_address_functions.md         |  2 +-
 .../functions/json_functions.md               |  2 +-
 .../functions/logical_functions.md            |  2 +-
 .../functions/math_functions.md               |  2 +-
 .../functions/other_functions.md              |  2 +-
 .../functions/random_functions.md             |  2 +-
 .../functions/rounding_functions.md           |  2 +-
 .../functions/splitting_merging_functions.md  |  2 +-
 .../functions/string_functions.md             |  2 +-
 .../functions/string_replace_functions.md     |  2 +-
 .../functions/string_search_functions.md      |  2 +-
 .../functions/type_conversion_functions.md    |  2 +-
 .../query_language/functions/url_functions.md |  2 +-
 .../functions/uuid_functions.md               |  2 +-
 .../functions/ym_dict_functions.md            |  2 +-
 docs/en/query_language/index.md               |  2 +-
 docs/en/query_language/insert_into.md         |  2 +-
 docs/en/query_language/misc.md                |  2 +-
 docs/en/query_language/operators.md           |  2 +-
 docs/en/query_language/select.md              |  2 +-
 docs/en/query_language/syntax.md              |  2 +-
 docs/en/query_language/system.md              |  2 +-
 .../en/query_language/table_functions/file.md |  4 +-
 .../en/query_language/table_functions/hdfs.md |  4 +-
 .../query_language/table_functions/index.md   |  2 +-
 .../query_language/table_functions/input.md   |  2 +-
 .../en/query_language/table_functions/jdbc.md |  2 +-
 .../query_language/table_functions/merge.md   |  2 +-
 .../query_language/table_functions/mysql.md   |  2 +-
 .../query_language/table_functions/numbers.md |  2 +-
 .../en/query_language/table_functions/odbc.md |  2 +-
 .../query_language/table_functions/remote.md  |  2 +-
 docs/en/query_language/table_functions/url.md |  2 +-
 docs/en/roadmap.md                            |  2 +-
 docs/en/security_changelog.md                 |  2 +-
 docs/fa/data_types/array.md                   |  2 +-
 docs/fa/data_types/boolean.md                 |  2 +-
 docs/fa/data_types/date.md                    |  2 +-
 docs/fa/data_types/datetime.md                |  2 +-
 docs/fa/data_types/enum.md                    |  2 +-
 docs/fa/data_types/fixedstring.md             |  2 +-
 docs/fa/data_types/float.md                   |  2 +-
 docs/fa/data_types/index.md                   |  2 +-
 docs/fa/data_types/int_uint.md                |  2 +-
 .../aggregatefunction.md                      |  2 +-
 .../nested_data_structures/index.md           |  2 +-
 .../nested_data_structures/nested.md          |  2 +-
 .../special_data_types/expression.md          |  2 +-
 .../fa/data_types/special_data_types/index.md |  2 +-
 docs/fa/data_types/special_data_types/set.md  |  2 +-
 docs/fa/data_types/string.md                  |  2 +-
 docs/fa/data_types/tuple.md                   |  2 +-
 .../example_datasets/amplab_benchmark.md      |  2 +-
 .../example_datasets/criteo.md                |  2 +-
 .../example_datasets/nyc_taxi.md              |  2 +-
 .../example_datasets/ontime.md                |  2 +-
 .../example_datasets/star_schema.md           |  2 +-
 .../example_datasets/wikistat.md              |  2 +-
 docs/fa/getting_started/index.md              |  2 +-
 docs/fa/getting_started/install.md            |  2 +-
 docs/fa/index.md                              |  2 +-
 docs/fa/interfaces/cli.md                     |  2 +-
 docs/fa/interfaces/cpp.md                     |  2 +-
 docs/fa/interfaces/formats.md                 |  2 +-
 docs/fa/interfaces/http.md                    |  2 +-
 docs/fa/interfaces/index.md                   |  2 +-
 docs/fa/interfaces/jdbc.md                    |  2 +-
 docs/fa/interfaces/odbc.md                    |  2 +-
 docs/fa/interfaces/tcp.md                     |  2 +-
 .../third-party/client_libraries.md           |  2 +-
 docs/fa/interfaces/third-party/gui.md         |  2 +-
 .../fa/interfaces/third-party/integrations.md |  2 +-
 docs/fa/interfaces/third-party/proxy.md       |  2 +-
 docs/fa/introduction/distinctive_features.md  |  2 +-
 .../features_considered_disadvantages.md      |  2 +-
 docs/fa/introduction/history.md               |  2 +-
 docs/fa/introduction/performance.md           |  6 +-
 docs/ja/index.md                              |  2 +-
 docs/ru/data_types/array.md                   |  2 +-
 docs/ru/data_types/boolean.md                 |  2 +-
 docs/ru/data_types/date.md                    |  2 +-
 docs/ru/data_types/datetime.md                |  2 +-
 docs/ru/data_types/decimal.md                 |  2 +-
 docs/ru/data_types/domains/ipv4.md            |  8 +--
 docs/ru/data_types/domains/ipv6.md            |  8 +--
 docs/ru/data_types/domains/overview.md        |  2 +-
 docs/ru/data_types/enum.md                    |  2 +-
 docs/ru/data_types/fixedstring.md             |  2 +-
 docs/ru/data_types/float.md                   |  2 +-
 docs/ru/data_types/index.md                   |  2 +-
 docs/ru/data_types/int_uint.md                |  2 +-
 .../aggregatefunction.md                      |  2 +-
 .../nested_data_structures/index.md           |  2 +-
 .../nested_data_structures/nested.md          |  2 +-
 docs/ru/data_types/nullable.md                |  2 +-
 .../special_data_types/expression.md          |  2 +-
 .../ru/data_types/special_data_types/index.md |  2 +-
 .../data_types/special_data_types/nothing.md  |  2 +-
 docs/ru/data_types/special_data_types/set.md  |  2 +-
 docs/ru/data_types/string.md                  |  2 +-
 docs/ru/data_types/tuple.md                   |  2 +-
 docs/ru/data_types/uuid.md                    |  2 +-
 docs/ru/development/developer_instruction.md  |  8 +--
 docs/ru/development/style.md                  |  2 +-
 docs/ru/extended_roadmap.md                   |  4 +-
 docs/ru/faq/general.md                        |  2 +-
 .../example_datasets/amplab_benchmark.md      |  2 +-
 .../example_datasets/criteo.md                |  2 +-
 .../example_datasets/nyc_taxi.md              |  2 +-
 .../example_datasets/ontime.md                |  2 +-
 .../example_datasets/star_schema.md           |  2 +-
 .../example_datasets/wikistat.md              |  2 +-
 docs/ru/getting_started/index.md              |  2 +-
 docs/ru/getting_started/install.md            |  4 +-
 docs/ru/index.md                              |  2 +-
 docs/ru/interfaces/cli.md                     |  2 +-
 docs/ru/interfaces/cpp.md                     |  2 +-
 docs/ru/interfaces/formats.md                 |  2 +-
 docs/ru/interfaces/http.md                    |  2 +-
 docs/ru/interfaces/index.md                   |  2 +-
 docs/ru/interfaces/jdbc.md                    |  2 +-
 docs/ru/interfaces/odbc.md                    |  2 +-
 docs/ru/interfaces/tcp.md                     |  2 +-
 .../third-party/client_libraries.md           |  2 +-
 docs/ru/interfaces/third-party/gui.md         |  2 +-
 .../ru/interfaces/third-party/integrations.md |  2 +-
 docs/ru/interfaces/third-party/proxy.md       |  2 +-
 docs/ru/introduction/distinctive_features.md  |  2 +-
 .../features_considered_disadvantages.md      |  2 +-
 docs/ru/introduction/history.md               |  2 +-
 docs/ru/introduction/info.md                  |  2 +-
 docs/ru/introduction/performance.md           |  6 +-
 docs/ru/operations/access_rights.md           |  2 +-
 docs/ru/operations/backup.md                  |  2 +-
 docs/ru/operations/configuration_files.md     |  2 +-
 docs/ru/operations/index.md                   |  2 +-
 docs/ru/operations/quotas.md                  |  2 +-
 docs/ru/operations/server_settings/index.md   |  2 +-
 .../ru/operations/server_settings/settings.md |  2 +-
 .../settings/constraints_on_settings.md       |  2 +-
 docs/ru/operations/settings/index.md          |  2 +-
 .../settings/permissions_for_queries.md       |  2 +-
 .../operations/settings/query_complexity.md   |  2 +-
 docs/ru/operations/settings/settings.md       |  2 +-
 .../operations/settings/settings_profiles.md  |  2 +-
 docs/ru/operations/settings/settings_users.md |  2 +-
 docs/ru/operations/system_tables.md           |  2 +-
 .../table_engines/aggregatingmergetree.md     |  2 +-
 docs/ru/operations/table_engines/buffer.md    |  2 +-
 .../table_engines/collapsingmergetree.md      |  2 +-
 .../table_engines/custom_partitioning_key.md  |  2 +-
 .../ru/operations/table_engines/dictionary.md |  2 +-
 .../operations/table_engines/distributed.md   |  2 +-
 .../operations/table_engines/external_data.md |  2 +-
 docs/ru/operations/table_engines/file.md      |  2 +-
 .../table_engines/graphitemergetree.md        |  2 +-
 docs/ru/operations/table_engines/hdfs.md      |  2 +-
 docs/ru/operations/table_engines/index.md     |  2 +-
 docs/ru/operations/table_engines/jdbc.md      |  2 +-
 docs/ru/operations/table_engines/join.md      |  2 +-
 docs/ru/operations/table_engines/kafka.md     |  2 +-
 docs/ru/operations/table_engines/log.md       |  2 +-
 .../ru/operations/table_engines/log_family.md |  2 +-
 .../table_engines/materializedview.md         |  2 +-
 docs/ru/operations/table_engines/memory.md    |  2 +-
 docs/ru/operations/table_engines/merge.md     |  2 +-
 docs/ru/operations/table_engines/mergetree.md |  2 +-
 docs/ru/operations/table_engines/mysql.md     |  2 +-
 docs/ru/operations/table_engines/null.md      |  2 +-
 docs/ru/operations/table_engines/odbc.md      |  2 +-
 .../table_engines/replacingmergetree.md       |  2 +-
 .../operations/table_engines/replication.md   |  2 +-
 docs/ru/operations/table_engines/set.md       |  2 +-
 docs/ru/operations/table_engines/stripelog.md |  2 +-
 .../table_engines/summingmergetree.md         |  2 +-
 docs/ru/operations/table_engines/tinylog.md   |  2 +-
 docs/ru/operations/table_engines/url.md       |  2 +-
 .../versionedcollapsingmergetree.md           |  2 +-
 docs/ru/operations/table_engines/view.md      |  2 +-
 docs/ru/operations/tips.md                    |  2 +-
 docs/ru/operations/utils/clickhouse-copier.md |  2 +-
 docs/ru/operations/utils/clickhouse-local.md  |  2 +-
 docs/ru/operations/utils/index.md             |  2 +-
 .../agg_functions/combinators.md              |  2 +-
 docs/ru/query_language/agg_functions/index.md |  2 +-
 .../agg_functions/parametric_functions.md     |  2 +-
 .../query_language/agg_functions/reference.md |  2 +-
 docs/ru/query_language/alter.md               |  2 +-
 docs/ru/query_language/create.md              |  2 +-
 .../ru/query_language/dicts/external_dicts.md |  2 +-
 .../dicts/external_dicts_dict.md              |  2 +-
 .../dicts/external_dicts_dict_layout.md       |  2 +-
 .../dicts/external_dicts_dict_lifetime.md     |  2 +-
 .../dicts/external_dicts_dict_sources.md      |  2 +-
 .../dicts/external_dicts_dict_structure.md    |  2 +-
 docs/ru/query_language/dicts/index.md         |  2 +-
 .../ru/query_language/dicts/internal_dicts.md |  2 +-
 .../functions/arithmetic_functions.md         |  2 +-
 .../functions/array_functions.md              |  2 +-
 .../ru/query_language/functions/array_join.md |  2 +-
 .../query_language/functions/bit_functions.md |  2 +-
 .../functions/bitmap_functions.md             |  2 +-
 .../functions/comparison_functions.md         |  2 +-
 .../functions/conditional_functions.md        |  2 +-
 .../functions/date_time_functions.md          |  2 +-
 .../functions/encoding_functions.md           |  2 +-
 .../functions/ext_dict_functions.md           |  2 +-
 .../functions/functions_for_nulls.md          |  2 +-
 docs/ru/query_language/functions/geo.md       |  2 +-
 .../functions/hash_functions.md               |  2 +-
 .../functions/higher_order_functions.md       |  2 +-
 .../query_language/functions/in_functions.md  |  2 +-
 docs/ru/query_language/functions/index.md     |  2 +-
 .../functions/ip_address_functions.md         |  2 +-
 .../functions/json_functions.md               |  2 +-
 .../functions/logical_functions.md            |  2 +-
 .../functions/math_functions.md               |  2 +-
 .../functions/other_functions.md              |  2 +-
 .../functions/random_functions.md             |  2 +-
 .../functions/rounding_functions.md           |  2 +-
 .../functions/splitting_merging_functions.md  |  2 +-
 .../functions/string_functions.md             |  2 +-
 .../functions/string_replace_functions.md     |  2 +-
 .../functions/string_search_functions.md      |  2 +-
 .../functions/type_conversion_functions.md    |  2 +-
 .../query_language/functions/url_functions.md |  2 +-
 .../functions/uuid_functions.md               |  2 +-
 .../functions/ym_dict_functions.md            |  2 +-
 docs/ru/query_language/index.md               |  2 +-
 docs/ru/query_language/insert_into.md         |  2 +-
 docs/ru/query_language/misc.md                |  2 +-
 docs/ru/query_language/operators.md           |  2 +-
 docs/ru/query_language/select.md              |  2 +-
 docs/ru/query_language/syntax.md              |  2 +-
 docs/ru/query_language/system.md              |  2 +-
 .../ru/query_language/table_functions/file.md |  2 +-
 .../ru/query_language/table_functions/hdfs.md |  2 +-
 .../query_language/table_functions/index.md   |  2 +-
 .../query_language/table_functions/input.md   |  2 +-
 .../ru/query_language/table_functions/jdbc.md |  2 +-
 .../query_language/table_functions/merge.md   |  2 +-
 .../query_language/table_functions/mysql.md   |  2 +-
 .../query_language/table_functions/numbers.md |  2 +-
 .../ru/query_language/table_functions/odbc.md |  2 +-
 .../query_language/table_functions/remote.md  |  2 +-
 docs/ru/query_language/table_functions/url.md |  2 +-
 docs/ru/security_changelog.md                 |  2 +-
 docs/tools/README.md                          |  4 +-
 docs/zh/data_types/domains/ipv4.md            |  8 +--
 docs/zh/data_types/domains/ipv6.md            |  8 +--
 docs/zh/data_types/domains/overview.md        |  2 +-
 docs/zh/data_types/fixedstring.md             |  2 +-
 .../aggregatefunction.md                      |  2 +-
 docs/zh/data_types/nullable.md                |  2 +-
 docs/zh/database_engines/index.md             |  2 +-
 docs/zh/database_engines/mysql.md             |  2 +-
 docs/zh/development/architecture.md           |  2 +-
 docs/zh/development/build.md                  |  2 +-
 docs/zh/development/build_cross_osx.md        |  2 +-
 docs/zh/development/build_osx.md              |  2 +-
 docs/zh/development/developer_instruction.md  |  8 +--
 docs/zh/development/index.md                  |  2 +-
 docs/zh/development/style.md                  |  2 +-
 docs/zh/development/tests.md                  |  4 +-
 docs/zh/faq/general.md                        |  2 +-
 .../example_datasets/amplab_benchmark.md      |  2 +-
 .../example_datasets/criteo.md                |  2 +-
 .../example_datasets/nyc_taxi.md              |  2 +-
 .../example_datasets/star_schema.md           |  2 +-
 .../example_datasets/wikistat.md              |  2 +-
 docs/zh/getting_started/index.md              |  2 +-
 docs/zh/getting_started/install.md            |  2 +-
 docs/zh/index.md                              |  2 +-
 docs/zh/interfaces/cli.md                     |  2 +-
 docs/zh/interfaces/cpp.md                     |  2 +-
 docs/zh/interfaces/formats.md                 |  2 +-
 docs/zh/interfaces/http.md                    |  2 +-
 docs/zh/interfaces/index.md                   |  2 +-
 docs/zh/interfaces/jdbc.md                    |  2 +-
 docs/zh/interfaces/odbc.md                    |  2 +-
 docs/zh/interfaces/tcp.md                     |  2 +-
 .../third-party/client_libraries.md           |  2 +-
 docs/zh/interfaces/third-party/gui.md         |  2 +-
 .../zh/interfaces/third-party/integrations.md |  2 +-
 docs/zh/interfaces/third-party/proxy.md       |  2 +-
 docs/zh/introduction/distinctive_features.md  |  2 +-
 .../features_considered_disadvantages.md      |  2 +-
 docs/zh/introduction/history.md               |  2 +-
 docs/zh/introduction/performance.md           |  6 +-
 docs/zh/operations/access_rights.md           |  2 +-
 docs/zh/operations/configuration_files.md     |  2 +-
 docs/zh/operations/index.md                   |  2 +-
 docs/zh/operations/quotas.md                  |  2 +-
 docs/zh/operations/server_settings/index.md   |  2 +-
 docs/zh/operations/settings/index.md          |  2 +-
 .../operations/settings/query_complexity.md   |  2 +-
 .../table_engines/aggregatingmergetree.md     |  2 +-
 docs/zh/operations/table_engines/buffer.md    |  2 +-
 .../table_engines/collapsingmergetree.md      |  2 +-
 .../table_engines/custom_partitioning_key.md  |  2 +-
 .../zh/operations/table_engines/dictionary.md |  2 +-
 .../operations/table_engines/distributed.md   |  2 +-
 .../operations/table_engines/external_data.md |  2 +-
 docs/zh/operations/table_engines/file.md      |  2 +-
 docs/zh/operations/table_engines/index.md     | 58 +++++++++----------
 docs/zh/operations/table_engines/join.md      |  2 +-
 docs/zh/operations/table_engines/kafka.md     |  2 +-
 docs/zh/operations/table_engines/log.md       |  2 +-
 .../zh/operations/table_engines/log_family.md |  2 +-
 .../table_engines/materializedview.md         |  2 +-
 docs/zh/operations/table_engines/memory.md    |  2 +-
 docs/zh/operations/table_engines/merge.md     |  2 +-
 docs/zh/operations/table_engines/mergetree.md |  2 +-
 docs/zh/operations/table_engines/mysql.md     |  2 +-
 docs/zh/operations/table_engines/null.md      |  2 +-
 .../table_engines/replacingmergetree.md       |  2 +-
 .../operations/table_engines/replication.md   |  2 +-
 docs/zh/operations/table_engines/set.md       |  2 +-
 docs/zh/operations/table_engines/stripelog.md |  2 +-
 .../table_engines/summingmergetree.md         |  2 +-
 docs/zh/operations/table_engines/tinylog.md   |  2 +-
 docs/zh/operations/table_engines/url.md       |  2 +-
 docs/zh/operations/table_engines/view.md      |  2 +-
 docs/zh/operations/tips.md                    |  2 +-
 docs/zh/operations/utils/clickhouse-copier.md |  2 +-
 docs/zh/operations/utils/clickhouse-local.md  |  2 +-
 docs/zh/operations/utils/index.md             |  2 +-
 docs/zh/query_language/create.md              |  2 +-
 .../functions/arithmetic_functions.md         |  2 +-
 .../functions/array_functions.md              |  2 +-
 .../zh/query_language/functions/array_join.md |  2 +-
 .../query_language/functions/bit_functions.md |  2 +-
 .../functions/bitmap_functions.md             |  2 +-
 .../functions/comparison_functions.md         |  2 +-
 .../functions/conditional_functions.md        |  2 +-
 .../functions/date_time_functions.md          |  2 +-
 .../functions/encoding_functions.md           |  2 +-
 .../functions/ext_dict_functions.md           |  2 +-
 .../functions/functions_for_nulls.md          |  2 +-
 docs/zh/query_language/functions/geo.md       |  2 +-
 .../functions/hash_functions.md               |  2 +-
 .../functions/higher_order_functions.md       |  2 +-
 .../query_language/functions/in_functions.md  |  2 +-
 docs/zh/query_language/functions/index.md     |  2 +-
 .../functions/ip_address_functions.md         |  2 +-
 .../functions/json_functions.md               |  2 +-
 .../functions/logical_functions.md            |  2 +-
 .../functions/math_functions.md               |  2 +-
 .../functions/other_functions.md              |  2 +-
 .../functions/random_functions.md             |  2 +-
 .../functions/rounding_functions.md           |  2 +-
 .../functions/splitting_merging_functions.md  |  2 +-
 .../functions/string_functions.md             |  2 +-
 .../functions/string_replace_functions.md     |  2 +-
 .../functions/string_search_functions.md      |  2 +-
 .../functions/type_conversion_functions.md    |  2 +-
 .../query_language/functions/url_functions.md |  2 +-
 .../functions/uuid_functions.md               |  2 +-
 .../functions/ym_dict_functions.md            |  2 +-
 docs/zh/query_language/insert_into.md         |  2 +-
 docs/zh/query_language/operators.md           |  2 +-
 docs/zh/query_language/select.md              |  2 +-
 docs/zh/roadmap.md                            |  2 +-
 docs/zh/security_changelog.md                 |  2 +-
 509 files changed, 588 insertions(+), 586 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2cecc15bdcf..d955392fe97 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,12 +13,14 @@
 /build_*
 /build-*
 /docs/build
+/docs/publish
 /docs/edit
 /docs/tools/venv/
-/docs/en/development/build/
-/docs/ru/development/build/
 /docs/en/single.md
 /docs/ru/single.md
+/docs/zh/single.md
+/docs/ja/single.md
+/docs/fa/single.md
 
 # callgrind files
 callgrind.out.*
diff --git a/docs/README.md b/docs/README.md
index 91883444459..f0071976443 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -38,7 +38,7 @@ Writing the docs is extremely useful for project's users and developers, and gro
 
 The documentation contains information about all the aspects of the ClickHouse lifecycle: developing, testing, installing, operating, and using. The base language of the documentation is English. The English version is the most actual. All other languages are supported as much as they can by contributors from different countries.
 
-At the moment, [documentation](https://clickhouse.yandex/docs) exists in English, Russian, Chinese, Japanese, and Farsi. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs).
+At the moment, [documentation](https://clickhouse.tech/docs) exists in English, Russian, Chinese, Japanese, and Farsi. We store the documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs).
 
 Each language lays in the corresponding folder. Files that are not translated from English are the symbolic links to the English ones.
 
@@ -54,7 +54,7 @@ You can contribute to the documentation in many ways, for example:
 
 - Open a required file in the ClickHouse repository and edit it from the GitHub web interface.
 
-    You can do it on GitHub, or on the [ClickHouse Documentation](https://clickhouse.yandex/docs/en/) site. Each page of ClickHouse Documentation site contains an "Edit this page" (🖋) element in the upper right corner. Clicking this symbol, you get to the ClickHouse docs file opened for editing.
+    You can do it on GitHub, or on the [ClickHouse Documentation](https://clickhouse.tech/docs/en/) site. Each page of ClickHouse Documentation site contains an "Edit this page" (🖋) element in the upper right corner. Clicking this symbol, you get to the ClickHouse docs file opened for editing.
 
     When you are saving a file, GitHub opens a pull-request for your contribution. Add the `documentation` label to this pull request for proper automatic checks applying. If you have no permissions for adding labels, the reviewer of your PR adds it.
 
@@ -161,7 +161,7 @@ When writing documentation, think about people who read it. Each audience has sp
 
 ClickHouse documentation can be divided by the audience for the following parts:
 
-- Conceptual topics in [Introduction](https://clickhouse.yandex/docs/en/), tutorials and overviews, changelog.
+- Conceptual topics in [Introduction](https://clickhouse.tech/docs/en/), tutorials and overviews, changelog.
 
     These topics are for the most common auditory. When editing text in them, use the most common terms that are comfortable for the audience with basic technical skills.
 
diff --git a/docs/en/data_types/array.md b/docs/en/data_types/array.md
index 4e9c7d5930e..d23fe60c327 100644
--- a/docs/en/data_types/array.md
+++ b/docs/en/data_types/array.md
@@ -66,4 +66,4 @@ Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/array/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/array/) 
diff --git a/docs/en/data_types/boolean.md b/docs/en/data_types/boolean.md
index 52d2c8e320c..d5fc88e45cc 100644
--- a/docs/en/data_types/boolean.md
+++ b/docs/en/data_types/boolean.md
@@ -3,4 +3,4 @@
 There isn't a separate type for boolean values. They use the UInt8 type, restricted to the values 0 or 1.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/boolean/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/boolean/) 
diff --git a/docs/en/data_types/date.md b/docs/en/data_types/date.md
index b5bac572a73..cb5f84d6545 100644
--- a/docs/en/data_types/date.md
+++ b/docs/en/data_types/date.md
@@ -6,4 +6,4 @@ The minimum value is output as 0000-00-00.
 The date is stored without the time zone.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/date/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/date/) 
diff --git a/docs/en/data_types/datetime.md b/docs/en/data_types/datetime.md
index 7e30f3ae426..1695b0b86e9 100644
--- a/docs/en/data_types/datetime.md
+++ b/docs/en/data_types/datetime.md
@@ -87,4 +87,4 @@ SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x
 - [Operators for working with dates and times](../query_language/operators.md#operators-datetime)
 - [The `Date` data type](date.md)
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/datetime/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/datetime/) 
diff --git a/docs/en/data_types/decimal.md b/docs/en/data_types/decimal.md
index 262330dcff9..8d3112c3dab 100644
--- a/docs/en/data_types/decimal.md
+++ b/docs/en/data_types/decimal.md
@@ -95,4 +95,4 @@ SELECT toDecimal32(1, 8) < 100
 DB::Exception: Can't compare.
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/decimal/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/decimal/) 
diff --git a/docs/en/data_types/domains/ipv4.md b/docs/en/data_types/domains/ipv4.md
index 1ee6903b897..f4c76a711c0 100644
--- a/docs/en/data_types/domains/ipv4.md
+++ b/docs/en/data_types/domains/ipv4.md
@@ -26,16 +26,16 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from;
 `IPv4` domain supports custom input format as IPv4-strings:
 
 ```sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.yandex', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
 
 SELECT * FROM hits;
 ```
 
 ```text
 ┌─url────────────────────────────────┬───────────from─┐
-│ https://clickhouse.yandex/docs/en/ │ 116.106.34.242 │
+│ https://clickhouse.tech/docs/en/ │ 116.106.34.242 │
 │ https://wikipedia.org              │ 116.253.40.133 │
-│ https://clickhouse.yandex          │ 183.247.232.58 │
+│ https://clickhouse.tech          │ 183.247.232.58 │
 └────────────────────────────────────┴────────────────┘
 ```
 
@@ -76,4 +76,4 @@ SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1;
 └──────────────────────────────────┴────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/domains/ipv4) 
+[Original article](https://clickhouse.tech/docs/en/data_types/domains/ipv4) 
diff --git a/docs/en/data_types/domains/ipv6.md b/docs/en/data_types/domains/ipv6.md
index e2abaff0172..1d5ebf598b3 100644
--- a/docs/en/data_types/domains/ipv6.md
+++ b/docs/en/data_types/domains/ipv6.md
@@ -26,15 +26,15 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from;
 `IPv6` domain supports custom input as IPv6-strings:
 
 ```sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.yandex', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
 
 SELECT * FROM hits;
 ```
 
 ```text
 ┌─url────────────────────────────────┬─from──────────────────────────┐
-│ https://clickhouse.yandex          │ 2001:44c8:129:2632:33:0:252:2 │
-│ https://clickhouse.yandex/docs/en/ │ 2a02:e980:1e::1               │
+│ https://clickhouse.tech          │ 2001:44c8:129:2632:33:0:252:2 │
+│ https://clickhouse.tech/docs/en/ │ 2a02:e980:1e::1               │
 │ https://wikipedia.org              │ 2a02:aa08:e000:3100::2        │
 └────────────────────────────────────┴───────────────────────────────┘
 ```
@@ -76,4 +76,4 @@ SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1;
 └───────────────────────────────────────────┴─────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/domains/ipv6) 
+[Original article](https://clickhouse.tech/docs/en/data_types/domains/ipv6) 
diff --git a/docs/en/data_types/domains/overview.md b/docs/en/data_types/domains/overview.md
index fe7cd82df28..103a48653fc 100644
--- a/docs/en/data_types/domains/overview.md
+++ b/docs/en/data_types/domains/overview.md
@@ -23,4 +23,4 @@ You can use domains anywhere corresponding base type can be used:
 * Can't implicitly convert string values into domain values when inserting data from another column or table.
 * Domain adds no constrains on stored values.
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/domains/overview) 
\ No newline at end of file
+[Original article](https://clickhouse.tech/docs/en/data_types/domains/overview) 
\ No newline at end of file
diff --git a/docs/en/data_types/enum.md b/docs/en/data_types/enum.md
index 3b27b2e6c7e..6fee1602e38 100644
--- a/docs/en/data_types/enum.md
+++ b/docs/en/data_types/enum.md
@@ -117,4 +117,4 @@ The Enum type can be changed without cost using ALTER, if only the set of values
 Using ALTER, it is possible to change an Enum8 to an Enum16 or vice versa, just like changing an Int8 to Int16.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/enum/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/enum/) 
diff --git a/docs/en/data_types/fixedstring.md b/docs/en/data_types/fixedstring.md
index fd2f1a17a37..9c40295bbb8 100644
--- a/docs/en/data_types/fixedstring.md
+++ b/docs/en/data_types/fixedstring.md
@@ -52,4 +52,4 @@ This behavior differs from MySQL behavior for the `CHAR` type (where strings are
 
 Note that the length of the `FixedString(N)` value is constant. The [length](../query_language/functions/array_functions.md#array_functions-length) function returns `N` even if the `FixedString(N)` value is filled only with null bytes, but the [empty](../query_language/functions/string_functions.md#string_functions-empty) function returns `1` in this case.
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/fixedstring/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/fixedstring/) 
diff --git a/docs/en/data_types/float.md b/docs/en/data_types/float.md
index a365c80b4d1..1531c8d5722 100644
--- a/docs/en/data_types/float.md
+++ b/docs/en/data_types/float.md
@@ -70,4 +70,4 @@ SELECT 0 / 0
   See the rules for `NaN` sorting in the section [ORDER BY clause](../query_language/select.md).
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/float/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/float/) 
diff --git a/docs/en/data_types/index.md b/docs/en/data_types/index.md
index b0f61096952..17a30842748 100644
--- a/docs/en/data_types/index.md
+++ b/docs/en/data_types/index.md
@@ -5,4 +5,4 @@ ClickHouse can store various types of data in table cells.
 This section describes the supported data types and special considerations when using and/or implementing them, if any.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/) 
diff --git a/docs/en/data_types/int_uint.md b/docs/en/data_types/int_uint.md
index 69528b36278..ae318445c1a 100644
--- a/docs/en/data_types/int_uint.md
+++ b/docs/en/data_types/int_uint.md
@@ -18,4 +18,4 @@ Fixed-length integers, with or without a sign.
 - UInt64 - [0 : 18446744073709551615]
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/int_uint/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/int_uint/) 
diff --git a/docs/en/data_types/nested_data_structures/aggregatefunction.md b/docs/en/data_types/nested_data_structures/aggregatefunction.md
index 46e43ea5d91..67520f75670 100644
--- a/docs/en/data_types/nested_data_structures/aggregatefunction.md
+++ b/docs/en/data_types/nested_data_structures/aggregatefunction.md
@@ -61,4 +61,4 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP
 See [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) engine description.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/aggregatefunction/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/nested_data_structures/aggregatefunction/) 
diff --git a/docs/en/data_types/nested_data_structures/index.md b/docs/en/data_types/nested_data_structures/index.md
index 1c4d2c902a8..7815b8229b9 100644
--- a/docs/en/data_types/nested_data_structures/index.md
+++ b/docs/en/data_types/nested_data_structures/index.md
@@ -1,4 +1,4 @@
 # Nested Data Structures
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/nested_data_structures/) 
diff --git a/docs/en/data_types/nested_data_structures/nested.md b/docs/en/data_types/nested_data_structures/nested.md
index 3083d0f915c..727150ce841 100644
--- a/docs/en/data_types/nested_data_structures/nested.md
+++ b/docs/en/data_types/nested_data_structures/nested.md
@@ -97,4 +97,4 @@ For a DESCRIBE query, the columns in a nested data structure are listed separate
 The ALTER query is very limited for elements in a nested data structure.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/nested/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/nested_data_structures/nested/) 
diff --git a/docs/en/data_types/nullable.md b/docs/en/data_types/nullable.md
index a46bcbcb3f2..3974091b4ce 100644
--- a/docs/en/data_types/nullable.md
+++ b/docs/en/data_types/nullable.md
@@ -33,4 +33,4 @@ SELECT x + y FROM t_null
 └────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/nullable/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/nullable/) 
diff --git a/docs/en/data_types/special_data_types/expression.md b/docs/en/data_types/special_data_types/expression.md
index 0bc7f635a0b..e3d461e94d2 100644
--- a/docs/en/data_types/special_data_types/expression.md
+++ b/docs/en/data_types/special_data_types/expression.md
@@ -3,4 +3,4 @@
 Used for representing lambda expressions in high-order functions.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/special_data_types/expression/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/expression/) 
diff --git a/docs/en/data_types/special_data_types/index.md b/docs/en/data_types/special_data_types/index.md
index c66857cf5df..14c2406edc0 100644
--- a/docs/en/data_types/special_data_types/index.md
+++ b/docs/en/data_types/special_data_types/index.md
@@ -3,4 +3,4 @@
 Special data type values can't be saved to a table or output in results, but are used as the intermediate result of running a query.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/special_data_types/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/) 
diff --git a/docs/en/data_types/special_data_types/nothing.md b/docs/en/data_types/special_data_types/nothing.md
index d0c34e1fecf..3540ecf861d 100644
--- a/docs/en/data_types/special_data_types/nothing.md
+++ b/docs/en/data_types/special_data_types/nothing.md
@@ -17,4 +17,4 @@ SELECT toTypeName(array())
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/special_data_types/nothing/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/nothing/) 
diff --git a/docs/en/data_types/special_data_types/set.md b/docs/en/data_types/special_data_types/set.md
index d2d61e7ac62..560a4b4d01f 100644
--- a/docs/en/data_types/special_data_types/set.md
+++ b/docs/en/data_types/special_data_types/set.md
@@ -3,4 +3,4 @@
 Used for the right half of an IN expression.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/special_data_types/set/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/special_data_types/set/) 
diff --git a/docs/en/data_types/string.md b/docs/en/data_types/string.md
index 1d5b0a7e86c..59dd88dbf73 100644
--- a/docs/en/data_types/string.md
+++ b/docs/en/data_types/string.md
@@ -12,4 +12,4 @@ Similarly, certain functions for working with strings have separate variations t
 For example, the 'length' function calculates the string length in bytes, while the 'lengthUTF8' function calculates the string length in Unicode code points, assuming that the value is UTF-8 encoded.
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/string/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/string/) 
diff --git a/docs/en/data_types/tuple.md b/docs/en/data_types/tuple.md
index 7726c25ce85..17acf696813 100644
--- a/docs/en/data_types/tuple.md
+++ b/docs/en/data_types/tuple.md
@@ -42,4 +42,4 @@ SELECT tuple(1, NULL) AS x, toTypeName(x)
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/tuple/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/tuple/) 
diff --git a/docs/en/data_types/uuid.md b/docs/en/data_types/uuid.md
index 81e6d11435a..c5ace976ef9 100644
--- a/docs/en/data_types/uuid.md
+++ b/docs/en/data_types/uuid.md
@@ -62,4 +62,4 @@ The UUID data type only supports functions which [String](string.md) data type a
 
 The UUID data type is not supported by arithmetic operations (for example, [abs](../query_language/functions/arithmetic_functions.md#arithm_func-abs)) or aggregate functions, such as [sum](../query_language/agg_functions/reference.md#agg_function-sum) and [avg](../query_language/agg_functions/reference.md#agg_function-avg).
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/uuid/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/uuid/) 
diff --git a/docs/en/database_engines/index.md b/docs/en/database_engines/index.md
index be00c77783e..184cd29982b 100644
--- a/docs/en/database_engines/index.md
+++ b/docs/en/database_engines/index.md
@@ -10,4 +10,4 @@ You can also use the following database engines:
 
 - [Lazy](lazy.md)
 
-[Original article](https://clickhouse.yandex/docs/en/database_engines/) 
+[Original article](https://clickhouse.tech/docs/en/database_engines/) 
diff --git a/docs/en/database_engines/lazy.md b/docs/en/database_engines/lazy.md
index 03c3dbb8a95..f0424b95bbb 100644
--- a/docs/en/database_engines/lazy.md
+++ b/docs/en/database_engines/lazy.md
@@ -10,4 +10,4 @@ It's optimized for storing many small \*Log tables, for which there is a long ti
 CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds);
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/database_engines/lazy/) 
+[Original article](https://clickhouse.tech/docs/en/database_engines/lazy/) 
diff --git a/docs/en/database_engines/mysql.md b/docs/en/database_engines/mysql.md
index 049d784faae..35809e00caf 100644
--- a/docs/en/database_engines/mysql.md
+++ b/docs/en/database_engines/mysql.md
@@ -119,4 +119,4 @@ SELECT * FROM mysql_db.mysql_table
 └────────┴───────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/database_engines/mysql/) 
+[Original article](https://clickhouse.tech/docs/en/database_engines/mysql/) 
diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md
index 3f98136e68c..28ddd71f4a2 100644
--- a/docs/en/development/architecture.md
+++ b/docs/en/development/architecture.md
@@ -194,4 +194,4 @@ In addition, each replica stores its state in ZooKeeper as the set of parts and
 > The ClickHouse cluster consists of independent shards, and each shard consists of replicas. The cluster is not elastic, so after adding a new shard, data is not rebalanced between shards automatically. Instead, the cluster load will be uneven. This implementation gives you more control, and it is fine for relatively small clusters such as tens of nodes. But for clusters with hundreds of nodes that we are using in production, this approach becomes a significant drawback. We should implement a table engine that will span its data across the cluster with dynamically replicated regions that could be split and balanced between clusters automatically.
 
 
-[Original article](https://clickhouse.yandex/docs/en/development/architecture/) 
+[Original article](https://clickhouse.tech/docs/en/development/architecture/) 
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index 1686f013559..107116e6a7c 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -83,4 +83,4 @@ To create an executable, run `ninja clickhouse`.
 This will create the `dbms/programs/clickhouse` executable, which can be used with `client` or `server` arguments.
 
 
-[Original article](https://clickhouse.yandex/docs/en/development/build/) 
+[Original article](https://clickhouse.tech/docs/en/development/build/) 
diff --git a/docs/en/development/build_cross_osx.md b/docs/en/development/build_cross_osx.md
index b6baabbb85d..b8e38b44e1e 100644
--- a/docs/en/development/build_cross_osx.md
+++ b/docs/en/development/build_cross_osx.md
@@ -1,6 +1,6 @@
 # How to Build ClickHouse on Linux for Mac OS X
 
-This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on OS X. This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on Mac OS X, then proceed with another instruction: https://clickhouse.yandex/docs/en/development/build_osx/
+This is for the case when you have Linux machine and want to use it to build `clickhouse` binary that will run on OS X. This is intended for continuous integration checks that run on Linux servers. If you want to build ClickHouse directly on Mac OS X, then proceed with another instruction: https://clickhouse.tech/docs/en/development/build_osx/
 
 The cross-build for Mac OS X is based on the [Build instructions](build.md), follow them first.
 
diff --git a/docs/en/development/build_osx.md b/docs/en/development/build_osx.md
index ae08d3abb3f..e0615e3ff9a 100644
--- a/docs/en/development/build_osx.md
+++ b/docs/en/development/build_osx.md
@@ -82,4 +82,4 @@ Reboot.
 To check if it's working, you can use `ulimit -n` command.
 
 
-[Original article](https://clickhouse.yandex/docs/en/development/build_osx/) 
+[Original article](https://clickhouse.tech/docs/en/development/build_osx/) 
diff --git a/docs/en/development/developer_instruction.md b/docs/en/development/developer_instruction.md
index 591fc7c486c..fe664728c4f 100644
--- a/docs/en/development/developer_instruction.md
+++ b/docs/en/development/developer_instruction.md
@@ -111,7 +111,7 @@ Official Yandex builds currently use GCC because it generates machine code of sl
 
 To install GCC on Ubuntu run: `sudo apt install gcc g++`
 
-Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.yandex/docs/en/development/build/#install-gcc-9.
+Check the version of gcc: `gcc --version`. If it is below 9, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-9.
 
 Mac OS X build is supported only for Clang. Just run `brew install llvm`
 
@@ -226,11 +226,11 @@ Just in case, it is worth mentioning that CLion creates `build` path on its own,
 
 # Writing Code
 
-The description of ClickHouse architecture can be found here: https://clickhouse.yandex/docs/en/development/architecture/
+The description of ClickHouse architecture can be found here: https://clickhouse.tech/docs/en/development/architecture/
 
-The Code Style Guide: https://clickhouse.yandex/docs/en/development/style/
+The Code Style Guide: https://clickhouse.tech/docs/en/development/style/
 
-Writing tests: https://clickhouse.yandex/docs/en/development/tests/
+Writing tests: https://clickhouse.tech/docs/en/development/tests/
 
 List of tasks: https://github.com/yandex/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_en.md
 
diff --git a/docs/en/development/index.md b/docs/en/development/index.md
index d9095f383b3..19074d6f102 100644
--- a/docs/en/development/index.md
+++ b/docs/en/development/index.md
@@ -1,4 +1,4 @@
 # ClickHouse Development
 
 
-[Original article](https://clickhouse.yandex/docs/en/development/) 
+[Original article](https://clickhouse.tech/docs/en/development/) 
diff --git a/docs/en/development/style.md b/docs/en/development/style.md
index df56559e68a..69e6a9919d7 100644
--- a/docs/en/development/style.md
+++ b/docs/en/development/style.md
@@ -835,4 +835,4 @@ function(
       size_t limit)
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/development/style/) 
+[Original article](https://clickhouse.tech/docs/en/development/style/) 
diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md
index e451d17e717..bc692ce4041 100644
--- a/docs/en/development/tests.md
+++ b/docs/en/development/tests.md
@@ -226,7 +226,7 @@ If you use `CLion` as an IDE, you can leverage some `clang-tidy` checks out of t
 
 ## Code Style
 
-Code style rules are described [here](https://clickhouse.yandex/docs/en/development/style/).
+Code style rules are described [here](https://clickhouse.tech/docs/en/development/style/).
 
 To check for some common style violations, you can use `utils/check-style` script.
 
@@ -258,4 +258,4 @@ Build jobs and tests are run in Sandbox on per commit basis. Resulting packages
 We don't use Travis CI due to the limit on time and computational power.
 We don't use Jenkins. It was used before and now we are happy we are not using Jenkins.
 
-[Original article](https://clickhouse.yandex/docs/en/development/tests/) 
+[Original article](https://clickhouse.tech/docs/en/development/tests/) 
diff --git a/docs/en/faq/general.md b/docs/en/faq/general.md
index 8afe995f724..eac1f130959 100644
--- a/docs/en/faq/general.md
+++ b/docs/en/faq/general.md
@@ -53,4 +53,4 @@ $ clickhouse-client --query "SELECT * from table" > result.txt
 
 See [clickhouse-client](../interfaces/cli.md).
 
-[Original article](https://clickhouse.yandex/docs/en/faq/general/) 
+[Original article](https://clickhouse.tech/docs/en/faq/general/) 
diff --git a/docs/en/getting_started/example_datasets/amplab_benchmark.md b/docs/en/getting_started/example_datasets/amplab_benchmark.md
index 29c404cedfe..67ac38406ee 100644
--- a/docs/en/getting_started/example_datasets/amplab_benchmark.md
+++ b/docs/en/getting_started/example_datasets/amplab_benchmark.md
@@ -120,4 +120,4 @@ LIMIT 1
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/amplab_benchmark/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/amplab_benchmark/) 
diff --git a/docs/en/getting_started/example_datasets/criteo.md b/docs/en/getting_started/example_datasets/criteo.md
index 8c57a2b0e9b..d9a7f5b56c1 100644
--- a/docs/en/getting_started/example_datasets/criteo.md
+++ b/docs/en/getting_started/example_datasets/criteo.md
@@ -72,4 +72,4 @@ DROP TABLE criteo_log;
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/criteo/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/criteo/) 
diff --git a/docs/en/getting_started/example_datasets/nyc_taxi.md b/docs/en/getting_started/example_datasets/nyc_taxi.md
index a3226e3d28a..ab91089b0dd 100644
--- a/docs/en/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/en/getting_started/example_datasets/nyc_taxi.md
@@ -382,4 +382,4 @@ We ran queries using a client located in a Yandex datacenter in Finland on a clu
 | 140 | 0.028 | 0.043 | 0.051 | 0.072 |
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/nyc_taxi/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/nyc_taxi/) 
diff --git a/docs/en/getting_started/example_datasets/ontime.md b/docs/en/getting_started/example_datasets/ontime.md
index 335532f1da9..4aea3985aab 100644
--- a/docs/en/getting_started/example_datasets/ontime.md
+++ b/docs/en/getting_started/example_datasets/ontime.md
@@ -402,4 +402,4 @@ This performance test was created by Vadim Tkachenko. See:
 - 
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/ontime/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/ontime/) 
diff --git a/docs/en/getting_started/example_datasets/star_schema.md b/docs/en/getting_started/example_datasets/star_schema.md
index 43c1b1df6fa..dea017e4d79 100644
--- a/docs/en/getting_started/example_datasets/star_schema.md
+++ b/docs/en/getting_started/example_datasets/star_schema.md
@@ -337,4 +337,4 @@ ORDER BY
     P_BRAND ASC;
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/star_schema/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) 
diff --git a/docs/en/getting_started/example_datasets/wikistat.md b/docs/en/getting_started/example_datasets/wikistat.md
index b9ab6f184f7..b6827375c57 100644
--- a/docs/en/getting_started/example_datasets/wikistat.md
+++ b/docs/en/getting_started/example_datasets/wikistat.md
@@ -26,4 +26,4 @@ $ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikis
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/wikistat/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/wikistat/) 
diff --git a/docs/en/getting_started/index.md b/docs/en/getting_started/index.md
index bfdcb0e108a..7ffdceebdea 100644
--- a/docs/en/getting_started/index.md
+++ b/docs/en/getting_started/index.md
@@ -5,4 +5,4 @@ If you are new to ClickHouse and want to get a hands-on feeling of it's performa
 * [Go through detailed tutorial](tutorial.md)
 * [Experiment with example datasets](example_datasets/ontime.md)
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/) 
diff --git a/docs/en/getting_started/install.md b/docs/en/getting_started/install.md
index f1ec980fa70..0d446a433f8 100644
--- a/docs/en/getting_started/install.md
+++ b/docs/en/getting_started/install.md
@@ -176,7 +176,7 @@ SELECT 1
 
 **Congratulations, the system works!**
 
-To continue experimenting, you can download one of test data sets or go through [tutorial](https://clickhouse.yandex/tutorial.html).
+To continue experimenting, you can download one of test data sets or go through [tutorial](https://clickhouse.tech/tutorial.html).
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/install/) 
diff --git a/docs/en/index.md b/docs/en/index.md
index 40158f524ec..0985a15d61b 100644
--- a/docs/en/index.md
+++ b/docs/en/index.md
@@ -129,4 +129,4 @@ This is not done in "normal" databases, because it doesn't make sense when runni
 
 Note that for CPU efficiency, the query language must be declarative (SQL or MDX), or at least a vector (J, K). The query should only contain implicit loops, allowing for optimization.
 
-[Original article](https://clickhouse.yandex/docs/en/) 
+[Original article](https://clickhouse.tech/docs/en/) 
diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index 0fcc5e8899f..48965d11062 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -147,4 +147,4 @@ Example of a config file:
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/cli/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/cli/) 
diff --git a/docs/en/interfaces/cpp.md b/docs/en/interfaces/cpp.md
index e10942ad8d5..cc5e670a00a 100644
--- a/docs/en/interfaces/cpp.md
+++ b/docs/en/interfaces/cpp.md
@@ -2,4 +2,4 @@
 
 See README at [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) repository.
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/cpp/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/cpp/) 
diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 9992467047a..f0d891169ae 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -1024,7 +1024,7 @@ If you input or output data via the [HTTP interface](../interfaces/http.md) the
 should be located in the directory specified in [format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path)
 in the server configuration.
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/formats/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/formats/) 
 
 ## Skipping Errors {#skippingerrors}
 
diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md
index f8f4b159a4e..aa545bfed97 100644
--- a/docs/en/interfaces/http.md
+++ b/docs/en/interfaces/http.md
@@ -264,4 +264,4 @@ You can create a query with parameters and pass values for them from the corresp
 $ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}"
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/http_interface/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/http_interface/) 
diff --git a/docs/en/interfaces/index.md b/docs/en/interfaces/index.md
index 57b22adaa69..9af7c9863dc 100644
--- a/docs/en/interfaces/index.md
+++ b/docs/en/interfaces/index.md
@@ -18,4 +18,4 @@ There are also a wide range of third-party libraries for working with ClickHouse
 * [Integrations](third-party/integrations.md)
 * [Visual interfaces](third-party/gui.md)
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/) 
diff --git a/docs/en/interfaces/jdbc.md b/docs/en/interfaces/jdbc.md
index 7eb8238fe65..039e3abfa31 100644
--- a/docs/en/interfaces/jdbc.md
+++ b/docs/en/interfaces/jdbc.md
@@ -6,4 +6,4 @@
   * [clickhouse4j](https://github.com/blynkkk/clickhouse4j)
 
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/jdbc/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/jdbc/) 
diff --git a/docs/en/interfaces/odbc.md b/docs/en/interfaces/odbc.md
index 07212df6687..9e65027b6a8 100644
--- a/docs/en/interfaces/odbc.md
+++ b/docs/en/interfaces/odbc.md
@@ -3,4 +3,4 @@
 - [Official driver](https://github.com/ClickHouse/clickhouse-odbc).
 
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/odbc/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/odbc/) 
diff --git a/docs/en/interfaces/tcp.md b/docs/en/interfaces/tcp.md
index 0c3e0e7885f..6194809db82 100644
--- a/docs/en/interfaces/tcp.md
+++ b/docs/en/interfaces/tcp.md
@@ -2,4 +2,4 @@
 
 The native protocol is used in the [command-line client](cli.md), for interserver communication during distributed query processing, and also in other C++ programs. Unfortunately, native ClickHouse protocol does not have formal specification yet, but it can be reverse engineered from ClickHouse source code (starting [around here](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/src/Client)) and/or by intercepting and analyzing TCP traffic.
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/tcp/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/tcp/) 
diff --git a/docs/en/interfaces/third-party/client_libraries.md b/docs/en/interfaces/third-party/client_libraries.md
index 514d2bd0305..167c23bd09c 100644
--- a/docs/en/interfaces/third-party/client_libraries.md
+++ b/docs/en/interfaces/third-party/client_libraries.md
@@ -48,4 +48,4 @@
     - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
 
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/third-party/client_libraries/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/client_libraries/) 
diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md
index 447173b3103..62424022eb3 100644
--- a/docs/en/interfaces/third-party/gui.md
+++ b/docs/en/interfaces/third-party/gui.md
@@ -126,4 +126,4 @@ Features:
 [Data Modeling](https://looker.com/platform/data-modeling) to support report writers and end users.
 - Powerful workflow integration via Looker's [Data Actions](https://looker.com/platform/actions).
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/third-party/gui/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) 
diff --git a/docs/en/interfaces/third-party/integrations.md b/docs/en/interfaces/third-party/integrations.md
index 692dfae9776..80a13ce846d 100644
--- a/docs/en/interfaces/third-party/integrations.md
+++ b/docs/en/interfaces/third-party/integrations.md
@@ -85,4 +85,4 @@
         - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/third-party/integrations/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/integrations/) 
diff --git a/docs/en/interfaces/third-party/proxy.md b/docs/en/interfaces/third-party/proxy.md
index 9ccd39040d4..7bce14748c8 100644
--- a/docs/en/interfaces/third-party/proxy.md
+++ b/docs/en/interfaces/third-party/proxy.md
@@ -36,4 +36,4 @@ Features:
 
 Implemented in Go.
 
-[Original article](https://clickhouse.yandex/docs/en/interfaces/third-party/proxy/) 
+[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/proxy/) 
diff --git a/docs/en/introduction/distinctive_features.md b/docs/en/introduction/distinctive_features.md
index ed3543ddf34..af4050fa3e2 100644
--- a/docs/en/introduction/distinctive_features.md
+++ b/docs/en/introduction/distinctive_features.md
@@ -61,4 +61,4 @@ Uses asynchronous multimaster replication. After being written to any available
 
 For more information, see the section [Data replication](../operations/table_engines/replication.md).
 
-[Original article](https://clickhouse.yandex/docs/en/introduction/distinctive_features/) 
+[Original article](https://clickhouse.tech/docs/en/introduction/distinctive_features/) 
diff --git a/docs/en/introduction/features_considered_disadvantages.md b/docs/en/introduction/features_considered_disadvantages.md
index 5071c9b9b8d..6717e32bfe4 100644
--- a/docs/en/introduction/features_considered_disadvantages.md
+++ b/docs/en/introduction/features_considered_disadvantages.md
@@ -4,4 +4,4 @@
 2. Lack of ability to modify or delete already inserted data with high rate and low latency. There are batch deletes and updates available to clean up or modify data, for example to comply with [GDPR](https://gdpr-info.eu).
 3. The sparse index makes ClickHouse not really suitable for point queries retrieving single rows by their keys.
 
-[Original article](https://clickhouse.yandex/docs/en/introduction/features_considered_disadvantages/) 
+[Original article](https://clickhouse.tech/docs/en/introduction/features_considered_disadvantages/) 
diff --git a/docs/en/introduction/history.md b/docs/en/introduction/history.md
index e8f373880f1..1d35a6df683 100644
--- a/docs/en/introduction/history.md
+++ b/docs/en/introduction/history.md
@@ -47,4 +47,4 @@ OLAPServer worked well for non-aggregated data, but it had many restrictions tha
 To remove the limitations of OLAPServer and solve the problem of working with non-aggregated data for all reports, we developed the ClickHouse DBMS.
 
 
-[Original article](https://clickhouse.yandex/docs/en/introduction/history/) 
+[Original article](https://clickhouse.tech/docs/en/introduction/history/) 
diff --git a/docs/en/introduction/performance.md b/docs/en/introduction/performance.md
index d2d67ce6cf9..0f42fa538e5 100644
--- a/docs/en/introduction/performance.md
+++ b/docs/en/introduction/performance.md
@@ -1,8 +1,8 @@
 # Performance
 
-According to internal testing results at Yandex, ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](https://clickhouse.yandex/benchmark.html).
+According to internal testing results at Yandex, ClickHouse shows the best performance (both the highest throughput for long queries and the lowest latency on short queries) for comparable operating scenarios among systems of its class that were available for testing. You can view the test results on a [separate page](/benchmark.html).
 
-This has also been confirmed by numerous independent benchmarks. They are not difficult to find using an internet search, or you can see [our small collection of related links](https://clickhouse.yandex/#independent-benchmarks).
+This has also been confirmed by numerous independent benchmarks. They are not difficult to find using an internet search, or you can see [our small collection of related links](/#independent-benchmarks).
 
 ## Throughput for a Single Large Query
 
@@ -22,4 +22,4 @@ Under the same conditions, ClickHouse can handle several hundred queries per sec
 
 We recommend inserting data in packets of at least 1000 rows, or no more than a single request per second. When inserting to a MergeTree table from a tab-separated dump, the insertion speed will be from 50 to 200 MB/s. If the inserted rows are around 1 Kb in size, the speed will be from 50,000 to 200,000 rows per second. If the rows are small, the performance will be higher in rows per second (on Banner System data -`>` 500,000 rows per second; on Graphite data -`>` 1,000,000 rows per second). To improve performance, you can make multiple INSERT queries in parallel, and performance will increase linearly.
 
-[Original article](https://clickhouse.yandex/docs/en/introduction/performance/) 
+[Original article](https://clickhouse.tech/docs/en/introduction/performance/) 
diff --git a/docs/en/operations/access_rights.md b/docs/en/operations/access_rights.md
index e5b1781f59f..8806aa11ebe 100644
--- a/docs/en/operations/access_rights.md
+++ b/docs/en/operations/access_rights.md
@@ -104,4 +104,4 @@ The user can get a list of all databases and tables in them by using `SHOW` quer
 
 Database access is not related to the [readonly](settings/permissions_for_queries.md#settings_readonly) setting. You can't grant full access to one database and `readonly` access to another one.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/access_rights/) 
+[Original article](https://clickhouse.tech/docs/en/operations/access_rights/) 
diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index fcbfb7fb40c..0b04f5aa1b8 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -31,4 +31,4 @@ For more information about queries related to partition manipulations, see the [
 
 A third-party tool is available to automate this approach: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup).
 
-[Original article](https://clickhouse.yandex/docs/en/operations/backup/) 
+[Original article](https://clickhouse.tech/docs/en/operations/backup/) 
diff --git a/docs/en/operations/configuration_files.md b/docs/en/operations/configuration_files.md
index ea10eb3bfe2..69204506c2d 100644
--- a/docs/en/operations/configuration_files.md
+++ b/docs/en/operations/configuration_files.md
@@ -42,4 +42,4 @@ For each config file, the server also generates `file-preprocessed.xml` files wh
 
 The server tracks changes in config files, as well as files and ZooKeeper nodes that were used when performing substitutions and overrides, and reloads the settings for users and clusters on the fly. This means that you can modify the cluster, users, and their settings without restarting the server.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/configuration_files/) 
+[Original article](https://clickhouse.tech/docs/en/operations/configuration_files/) 
diff --git a/docs/en/operations/index.md b/docs/en/operations/index.md
index 3ce48c8e57e..de35dfbad23 100644
--- a/docs/en/operations/index.md
+++ b/docs/en/operations/index.md
@@ -17,4 +17,4 @@ ClickHouse operations manual consists of the following major sections:
   - [Settings](settings/index.md)
   - [Utilities](utils/index.md)
 
-[Original article](https://clickhouse.yandex/docs/en/operations/) 
+[Original article](https://clickhouse.tech/docs/en/operations/) 
diff --git a/docs/en/operations/performance_test.md b/docs/en/operations/performance_test.md
index f5f249a75e8..dfdabe08395 100644
--- a/docs/en/operations/performance_test.md
+++ b/docs/en/operations/performance_test.md
@@ -38,7 +38,7 @@ wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/dbms/benchma
 
 According to the instruction:
 
-[https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/](https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/)
+[https://clickhouse.tech/docs/en/getting_started/example_datasets/metrica/](https://clickhouse.yandex/docs/en/getting_started/example_datasets/metrica/)
 
 ("hits" table containing 100 million rows)
 
@@ -73,4 +73,4 @@ mcedit benchmark-new.sh
 
 \12. Send the numbers and the info about your hardware configuration to clickhouse-feedback@yandex-team.com
 
-All the results are published here: [https://clickhouse.yandex/benchmark_hardware.html](https://clickhouse.yandex/benchmark_hardware.html)
+All the results are published here: [https://clickhouse.tech/benchmark_hardware.html](https://clickhouse.yandex/benchmark_hardware.html)
diff --git a/docs/en/operations/quotas.md b/docs/en/operations/quotas.md
index d5a48408d26..608e7f39034 100644
--- a/docs/en/operations/quotas.md
+++ b/docs/en/operations/quotas.md
@@ -103,4 +103,4 @@ For distributed query processing, the accumulated amounts are stored on the requ
 When the server is restarted, quotas are reset.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/quotas/) 
+[Original article](https://clickhouse.tech/docs/en/operations/quotas/) 
diff --git a/docs/en/operations/server_settings/index.md b/docs/en/operations/server_settings/index.md
index 3b721f95ef2..39f3a5bb6d4 100644
--- a/docs/en/operations/server_settings/index.md
+++ b/docs/en/operations/server_settings/index.md
@@ -9,4 +9,4 @@ Other settings are described in the "[Settings](../settings/index.md#settings)"
 Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/server_settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/server_settings/) 
diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md
index 7766be91fd5..9c1c772c642 100644
--- a/docs/en/operations/server_settings/settings.md
+++ b/docs/en/operations/server_settings/settings.md
@@ -882,4 +882,4 @@ The update is performed asynchronously, in a separate system thread.
 
 **Default value**: 15.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/server_settings/settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/server_settings/settings/) 
diff --git a/docs/en/operations/settings/constraints_on_settings.md b/docs/en/operations/settings/constraints_on_settings.md
index c3bc4f05f22..e5dfde7341b 100644
--- a/docs/en/operations/settings/constraints_on_settings.md
+++ b/docs/en/operations/settings/constraints_on_settings.md
@@ -65,4 +65,4 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n
 
 **Note:** the `default` profile has a special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they're overriden explicitly for these users.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/constraints_on_settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/constraints_on_settings/) 
diff --git a/docs/en/operations/settings/index.md b/docs/en/operations/settings/index.md
index 16487c2a558..3dd02876955 100644
--- a/docs/en/operations/settings/index.md
+++ b/docs/en/operations/settings/index.md
@@ -21,4 +21,4 @@ Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you
 Settings that can only be made in the server config file are not covered in this section.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/) 
diff --git a/docs/en/operations/settings/permissions_for_queries.md b/docs/en/operations/settings/permissions_for_queries.md
index 892e1cdd9ea..e6dcd490e97 100644
--- a/docs/en/operations/settings/permissions_for_queries.md
+++ b/docs/en/operations/settings/permissions_for_queries.md
@@ -51,4 +51,4 @@ You can't execute `SET allow_ddl = 1` if `allow_ddl = 0` for the current session
 
 Default value: 1
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/permissions_for_queries/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/permissions_for_queries/) 
diff --git a/docs/en/operations/settings/query_complexity.md b/docs/en/operations/settings/query_complexity.md
index 1e71fb5e333..e5f993540a7 100644
--- a/docs/en/operations/settings/query_complexity.md
+++ b/docs/en/operations/settings/query_complexity.md
@@ -274,4 +274,4 @@ When inserting data, ClickHouse calculates the number of partitions in the inser
 
 > "Too many partitions for single INSERT block (more than " + toString(max_parts) + "). The limit is controlled by 'max_partitions_per_insert_block' setting. Large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc)."
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/query_complexity/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/query_complexity/) 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index f380cd2d4d7..75c3b8b0bf3 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1143,4 +1143,4 @@ Enable order-preserving parallel parsing of data formats. Supported only for TSV
 
 The minimum chunk size in bytes, which each thread will parse in parallel.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) 
diff --git a/docs/en/operations/settings/settings_profiles.md b/docs/en/operations/settings/settings_profiles.md
index 095bb362936..604206bf9da 100644
--- a/docs/en/operations/settings/settings_profiles.md
+++ b/docs/en/operations/settings/settings_profiles.md
@@ -63,4 +63,4 @@ The example specifies two profiles: `default` and `web`. The `default` profile h
 Settings profiles can inherit from each other. To use inheritance, indicate one or multiple `profile` settings before the other settings that are listed in the profile. In case when one setting is defined in different profiles, the latest defined is used.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_profiles/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/settings_profiles/) 
diff --git a/docs/en/operations/settings/settings_users.md b/docs/en/operations/settings/settings_users.md
index 04f7900382c..c30786759e6 100644
--- a/docs/en/operations/settings/settings_users.md
+++ b/docs/en/operations/settings/settings_users.md
@@ -143,4 +143,4 @@ The following configuration forces that user `user1` can only see the rows of `t
 
 The `filter` can be any expression resulting in a [UInt8](../../data_types/int_uint.md)-type value. It usually contains comparisons and logical operators. Rows from `database_name.table1` where filter results to 0 are not returned for this user. The filtering is incompatible with `PREWHERE` operations and disables `WHERE→PREWHERE` optimization.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_users/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/settings_users/) 
diff --git a/docs/en/operations/system_tables.md b/docs/en/operations/system_tables.md
index a2fc6c44d06..802d42e8294 100644
--- a/docs/en/operations/system_tables.md
+++ b/docs/en/operations/system_tables.md
@@ -960,4 +960,4 @@ Columns:
 
 If the storage policy contains more then one volume, then information for each volume is stored in the individual row of the table.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/system_tables/) 
+[Original article](https://clickhouse.tech/docs/en/operations/system_tables/) 
diff --git a/docs/en/operations/table_engines/aggregatingmergetree.md b/docs/en/operations/table_engines/aggregatingmergetree.md
index 006614eec1c..f45ded4067c 100644
--- a/docs/en/operations/table_engines/aggregatingmergetree.md
+++ b/docs/en/operations/table_engines/aggregatingmergetree.md
@@ -91,4 +91,4 @@ GROUP BY StartDate
 ORDER BY StartDate;
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/aggregatingmergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/aggregatingmergetree/) 
diff --git a/docs/en/operations/table_engines/buffer.md b/docs/en/operations/table_engines/buffer.md
index 2c9e0dd8dc0..f574fcb9f74 100644
--- a/docs/en/operations/table_engines/buffer.md
+++ b/docs/en/operations/table_engines/buffer.md
@@ -62,4 +62,4 @@ A Buffer table is used when too many INSERTs are received from a large number of
 Note that it doesn't make sense to insert data one row at a time, even for Buffer tables. This will only produce a speed of a few thousand rows per second, while inserting larger blocks of data can produce over a million rows per second (see the section "Performance").
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/buffer/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/buffer/) 
diff --git a/docs/en/operations/table_engines/collapsingmergetree.md b/docs/en/operations/table_engines/collapsingmergetree.md
index 42c1bf91860..56f10f4c988 100644
--- a/docs/en/operations/table_engines/collapsingmergetree.md
+++ b/docs/en/operations/table_engines/collapsingmergetree.md
@@ -286,4 +286,4 @@ select * FROM UAct
 
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/collapsingmergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/collapsingmergetree/) 
diff --git a/docs/en/operations/table_engines/custom_partitioning_key.md b/docs/en/operations/table_engines/custom_partitioning_key.md
index ba9c551ce2a..d5e80e30f7e 100644
--- a/docs/en/operations/table_engines/custom_partitioning_key.md
+++ b/docs/en/operations/table_engines/custom_partitioning_key.md
@@ -117,4 +117,4 @@ Note that on the operating server, you cannot manually change the set of parts o
 
 ClickHouse allows you to perform operations with the partitions: delete them, copy from one table to another, or create a backup. See the list of all operations in the section [Manipulations With Partitions and Parts](../../query_language/alter.md#alter_manipulations-with-partitions).
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/custom_partitioning_key/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) 
diff --git a/docs/en/operations/table_engines/dictionary.md b/docs/en/operations/table_engines/dictionary.md
index 045c013f1c7..c3de79f69cf 100644
--- a/docs/en/operations/table_engines/dictionary.md
+++ b/docs/en/operations/table_engines/dictionary.md
@@ -90,4 +90,4 @@ select * from products limit 1;
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/dictionary/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/dictionary/) 
diff --git a/docs/en/operations/table_engines/distributed.md b/docs/en/operations/table_engines/distributed.md
index e26e83dbf5e..39b02310575 100644
--- a/docs/en/operations/table_engines/distributed.md
+++ b/docs/en/operations/table_engines/distributed.md
@@ -142,4 +142,4 @@ When the max_parallel_replicas option is enabled, query processing is paralleliz
 - [Virtual columns](index.md#table_engines-virtual_columns)
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/distributed/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/distributed/) 
diff --git a/docs/en/operations/table_engines/external_data.md b/docs/en/operations/table_engines/external_data.md
index c57794df565..12fc34f516e 100644
--- a/docs/en/operations/table_engines/external_data.md
+++ b/docs/en/operations/table_engines/external_data.md
@@ -60,4 +60,4 @@ $ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+coun
 For distributed query processing, the temporary tables are sent to all the remote servers.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/external_data/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/external_data/) 
diff --git a/docs/en/operations/table_engines/file.md b/docs/en/operations/table_engines/file.md
index ba106856968..d45eb596ea1 100644
--- a/docs/en/operations/table_engines/file.md
+++ b/docs/en/operations/table_engines/file.md
@@ -80,4 +80,4 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64
     - Indices
     - Replication
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/file/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/file/) 
diff --git a/docs/en/operations/table_engines/graphitemergetree.md b/docs/en/operations/table_engines/graphitemergetree.md
index 7a47eabfe22..c260acca315 100644
--- a/docs/en/operations/table_engines/graphitemergetree.md
+++ b/docs/en/operations/table_engines/graphitemergetree.md
@@ -161,4 +161,4 @@ Fields for `pattern` and `default` sections:
 
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/graphitemergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) 
diff --git a/docs/en/operations/table_engines/hdfs.md b/docs/en/operations/table_engines/hdfs.md
index 2f98e4dc452..fb254f11112 100644
--- a/docs/en/operations/table_engines/hdfs.md
+++ b/docs/en/operations/table_engines/hdfs.md
@@ -107,6 +107,6 @@ CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9
 
 **See Also**
 
-- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns)
+- [Virtual columns](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns)
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/hdfs/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/hdfs/) 
diff --git a/docs/en/operations/table_engines/index.md b/docs/en/operations/table_engines/index.md
index 8da7288b35a..4ee5eb249e4 100644
--- a/docs/en/operations/table_engines/index.md
+++ b/docs/en/operations/table_engines/index.md
@@ -74,4 +74,4 @@ To select data from a virtual column, you must specify its name in the `SELECT`
 
 If you create a table with a column that has the same name as one of the table virtual columns, the virtual column becomes inaccessible. We don't recommend doing this. To help avoid conflicts, virtual column names are usually prefixed with an underscore.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/) 
diff --git a/docs/en/operations/table_engines/jdbc.md b/docs/en/operations/table_engines/jdbc.md
index 293f3c10dad..abb26aea0e2 100644
--- a/docs/en/operations/table_engines/jdbc.md
+++ b/docs/en/operations/table_engines/jdbc.md
@@ -82,4 +82,4 @@ FROM jdbc_table
 
 - [JDBC table function](../../query_language/table_functions/jdbc.md).
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/jdbc/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/jdbc/) 
diff --git a/docs/en/operations/table_engines/join.md b/docs/en/operations/table_engines/join.md
index 385a64747c2..fd0f3658a52 100644
--- a/docs/en/operations/table_engines/join.md
+++ b/docs/en/operations/table_engines/join.md
@@ -95,4 +95,4 @@ The `Join`-engine tables can't be used in `GLOBAL JOIN` operations.
 
 If the server restarts incorrectly, the data block on the disk might get lost or damaged. In this case, you may need to manually delete the file with damaged data.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/join/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/join/) 
diff --git a/docs/en/operations/table_engines/kafka.md b/docs/en/operations/table_engines/kafka.md
index 16c53d786c6..5731d68a61d 100644
--- a/docs/en/operations/table_engines/kafka.md
+++ b/docs/en/operations/table_engines/kafka.md
@@ -168,4 +168,4 @@ For a list of possible configuration options, see the [librdkafka configuration
 - [Virtual columns](index.md#table_engines-virtual_columns)
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) 
diff --git a/docs/en/operations/table_engines/log.md b/docs/en/operations/table_engines/log.md
index f59fc4fe46c..4e7b4703f42 100644
--- a/docs/en/operations/table_engines/log.md
+++ b/docs/en/operations/table_engines/log.md
@@ -8,4 +8,4 @@ For concurrent data access, the read operations can be performed simultaneously,
 The Log engine does not support indexes. Similarly, if writing to a table failed, the table is broken, and reading from it returns an error. The Log engine is appropriate for temporary data, write-once tables, and for testing or demonstration purposes.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/log/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/log/) 
diff --git a/docs/en/operations/table_engines/log_family.md b/docs/en/operations/table_engines/log_family.md
index 9353ea796b6..68ea24542bb 100644
--- a/docs/en/operations/table_engines/log_family.md
+++ b/docs/en/operations/table_engines/log_family.md
@@ -35,4 +35,4 @@ The `TinyLog` engine is the simplest in the family and provides the poorest func
 The `Log` and `StripeLog` engines support parallel data reading. When reading data, ClickHouse uses multiple threads. Each thread processes a separate data block. The `Log` engine uses a separate file for each column of the table. `StripeLog` stores all the data in one file. As a result, the `StripeLog` engine uses fewer descriptors in the operating system, but the `Log` engine provides higher efficiency when reading data.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/log_family/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) 
diff --git a/docs/en/operations/table_engines/materializedview.md b/docs/en/operations/table_engines/materializedview.md
index 1e0a1615874..bc4d2288542 100644
--- a/docs/en/operations/table_engines/materializedview.md
+++ b/docs/en/operations/table_engines/materializedview.md
@@ -3,4 +3,4 @@
 Used for implementing materialized views (for more information, see [CREATE TABLE](../../query_language/create.md)). For storing data, it uses a different engine that was specified when creating the view. When reading from a table, it just uses this engine.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/materializedview/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) 
diff --git a/docs/en/operations/table_engines/memory.md b/docs/en/operations/table_engines/memory.md
index d68ae923684..dded3ca8d41 100644
--- a/docs/en/operations/table_engines/memory.md
+++ b/docs/en/operations/table_engines/memory.md
@@ -10,4 +10,4 @@ Normally, using this table engine is not justified. However, it can be used for
 The Memory engine is used by the system for temporary tables with external query data (see the section "External data for processing a query"), and for implementing GLOBAL IN (see the section "IN operators").
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/memory/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/memory/) 
diff --git a/docs/en/operations/table_engines/merge.md b/docs/en/operations/table_engines/merge.md
index f2347d9777d..e081963cc87 100644
--- a/docs/en/operations/table_engines/merge.md
+++ b/docs/en/operations/table_engines/merge.md
@@ -60,4 +60,4 @@ FROM WatchLog
 - [Virtual columns](index.md#table_engines-virtual_columns)
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/merge/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/merge/) 
diff --git a/docs/en/operations/table_engines/mergetree.md b/docs/en/operations/table_engines/mergetree.md
index d2cf9e43857..a8f0cabb498 100644
--- a/docs/en/operations/table_engines/mergetree.md
+++ b/docs/en/operations/table_engines/mergetree.md
@@ -477,7 +477,7 @@ When ClickHouse see that data is expired, it performs an off-schedule merge. To
 
 If you perform the `SELECT` query between merges, you may get expired data. To avoid it, use the [OPTIMIZE](../../query_language/misc.md#misc_operations-optimize) query before `SELECT`.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/mergetree/) 
 
 
 ## Using Multiple Block Devices for Data Storage {#table_engine-mergetree-multiple-volumes}
@@ -651,4 +651,4 @@ Moving data does not interfere with data replication. Therefore, different stora
 After the completion of background merges and mutations, old parts are removed only after a certain amount of time (`old_parts_lifetime`).
 During this time, they are not moved to other volumes or disks. Therefore, until the parts are finally removed, they are still taken into account for evaluation of the occupied disk space.
 
-[Original article](https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/) 
+[Original article](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) 
diff --git a/docs/en/operations/table_engines/mysql.md b/docs/en/operations/table_engines/mysql.md
index 75bf23d3f40..d6d1dbff595 100644
--- a/docs/en/operations/table_engines/mysql.md
+++ b/docs/en/operations/table_engines/mysql.md
@@ -88,4 +88,4 @@ SELECT * FROM mysql_table
 - [The 'mysql' table function](../../query_language/table_functions/mysql.md)
 - [Using MySQL as a source of external dictionary](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/mysql/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/mysql/) 
diff --git a/docs/en/operations/table_engines/null.md b/docs/en/operations/table_engines/null.md
index a236de2f180..a0bc65a287f 100644
--- a/docs/en/operations/table_engines/null.md
+++ b/docs/en/operations/table_engines/null.md
@@ -6,4 +6,4 @@ When writing to a Null table, data is ignored. When reading from a Null table, t
 However, you can create a materialized view on a Null table. So the data written to the table will end up in the view.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/null/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/null/) 
diff --git a/docs/en/operations/table_engines/odbc.md b/docs/en/operations/table_engines/odbc.md
index afcedd4849e..8bce818d5ed 100644
--- a/docs/en/operations/table_engines/odbc.md
+++ b/docs/en/operations/table_engines/odbc.md
@@ -119,4 +119,4 @@ SELECT * FROM odbc_t
 - [ODBC external dictionaries](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc)
 - [ODBC table function](../../query_language/table_functions/odbc.md)
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/odbc/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/odbc/) 
diff --git a/docs/en/operations/table_engines/replacingmergetree.md b/docs/en/operations/table_engines/replacingmergetree.md
index d630e3b2e70..ee552544f69 100644
--- a/docs/en/operations/table_engines/replacingmergetree.md
+++ b/docs/en/operations/table_engines/replacingmergetree.md
@@ -56,4 +56,4 @@ All of the parameters excepting `ver` have the same meaning as in `MergeTree`.
 - `ver` - column with the version. Optional parameter. For a description, see the text above.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/replacingmergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/replacingmergetree/) 
diff --git a/docs/en/operations/table_engines/replication.md b/docs/en/operations/table_engines/replication.md
index bbb21cc2e81..2cce677267e 100644
--- a/docs/en/operations/table_engines/replication.md
+++ b/docs/en/operations/table_engines/replication.md
@@ -208,4 +208,4 @@ After this, you can launch the server, create a `MergeTree` table, move the data
 
 If the data in ZooKeeper was lost or damaged, you can save data by moving it to an unreplicated table as described above.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/replication/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) 
diff --git a/docs/en/operations/table_engines/set.md b/docs/en/operations/table_engines/set.md
index 75c1f3072d4..ee369aa257e 100644
--- a/docs/en/operations/table_engines/set.md
+++ b/docs/en/operations/table_engines/set.md
@@ -10,4 +10,4 @@ Data is always located in RAM. For INSERT, the blocks of inserted data are also
 For a rough server restart, the block of data on the disk might be lost or damaged. In the latter case, you may need to manually delete the file with damaged data.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/set/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/set/) 
diff --git a/docs/en/operations/table_engines/stripelog.md b/docs/en/operations/table_engines/stripelog.md
index 11ba0ae3ace..59c918defc1 100644
--- a/docs/en/operations/table_engines/stripelog.md
+++ b/docs/en/operations/table_engines/stripelog.md
@@ -83,4 +83,4 @@ SELECT * FROM stripe_log_table ORDER BY timestamp
 └─────────────────────┴──────────────┴────────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/stripelog/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/stripelog/) 
diff --git a/docs/en/operations/table_engines/summingmergetree.md b/docs/en/operations/table_engines/summingmergetree.md
index 11a7b73bc44..332daa60a92 100644
--- a/docs/en/operations/table_engines/summingmergetree.md
+++ b/docs/en/operations/table_engines/summingmergetree.md
@@ -130,4 +130,4 @@ When requesting data, use the [sumMap(key, value)](../../query_language/agg_func
 
 For nested data structure, you do not need to specify its columns in the tuple of columns for summation.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/summingmergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) 
diff --git a/docs/en/operations/table_engines/tinylog.md b/docs/en/operations/table_engines/tinylog.md
index 7be5e36c0b6..d2bcb5356d6 100644
--- a/docs/en/operations/table_engines/tinylog.md
+++ b/docs/en/operations/table_engines/tinylog.md
@@ -6,4 +6,4 @@ This table engine is typically used with the write-once method: write data one t
 
 Queries are executed in a single stream. In other words, this engine is intended for relatively small tables (up to about 1,000,000 rows). It makes sense to use this table engine if you have many small tables, since it's simpler than the [Log](log.md) engine (fewer files need to be opened).
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/tinylog/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/tinylog/) 
diff --git a/docs/en/operations/table_engines/url.md b/docs/en/operations/table_engines/url.md
index 4e46c58b843..60d8b5e1f2a 100644
--- a/docs/en/operations/table_engines/url.md
+++ b/docs/en/operations/table_engines/url.md
@@ -72,4 +72,4 @@ SELECT * FROM url_engine_table
     - Indexes.
     - Replication.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/url/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/url/) 
diff --git a/docs/en/operations/table_engines/versionedcollapsingmergetree.md b/docs/en/operations/table_engines/versionedcollapsingmergetree.md
index 547e6625744..4e7747ffae1 100644
--- a/docs/en/operations/table_engines/versionedcollapsingmergetree.md
+++ b/docs/en/operations/table_engines/versionedcollapsingmergetree.md
@@ -224,4 +224,4 @@ SELECT * FROM UAct FINAL
 
 This is a very inefficient way to select data. Don't use it for large tables.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/versionedcollapsingmergetree/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/versionedcollapsingmergetree/) 
diff --git a/docs/en/operations/table_engines/view.md b/docs/en/operations/table_engines/view.md
index 42e46148ce5..8c2e1295156 100644
--- a/docs/en/operations/table_engines/view.md
+++ b/docs/en/operations/table_engines/view.md
@@ -3,4 +3,4 @@
 Used for implementing views (for more information, see the `CREATE VIEW query`). It does not store data, but only stores the specified `SELECT` query. When reading from a table, it runs this query (and deletes all unnecessary columns from the query).
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/view/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/view/) 
diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md
index c3418de4be1..3cb64627f2a 100644
--- a/docs/en/operations/tips.md
+++ b/docs/en/operations/tips.md
@@ -241,4 +241,4 @@ end script
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/tips/) 
+[Original article](https://clickhouse.tech/docs/en/operations/tips/) 
diff --git a/docs/en/operations/utils/clickhouse-copier.md b/docs/en/operations/utils/clickhouse-copier.md
index 2a75667cb50..795edc9748d 100644
--- a/docs/en/operations/utils/clickhouse-copier.md
+++ b/docs/en/operations/utils/clickhouse-copier.md
@@ -167,4 +167,4 @@ Parameters:
 `clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/clickhouse-copier/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/clickhouse-copier/) 
diff --git a/docs/en/operations/utils/clickhouse-local.md b/docs/en/operations/utils/clickhouse-local.md
index 618a90585d2..e6ec112b0c3 100644
--- a/docs/en/operations/utils/clickhouse-local.md
+++ b/docs/en/operations/utils/clickhouse-local.md
@@ -73,4 +73,4 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec.
 ...
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/clickhouse-local/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/clickhouse-local/) 
diff --git a/docs/en/operations/utils/index.md b/docs/en/operations/utils/index.md
index 3fcd66da8d8..71d46f3f7c8 100644
--- a/docs/en/operations/utils/index.md
+++ b/docs/en/operations/utils/index.md
@@ -4,4 +4,4 @@
 - [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
 - [clickhouse-benchmark](clickhouse-benchmark.md) — Loads server with the custom queries and settings.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/) 
diff --git a/docs/en/query_language/agg_functions/combinators.md b/docs/en/query_language/agg_functions/combinators.md
index 6a1f347cc8a..9cb6c4be4fa 100644
--- a/docs/en/query_language/agg_functions/combinators.md
+++ b/docs/en/query_language/agg_functions/combinators.md
@@ -152,4 +152,4 @@ FROM people
 └────────┴───────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/combinators/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/combinators/) 
diff --git a/docs/en/query_language/agg_functions/index.md b/docs/en/query_language/agg_functions/index.md
index 42b9bf2a511..613518611f6 100644
--- a/docs/en/query_language/agg_functions/index.md
+++ b/docs/en/query_language/agg_functions/index.md
@@ -53,4 +53,4 @@ SELECT groupArray(y) FROM t_null_big
 `groupArray` does not include `NULL` in the resulting array.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/) 
diff --git a/docs/en/query_language/agg_functions/parametric_functions.md b/docs/en/query_language/agg_functions/parametric_functions.md
index 8784811a311..936afa82c6f 100644
--- a/docs/en/query_language/agg_functions/parametric_functions.md
+++ b/docs/en/query_language/agg_functions/parametric_functions.md
@@ -327,7 +327,7 @@ Problem: Generate a report that shows only keywords that produced at least 5 uni
 Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/parametric_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) 
 
 
 ## sumMapFiltered(keys_to_keep)(keys, values)
diff --git a/docs/en/query_language/agg_functions/reference.md b/docs/en/query_language/agg_functions/reference.md
index beae49fca76..c030cb064f6 100644
--- a/docs/en/query_language/agg_functions/reference.md
+++ b/docs/en/query_language/agg_functions/reference.md
@@ -1322,4 +1322,4 @@ SELECT arraySort(bitmapToArray(groupBitmapXorState(z))) FROM bitmap_column_expr_
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/agg_functions/reference/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/reference/) 
diff --git a/docs/en/query_language/alter.md b/docs/en/query_language/alter.md
index e9eab89bf9c..5d1c057f9c3 100644
--- a/docs/en/query_language/alter.md
+++ b/docs/en/query_language/alter.md
@@ -476,4 +476,4 @@ A mutation query returns immediately after the mutation entry is added (in case
 
 Entries for finished mutations are not deleted right away (the number of preserved entries is determined by the `finished_mutations_to_keep` storage engine parameter). Older mutation entries are deleted.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/alter/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/alter/) 
diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md
index 2705fd123c1..724ae204247 100644
--- a/docs/en/query_language/create.md
+++ b/docs/en/query_language/create.md
@@ -275,7 +275,7 @@ Views look the same as normal tables. For example, they are listed in the result
 
 There isn't a separate query for deleting views. To delete a view, use `DROP TABLE`.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/create/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/create/) 
 
 ## CREATE DICTIONARY {#create-dictionary-query}
 
diff --git a/docs/en/query_language/dicts/external_dicts.md b/docs/en/query_language/dicts/external_dicts.md
index abae466fcb4..19a45e40e73 100644
--- a/docs/en/query_language/dicts/external_dicts.md
+++ b/docs/en/query_language/dicts/external_dicts.md
@@ -42,4 +42,4 @@ You can [configure](external_dicts_dict.md) any number of dictionaries in the sa
 - [Functions for Working with External Dictionaries](../functions/ext_dict_functions.md)
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts/) 
diff --git a/docs/en/query_language/dicts/external_dicts_dict.md b/docs/en/query_language/dicts/external_dicts_dict.md
index 9ff58c4f72d..03963c2d3b2 100644
--- a/docs/en/query_language/dicts/external_dicts_dict.md
+++ b/docs/en/query_language/dicts/external_dicts_dict.md
@@ -43,4 +43,4 @@ LIFETIME(...) -- Lifetime of dictionary in memory
 - [structure](external_dicts_dict_structure.md) — Structure of the dictionary . A key and attributes that can be retrieved by this key.
 - [lifetime](external_dicts_dict_lifetime.md) — Frequency of dictionary updates.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict/) 
diff --git a/docs/en/query_language/dicts/external_dicts_dict_layout.md b/docs/en/query_language/dicts/external_dicts_dict_layout.md
index 3da10ef660c..5a587b5408c 100644
--- a/docs/en/query_language/dicts/external_dicts_dict_layout.md
+++ b/docs/en/query_language/dicts/external_dicts_dict_layout.md
@@ -366,4 +366,4 @@ Other types are not supported yet. The function returns the attribute for the pr
 
 Data is stored in a `trie`. It must completely fit into RAM.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_layout/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) 
diff --git a/docs/en/query_language/dicts/external_dicts_dict_lifetime.md b/docs/en/query_language/dicts/external_dicts_dict_lifetime.md
index 0cebf459698..dabb07f82af 100644
--- a/docs/en/query_language/dicts/external_dicts_dict_lifetime.md
+++ b/docs/en/query_language/dicts/external_dicts_dict_lifetime.md
@@ -78,4 +78,4 @@ SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source wher
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_lifetime/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_lifetime/) 
diff --git a/docs/en/query_language/dicts/external_dicts_dict_sources.md b/docs/en/query_language/dicts/external_dicts_dict_sources.md
index ecb1a94e002..9a3de191bc9 100644
--- a/docs/en/query_language/dicts/external_dicts_dict_sources.md
+++ b/docs/en/query_language/dicts/external_dicts_dict_sources.md
@@ -605,4 +605,4 @@ Setting fields:
 - `storage_type` – The structure of internal Redis storage using for work with keys. `simple` is for simple sources and for hashed single key sources, `hash_map` is for hashed sources with two keys. Ranged sources and cache sources with complex key are unsupported. May be omitted, default value is `simple`.
 - `db_index` – The specific numeric index of Redis logical database. May be omitted, default value is 0.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_sources/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) 
diff --git a/docs/en/query_language/dicts/external_dicts_dict_structure.md b/docs/en/query_language/dicts/external_dicts_dict_structure.md
index bc960f19596..8cb4090a310 100644
--- a/docs/en/query_language/dicts/external_dicts_dict_structure.md
+++ b/docs/en/query_language/dicts/external_dicts_dict_structure.md
@@ -161,4 +161,4 @@ Tag | Description | Required
 `injective` | Flag that shows whether the `id -> attribute` image is [injective](https://en.wikipedia.org/wiki/Injective_function).
If `true`, ClickHouse can automatically place after the `GROUP BY` clause the requests to dictionaries with injection. Usually it significantly reduces the amount of such requests.

Default value: `false`. | No
 `is_object_id` | Flag that shows whether the query is executed for a MongoDB document by `ObjectID`.

Default value: `false`. | No
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/external_dicts_dict_structure/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_structure/) 
diff --git a/docs/en/query_language/dicts/index.md b/docs/en/query_language/dicts/index.md
index 8b6988b0504..d189712c490 100644
--- a/docs/en/query_language/dicts/index.md
+++ b/docs/en/query_language/dicts/index.md
@@ -12,4 +12,4 @@ ClickHouse supports:
 - [Plug-in (external) dictionaries](external_dicts.md) with a [set of functions](../functions/ext_dict_functions.md).
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/) 
diff --git a/docs/en/query_language/dicts/internal_dicts.md b/docs/en/query_language/dicts/internal_dicts.md
index 627719ea05a..7df915dd998 100644
--- a/docs/en/query_language/dicts/internal_dicts.md
+++ b/docs/en/query_language/dicts/internal_dicts.md
@@ -45,4 +45,4 @@ We recommend periodically updating the dictionaries with the geobase. During an
 
 There are also functions for working with OS identifiers and Yandex.Metrica search engines, but they shouldn't be used.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/dicts/internal_dicts/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/dicts/internal_dicts/) 
diff --git a/docs/en/query_language/functions/arithmetic_functions.md b/docs/en/query_language/functions/arithmetic_functions.md
index fb0c3939a2a..ee6a122a008 100644
--- a/docs/en/query_language/functions/arithmetic_functions.md
+++ b/docs/en/query_language/functions/arithmetic_functions.md
@@ -74,4 +74,4 @@ Returns the least common multiple of the numbers.
 An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/arithmetic_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/arithmetic_functions/) 
diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md
index 6d5d9ccd6bf..c3ae0b979ec 100644
--- a/docs/en/query_language/functions/array_functions.md
+++ b/docs/en/query_language/functions/array_functions.md
@@ -897,4 +897,4 @@ Result:
 └────────────────────────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) 
diff --git a/docs/en/query_language/functions/array_join.md b/docs/en/query_language/functions/array_join.md
index 25dfc626581..5db4b0e601e 100644
--- a/docs/en/query_language/functions/array_join.md
+++ b/docs/en/query_language/functions/array_join.md
@@ -28,4 +28,4 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_join/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/array_join/) 
diff --git a/docs/en/query_language/functions/bit_functions.md b/docs/en/query_language/functions/bit_functions.md
index 3c36a1b28bc..941ba44cf30 100644
--- a/docs/en/query_language/functions/bit_functions.md
+++ b/docs/en/query_language/functions/bit_functions.md
@@ -199,4 +199,4 @@ Result:
 └──────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/bit_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/bit_functions/) 
diff --git a/docs/en/query_language/functions/bitmap_functions.md b/docs/en/query_language/functions/bitmap_functions.md
index a4aba098dbb..1d585785b9b 100644
--- a/docs/en/query_language/functions/bitmap_functions.md
+++ b/docs/en/query_language/functions/bitmap_functions.md
@@ -495,4 +495,4 @@ SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/bitmap_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/bitmap_functions/) 
diff --git a/docs/en/query_language/functions/comparison_functions.md b/docs/en/query_language/functions/comparison_functions.md
index 337a213673d..0c4bf8324f6 100644
--- a/docs/en/query_language/functions/comparison_functions.md
+++ b/docs/en/query_language/functions/comparison_functions.md
@@ -30,4 +30,4 @@ Note. Up until version 1.1.54134, signed and unsigned numbers were compared the
 ## greaterOrEquals, `>= operator` {#function-greaterorequals}
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/comparison_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) 
diff --git a/docs/en/query_language/functions/conditional_functions.md b/docs/en/query_language/functions/conditional_functions.md
index 8b46566af31..0f883d5b884 100644
--- a/docs/en/query_language/functions/conditional_functions.md
+++ b/docs/en/query_language/functions/conditional_functions.md
@@ -148,4 +148,4 @@ FROM LEFT_RIGHT
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/conditional_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) 
diff --git a/docs/en/query_language/functions/date_time_functions.md b/docs/en/query_language/functions/date_time_functions.md
index 26c44a6a391..9e145976101 100644
--- a/docs/en/query_language/functions/date_time_functions.md
+++ b/docs/en/query_language/functions/date_time_functions.md
@@ -366,4 +366,4 @@ Supported modifiers for Format:
 |%Y|Year|2018|
 |%%|a % sign|%|
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/date_time_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) 
diff --git a/docs/en/query_language/functions/encoding_functions.md b/docs/en/query_language/functions/encoding_functions.md
index 7cf93071b93..c3905e401b8 100644
--- a/docs/en/query_language/functions/encoding_functions.md
+++ b/docs/en/query_language/functions/encoding_functions.md
@@ -29,4 +29,4 @@ Accepts an integer. Returns a string containing the list of powers of two that t
 Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/encoding_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/encoding_functions/) 
diff --git a/docs/en/query_language/functions/ext_dict_functions.md b/docs/en/query_language/functions/ext_dict_functions.md
index 5af21d6014a..aaf68a36775 100644
--- a/docs/en/query_language/functions/ext_dict_functions.md
+++ b/docs/en/query_language/functions/ext_dict_functions.md
@@ -191,4 +191,4 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr)
 
 ClickHouse throws an exception if it cannot parse the value of the attribute or the value doesn't match the attribute data type.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/ext_dict_functions/) 
diff --git a/docs/en/query_language/functions/functions_for_nulls.md b/docs/en/query_language/functions/functions_for_nulls.md
index a5116096947..6abf94ad289 100644
--- a/docs/en/query_language/functions/functions_for_nulls.md
+++ b/docs/en/query_language/functions/functions_for_nulls.md
@@ -293,4 +293,4 @@ SELECT toTypeName(toNullable(10))
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/functions_for_nulls/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/functions_for_nulls/) 
diff --git a/docs/en/query_language/functions/geo.md b/docs/en/query_language/functions/geo.md
index 8f79dddac8f..bce764588ae 100644
--- a/docs/en/query_language/functions/geo.md
+++ b/docs/en/query_language/functions/geo.md
@@ -227,4 +227,4 @@ SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos
 └─────────────────────────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/geo/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/geo/) 
diff --git a/docs/en/query_language/functions/hash_functions.md b/docs/en/query_language/functions/hash_functions.md
index d98c56cd584..5c51bf33b20 100644
--- a/docs/en/query_language/functions/hash_functions.md
+++ b/docs/en/query_language/functions/hash_functions.md
@@ -429,4 +429,4 @@ Result:
 
 - [xxHash](http://cyan4973.github.io/xxHash/).
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/hash_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) 
diff --git a/docs/en/query_language/functions/higher_order_functions.md b/docs/en/query_language/functions/higher_order_functions.md
index fdf39b4403f..ef5fd0c398a 100644
--- a/docs/en/query_language/functions/higher_order_functions.md
+++ b/docs/en/query_language/functions/higher_order_functions.md
@@ -250,4 +250,4 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 
 For more information about the `arrayReverseSort` method, see the [Functions for Working With Arrays](array_functions.md#array_functions-reverse-sort) section.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/higher_order_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/higher_order_functions/) 
diff --git a/docs/en/query_language/functions/in_functions.md b/docs/en/query_language/functions/in_functions.md
index 5886dcc3bc7..287d401a0a5 100644
--- a/docs/en/query_language/functions/in_functions.md
+++ b/docs/en/query_language/functions/in_functions.md
@@ -17,4 +17,4 @@ A function that allows getting a column from a tuple.
 There is no cost to execute the function.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/in_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/in_functions/) 
diff --git a/docs/en/query_language/functions/index.md b/docs/en/query_language/functions/index.md
index 2e3c46779db..a25ee88dea1 100644
--- a/docs/en/query_language/functions/index.md
+++ b/docs/en/query_language/functions/index.md
@@ -64,4 +64,4 @@ Another example is the `hostName` function, which returns the name of the server
 If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an 'any' aggregate function or add it to a key in `GROUP BY`.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/) 
diff --git a/docs/en/query_language/functions/ip_address_functions.md b/docs/en/query_language/functions/ip_address_functions.md
index 411b882d6b2..89fabc04b0d 100644
--- a/docs/en/query_language/functions/ip_address_functions.md
+++ b/docs/en/query_language/functions/ip_address_functions.md
@@ -241,4 +241,4 @@ SELECT
 └───────────────────────────────────┴──────────────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/ip_address_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/ip_address_functions/) 
diff --git a/docs/en/query_language/functions/json_functions.md b/docs/en/query_language/functions/json_functions.md
index eeb41870112..90fd687f1ed 100644
--- a/docs/en/query_language/functions/json_functions.md
+++ b/docs/en/query_language/functions/json_functions.md
@@ -218,4 +218,4 @@ Example:
 SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']'
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/json_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) 
diff --git a/docs/en/query_language/functions/logical_functions.md b/docs/en/query_language/functions/logical_functions.md
index 45c722f52d7..e9c6d2864c0 100644
--- a/docs/en/query_language/functions/logical_functions.md
+++ b/docs/en/query_language/functions/logical_functions.md
@@ -13,4 +13,4 @@ Zero as an argument is considered "false," while any non-zero value is considere
 ## xor
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/logical_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) 
diff --git a/docs/en/query_language/functions/math_functions.md b/docs/en/query_language/functions/math_functions.md
index d2673ddda41..6da8b4fb55f 100644
--- a/docs/en/query_language/functions/math_functions.md
+++ b/docs/en/query_language/functions/math_functions.md
@@ -106,4 +106,4 @@ Accepts a numeric argument and returns a UInt64 number close to 2 to the power o
 
 Accepts a numeric argument and returns a UInt64 number close to 10 to the power of x.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/math_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) 
diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md
index 5542369e738..98c42a911b4 100644
--- a/docs/en/query_language/functions/other_functions.md
+++ b/docs/en/query_language/functions/other_functions.md
@@ -933,4 +933,4 @@ SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers
 └────────┴────────────────────────────────┴──────────────────────────────────┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/other_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) 
diff --git a/docs/en/query_language/functions/random_functions.md b/docs/en/query_language/functions/random_functions.md
index 7e8649990d5..91bd5a5a725 100644
--- a/docs/en/query_language/functions/random_functions.md
+++ b/docs/en/query_language/functions/random_functions.md
@@ -20,4 +20,4 @@ Uses a linear congruential generator.
 
 Returns a pseudo-random UInt32 number, The value is one for different blocks.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/random_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) 
diff --git a/docs/en/query_language/functions/rounding_functions.md b/docs/en/query_language/functions/rounding_functions.md
index ec365235381..a1460b054fa 100644
--- a/docs/en/query_language/functions/rounding_functions.md
+++ b/docs/en/query_language/functions/rounding_functions.md
@@ -177,4 +177,4 @@ Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rou
 
 Accept a number, round it down to an element in the specified array. If the value is less than the lowest bound, the lowest bound is returned.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/rounding_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/rounding_functions/) 
diff --git a/docs/en/query_language/functions/splitting_merging_functions.md b/docs/en/query_language/functions/splitting_merging_functions.md
index 8c4e1ff8b45..514c2165376 100644
--- a/docs/en/query_language/functions/splitting_merging_functions.md
+++ b/docs/en/query_language/functions/splitting_merging_functions.md
@@ -28,4 +28,4 @@ SELECT alphaTokens('abca1abc')
 │ ['abca','abc']          │
 └─────────────────────────┘
 ```
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/splitting_merging_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/splitting_merging_functions/) 
diff --git a/docs/en/query_language/functions/string_functions.md b/docs/en/query_language/functions/string_functions.md
index 8b73f1b3c19..d85b3f51d67 100644
--- a/docs/en/query_language/functions/string_functions.md
+++ b/docs/en/query_language/functions/string_functions.md
@@ -387,4 +387,4 @@ Returns the CRC64 checksum of a string, using CRC-64-ECMA polynomial.
 
 The result type is UInt64.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/string_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) 
diff --git a/docs/en/query_language/functions/string_replace_functions.md b/docs/en/query_language/functions/string_replace_functions.md
index 22c0a27bd41..258c4459f01 100644
--- a/docs/en/query_language/functions/string_replace_functions.md
+++ b/docs/en/query_language/functions/string_replace_functions.md
@@ -86,4 +86,4 @@ This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero b
 For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473)
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/string_replace_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_replace_functions/) 
diff --git a/docs/en/query_language/functions/string_search_functions.md b/docs/en/query_language/functions/string_search_functions.md
index bcf0689eb2c..6cfff7b694e 100644
--- a/docs/en/query_language/functions/string_search_functions.md
+++ b/docs/en/query_language/functions/string_search_functions.md
@@ -127,4 +127,4 @@ For case-insensitive search or/and in UTF-8 format use functions `ngramSearchCas
 !!! note "Note"
     For UTF-8 case we use 3-gram distance. All these are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables -- collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function -- we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one -- this works for Latin and mostly for all Cyrillic letters.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/string_search_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) 
diff --git a/docs/en/query_language/functions/type_conversion_functions.md b/docs/en/query_language/functions/type_conversion_functions.md
index ec0ddbe3e9d..ffb757c17a2 100644
--- a/docs/en/query_language/functions/type_conversion_functions.md
+++ b/docs/en/query_language/functions/type_conversion_functions.md
@@ -406,4 +406,4 @@ Same as for [parseDateTimeBestEffort](#type_conversion_functions-parsedatetimebe
 
 Same as for [parseDateTimeBestEffort](#type_conversion_functions-parsedatetimebesteffort) except that it returns zero date or zero date time when it encounters a date format that cannot be processed.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/type_conversion_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) 
diff --git a/docs/en/query_language/functions/url_functions.md b/docs/en/query_language/functions/url_functions.md
index 43b92633653..1f5f4d32b03 100644
--- a/docs/en/query_language/functions/url_functions.md
+++ b/docs/en/query_language/functions/url_functions.md
@@ -199,4 +199,4 @@ Removes the query string and fragment identifier. The question mark and number s
 Removes the 'name' URL parameter, if present. This function works under the assumption that the parameter name is encoded in the URL exactly the same way as in the passed argument.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/url_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/url_functions/) 
diff --git a/docs/en/query_language/functions/uuid_functions.md b/docs/en/query_language/functions/uuid_functions.md
index 2ab89dabe9f..af8ba7f84f2 100644
--- a/docs/en/query_language/functions/uuid_functions.md
+++ b/docs/en/query_language/functions/uuid_functions.md
@@ -109,4 +109,4 @@ SELECT
 
 - [dictGetUUID](ext_dict_functions.md#ext_dict_functions-other)
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/uuid_function/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) 
diff --git a/docs/en/query_language/functions/ym_dict_functions.md b/docs/en/query_language/functions/ym_dict_functions.md
index b04c16dab33..e83e86d73f9 100644
--- a/docs/en/query_language/functions/ym_dict_functions.md
+++ b/docs/en/query_language/functions/ym_dict_functions.md
@@ -124,4 +124,4 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. A string with
 `ua` and `uk` both mean Ukrainian.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/ym_dict_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/ym_dict_functions/) 
diff --git a/docs/en/query_language/index.md b/docs/en/query_language/index.md
index 2d457fdaad3..d292f2d92db 100644
--- a/docs/en/query_language/index.md
+++ b/docs/en/query_language/index.md
@@ -7,4 +7,4 @@
 * [Other types of queries](misc.md)
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/) 
diff --git a/docs/en/query_language/insert_into.md b/docs/en/query_language/insert_into.md
index 3383e90620e..e2bf226c298 100644
--- a/docs/en/query_language/insert_into.md
+++ b/docs/en/query_language/insert_into.md
@@ -72,4 +72,4 @@ Performance will not decrease if:
 - You upload data that is usually sorted by time.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/insert_into/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/insert_into/) 
diff --git a/docs/en/query_language/misc.md b/docs/en/query_language/misc.md
index a9d8e7339e9..a11e90366e1 100644
--- a/docs/en/query_language/misc.md
+++ b/docs/en/query_language/misc.md
@@ -242,4 +242,4 @@ Lets you set the current database for the session.
 The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name.
 This query can't be made when using the HTTP protocol, since there is no concept of a session.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/misc/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/misc/) 
diff --git a/docs/en/query_language/operators.md b/docs/en/query_language/operators.md
index 571685e61d0..3c280a3a399 100644
--- a/docs/en/query_language/operators.md
+++ b/docs/en/query_language/operators.md
@@ -260,4 +260,4 @@ SELECT * FROM t_null WHERE y IS NOT NULL
 └───┴───┘
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/operators/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/operators/) 
diff --git a/docs/en/query_language/select.md b/docs/en/query_language/select.md
index 7a80557c393..95c3df8bb0f 100644
--- a/docs/en/query_language/select.md
+++ b/docs/en/query_language/select.md
@@ -1365,4 +1365,4 @@ You can put an asterisk in any part of a query instead of an expression. When th
 
 In all other cases, we don't recommend using the asterisk, since it only gives you the drawbacks of a columnar DBMS instead of the advantages. In other words using the asterisk is not recommended.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/select/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/select/) 
diff --git a/docs/en/query_language/syntax.md b/docs/en/query_language/syntax.md
index a67a0983961..640114741c3 100644
--- a/docs/en/query_language/syntax.md
+++ b/docs/en/query_language/syntax.md
@@ -173,4 +173,4 @@ A list of expressions is one or more expressions separated by commas.
 Functions and operators, in turn, can have expressions as arguments.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/syntax/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/syntax/) 
diff --git a/docs/en/query_language/system.md b/docs/en/query_language/system.md
index 0b08183afe8..6bff8381f0e 100644
--- a/docs/en/query_language/system.md
+++ b/docs/en/query_language/system.md
@@ -107,4 +107,4 @@ Provides possibility to start background merges for tables in the MergeTree fami
 SYSTEM START MERGES [[db.]merge_tree_family_table_name]
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/system/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/system/) 
diff --git a/docs/en/query_language/table_functions/file.md b/docs/en/query_language/table_functions/file.md
index 6ad324bb0d9..48a7907ac80 100644
--- a/docs/en/query_language/table_functions/file.md
+++ b/docs/en/query_language/table_functions/file.md
@@ -105,6 +105,6 @@ FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32')
 
 **See Also**
 
-- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns)
+- [Virtual columns](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns)
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/file/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/file/) 
diff --git a/docs/en/query_language/table_functions/hdfs.md b/docs/en/query_language/table_functions/hdfs.md
index 2c13ef0c95b..6acaf9842ac 100644
--- a/docs/en/query_language/table_functions/hdfs.md
+++ b/docs/en/query_language/table_functions/hdfs.md
@@ -90,6 +90,6 @@ FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name Strin
 
 **See Also**
 
-- [Virtual columns](https://clickhouse.yandex/docs/en/operations/table_engines/#table_engines-virtual_columns)
+- [Virtual columns](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns)
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/hdfs/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/hdfs/) 
diff --git a/docs/en/query_language/table_functions/index.md b/docs/en/query_language/table_functions/index.md
index 0e27ba7b497..9718f2b1439 100644
--- a/docs/en/query_language/table_functions/index.md
+++ b/docs/en/query_language/table_functions/index.md
@@ -27,4 +27,4 @@ Function | Description
 [odbc](odbc.md) | Creates a [ODBC](../../operations/table_engines/odbc.md)-engine table.
 [hdfs](hdfs.md) | Creates a [HDFS](../../operations/table_engines/hdfs.md)-engine table.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/) 
diff --git a/docs/en/query_language/table_functions/input.md b/docs/en/query_language/table_functions/input.md
index 408ff7939ac..f1cca3bbc50 100644
--- a/docs/en/query_language/table_functions/input.md
+++ b/docs/en/query_language/table_functions/input.md
@@ -32,4 +32,4 @@ $ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV"
 $ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV"
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/input/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/input/) 
diff --git a/docs/en/query_language/table_functions/jdbc.md b/docs/en/query_language/table_functions/jdbc.md
index 03cd1710e6b..97f3b5af666 100644
--- a/docs/en/query_language/table_functions/jdbc.md
+++ b/docs/en/query_language/table_functions/jdbc.md
@@ -20,4 +20,4 @@ SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema',
 SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table')
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/jdbc/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) 
diff --git a/docs/en/query_language/table_functions/merge.md b/docs/en/query_language/table_functions/merge.md
index 37a206d36a5..beb12737e53 100644
--- a/docs/en/query_language/table_functions/merge.md
+++ b/docs/en/query_language/table_functions/merge.md
@@ -5,4 +5,4 @@
 The table structure is taken from the first table encountered that matches the regular expression.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/merge/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/merge/) 
diff --git a/docs/en/query_language/table_functions/mysql.md b/docs/en/query_language/table_functions/mysql.md
index 4d643ba286d..8a80c1167f1 100644
--- a/docs/en/query_language/table_functions/mysql.md
+++ b/docs/en/query_language/table_functions/mysql.md
@@ -69,4 +69,4 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
 - [The 'MySQL' table engine](../../operations/table_engines/mysql.md)
 - [Using MySQL as a source of external dictionary](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/mysql/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/mysql/) 
diff --git a/docs/en/query_language/table_functions/numbers.md b/docs/en/query_language/table_functions/numbers.md
index f3f64cf6399..112b4bf474a 100644
--- a/docs/en/query_language/table_functions/numbers.md
+++ b/docs/en/query_language/table_functions/numbers.md
@@ -21,4 +21,4 @@ select toDate('2010-01-01') + number as d FROM numbers(365);
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/numbers/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/numbers/) 
diff --git a/docs/en/query_language/table_functions/odbc.md b/docs/en/query_language/table_functions/odbc.md
index 65036a6219d..d9115557f1e 100644
--- a/docs/en/query_language/table_functions/odbc.md
+++ b/docs/en/query_language/table_functions/odbc.md
@@ -96,4 +96,4 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test')
 - [ODBC external dictionaries](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc)
 - [ODBC table engine](../../operations/table_engines/odbc.md).
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/jdbc/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) 
diff --git a/docs/en/query_language/table_functions/remote.md b/docs/en/query_language/table_functions/remote.md
index 0543d17cade..79b5a93eb68 100644
--- a/docs/en/query_language/table_functions/remote.md
+++ b/docs/en/query_language/table_functions/remote.md
@@ -74,4 +74,4 @@ If the password is not specified, an empty password is used.
 
 `remoteSecure` - same as `remote` but with secured connection. Default port — [tcp_port_secure](../../operations/server_settings/settings.md#server_settings-tcp_port_secure) from config or 9440.
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/remote/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/remote/) 
diff --git a/docs/en/query_language/table_functions/url.md b/docs/en/query_language/table_functions/url.md
index 005a23b6bdc..59ccdc1c41d 100644
--- a/docs/en/query_language/table_functions/url.md
+++ b/docs/en/query_language/table_functions/url.md
@@ -17,4 +17,4 @@ structure - table structure in `'UserID UInt64, Name String'` format. Determines
 SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/url/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/url/) 
diff --git a/docs/en/roadmap.md b/docs/en/roadmap.md
index 1628f046c6f..8fc91286dfe 100644
--- a/docs/en/roadmap.md
+++ b/docs/en/roadmap.md
@@ -6,4 +6,4 @@
 - Integration with external authentication services
 - Resource pools for more precise distribution of cluster capacity between users
 
-[Original article](https://clickhouse.yandex/docs/en/roadmap/) 
+[Original article](https://clickhouse.tech/docs/en/roadmap/) 
diff --git a/docs/en/security_changelog.md b/docs/en/security_changelog.md
index 92b35868f94..dbd5690499a 100644
--- a/docs/en/security_changelog.md
+++ b/docs/en/security_changelog.md
@@ -66,4 +66,4 @@ Incorrect configuration in deb package could lead to unauthorized use of the dat
 
 Credits: the UK's National Cyber Security Centre (NCSC)
 
-[Original article](https://clickhouse.yandex/docs/en/security_changelog/) 
+[Original article](https://clickhouse.tech/docs/en/security_changelog/) 
diff --git a/docs/fa/data_types/array.md b/docs/fa/data_types/array.md
index e14f698f6ed..c4c5d409860 100644
--- a/docs/fa/data_types/array.md
+++ b/docs/fa/data_types/array.md
@@ -5,4 +5,4 @@
 آرایه ای از عناصر با تایپ T. تایپ T می تواند هر Type باشد، از جمله یک آرایه. ما توصیه به استفاده از آرایه های multidimensional نمی کنیم، چون آنها به خوبی پشتیبانی نمی شوند (برای مثال، شما نمی تونید در جداولی که موتور آنها MergeTree است، آرایه های multidimensional ذخیره سازی کنید).
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/array/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/array/) 
diff --git a/docs/fa/data_types/boolean.md b/docs/fa/data_types/boolean.md
index 90f540468c7..c4ac8496383 100644
--- a/docs/fa/data_types/boolean.md
+++ b/docs/fa/data_types/boolean.md
@@ -6,4 +6,4 @@ type مخصوص مقادیر boolean وجود ندارد. از Uint8 و محدو
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/boolean/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/boolean/) 
diff --git a/docs/fa/data_types/date.md b/docs/fa/data_types/date.md
index 2a71cbd24bd..21e84f2be7b 100644
--- a/docs/fa/data_types/date.md
+++ b/docs/fa/data_types/date.md
@@ -7,4 +7,4 @@ Date، دو بایت به ازای هر تاریخ که به صورت عددی 
 Date بدون time zone ذخیره می شود.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/date/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/date/) 
diff --git a/docs/fa/data_types/datetime.md b/docs/fa/data_types/datetime.md
index d36243f7a4f..0152fb4e774 100644
--- a/docs/fa/data_types/datetime.md
+++ b/docs/fa/data_types/datetime.md
@@ -14,4 +14,4 @@
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/datetime/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/datetime/) 
diff --git a/docs/fa/data_types/enum.md b/docs/fa/data_types/enum.md
index d84f2a1aa68..dddd6ace97b 100644
--- a/docs/fa/data_types/enum.md
+++ b/docs/fa/data_types/enum.md
@@ -31,4 +31,4 @@ Enum8('hello' = 1, 'world' = 2)
 استفاده از ALTER برای تبدیل Enum8 به Enum16 یا برعکس، ممکن است، دقیقا شبیه به Int8 به Int16.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/enum/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/enum/) 
diff --git a/docs/fa/data_types/fixedstring.md b/docs/fa/data_types/fixedstring.md
index 96234069307..f006834d9bb 100644
--- a/docs/fa/data_types/fixedstring.md
+++ b/docs/fa/data_types/fixedstring.md
@@ -7,4 +7,4 @@
 توابع کمتری نسبت به String برای FixedString(N) وجود دارد، و برای استفاده کمتر مناسب است.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/fixedstring/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/fixedstring/) 
diff --git a/docs/fa/data_types/float.md b/docs/fa/data_types/float.md
index df83dabd206..8c4466663a8 100644
--- a/docs/fa/data_types/float.md
+++ b/docs/fa/data_types/float.md
@@ -87,4 +87,4 @@ SELECT 0 / 0
 قوانین مربوط به مرتب سازی ` Nan ` را در بخش [ORDER BY clause](../query_language/select.md) ببینید.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/float/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/float/) 
diff --git a/docs/fa/data_types/index.md b/docs/fa/data_types/index.md
index c6073522c5c..d8910be367c 100644
--- a/docs/fa/data_types/index.md
+++ b/docs/fa/data_types/index.md
@@ -8,4 +8,4 @@ ClickHouse قابلیت ذخیره سازی انواع type های مختلف ب
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/) 
diff --git a/docs/fa/data_types/int_uint.md b/docs/fa/data_types/int_uint.md
index 0a1bbfe4a82..383eb978a4b 100644
--- a/docs/fa/data_types/int_uint.md
+++ b/docs/fa/data_types/int_uint.md
@@ -20,4 +20,4 @@
 - UInt64 - [0 : 18446744073709551615]
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/int_uint/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/int_uint/) 
diff --git a/docs/fa/data_types/nested_data_structures/aggregatefunction.md b/docs/fa/data_types/nested_data_structures/aggregatefunction.md
index 3ebcde1b6ae..e0345142080 100644
--- a/docs/fa/data_types/nested_data_structures/aggregatefunction.md
+++ b/docs/fa/data_types/nested_data_structures/aggregatefunction.md
@@ -5,4 +5,4 @@
 حالت متوسط از توابع aggregate. برای دریافت آن، از توابع aggregate به همراه پسوند '-State' استفاده کنید. برای اطلاعات بیشتر قسمت "AggregatingMergeTree" را ببینید.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/nested_data_structures/aggregatefunction/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/nested_data_structures/aggregatefunction/) 
diff --git a/docs/fa/data_types/nested_data_structures/index.md b/docs/fa/data_types/nested_data_structures/index.md
index b5a3818646a..ed3d9119815 100644
--- a/docs/fa/data_types/nested_data_structures/index.md
+++ b/docs/fa/data_types/nested_data_structures/index.md
@@ -4,4 +4,4 @@
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/nested_data_structures/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/nested_data_structures/) 
diff --git a/docs/fa/data_types/nested_data_structures/nested.md b/docs/fa/data_types/nested_data_structures/nested.md
index a7a4be5b1ea..b97096b49c8 100644
--- a/docs/fa/data_types/nested_data_structures/nested.md
+++ b/docs/fa/data_types/nested_data_structures/nested.md
@@ -111,4 +111,4 @@ LIMIT 10
 دستور ALTER برای عناصر داخل nested بسیار محدود است.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/nested_data_structures/nested/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/nested_data_structures/nested/) 
diff --git a/docs/fa/data_types/special_data_types/expression.md b/docs/fa/data_types/special_data_types/expression.md
index ada0eb0856e..07de3db7201 100644
--- a/docs/fa/data_types/special_data_types/expression.md
+++ b/docs/fa/data_types/special_data_types/expression.md
@@ -5,4 +5,4 @@
 برای نشان دادن توابع لامبدا در توابع high-order استفاده می شود.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/special_data_types/expression/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/special_data_types/expression/) 
diff --git a/docs/fa/data_types/special_data_types/index.md b/docs/fa/data_types/special_data_types/index.md
index 257b7508753..8b71fa9e124 100644
--- a/docs/fa/data_types/special_data_types/index.md
+++ b/docs/fa/data_types/special_data_types/index.md
@@ -5,4 +5,4 @@
 مقادیر نوع داده special، نمیتوانند در در جدول ذخیره و یا در نتایج خروجی قرار بگیرند، اما در نتایج متوسط (intermediate) یک query در حال اجرا استفاده می شوند.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/special_data_types/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/special_data_types/) 
diff --git a/docs/fa/data_types/special_data_types/set.md b/docs/fa/data_types/special_data_types/set.md
index f2332e9eb86..376c9615514 100644
--- a/docs/fa/data_types/special_data_types/set.md
+++ b/docs/fa/data_types/special_data_types/set.md
@@ -5,4 +5,4 @@
 برای نصف سمت راست IN استفاده می شود.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/special_data_types/set/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/special_data_types/set/) 
diff --git a/docs/fa/data_types/string.md b/docs/fa/data_types/string.md
index 71685574cc3..87500a3c882 100644
--- a/docs/fa/data_types/string.md
+++ b/docs/fa/data_types/string.md
@@ -9,4 +9,4 @@ String یک type برای قرار دادن رشته با طول دلخواه م
 ClickHouse مفهومی به نام encoding ندارد. String ها می توانند شامل مجموعه ای بایت ها باشند که با همان شکل که نوشته می شوند به همان شکل هم در خروجی دیده شوند. اگر شما نیاز به ذخیره سازی متن دارید، توصیه می کنیم از UTF-8 استفاده کنید. حداقل اگر ترمینال شما از UTF-8 (پیشنهاد شده)، استفاده می کند، شما می توانید به راحتی مقادیر خود را نوشته و بخوانید.به طور مشابه توابع خاصی برای کار با رشته های متنوع وجود دارند که تخل این فرضیه عمل می کنند که رشته شامل مجوعه ای از بایت ها می باشند که نماینده ی متن های UTF-8 هستند. برای مثال تابع 'length' برای محاسبه طول رشته براساس بایت است، در حالی که تابع 'lengthUTF8' برای محاسبه طول رشته بر اساس UNICODE می باشد.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/string/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/string/) 
diff --git a/docs/fa/data_types/tuple.md b/docs/fa/data_types/tuple.md
index 3d0a22cb16c..561180c6804 100644
--- a/docs/fa/data_types/tuple.md
+++ b/docs/fa/data_types/tuple.md
@@ -7,4 +7,4 @@ Tuple ها نمیتوانند در جدول نوشته شوند (به غیر ج
 Tuple می توانند در خروجی نتیجه query در حال اجرا باشند. در این مورد، برای فرمت های text به غیر از JSON\*، مقادیر به صورت comma-separate داخل براکت قرار میگیرند. در فرمت های JSON\* مقادیر tuple به صورت آرایه در خروجی می آیند.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/data_types/tuple/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/data_types/tuple/) 
diff --git a/docs/fa/getting_started/example_datasets/amplab_benchmark.md b/docs/fa/getting_started/example_datasets/amplab_benchmark.md
index 6718b358f49..991622a9064 100644
--- a/docs/fa/getting_started/example_datasets/amplab_benchmark.md
+++ b/docs/fa/getting_started/example_datasets/amplab_benchmark.md
@@ -135,4 +135,4 @@ ORDER BY totalRevenue DESC
 LIMIT 1
 ```
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/amplab_benchmark/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/amplab_benchmark/) 
diff --git a/docs/fa/getting_started/example_datasets/criteo.md b/docs/fa/getting_started/example_datasets/criteo.md
index 47013f7aaab..5925776a58b 100644
--- a/docs/fa/getting_started/example_datasets/criteo.md
+++ b/docs/fa/getting_started/example_datasets/criteo.md
@@ -88,4 +88,4 @@ DROP TABLE criteo_log;
 ```
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/criteo/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/criteo/) 
diff --git a/docs/fa/getting_started/example_datasets/nyc_taxi.md b/docs/fa/getting_started/example_datasets/nyc_taxi.md
index 1f7c5c1a573..70101930c05 100644
--- a/docs/fa/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/fa/getting_started/example_datasets/nyc_taxi.md
@@ -409,4 +409,4 @@ Q4: 0.072 ثانیه.
 |   140 | 0.028 | 0.043 | 0.051 | 0.072 |
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/nyc_taxi/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/nyc_taxi/) 
diff --git a/docs/fa/getting_started/example_datasets/ontime.md b/docs/fa/getting_started/example_datasets/ontime.md
index b322fd97a83..e07ce8572e2 100644
--- a/docs/fa/getting_started/example_datasets/ontime.md
+++ b/docs/fa/getting_started/example_datasets/ontime.md
@@ -391,4 +391,4 @@ SELECT OriginCityName, count() AS c FROM ontime GROUP BY OriginCityName ORDER BY
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/ontime/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/ontime/) 
diff --git a/docs/fa/getting_started/example_datasets/star_schema.md b/docs/fa/getting_started/example_datasets/star_schema.md
index fe7f861dd15..f2cd4978608 100644
--- a/docs/fa/getting_started/example_datasets/star_schema.md
+++ b/docs/fa/getting_started/example_datasets/star_schema.md
@@ -97,4 +97,4 @@ cat customer.tbl | sed 's/$/2000-01-01/' | clickhouse-client --query "INSERT INT
 cat lineorder.tbl | clickhouse-client --query "INSERT INTO lineorder FORMAT CSV"
 ```
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/star_schema/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/star_schema/) 
diff --git a/docs/fa/getting_started/example_datasets/wikistat.md b/docs/fa/getting_started/example_datasets/wikistat.md
index c49fd4ae271..4232815d794 100644
--- a/docs/fa/getting_started/example_datasets/wikistat.md
+++ b/docs/fa/getting_started/example_datasets/wikistat.md
@@ -34,4 +34,4 @@ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/pageco
 ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
 ```
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/example_datasets/wikistat/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/example_datasets/wikistat/) 
diff --git a/docs/fa/getting_started/index.md b/docs/fa/getting_started/index.md
index 57496c474e2..f5492a490fc 100644
--- a/docs/fa/getting_started/index.md
+++ b/docs/fa/getting_started/index.md
@@ -7,5 +7,5 @@
 * [ﺪﯿﻨﮐ ﯽﻃ ﺍﺭ ﻞﺼﻔﻣ ﺵﺯﻮﻣﺁ](tutorial.md)
 * [ﺪﯿﻨﮐ ﺶﯾﺎﻣﺯﺁ ﻪﻧﻮﻤﻧ ﯼﺎﻫ ﻩﺩﺍﺩ ﺎﺑ](example_datasets/ontime.md)
 
-[ﯽﻠﺻﺍ ﻪﻟﺎﻘﻣ](https://clickhouse.yandex/docs/fa/getting_started/) 
+[ﯽﻠﺻﺍ ﻪﻟﺎﻘﻣ](https://clickhouse.tech/docs/fa/getting_started/) 
 
diff --git a/docs/fa/getting_started/install.md b/docs/fa/getting_started/install.md
index 790c9381007..2ea9537b893 100644
--- a/docs/fa/getting_started/install.md
+++ b/docs/fa/getting_started/install.md
@@ -196,4 +196,4 @@ SELECT 1
 برای ادامه آزمایشات، شما میتوانید دیتاست های تستی را دریافت و امتحان کنید.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/getting_started/install/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/getting_started/install/) 
diff --git a/docs/fa/index.md b/docs/fa/index.md
index d5f6c20d2a4..51eca317237 100644
--- a/docs/fa/index.md
+++ b/docs/fa/index.md
@@ -146,4 +146,4 @@ LIMIT 20
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/) 
diff --git a/docs/fa/interfaces/cli.md b/docs/fa/interfaces/cli.md
index 4c366ac0104..61588744728 100644
--- a/docs/fa/interfaces/cli.md
+++ b/docs/fa/interfaces/cli.md
@@ -119,4 +119,4 @@ command line برا پایه 'replxx' می باشد. به عبارت دیگر،
 ```
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/cli/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/cli/) 
diff --git a/docs/fa/interfaces/cpp.md b/docs/fa/interfaces/cpp.md
index 29d53571e94..490f852792a 100644
--- a/docs/fa/interfaces/cpp.md
+++ b/docs/fa/interfaces/cpp.md
@@ -2,4 +2,4 @@
 
 See README at [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp) repository.
 
-[Original article](https://clickhouse.yandex/docs/fa/interfaces/cpp/) 
+[Original article](https://clickhouse.tech/docs/fa/interfaces/cpp/) 
diff --git a/docs/fa/interfaces/formats.md b/docs/fa/interfaces/formats.md
index d01dc1d1be0..2253ab3d3d7 100644
--- a/docs/fa/interfaces/formats.md
+++ b/docs/fa/interfaces/formats.md
@@ -750,4 +750,4 @@ struct Message {
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/formats/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/formats/) 
diff --git a/docs/fa/interfaces/http.md b/docs/fa/interfaces/http.md
index ed11cf53eac..844558ad43a 100644
--- a/docs/fa/interfaces/http.md
+++ b/docs/fa/interfaces/http.md
@@ -281,4 +281,4 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/http_interface/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/http_interface/) 
diff --git a/docs/fa/interfaces/index.md b/docs/fa/interfaces/index.md
index 57d8e673748..6d99795f264 100644
--- a/docs/fa/interfaces/index.md
+++ b/docs/fa/interfaces/index.md
@@ -19,4 +19,4 @@ ClickHouse دو اینترفیس شبکه را فراهم می کند (هر دو
 * [رابط های بصری](third-party/gui.md)
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/) 
diff --git a/docs/fa/interfaces/jdbc.md b/docs/fa/interfaces/jdbc.md
index 7a498a32c63..5c47948335b 100644
--- a/docs/fa/interfaces/jdbc.md
+++ b/docs/fa/interfaces/jdbc.md
@@ -10,4 +10,4 @@
 - [ClickHouse-Native-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC)
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/jdbc/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/jdbc/) 
diff --git a/docs/fa/interfaces/odbc.md b/docs/fa/interfaces/odbc.md
index da96813ce8d..35b1cc508ab 100644
--- a/docs/fa/interfaces/odbc.md
+++ b/docs/fa/interfaces/odbc.md
@@ -4,4 +4,4 @@
 درایور رسمی ODBC برای ClickHouse وجود دارد. برای اطلاعات بیشتر [اینجا](https://github.com/ClickHouse/clickhouse-odbc) را ببینید.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/odbc/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/odbc/) 
diff --git a/docs/fa/interfaces/tcp.md b/docs/fa/interfaces/tcp.md
index d6fad9bd5c7..bf226fa7bf0 100644
--- a/docs/fa/interfaces/tcp.md
+++ b/docs/fa/interfaces/tcp.md
@@ -5,4 +5,4 @@
 پروتکل بومی در [خط فرمان خط] (cli.md)، برای برقراری ارتباط بین سرور در طی پردازش پرس و جو توزیع شده، و همچنین در سایر برنامه های C ++ استفاده می شود. متاسفانه، پروتکل ClickHouse بومی هنوز مشخصات رسمی ندارد، اما می توان آن را از کد منبع ClickHouse (شروع [از اینجا](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/src/Client)) و / یا با رهگیری و تجزیه و تحلیل ترافیک TCP.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/tcp/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/tcp/) 
diff --git a/docs/fa/interfaces/third-party/client_libraries.md b/docs/fa/interfaces/third-party/client_libraries.md
index c6ee8785499..0057230781a 100644
--- a/docs/fa/interfaces/third-party/client_libraries.md
+++ b/docs/fa/interfaces/third-party/client_libraries.md
@@ -49,4 +49,4 @@
 ما این کتابخانه ها را تست نکردیم. آنها به صورت تصادفی انتخاب شده اند.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/third-party/client_libraries/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/third-party/client_libraries/) 
diff --git a/docs/fa/interfaces/third-party/gui.md b/docs/fa/interfaces/third-party/gui.md
index 38378f450e6..cd6862d1091 100644
--- a/docs/fa/interfaces/third-party/gui.md
+++ b/docs/fa/interfaces/third-party/gui.md
@@ -89,4 +89,4 @@ interface تحت وب برای ClickHouse در پروژه [Tabix](https://github
 - جستجو و ناوبری
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/third-party/gui/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/third-party/gui/) 
diff --git a/docs/fa/interfaces/third-party/integrations.md b/docs/fa/interfaces/third-party/integrations.md
index 7aed10d3762..fc222e6c421 100644
--- a/docs/fa/interfaces/third-party/integrations.md
+++ b/docs/fa/interfaces/third-party/integrations.md
@@ -85,4 +85,4 @@
         - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/third-party/integrations/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/third-party/integrations/) 
diff --git a/docs/fa/interfaces/third-party/proxy.md b/docs/fa/interfaces/third-party/proxy.md
index 74405201b8f..783af4749e0 100644
--- a/docs/fa/interfaces/third-party/proxy.md
+++ b/docs/fa/interfaces/third-party/proxy.md
@@ -37,4 +37,4 @@
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/interfaces/third-party/proxy/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/interfaces/third-party/proxy/) 
diff --git a/docs/fa/introduction/distinctive_features.md b/docs/fa/introduction/distinctive_features.md
index 0dbe60db7f3..189cc504f94 100644
--- a/docs/fa/introduction/distinctive_features.md
+++ b/docs/fa/introduction/distinctive_features.md
@@ -64,4 +64,4 @@ ClickHouse از روش asynchronous multimaster replication استفاده می
 برای اطلاعات بیشتر، به بخش [replication داده ها](../operations/table_engines/replication.md) مراجعه کنید.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/distinctive_features/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/distinctive_features/) 
diff --git a/docs/fa/introduction/features_considered_disadvantages.md b/docs/fa/introduction/features_considered_disadvantages.md
index f6066bbeadf..13a9828e6b9 100644
--- a/docs/fa/introduction/features_considered_disadvantages.md
+++ b/docs/fa/introduction/features_considered_disadvantages.md
@@ -8,4 +8,4 @@
 
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/features_considered_disadvantages/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/features_considered_disadvantages/) 
diff --git a/docs/fa/introduction/history.md b/docs/fa/introduction/history.md
index abde10aa6f3..db523df31c8 100644
--- a/docs/fa/introduction/history.md
+++ b/docs/fa/introduction/history.md
@@ -46,4 +46,4 @@ Yandex.Metrica دارای یک سیستم تخصصی برای aggregate کردن
 برای حذف محدودیت های OLAPServer و حل مشکلات کار با داده های Non-Aggregate برای تمام گزارش ها، ما مدیریت دیتابیس ClicHouse را توسعه دادیم..
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/ya_metrika_task/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/ya_metrika_task/) 
diff --git a/docs/fa/introduction/performance.md b/docs/fa/introduction/performance.md
index 2fb2a9dc036..a0ebdc34af4 100644
--- a/docs/fa/introduction/performance.md
+++ b/docs/fa/introduction/performance.md
@@ -2,9 +2,9 @@
 
 # Performance
 
-با توجه به نتایج تست های Yandex، ClickHouse بهترین عملکرد را برای سناریوهای عملیاتی قابل مقایسه با دیگر سیستم های در کلاس خود را از خود نشان داد. این تست ها شامل بالاترین توان عملیاتی برای query های طولانی، و کمترین latency برای query های کوتاه بود. نتایج این تست های در [صفحه ی جدا](https://clickhouse.yandex/benchmark.html) موجود است.
+با توجه به نتایج تست های Yandex، ClickHouse بهترین عملکرد را برای سناریوهای عملیاتی قابل مقایسه با دیگر سیستم های در کلاس خود را از خود نشان داد. این تست ها شامل بالاترین توان عملیاتی برای query های طولانی، و کمترین latency برای query های کوتاه بود. نتایج این تست های در [صفحه ی جدا](/benchmark.html) موجود است.
 
-benchmark های زیادی وجود دارند که این نتایج را تایید می کنند. میتوانید این نتایج را جستجو کنید و یا [این لینک های benchmark](https://clickhouse.yandex/#independent-benchmarks). مستقل را ببینید.
+benchmark های زیادی وجود دارند که این نتایج را تایید می کنند. میتوانید این نتایج را جستجو کنید و یا [این لینک های benchmark](/#independent-benchmarks). مستقل را ببینید.
 
 ## توان عملیاتی برای یک query بزرگ
 
@@ -25,4 +25,4 @@ benchmark های زیادی وجود دارند که این نتایج را تا
 پیشنهاد می کنیم درج داده ها را به صورت دسته ای و حداقل 100 سطر در هر دسته انجام دهید و یا بیش از یک درخواست insert در ثانیه را نداشته باشید. در هنگام درج داده در جدول MergeTree از یک dump جدا شده با tab، سرعت درج داده از 50 تا 200 مگابایت در ثانیه می باشد. اگر سطر های درج شده حدود 1 کیلوبایت باشند، سرعت حدود 50 هزار تا 200 هزار سطر در ثانیه می باشد. اگر سطر ها کوچک باشند بازدهی بالایی در تعداد سطر در ثانیه خواهیم داشت. در Banner System Data -`>` 500 هزار سطر در ثانیه، در Graphite data -`>` 1 میلیون سطر در ثانیه). برای بهبود کارایی، شما می توانید چندین insert را به صورت موازی اجرا کنید، که در این حالت کارایی سیستم به صورت خطی افزایش می یابد.
 
 
-[مقاله اصلی](https://clickhouse.yandex/docs/fa/introduction/performance/) 
+[مقاله اصلی](https://clickhouse.tech/docs/fa/introduction/performance/) 
diff --git a/docs/ja/index.md b/docs/ja/index.md
index a7f8681a2bf..b0aca897670 100644
--- a/docs/ja/index.md
+++ b/docs/ja/index.md
@@ -139,4 +139,4 @@ LIMIT 20
 
 CPU効率のために、クエリ言語は宣言型（SQLまたはMDX）、または少なくともベクトル（J、K）でなければなりません。 クエリには、最適化を可能にする暗黙的なループのみを含める必要があります。
 
-[Original article](https://clickhouse.yandex/docs/ja/) 
+[Original article](https://clickhouse.tech/docs/ja/) 
diff --git a/docs/ru/data_types/array.md b/docs/ru/data_types/array.md
index c6cc6af4b13..c9b6df35ede 100644
--- a/docs/ru/data_types/array.md
+++ b/docs/ru/data_types/array.md
@@ -63,4 +63,4 @@ Received exception from server (version 1.1.54388):
 Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not.
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/array/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/array/) 
diff --git a/docs/ru/data_types/boolean.md b/docs/ru/data_types/boolean.md
index b85f1393a80..f4a908840ae 100644
--- a/docs/ru/data_types/boolean.md
+++ b/docs/ru/data_types/boolean.md
@@ -2,4 +2,4 @@
 
 Отдельного типа для булевых значений нет. Для них используется тип UInt8, в котором используются только значения 0 и 1.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/boolean/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/boolean/) 
diff --git a/docs/ru/data_types/date.md b/docs/ru/data_types/date.md
index 8e747eeb554..3e39f72d5ed 100644
--- a/docs/ru/data_types/date.md
+++ b/docs/ru/data_types/date.md
@@ -5,4 +5,4 @@
 
 Дата хранится без учёта часового пояса.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/date/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/date/) 
diff --git a/docs/ru/data_types/datetime.md b/docs/ru/data_types/datetime.md
index e2bc3e1482b..90314c1ba7a 100644
--- a/docs/ru/data_types/datetime.md
+++ b/docs/ru/data_types/datetime.md
@@ -80,4 +80,4 @@ SELECT toDateTime(now(), 'Europe/Moscow') AS column, toTypeName(column) AS x
 - [Операторы для работы с датой и временем](../query_language/operators.md#operators-datetime)
 - [Тип данных `Date`](date.md)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/datetime/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) 
diff --git a/docs/ru/data_types/decimal.md b/docs/ru/data_types/decimal.md
index d6faa745723..e8e8a15fe49 100644
--- a/docs/ru/data_types/decimal.md
+++ b/docs/ru/data_types/decimal.md
@@ -93,4 +93,4 @@ SELECT toDecimal32(1, 8) < 100
 DB::Exception: Can't compare.
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/decimal/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/decimal/) 
diff --git a/docs/ru/data_types/domains/ipv4.md b/docs/ru/data_types/domains/ipv4.md
index 8d8b5bf9fc5..cce7c17d21b 100644
--- a/docs/ru/data_types/domains/ipv4.md
+++ b/docs/ru/data_types/domains/ipv4.md
@@ -26,16 +26,16 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from;
 `IPv4` поддерживает вставку в виде строк с текстовым представлением IPv4 адреса:
 
 ```sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.yandex', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
 
 SELECT * FROM hits;
 ```
 
 ```text
 ┌─url────────────────────────────────┬───────────from─┐
-│ https://clickhouse.yandex/docs/en/ │ 116.106.34.242 │
+│ https://clickhouse.tech/docs/en/ │ 116.106.34.242 │
 │ https://wikipedia.org              │ 116.253.40.133 │
-│ https://clickhouse.yandex          │ 183.247.232.58 │
+│ https://clickhouse.tech          │ 183.247.232.58 │
 └────────────────────────────────────┴────────────────┘
 ```
 
@@ -76,4 +76,4 @@ SELECT toTypeName(i), CAST(from AS UInt32) AS i FROM hits LIMIT 1;
 └──────────────────────────────────┴────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/domains/ipv4) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/ipv4) 
diff --git a/docs/ru/data_types/domains/ipv6.md b/docs/ru/data_types/domains/ipv6.md
index 796f8ef4040..e25805b7931 100644
--- a/docs/ru/data_types/domains/ipv6.md
+++ b/docs/ru/data_types/domains/ipv6.md
@@ -26,15 +26,15 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from;
 `IPv6` поддерживает вставку в виде строк с текстовым представлением IPv6 адреса:
 
 ```sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.yandex', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
 
 SELECT * FROM hits;
 ```
 
 ```text
 ┌─url────────────────────────────────┬─from──────────────────────────┐
-│ https://clickhouse.yandex          │ 2001:44c8:129:2632:33:0:252:2 │
-│ https://clickhouse.yandex/docs/en/ │ 2a02:e980:1e::1               │
+│ https://clickhouse.tech          │ 2001:44c8:129:2632:33:0:252:2 │
+│ https://clickhouse.tech/docs/en/ │ 2a02:e980:1e::1               │
 │ https://wikipedia.org              │ 2a02:aa08:e000:3100::2        │
 └────────────────────────────────────┴───────────────────────────────┘
 ```
@@ -76,4 +76,4 @@ SELECT toTypeName(i), CAST(from AS FixedString(16)) AS i FROM hits LIMIT 1;
 └───────────────────────────────────────────┴─────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/domains/ipv6) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/ipv6) 
diff --git a/docs/ru/data_types/domains/overview.md b/docs/ru/data_types/domains/overview.md
index c37555c0cea..03818d03a20 100644
--- a/docs/ru/data_types/domains/overview.md
+++ b/docs/ru/data_types/domains/overview.md
@@ -23,4 +23,4 @@
 * Невозможно неявно преобразовывать строковые значение в значения с доменным типом данных при вставке данных из другого столбца или таблицы.
 * Домен не добавляет ограничения на хранимые значения.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/domains/overview) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/overview) 
diff --git a/docs/ru/data_types/enum.md b/docs/ru/data_types/enum.md
index 8127b9db180..b157c745602 100644
--- a/docs/ru/data_types/enum.md
+++ b/docs/ru/data_types/enum.md
@@ -115,4 +115,4 @@ INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL)
 
 При ALTER, есть возможность поменять Enum8 на Enum16 и обратно - так же, как можно поменять Int8 на Int16.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/enum/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/enum/) 
diff --git a/docs/ru/data_types/fixedstring.md b/docs/ru/data_types/fixedstring.md
index 64b8dbf6409..07e45f71cab 100644
--- a/docs/ru/data_types/fixedstring.md
+++ b/docs/ru/data_types/fixedstring.md
@@ -53,4 +53,4 @@ WHERE a = 'b\0'
 
 Обратите внимание, что длина значения `FixedString(N)` постоянна. Функция [length](../query_language/functions/array_functions.md#array_functions-length) возвращает `N` даже если значение `FixedString(N)` заполнено только нулевыми байтами, однако функция [empty](../query_language/functions/string_functions.md#string_functions-empty) в этом же случае возвращает `1`.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/fixedstring/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/fixedstring/) 
diff --git a/docs/ru/data_types/float.md b/docs/ru/data_types/float.md
index eb4ada0e24c..23c0097cd19 100644
--- a/docs/ru/data_types/float.md
+++ b/docs/ru/data_types/float.md
@@ -67,4 +67,4 @@ SELECT 0 / 0
 
   Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY](../query_language/select.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/float/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/float/) 
diff --git a/docs/ru/data_types/index.md b/docs/ru/data_types/index.md
index e6c06152b9c..2ed9b9b6a4f 100644
--- a/docs/ru/data_types/index.md
+++ b/docs/ru/data_types/index.md
@@ -3,4 +3,4 @@
 ClickHouse может сохранять в ячейках таблиц данные различных типов. 
 
 Раздел содержит описания поддерживаемых типов данных и специфику их использования и/или реализации, если таковые имеются.
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/) 
diff --git a/docs/ru/data_types/int_uint.md b/docs/ru/data_types/int_uint.md
index 630980d641a..1b74432c013 100644
--- a/docs/ru/data_types/int_uint.md
+++ b/docs/ru/data_types/int_uint.md
@@ -19,4 +19,4 @@
 - UInt32 - [ 0 : 4294967295 ]
 - UInt64 - [ 0 : 18446744073709551615 ]
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/int_uint/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/int_uint/) 
diff --git a/docs/ru/data_types/nested_data_structures/aggregatefunction.md b/docs/ru/data_types/nested_data_structures/aggregatefunction.md
index b772aaf935b..3c71db5e8a4 100644
--- a/docs/ru/data_types/nested_data_structures/aggregatefunction.md
+++ b/docs/ru/data_types/nested_data_structures/aggregatefunction.md
@@ -60,4 +60,4 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP
 
 Смотрите в описании движка [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/nested_data_structures/aggregatefunction/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/aggregatefunction/) 
diff --git a/docs/ru/data_types/nested_data_structures/index.md b/docs/ru/data_types/nested_data_structures/index.md
index 3e5ea1bd69e..ec665f2dbb5 100644
--- a/docs/ru/data_types/nested_data_structures/index.md
+++ b/docs/ru/data_types/nested_data_structures/index.md
@@ -1,2 +1,2 @@
 # Вложенные структуры данных
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/nested_data_structures/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/) 
diff --git a/docs/ru/data_types/nested_data_structures/nested.md b/docs/ru/data_types/nested_data_structures/nested.md
index 8b4f7d247cf..face89c02d1 100644
--- a/docs/ru/data_types/nested_data_structures/nested.md
+++ b/docs/ru/data_types/nested_data_structures/nested.md
@@ -96,4 +96,4 @@ LIMIT 10
 
 Работоспособность запроса ALTER для элементов вложенных структур данных, является сильно ограниченной.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/nested_data_structures/nested/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/nested/) 
diff --git a/docs/ru/data_types/nullable.md b/docs/ru/data_types/nullable.md
index a953f6bb2b7..b379c619383 100644
--- a/docs/ru/data_types/nullable.md
+++ b/docs/ru/data_types/nullable.md
@@ -41,4 +41,4 @@ SELECT x + y from t_null
 
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/nullable/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nullable/) 
diff --git a/docs/ru/data_types/special_data_types/expression.md b/docs/ru/data_types/special_data_types/expression.md
index 438492c5399..6242805927e 100644
--- a/docs/ru/data_types/special_data_types/expression.md
+++ b/docs/ru/data_types/special_data_types/expression.md
@@ -2,4 +2,4 @@
 
 Используется для представления лямбда-выражений в функциях высшего порядка.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/special_data_types/expression/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/expression/) 
diff --git a/docs/ru/data_types/special_data_types/index.md b/docs/ru/data_types/special_data_types/index.md
index b9eda72c556..e4f7a418d7c 100644
--- a/docs/ru/data_types/special_data_types/index.md
+++ b/docs/ru/data_types/special_data_types/index.md
@@ -2,4 +2,4 @@
 
 Значения служебных типов данных не могут сохраняться в таблицу и выводиться в качестве результата, а возникают как промежуточный результат выполнения запроса.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/special_data_types/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/) 
diff --git a/docs/ru/data_types/special_data_types/nothing.md b/docs/ru/data_types/special_data_types/nothing.md
index 65b90ca8c89..fde3722e684 100644
--- a/docs/ru/data_types/special_data_types/nothing.md
+++ b/docs/ru/data_types/special_data_types/nothing.md
@@ -14,4 +14,4 @@ SELECT toTypeName(Array())
 └─────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/special_data_types/nothing/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/nothing/) 
diff --git a/docs/ru/data_types/special_data_types/set.md b/docs/ru/data_types/special_data_types/set.md
index fe31450ab29..6326cf6e9ce 100644
--- a/docs/ru/data_types/special_data_types/set.md
+++ b/docs/ru/data_types/special_data_types/set.md
@@ -2,4 +2,4 @@
 
 Используется для представления правой части выражения IN.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/special_data_types/set/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/set/) 
diff --git a/docs/ru/data_types/string.md b/docs/ru/data_types/string.md
index 2085bda8189..ec03eff41e7 100644
--- a/docs/ru/data_types/string.md
+++ b/docs/ru/data_types/string.md
@@ -11,4 +11,4 @@
 Также, некоторые функции по работе со строками, имеют отдельные варианты, которые работают при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8.
 Например, функция length вычисляет длину строки в байтах, а функция lengthUTF8 - длину строки в кодовых точках Unicode, при допущении, что значение в кодировке UTF-8.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/string/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/string/) 
diff --git a/docs/ru/data_types/tuple.md b/docs/ru/data_types/tuple.md
index 7b6d11ca168..5f7e4063690 100644
--- a/docs/ru/data_types/tuple.md
+++ b/docs/ru/data_types/tuple.md
@@ -41,4 +41,4 @@ SELECT tuple(1,NULL) AS x, toTypeName(x)
 └──────────┴─────────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/data_types/tuple/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/tuple/) 
diff --git a/docs/ru/data_types/uuid.md b/docs/ru/data_types/uuid.md
index 8c583bf72bf..523f4f8c64a 100644
--- a/docs/ru/data_types/uuid.md
+++ b/docs/ru/data_types/uuid.md
@@ -64,5 +64,5 @@ SELECT * FROM t_uuid
 
 Тип данных UUID не поддерживается арифметическими операциями  (например, [abs](../query_language/functions/arithmetic_functions.md#arithm_func-abs)) или агрегатными функциями, такими как [sum](../query_language/agg_functions/reference.md#agg_function-sum) и [avg](../query_language/agg_functions/reference.md#agg_function-avg).
 
-[Original article](https://clickhouse.yandex/docs/en/data_types/uuid/) 
+[Original article](https://clickhouse.tech/docs/en/data_types/uuid/) 
 
diff --git a/docs/ru/development/developer_instruction.md b/docs/ru/development/developer_instruction.md
index 4686847041d..3fc9f54ac3f 100644
--- a/docs/ru/development/developer_instruction.md
+++ b/docs/ru/development/developer_instruction.md
@@ -111,7 +111,7 @@ ClickHouse использует для сборки некоторое коли
 
 Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`.
 
-Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.yandex/docs/en/development/build/#install-gcc-9
+Проверьте версию gcc: `gcc --version`. Если версия меньше 9, то следуйте инструкции: https://clickhouse.tech/docs/en/development/build/#install-gcc-9
 
 Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm`
 
@@ -226,11 +226,11 @@ sudo -u clickhouse ClickHouse/build/dbms/programs/clickhouse server --config-fil
 
 # Написание кода
 
-Описание архитектуры ClickHouse: https://clickhouse.yandex/docs/ru/development/architecture/
+Описание архитектуры ClickHouse: https://clickhouse.tech/docs/ru/development/architecture/
 
-Стиль кода: https://clickhouse.yandex/docs/ru/development/style/
+Стиль кода: https://clickhouse.tech/docs/ru/development/style/
 
-Разработка тестов: https://clickhouse.yandex/docs/ru/development/tests/
+Разработка тестов: https://clickhouse.tech/docs/ru/development/tests/
 
 Список задач: https://github.com/ClickHouse/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_ru.md
 
diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md
index 139e69f375f..d47eb1c0d12 100644
--- a/docs/ru/development/style.md
+++ b/docs/ru/development/style.md
@@ -838,4 +838,4 @@ function(
       size_t limit)
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/development/style/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/development/style/) 
diff --git a/docs/ru/extended_roadmap.md b/docs/ru/extended_roadmap.md
index 145779cfccb..9696558916f 100644
--- a/docs/ru/extended_roadmap.md
+++ b/docs/ru/extended_roadmap.md
@@ -64,7 +64,7 @@ Q1. Закоммичено, но есть технический долг, ко
 
 Будет делать Сорокин Николай, ВШЭ и Яндекс.
 
-Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/#table_engine-mergetree-ttl
+Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/#table_engine-mergetree-ttl
 
 Это может быть задано для всей таблицы (тогда строки целиком удаляются после указанного времени) или для отдельных столбцов (тогда данные столбца физически удаляются с диска, а строки в таблице остаются; при чтении значений столбца, они читаются как значения по-умолчанию).
 
@@ -78,7 +78,7 @@ Q1. Закоммичено, но есть технический долг, ко
 
 А вот пункт 2 требуется продумать. Не очевидно даже, какой лучше использовать синтаксис для этого при создании таблицы. Но мы придумаем - сразу видно несколько вариантов.
 
-Частный случай такой задачи уже есть в https://clickhouse.yandex/docs/ru/operations/table_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить.
+Частный случай такой задачи уже есть в https://clickhouse.tech/docs/ru/operations/table_engines/graphitemergetree/ Но это было сделано для конкретной задачи. А надо обобщить.
 
 ### 1.10. Пережатие старых данных в фоне.
 
diff --git a/docs/ru/faq/general.md b/docs/ru/faq/general.md
index 970069bc641..65a388f6706 100644
--- a/docs/ru/faq/general.md
+++ b/docs/ru/faq/general.md
@@ -54,4 +54,4 @@ $ clickhouse-client --query "SELECT * from table" > result.txt
 Смотрите [clickhouse-client](../interfaces/cli.md).
 
 
-[Оригинальная статья ](https://clickhouse.yandex/docs/en/faq/general/) 
+[Оригинальная статья ](https://clickhouse.tech/docs/en/faq/general/) 
diff --git a/docs/ru/getting_started/example_datasets/amplab_benchmark.md b/docs/ru/getting_started/example_datasets/amplab_benchmark.md
index f1ef7230ed4..213ce406e05 100644
--- a/docs/ru/getting_started/example_datasets/amplab_benchmark.md
+++ b/docs/ru/getting_started/example_datasets/amplab_benchmark.md
@@ -120,4 +120,4 @@ ORDER BY totalRevenue DESC
 LIMIT 1
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/example_datasets/amplab_benchmark/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/amplab_benchmark/) 
diff --git a/docs/ru/getting_started/example_datasets/criteo.md b/docs/ru/getting_started/example_datasets/criteo.md
index 5afb8046a57..30858d80e64 100644
--- a/docs/ru/getting_started/example_datasets/criteo.md
+++ b/docs/ru/getting_started/example_datasets/criteo.md
@@ -71,4 +71,4 @@ INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int
 DROP TABLE criteo_log;
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/example_datasets/criteo/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/criteo/) 
diff --git a/docs/ru/getting_started/example_datasets/nyc_taxi.md b/docs/ru/getting_started/example_datasets/nyc_taxi.md
index 5f8935b14ef..dfe82a4a679 100644
--- a/docs/ru/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/ru/getting_started/example_datasets/nyc_taxi.md
@@ -383,4 +383,4 @@ Q4: 0.072 sec.
 |       3 | 0.212 | 0.438 | 0.733 | 1.241 |
 |     140 | 0.028 | 0.043 | 0.051 | 0.072 |
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/example_datasets/nyc_taxi/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/nyc_taxi/) 
diff --git a/docs/ru/getting_started/example_datasets/ontime.md b/docs/ru/getting_started/example_datasets/ontime.md
index cfaf959464c..577341abbaa 100644
--- a/docs/ru/getting_started/example_datasets/ontime.md
+++ b/docs/ru/getting_started/example_datasets/ontime.md
@@ -402,4 +402,4 @@ LIMIT 10;
 -   
 -   
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/example_datasets/ontime/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/ontime/) 
diff --git a/docs/ru/getting_started/example_datasets/star_schema.md b/docs/ru/getting_started/example_datasets/star_schema.md
index 28ab1e0fd2b..00fc083eb22 100644
--- a/docs/ru/getting_started/example_datasets/star_schema.md
+++ b/docs/ru/getting_started/example_datasets/star_schema.md
@@ -337,4 +337,4 @@ ORDER BY
     P_BRAND ASC;
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/star_schema/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) 
diff --git a/docs/ru/getting_started/example_datasets/wikistat.md b/docs/ru/getting_started/example_datasets/wikistat.md
index 12469694c53..dc236a403f1 100644
--- a/docs/ru/getting_started/example_datasets/wikistat.md
+++ b/docs/ru/getting_started/example_datasets/wikistat.md
@@ -25,4 +25,4 @@ $ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/page
 $ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/example_datasets/wikistat/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/wikistat/) 
diff --git a/docs/ru/getting_started/index.md b/docs/ru/getting_started/index.md
index a8d0fbaa5b1..c73b48a3b84 100644
--- a/docs/ru/getting_started/index.md
+++ b/docs/ru/getting_started/index.md
@@ -7,4 +7,4 @@
 * [Пройти подробное руководство для начинающих](tutorial.md)
 * [Поэкспериментировать с тестовыми наборами данных](example_datasets/ontime.md)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/) 
diff --git a/docs/ru/getting_started/install.md b/docs/ru/getting_started/install.md
index cd1a04b6192..c1bf1b74759 100644
--- a/docs/ru/getting_started/install.md
+++ b/docs/ru/getting_started/install.md
@@ -167,6 +167,6 @@ SELECT 1
 
 **Поздравляем, система работает!**
 
-Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.yandex/tutorial.html).
+Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.tech/tutorial.html).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/getting_started/install/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/install/) 
diff --git a/docs/ru/index.md b/docs/ru/index.md
index 15ad4c5853c..c1c864d921c 100644
--- a/docs/ru/index.md
+++ b/docs/ru/index.md
@@ -129,4 +129,4 @@ SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIM
 
 Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/) 
diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md
index 71742d02740..91720c5d5ec 100644
--- a/docs/ru/interfaces/cli.md
+++ b/docs/ru/interfaces/cli.md
@@ -147,4 +147,4 @@ $ clickhouse-client --param_tuple_in_tuple="(10, ('dt', 10))" -q "SELECT * FROM
 
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/cli/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/cli/) 
diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md
index 00cbbcb411a..f910aa0eab2 100644
--- a/docs/ru/interfaces/cpp.md
+++ b/docs/ru/interfaces/cpp.md
@@ -2,4 +2,4 @@
 
 См. README в репозитории [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/cpp/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/cpp/) 
diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md
index ad941b23345..e5fa485f9e1 100644
--- a/docs/ru/interfaces/formats.md
+++ b/docs/ru/interfaces/formats.md
@@ -1013,4 +1013,4 @@ $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT OR
 Если для ввода/вывода данных используется [HTTP-интерфейс](../interfaces/http.md), то файл со схемой должен располагаться на сервере в каталоге,
 указанном в параметре [format_schema_path](../operations/server_settings/settings.md#server_settings-format_schema_path) конфигурации сервера.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/formats/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) 
diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md
index 4779388cde7..7fec0f4eab3 100644
--- a/docs/ru/interfaces/http.md
+++ b/docs/ru/interfaces/http.md
@@ -265,5 +265,5 @@ $ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&
 $ curl -sS "
?param_id=2¶m_phrase=test" -d "SELECT * FROM table WHERE int_column = {id:UInt8} and string_column = {phrase:String}"
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/http_interface/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/http_interface/) 
 
diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md
index 4679641872e..3718c499f3e 100644
--- a/docs/ru/interfaces/index.md
+++ b/docs/ru/interfaces/index.md
@@ -18,4 +18,4 @@ ClickHouse предоставляет два сетевых интерфейса
 * [Библиотеки для интеграции](third-party/integrations.md);
 * [Визуальные интерфейсы](third-party/gui.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/) 
diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md
index 9c6d594bc1c..3dbdc28d49b 100644
--- a/docs/ru/interfaces/jdbc.md
+++ b/docs/ru/interfaces/jdbc.md
@@ -5,4 +5,4 @@
   * [ClickHouse-Native-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC)
   * [clickhouse4j](https://github.com/blynkkk/clickhouse4j)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/jdbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/jdbc/) 
diff --git a/docs/ru/interfaces/odbc.md b/docs/ru/interfaces/odbc.md
index 7605d6061b2..e6a3d2826b2 100644
--- a/docs/ru/interfaces/odbc.md
+++ b/docs/ru/interfaces/odbc.md
@@ -2,4 +2,4 @@
 
 - [Официальный драйвер](https://github.com/ClickHouse/clickhouse-odbc).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/odbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/odbc/) 
diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md
index 449c15b1bcc..b18012d1120 100644
--- a/docs/ru/interfaces/tcp.md
+++ b/docs/ru/interfaces/tcp.md
@@ -2,4 +2,4 @@
 
 Нативный протокол используется в [клиенте командной строки](cli.md), для взаимодействия между серверами во время обработки распределенных запросов, а также в других программах на C++. К сожалению, у родного протокола ClickHouse пока нет формальной спецификации, но в нем можно разобраться с использованием исходного кода ClickHouse (начиная с [примерно этого места](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/src/Client)) и/или путем перехвата и анализа TCP трафика.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/tcp/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/tcp/) 
diff --git a/docs/ru/interfaces/third-party/client_libraries.md b/docs/ru/interfaces/third-party/client_libraries.md
index 1860b074123..d911b2a1005 100644
--- a/docs/ru/interfaces/third-party/client_libraries.md
+++ b/docs/ru/interfaces/third-party/client_libraries.md
@@ -45,4 +45,4 @@
 - Nim
     - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/third-party/client_libraries/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/client_libraries/) 
diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md
index c618fd3a6e1..a697e822cf7 100644
--- a/docs/ru/interfaces/third-party/gui.md
+++ b/docs/ru/interfaces/third-party/gui.md
@@ -116,4 +116,4 @@
 - Подготовка данных и возможности ETL.
 - Моделирование данных с помощью SQL для их реляционного отображения.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/third-party/gui/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/gui/) 
diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md
index 470d02bea7d..ab018edf913 100644
--- a/docs/ru/interfaces/third-party/integrations.md
+++ b/docs/ru/interfaces/third-party/integrations.md
@@ -83,4 +83,4 @@
     - [Ecto](https://github.com/elixir-ecto/ecto)
         - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/third-party/integrations/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/integrations/) 
diff --git a/docs/ru/interfaces/third-party/proxy.md b/docs/ru/interfaces/third-party/proxy.md
index 9292d937330..1bce6b12631 100644
--- a/docs/ru/interfaces/third-party/proxy.md
+++ b/docs/ru/interfaces/third-party/proxy.md
@@ -36,4 +36,4 @@
 
 Реализован на Go.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/interfaces/third-party/proxy/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/proxy/) 
diff --git a/docs/ru/introduction/distinctive_features.md b/docs/ru/introduction/distinctive_features.md
index 9a01388f630..7570c0e58c9 100644
--- a/docs/ru/introduction/distinctive_features.md
+++ b/docs/ru/introduction/distinctive_features.md
@@ -61,4 +61,4 @@ ClickHouse предоставляет различные способы разм
 
 Подробнее смотрите раздел [Репликация данных](../operations/table_engines/replication.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/distinctive_features/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/distinctive_features/) 
diff --git a/docs/ru/introduction/features_considered_disadvantages.md b/docs/ru/introduction/features_considered_disadvantages.md
index f7425efa4b3..440311a199d 100644
--- a/docs/ru/introduction/features_considered_disadvantages.md
+++ b/docs/ru/introduction/features_considered_disadvantages.md
@@ -5,4 +5,4 @@
 3. Разреженный индекс делает ClickHouse плохо пригодным для точечных чтений одиночных строк по своим
 ключам.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/features_considered_disadvantages/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/features_considered_disadvantages/) 
diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md
index c0035b51f82..c2d2a85ea09 100644
--- a/docs/ru/introduction/history.md
+++ b/docs/ru/introduction/history.md
@@ -46,4 +46,4 @@ OLAPServer хорошо подходил для неагрегированных
 
 Чтобы снять ограничения OLAPServer-а и решить задачу работы с неагрегированными данными для всех отчётов, разработана СУБД ClickHouse.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/ya_metrika_task/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/ya_metrika_task/) 
diff --git a/docs/ru/introduction/info.md b/docs/ru/introduction/info.md
index 69b40c1776d..7f4e7c9e479 100644
--- a/docs/ru/introduction/info.md
+++ b/docs/ru/introduction/info.md
@@ -6,4 +6,4 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/info/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/info/) 
diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md
index 09381117764..e99edbbb5a9 100644
--- a/docs/ru/introduction/performance.md
+++ b/docs/ru/introduction/performance.md
@@ -1,8 +1,8 @@
 # Производительность
 
-По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](https://clickhouse.yandex/benchmark.html).
+По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](/benchmark.html).
 
-Также это подтверждают многочисленные независимые бенчмарки. Их не сложно найти в Интернете самостоятельно, либо можно воспользоваться [небольшой коллекцией ссылок по теме](https://clickhouse.yandex/).
+Также это подтверждают многочисленные независимые бенчмарки. Их не сложно найти в Интернете самостоятельно, либо можно воспользоваться [небольшой коллекцией ссылок по теме](/#independent-benchmarks).
 
 ## Пропускная способность при обработке одного большого запроса
 
@@ -22,4 +22,4 @@
 
 Данные рекомендуется вставлять пачками не менее 1000 строк или не более одного запроса в секунду. При вставке в таблицу типа MergeTree из tab-separated дампа, скорость вставки будет в районе 50-200 МБ/сек. Если вставляются строчки размером около 1 КБ, то скорость будет в районе 50 000 - 200 000 строчек в секунду. Если строчки маленькие - производительность в строчках в секунду будет выше (на данных БК - `>` 500 000 строк в секунду, на данных Graphite - `>` 1 000 000 строк в секунду). Для увеличения производительности, можно производить несколько запросов INSERT параллельно - при этом производительность растёт линейно.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/introduction/performance/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/performance/) 
diff --git a/docs/ru/operations/access_rights.md b/docs/ru/operations/access_rights.md
index d6a98b7b594..ac99c2b0449 100644
--- a/docs/ru/operations/access_rights.md
+++ b/docs/ru/operations/access_rights.md
@@ -98,4 +98,4 @@
 
 Доступ к БД не связан с настройкой [readonly](settings/permissions_for_queries.md#settings_readonly). Невозможно дать полный доступ к одной БД и `readonly` к другой.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/access_rights/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/access_rights/) 
diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md
index 6892483565e..dde55c7de2c 100644
--- a/docs/ru/operations/backup.md
+++ b/docs/ru/operations/backup.md
@@ -31,4 +31,4 @@ ClickHouse позволяет использовать запрос `ALTER TABLE
 
 Для автоматизации этого подхода доступен инструмент от сторонних разработчиков: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup).
 
-[Оригинальная статья ](https://clickhouse.yandex/docs/ru/operations/backup/) 
+[Оригинальная статья ](https://clickhouse.tech/docs/ru/operations/backup/) 
diff --git a/docs/ru/operations/configuration_files.md b/docs/ru/operations/configuration_files.md
index 78b803775e1..e5fc0fc0cec 100644
--- a/docs/ru/operations/configuration_files.md
+++ b/docs/ru/operations/configuration_files.md
@@ -41,4 +41,4 @@ $ cat /etc/clickhouse-server/users.d/alice.xml
 
 Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/configuration_files/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/configuration_files/) 
diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md
index 371afaf2af0..7e936a4f063 100644
--- a/docs/ru/operations/index.md
+++ b/docs/ru/operations/index.md
@@ -17,4 +17,4 @@
   - [Настройки](settings/index.md)
   - [Утилиты](utils/index.md)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/) 
diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md
index 57f1ad17b11..46750231161 100644
--- a/docs/ru/operations/quotas.md
+++ b/docs/ru/operations/quotas.md
@@ -102,4 +102,4 @@
 
 При перезапуске сервера, квоты сбрасываются.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/quotas/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/quotas/) 
diff --git a/docs/ru/operations/server_settings/index.md b/docs/ru/operations/server_settings/index.md
index b7a67bacaf9..1ff4d9ae84b 100644
--- a/docs/ru/operations/server_settings/index.md
+++ b/docs/ru/operations/server_settings/index.md
@@ -8,4 +8,4 @@
 
 Перед изучением настроек ознакомьтесь с разделом [Конфигурационные файлы](../configuration_files.md#configuration_files), обратите внимание на использование подстановок (атрибуты `incl` и `optional`).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/server_settings/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_settings/) 
diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md
index de0cf7d2a41..44bd04a0e35 100644
--- a/docs/ru/operations/server_settings/settings.md
+++ b/docs/ru/operations/server_settings/settings.md
@@ -817,4 +817,4 @@ ClickHouse использует ZooKeeper для хранения метадан
 
 **Значение по умолчанию**: 15.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/server_settings/settings/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_settings/settings/) 
diff --git a/docs/ru/operations/settings/constraints_on_settings.md b/docs/ru/operations/settings/constraints_on_settings.md
index 5a4442295d2..18e67f1729b 100644
--- a/docs/ru/operations/settings/constraints_on_settings.md
+++ b/docs/ru/operations/settings/constraints_on_settings.md
@@ -66,4 +66,4 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n
 
 **Примечание:** профиль с именем `default` обрабатывается специальным образом: все ограничения на изменение настроек из этого профиля становятся дефолтными и влияют на всех пользователей, кроме тех, где эти ограничения явно переопределены.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/constraints_on_settings/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/constraints_on_settings/) 
diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md
index 9ace3824ac4..6c74054f933 100644
--- a/docs/ru/operations/settings/index.md
+++ b/docs/ru/operations/settings/index.md
@@ -21,4 +21,4 @@
 
 Настройки, которые можно задать только в конфигурационном файле сервера, в разделе не рассматриваются.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/) 
diff --git a/docs/ru/operations/settings/permissions_for_queries.md b/docs/ru/operations/settings/permissions_for_queries.md
index 100da20670f..d86f5bb3344 100644
--- a/docs/ru/operations/settings/permissions_for_queries.md
+++ b/docs/ru/operations/settings/permissions_for_queries.md
@@ -54,4 +54,4 @@
 
 1
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/permissions_for_queries/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/permissions_for_queries/) 
diff --git a/docs/ru/operations/settings/query_complexity.md b/docs/ru/operations/settings/query_complexity.md
index d1372d8c38f..2fef7b4253f 100644
--- a/docs/ru/operations/settings/query_complexity.md
+++ b/docs/ru/operations/settings/query_complexity.md
@@ -274,4 +274,4 @@
 
 > "Too many partitions for single INSERT block (more than " + toString(max_parts) + "). The limit is controlled by 'max_partitions_per_insert_block' setting. Large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc)."
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/query_complexity/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/query_complexity/) 
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index cb2375ba25e..21e949a5cb2 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -1005,4 +1005,4 @@ ClickHouse генерирует исключение
 Значение по умолчанию: 0.
 
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/settings/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) 
diff --git a/docs/ru/operations/settings/settings_profiles.md b/docs/ru/operations/settings/settings_profiles.md
index 8b4e2316fe6..91cf60b5b2b 100644
--- a/docs/ru/operations/settings/settings_profiles.md
+++ b/docs/ru/operations/settings/settings_profiles.md
@@ -62,4 +62,4 @@ SET profile = 'web'
 
 Профили настроек могут наследоваться от друг-друга - это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/settings_profiles/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_profiles/) 
diff --git a/docs/ru/operations/settings/settings_users.md b/docs/ru/operations/settings/settings_users.md
index 2f39e8e86de..55a696e79eb 100644
--- a/docs/ru/operations/settings/settings_users.md
+++ b/docs/ru/operations/settings/settings_users.md
@@ -141,4 +141,4 @@
 
 Элемент `filter` содержать любое выражение, возвращающее значение типа [UInt8](../../data_types/int_uint.md). Обычно он содержит сравнения и логические операторы. Строки `database_name.table1`, для которых фильтр возвращает 0 не выдаются пользователю. Фильтрация несовместима с операциями `PREWHERE` и отключает оптимизацию `WHERE→PREWHERE`.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/settings_users/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_users/) 
diff --git a/docs/ru/operations/system_tables.md b/docs/ru/operations/system_tables.md
index c0a1de519bf..a36a23f936d 100644
--- a/docs/ru/operations/system_tables.md
+++ b/docs/ru/operations/system_tables.md
@@ -882,4 +882,4 @@ Cодержит информацию о дисках, заданных в [ко
 
 Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/system_tables/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/) 
diff --git a/docs/ru/operations/table_engines/aggregatingmergetree.md b/docs/ru/operations/table_engines/aggregatingmergetree.md
index fa452829a5b..74e1d9b9e2a 100644
--- a/docs/ru/operations/table_engines/aggregatingmergetree.md
+++ b/docs/ru/operations/table_engines/aggregatingmergetree.md
@@ -92,4 +92,4 @@ GROUP BY StartDate
 ORDER BY StartDate;
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/aggregatingmergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/aggregatingmergetree/) 
diff --git a/docs/ru/operations/table_engines/buffer.md b/docs/ru/operations/table_engines/buffer.md
index 964897162c2..b162df2b8b1 100644
--- a/docs/ru/operations/table_engines/buffer.md
+++ b/docs/ru/operations/table_engines/buffer.md
@@ -61,4 +61,4 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10
 
 Заметим, что даже для таблиц типа Buffer не имеет смысла вставлять данные по одной строке, так как таким образом будет достигнута скорость всего лишь в несколько тысяч строк в секунду, тогда как при вставке более крупными блоками, достижимо более миллиона строк в секунду (смотрите раздел "Производительность").
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/buffer/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/buffer/) 
diff --git a/docs/ru/operations/table_engines/collapsingmergetree.md b/docs/ru/operations/table_engines/collapsingmergetree.md
index 8260ce1bc76..e027ad774be 100644
--- a/docs/ru/operations/table_engines/collapsingmergetree.md
+++ b/docs/ru/operations/table_engines/collapsingmergetree.md
@@ -291,5 +291,5 @@ select * FROM UAct
 └─────────────────────┴───────────┴──────────┴──────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/collapsingmergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/collapsingmergetree/) 
 
diff --git a/docs/ru/operations/table_engines/custom_partitioning_key.md b/docs/ru/operations/table_engines/custom_partitioning_key.md
index 55904d7dbe0..9376000a3b5 100644
--- a/docs/ru/operations/table_engines/custom_partitioning_key.md
+++ b/docs/ru/operations/table_engines/custom_partitioning_key.md
@@ -123,5 +123,5 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  1 16:48 detached
 
 ClickHouse позволяет производить различные манипуляции с кусками: удалять, копировать из одной таблицы в другую или создавать их резервные копии. Подробнее см. в разделе [Манипуляции с партициями и кусками](../../query_language/alter.md#alter_manipulations-with-partitions).
 
-[Оригинальная статья: ](https://clickhouse.yandex/docs/ru/operations/table_engines/custom_partitioning_key/) 
+[Оригинальная статья: ](https://clickhouse.tech/docs/ru/operations/table_engines/custom_partitioning_key/) 
 
diff --git a/docs/ru/operations/table_engines/dictionary.md b/docs/ru/operations/table_engines/dictionary.md
index d24dca27a07..0bf34c16e93 100644
--- a/docs/ru/operations/table_engines/dictionary.md
+++ b/docs/ru/operations/table_engines/dictionary.md
@@ -86,4 +86,4 @@ select * from products limit 1;
 └───────────────┴─────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/dictionary/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/dictionary/) 
diff --git a/docs/ru/operations/table_engines/distributed.md b/docs/ru/operations/table_engines/distributed.md
index 53391fe8125..259841aebf1 100644
--- a/docs/ru/operations/table_engines/distributed.md
+++ b/docs/ru/operations/table_engines/distributed.md
@@ -118,4 +118,4 @@ logs - имя кластера в конфигурационном файле с
 
 При выставлении опции max_parallel_replicas выполнение запроса распараллеливается по всем репликам внутри одного шарда. Подробнее смотрите раздел [max_parallel_replicas](../settings/settings.md#settings-max_parallel_replicas).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/distributed/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/distributed/) 
diff --git a/docs/ru/operations/table_engines/external_data.md b/docs/ru/operations/table_engines/external_data.md
index 5d297b246b0..f5ce57529ca 100644
--- a/docs/ru/operations/table_engines/external_data.md
+++ b/docs/ru/operations/table_engines/external_data.md
@@ -60,4 +60,4 @@ $ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+coun
 
 При распределённой обработке запроса, временные таблицы передаются на все удалённые серверы.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/external_data/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/external_data/) 
diff --git a/docs/ru/operations/table_engines/file.md b/docs/ru/operations/table_engines/file.md
index 4d2d3d4d6fb..7391ff297dc 100644
--- a/docs/ru/operations/table_engines/file.md
+++ b/docs/ru/operations/table_engines/file.md
@@ -76,4 +76,4 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64
     - индексы;
     - репликация.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/file/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/file/) 
diff --git a/docs/ru/operations/table_engines/graphitemergetree.md b/docs/ru/operations/table_engines/graphitemergetree.md
index cbb4cc746df..e4c6a9baef0 100644
--- a/docs/ru/operations/table_engines/graphitemergetree.md
+++ b/docs/ru/operations/table_engines/graphitemergetree.md
@@ -164,4 +164,4 @@ default
 
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/graphitemergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/graphitemergetree/) 
diff --git a/docs/ru/operations/table_engines/hdfs.md b/docs/ru/operations/table_engines/hdfs.md
index ca889a5d4ee..2aad0902eb7 100644
--- a/docs/ru/operations/table_engines/hdfs.md
+++ b/docs/ru/operations/table_engines/hdfs.md
@@ -106,4 +106,4 @@ CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9
 
 - [Виртуальные столбцы](index.md#table_engines-virtual_columns)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/hdfs/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/hdfs/) 
diff --git a/docs/ru/operations/table_engines/index.md b/docs/ru/operations/table_engines/index.md
index ce414a3c0d1..d22c88a1b04 100644
--- a/docs/ru/operations/table_engines/index.md
+++ b/docs/ru/operations/table_engines/index.md
@@ -73,5 +73,5 @@
 
 При создании таблицы со столбцом, имя которого совпадает с именем одного из виртуальных столбцов таблицы, виртуальный столбец становится недоступным. Не делайте так. Чтобы помочь избежать конфликтов, имена виртуальных столбцов обычно предваряются подчеркиванием.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/) 
 
diff --git a/docs/ru/operations/table_engines/jdbc.md b/docs/ru/operations/table_engines/jdbc.md
index 08b1bef87d5..0bdf7c765ed 100644
--- a/docs/ru/operations/table_engines/jdbc.md
+++ b/docs/ru/operations/table_engines/jdbc.md
@@ -78,4 +78,4 @@ FROM jdbc_table
 
 - [Табличная функция JDBC](../../query_language/table_functions/jdbc.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/jdbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/jdbc/) 
diff --git a/docs/ru/operations/table_engines/join.md b/docs/ru/operations/table_engines/join.md
index f10f2dd60b0..f06e1af69ef 100644
--- a/docs/ru/operations/table_engines/join.md
+++ b/docs/ru/operations/table_engines/join.md
@@ -97,4 +97,4 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1))
 
 При аварийном перезапуске сервера блок данных на диске может быть потерян или повреждён. В последнем случае, может потребоваться вручную удалить файл с повреждёнными данными.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/join/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/join/) 
diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/operations/table_engines/kafka.md
index b79520cfd61..90549f3afa1 100644
--- a/docs/ru/operations/table_engines/kafka.md
+++ b/docs/ru/operations/table_engines/kafka.md
@@ -164,5 +164,5 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format
 
 - [Виртуальные столбцы](index.md#table_engines-virtual_columns)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/kafka/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/kafka/) 
 
diff --git a/docs/ru/operations/table_engines/log.md b/docs/ru/operations/table_engines/log.md
index a3211887b7e..c0e40f3ba38 100644
--- a/docs/ru/operations/table_engines/log.md
+++ b/docs/ru/operations/table_engines/log.md
@@ -6,4 +6,4 @@
 При конкурентном доступе к данным, чтения могут выполняться одновременно, а записи блокируют чтения и друг друга.
 Движок Log не поддерживает индексы. Также, если при записи в таблицу произошёл сбой, то таблица станет битой, и чтения из неё будут возвращать ошибку. Движок Log подходит для временных данных, write-once таблиц, а также для тестовых и демонстрационных целей.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/log/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/log/) 
diff --git a/docs/ru/operations/table_engines/log_family.md b/docs/ru/operations/table_engines/log_family.md
index ef59b79abeb..90afd645cba 100644
--- a/docs/ru/operations/table_engines/log_family.md
+++ b/docs/ru/operations/table_engines/log_family.md
@@ -33,4 +33,4 @@
 
 Движки `Log` и `StripeLog` поддерживают параллельное чтение. При чтении данных, ClickHouse использует множество потоков. Каждый поток обрабатывает отдельный блок данных. Движок `Log` сохраняет каждый столбец таблицы в отдельном файле. Движок `StripeLog` хранит все данные в одном файле. Таким образом, движок `StripeLog` использует меньше дескрипторов в операционной системе, а движок `Log` обеспечивает более эффективное считывание данных.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/log_family/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/log_family/) 
diff --git a/docs/ru/operations/table_engines/materializedview.md b/docs/ru/operations/table_engines/materializedview.md
index b607dc08c09..beee3adf48f 100644
--- a/docs/ru/operations/table_engines/materializedview.md
+++ b/docs/ru/operations/table_engines/materializedview.md
@@ -2,4 +2,4 @@
 
 Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../query_language/create.md)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/materializedview/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/materializedview/) 
diff --git a/docs/ru/operations/table_engines/memory.md b/docs/ru/operations/table_engines/memory.md
index 70cf0c8db20..1fa92e636ab 100644
--- a/docs/ru/operations/table_engines/memory.md
+++ b/docs/ru/operations/table_engines/memory.md
@@ -9,4 +9,4 @@
 
 Движок Memory используется системой для временных таблиц - внешних данных запроса (смотрите раздел "Внешние данные для обработки запроса"), для реализации `GLOBAL IN` (смотрите раздел "Операторы IN").
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/memory/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/memory/) 
diff --git a/docs/ru/operations/table_engines/merge.md b/docs/ru/operations/table_engines/merge.md
index acd7618f9a0..712a6379a74 100644
--- a/docs/ru/operations/table_engines/merge.md
+++ b/docs/ru/operations/table_engines/merge.md
@@ -60,4 +60,4 @@ FROM WatchLog
 
 - [Виртуальные столбцы](index.md#table_engines-virtual_columns)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/merge/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/merge/) 
diff --git a/docs/ru/operations/table_engines/mergetree.md b/docs/ru/operations/table_engines/mergetree.md
index f1815290064..944562f49a4 100644
--- a/docs/ru/operations/table_engines/mergetree.md
+++ b/docs/ru/operations/table_engines/mergetree.md
@@ -635,4 +635,4 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
 
 После выполнения фоновых слияний или мутаций старые куски не удаляются сразу, а через некоторое время (табличная настройка `old_parts_lifetime`). Также они не перемещаются на другие тома или диски, поэтому до момента удаления они продолжают учитываться при подсчёте занятого дискового пространства.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/mergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) 
diff --git a/docs/ru/operations/table_engines/mysql.md b/docs/ru/operations/table_engines/mysql.md
index ef7dd5b9d5d..79b000c0e12 100644
--- a/docs/ru/operations/table_engines/mysql.md
+++ b/docs/ru/operations/table_engines/mysql.md
@@ -89,4 +89,4 @@ SELECT * FROM mysql_table
 - [Табличная функция 'mysql'](../../query_language/table_functions/mysql.md)
 - [Использование MySQL в качестве источника для внешнего словаря](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/mysql/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/mysql/) 
diff --git a/docs/ru/operations/table_engines/null.md b/docs/ru/operations/table_engines/null.md
index a67b9bb16a8..949f101e68c 100644
--- a/docs/ru/operations/table_engines/null.md
+++ b/docs/ru/operations/table_engines/null.md
@@ -4,4 +4,4 @@
 
 Тем не менее, есть возможность создать материализованное представление над таблицей типа Null. Тогда данные, записываемые в таблицу, будут попадать в представление.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/null/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/null/) 
diff --git a/docs/ru/operations/table_engines/odbc.md b/docs/ru/operations/table_engines/odbc.md
index 34d6bec239d..91ac7e72876 100644
--- a/docs/ru/operations/table_engines/odbc.md
+++ b/docs/ru/operations/table_engines/odbc.md
@@ -121,4 +121,4 @@ SELECT * FROM odbc_t
 - [Внешние словари ODBC](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc)
 - [Табличная функция odbc](../../query_language/table_functions/odbc.md)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/odbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/odbc/) 
diff --git a/docs/ru/operations/table_engines/replacingmergetree.md b/docs/ru/operations/table_engines/replacingmergetree.md
index c364859cf67..b63d6582017 100644
--- a/docs/ru/operations/table_engines/replacingmergetree.md
+++ b/docs/ru/operations/table_engines/replacingmergetree.md
@@ -55,4 +55,4 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 - `ver` — столбец с версией. Необязательный параметр. Описание смотрите выше по тексту.
 
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/replacingmergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/replacingmergetree/) 
diff --git a/docs/ru/operations/table_engines/replication.md b/docs/ru/operations/table_engines/replication.md
index cd55296cc07..496050cceb6 100644
--- a/docs/ru/operations/table_engines/replication.md
+++ b/docs/ru/operations/table_engines/replication.md
@@ -206,4 +206,4 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
 
 Если данные в ZooKeeper оказались утеряны или повреждены, то вы можете сохранить данные, переместив их в нереплицируемую таблицу, как описано в пункте выше.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/replication/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/replication/) 
diff --git a/docs/ru/operations/table_engines/set.md b/docs/ru/operations/table_engines/set.md
index b4371e1a9c2..b7e7046efc3 100644
--- a/docs/ru/operations/table_engines/set.md
+++ b/docs/ru/operations/table_engines/set.md
@@ -9,4 +9,4 @@
 
 При грубом перезапуске сервера, блок данных на диске может быть потерян или повреждён. В последнем случае, может потребоваться вручную удалить файл с повреждёнными данными.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/set/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/set/) 
diff --git a/docs/ru/operations/table_engines/stripelog.md b/docs/ru/operations/table_engines/stripelog.md
index 7e80f79255f..d8659b08d28 100644
--- a/docs/ru/operations/table_engines/stripelog.md
+++ b/docs/ru/operations/table_engines/stripelog.md
@@ -85,4 +85,4 @@ SELECT * FROM stripe_log_table ORDER BY timestamp
 └─────────────────────┴──────────────┴────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/stripelog/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/stripelog/) 
diff --git a/docs/ru/operations/table_engines/summingmergetree.md b/docs/ru/operations/table_engines/summingmergetree.md
index f1fbdac92f4..1caa12a3f95 100644
--- a/docs/ru/operations/table_engines/summingmergetree.md
+++ b/docs/ru/operations/table_engines/summingmergetree.md
@@ -129,4 +129,4 @@ ClickHouse может слить куски данных таким образо
 
 Для вложенной структуры данных не нужно указывать её столбцы в кортеже столбцов для суммирования.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/summingmergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/summingmergetree/) 
diff --git a/docs/ru/operations/table_engines/tinylog.md b/docs/ru/operations/table_engines/tinylog.md
index 673647f628a..6d3d2441051 100644
--- a/docs/ru/operations/table_engines/tinylog.md
+++ b/docs/ru/operations/table_engines/tinylog.md
@@ -6,4 +6,4 @@
 
 Запросы выполняются в один поток. То есть, этот движок предназначен для сравнительно маленьких таблиц (до 1 000 000 строк). Этот движок таблиц имеет смысл использовать в том случае, когда у вас есть много маленьких таблиц, так как он проще, чем движок [Log](log.md) (требуется открывать меньше файлов).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/tinylog/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/tinylog/) 
diff --git a/docs/ru/operations/table_engines/url.md b/docs/ru/operations/table_engines/url.md
index 8a6a6790a15..0a4142f2f95 100644
--- a/docs/ru/operations/table_engines/url.md
+++ b/docs/ru/operations/table_engines/url.md
@@ -73,4 +73,4 @@ SELECT * FROM url_engine_table
     - репликация.
 
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/url/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/url/) 
diff --git a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md b/docs/ru/operations/table_engines/versionedcollapsingmergetree.md
index 0de677e726b..1f72e2cbecf 100644
--- a/docs/ru/operations/table_engines/versionedcollapsingmergetree.md
+++ b/docs/ru/operations/table_engines/versionedcollapsingmergetree.md
@@ -226,4 +226,4 @@ SELECT * FROM UAct FINAL
 
 Это очень неэффективный способ выбора данных. Не используйте его для больших таблиц.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/versionedcollapsingmergetree/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/versionedcollapsingmergetree/) 
diff --git a/docs/ru/operations/table_engines/view.md b/docs/ru/operations/table_engines/view.md
index 874569e3af4..865b666553d 100644
--- a/docs/ru/operations/table_engines/view.md
+++ b/docs/ru/operations/table_engines/view.md
@@ -2,4 +2,4 @@
 
 Используется для реализации представлений (подробнее см. запрос `CREATE VIEW`). Не хранит данные, а хранит только указанный запрос `SELECT`. При чтении из таблицы, выполняет его (с удалением из запроса всех ненужных столбцов).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/view/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/view/) 
diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md
index 821d7d735aa..f09a64ecdeb 100644
--- a/docs/ru/operations/tips.md
+++ b/docs/ru/operations/tips.md
@@ -241,4 +241,4 @@ script
 end script
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/tips/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/tips/) 
diff --git a/docs/ru/operations/utils/clickhouse-copier.md b/docs/ru/operations/utils/clickhouse-copier.md
index d36de755c59..1f8ba4bc5e5 100644
--- a/docs/ru/operations/utils/clickhouse-copier.md
+++ b/docs/ru/operations/utils/clickhouse-copier.md
@@ -165,4 +165,4 @@ $ clickhouse-copier copier --daemon --config zookeeper.xml --task-path /task/pat
 
 `clickhouse-copier` отслеживает изменения `/task/path/description` и применяет их "на лету". Если вы поменяете, например, значение `max_workers`, то количество процессов, выполняющих задания, также изменится.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/utils/clickhouse-copier/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/clickhouse-copier/) 
diff --git a/docs/ru/operations/utils/clickhouse-local.md b/docs/ru/operations/utils/clickhouse-local.md
index a9c32e49606..9c262b06bcf 100644
--- a/docs/ru/operations/utils/clickhouse-local.md
+++ b/docs/ru/operations/utils/clickhouse-local.md
@@ -66,4 +66,4 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec.
 ...
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/utils/clickhouse-local/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/clickhouse-local/) 
diff --git a/docs/ru/operations/utils/index.md b/docs/ru/operations/utils/index.md
index a0c90841c30..ef17f5d99c2 100644
--- a/docs/ru/operations/utils/index.md
+++ b/docs/ru/operations/utils/index.md
@@ -3,4 +3,4 @@
 * [clickhouse-local](clickhouse-local.md)
 * [clickhouse-copier](clickhouse-copier.md) - копирует (и перешардирует) данные с одного кластера на другой.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/utils/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/) 
diff --git a/docs/ru/query_language/agg_functions/combinators.md b/docs/ru/query_language/agg_functions/combinators.md
index 68e32fb7032..18fd5fabe55 100644
--- a/docs/ru/query_language/agg_functions/combinators.md
+++ b/docs/ru/query_language/agg_functions/combinators.md
@@ -117,4 +117,4 @@ FROM people
 └────────┴───────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/agg_functions/combinators/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/combinators/) 
diff --git a/docs/ru/query_language/agg_functions/index.md b/docs/ru/query_language/agg_functions/index.md
index 54d1bdb6f73..f1adc7f1c78 100644
--- a/docs/ru/query_language/agg_functions/index.md
+++ b/docs/ru/query_language/agg_functions/index.md
@@ -53,4 +53,4 @@ SELECT groupArray(y) FROM t_null_big
 
 `groupArray` не включает `NULL` в результирующий массив.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/agg_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/) 
diff --git a/docs/ru/query_language/agg_functions/parametric_functions.md b/docs/ru/query_language/agg_functions/parametric_functions.md
index 0b018d8876c..d4c942dcabd 100644
--- a/docs/ru/query_language/agg_functions/parametric_functions.md
+++ b/docs/ru/query_language/agg_functions/parametric_functions.md
@@ -292,4 +292,4 @@ ORDER BY level
 Решение: пишем в запросе GROUP BY SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/agg_functions/parametric_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/parametric_functions/) 
diff --git a/docs/ru/query_language/agg_functions/reference.md b/docs/ru/query_language/agg_functions/reference.md
index 758d762e26b..a3720393783 100644
--- a/docs/ru/query_language/agg_functions/reference.md
+++ b/docs/ru/query_language/agg_functions/reference.md
@@ -1193,4 +1193,4 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD')
 - [stochasticLinearRegression](#agg_functions-stochasticlinearregression)
 - [Отличие линейной от логистической регрессии](https://moredez.ru/q/51225972/)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/agg_functions/reference/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/reference/) 
diff --git a/docs/ru/query_language/alter.md b/docs/ru/query_language/alter.md
index 9cba92f23fd..743791592c2 100644
--- a/docs/ru/query_language/alter.md
+++ b/docs/ru/query_language/alter.md
@@ -460,4 +460,4 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name
 
 Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/alter/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/) 
diff --git a/docs/ru/query_language/create.md b/docs/ru/query_language/create.md
index fbdf5ce5265..7008e8588db 100644
--- a/docs/ru/query_language/create.md
+++ b/docs/ru/query_language/create.md
@@ -274,4 +274,4 @@ SELECT a, b, c FROM (SELECT ...)
 
 Отсутствует отдельный запрос для удаления представлений. Чтобы удалить представление, следует использовать `DROP TABLE`.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/create/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/create/) 
diff --git a/docs/ru/query_language/dicts/external_dicts.md b/docs/ru/query_language/dicts/external_dicts.md
index 6df7bb501b2..4ebb8f5143d 100644
--- a/docs/ru/query_language/dicts/external_dicts.md
+++ b/docs/ru/query_language/dicts/external_dicts.md
@@ -44,4 +44,4 @@ ClickHouse полностью или частично хранит словар
 - [Ключ и поля словаря](external_dicts_dict_structure.md)
 - [Функции для работы с внешними словарями](../functions/ext_dict_functions.md#ext_dict_functions)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) 
diff --git a/docs/ru/query_language/dicts/external_dicts_dict.md b/docs/ru/query_language/dicts/external_dicts_dict.md
index 94d923a6a75..b004b2d8100 100644
--- a/docs/ru/query_language/dicts/external_dicts_dict.md
+++ b/docs/ru/query_language/dicts/external_dicts_dict.md
@@ -30,4 +30,4 @@
 - [structure](external_dicts_dict_structure.md) - Структура словаря. Ключ и атрибуты, которые можно получить по ключу.
 - [lifetime](external_dicts_dict_lifetime.md) - Периодичность обновления словарей.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts_dict/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict/) 
diff --git a/docs/ru/query_language/dicts/external_dicts_dict_layout.md b/docs/ru/query_language/dicts/external_dicts_dict_layout.md
index 7d6919347ab..510b400f862 100644
--- a/docs/ru/query_language/dicts/external_dicts_dict_layout.md
+++ b/docs/ru/query_language/dicts/external_dicts_dict_layout.md
@@ -283,4 +283,4 @@ dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
 
 Данные хранятся в побитовом дереве (`trie`), он должен полностью помещаться в оперативной памяти.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts_dict_layout/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_layout/) 
diff --git a/docs/ru/query_language/dicts/external_dicts_dict_lifetime.md b/docs/ru/query_language/dicts/external_dicts_dict_lifetime.md
index f2774e63b0a..2108d3e49ff 100644
--- a/docs/ru/query_language/dicts/external_dicts_dict_lifetime.md
+++ b/docs/ru/query_language/dicts/external_dicts_dict_lifetime.md
@@ -56,4 +56,4 @@ ClickHouse периодически обновляет словари. Инте
 
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts_dict_lifetime/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_lifetime/) 
diff --git a/docs/ru/query_language/dicts/external_dicts_dict_sources.md b/docs/ru/query_language/dicts/external_dicts_dict_sources.md
index 65c98fa18b9..af0fc245e01 100644
--- a/docs/ru/query_language/dicts/external_dicts_dict_sources.md
+++ b/docs/ru/query_language/dicts/external_dicts_dict_sources.md
@@ -441,4 +441,4 @@ MySQL можно подключить на локальном хосте чер
 - `storage_type` – способ хранения ключей. Необходимо использовать `simple` для источников с одним столбцом ключей, `hash_map` -- для источников с двумя столбцами ключей. Источники с более, чем двумя столбцами ключей, не поддерживаются. Может отсутствовать, значение по умолчанию `simple`.
 - `db_index` – номер базы данных. Может отсутствовать, значение по умолчанию 0.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts_dict_sources/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_sources/) 
diff --git a/docs/ru/query_language/dicts/external_dicts_dict_structure.md b/docs/ru/query_language/dicts/external_dicts_dict_structure.md
index 6423451c1be..453de13eea6 100644
--- a/docs/ru/query_language/dicts/external_dicts_dict_structure.md
+++ b/docs/ru/query_language/dicts/external_dicts_dict_structure.md
@@ -113,4 +113,4 @@ ClickHouse поддерживает следующие виды ключей:
 | `injective` | Признак [инъективности](https://ru.wikipedia.org/wiki/Инъекция_(математика)) отображения `id -> attribute`. 
Если `true`, то обращения к словарям с включенной инъективностью могут быть автоматически переставлены ClickHouse за стадию `GROUP BY`, что как правило существенно сокращает их количество.

Значение по умолчанию: `false`. | Нет |
 | `is_object_id` | Признак того, что запрос выполняется к документу MongoDB по `ObjectID`.

Значение по умолчанию: `false`. | Нет |
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/external_dicts_dict_structure/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) 
diff --git a/docs/ru/query_language/dicts/index.md b/docs/ru/query_language/dicts/index.md
index 3820a0d49c9..815029c245c 100644
--- a/docs/ru/query_language/dicts/index.md
+++ b/docs/ru/query_language/dicts/index.md
@@ -11,4 +11,4 @@ ClickHouse поддерживает:
 - [Встроенные словари](internal_dicts.md#internal_dicts) со специфическим [набором функций](../functions/ym_dict_functions.md).
 - [Подключаемые (внешние) словари](external_dicts.md) с [набором функций](../functions/ext_dict_functions.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/) 
diff --git a/docs/ru/query_language/dicts/internal_dicts.md b/docs/ru/query_language/dicts/internal_dicts.md
index 2cd55d71f99..4773b03e634 100644
--- a/docs/ru/query_language/dicts/internal_dicts.md
+++ b/docs/ru/query_language/dicts/internal_dicts.md
@@ -45,4 +45,4 @@ ClickHouse содержит встроенную возможность рабо
 
 Также имеются функции для работы с идентификаторами операционных систем и поисковых систем Яндекс.Метрики, пользоваться которыми не нужно.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/dicts/internal_dicts/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/internal_dicts/) 
diff --git a/docs/ru/query_language/functions/arithmetic_functions.md b/docs/ru/query_language/functions/arithmetic_functions.md
index 3c541b50a99..7298d6ac6f8 100644
--- a/docs/ru/query_language/functions/arithmetic_functions.md
+++ b/docs/ru/query_language/functions/arithmetic_functions.md
@@ -72,4 +72,4 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0
 Вычисляет наименьшее общее кратное чисел.
 При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/arithmetic_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/arithmetic_functions/) 
diff --git a/docs/ru/query_language/functions/array_functions.md b/docs/ru/query_language/functions/array_functions.md
index d8eaae99c64..ade65ed1ebe 100644
--- a/docs/ru/query_language/functions/array_functions.md
+++ b/docs/ru/query_language/functions/array_functions.md
@@ -869,4 +869,4 @@ SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])
 └────────────────────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/array_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/array_functions/) 
diff --git a/docs/ru/query_language/functions/array_join.md b/docs/ru/query_language/functions/array_join.md
index 1317d4329f2..56a5fa40a9b 100644
--- a/docs/ru/query_language/functions/array_join.md
+++ b/docs/ru/query_language/functions/array_join.md
@@ -27,4 +27,4 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src
 └─────┴───────────┴─────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/array_join/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/array_join/) 
diff --git a/docs/ru/query_language/functions/bit_functions.md b/docs/ru/query_language/functions/bit_functions.md
index e4af747fa47..3b8563f3587 100644
--- a/docs/ru/query_language/functions/bit_functions.md
+++ b/docs/ru/query_language/functions/bit_functions.md
@@ -195,4 +195,4 @@ SELECT bitTestAny(43, 4, 2)
 └──────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/bit_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/bit_functions/) 
diff --git a/docs/ru/query_language/functions/bitmap_functions.md b/docs/ru/query_language/functions/bitmap_functions.md
index f54d88ab35f..795ec58dcb3 100644
--- a/docs/ru/query_language/functions/bitmap_functions.md
+++ b/docs/ru/query_language/functions/bitmap_functions.md
@@ -351,4 +351,4 @@ SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res
 └─────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/bitmap_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/bitmap_functions/) 
diff --git a/docs/ru/query_language/functions/comparison_functions.md b/docs/ru/query_language/functions/comparison_functions.md
index b69e272133b..62d889f72ce 100644
--- a/docs/ru/query_language/functions/comparison_functions.md
+++ b/docs/ru/query_language/functions/comparison_functions.md
@@ -30,4 +30,4 @@
 
 ## greaterOrEquals, оператор `>=` {#function-greaterorequals}
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/comparison_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/comparison_functions/) 
diff --git a/docs/ru/query_language/functions/conditional_functions.md b/docs/ru/query_language/functions/conditional_functions.md
index a648314deb4..6a8a7c23742 100644
--- a/docs/ru/query_language/functions/conditional_functions.md
+++ b/docs/ru/query_language/functions/conditional_functions.md
@@ -47,4 +47,4 @@ multiIf(cond_1, then_1, cond_2, then_2...else)
 └────────────────────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/conditional_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/conditional_functions/) 
diff --git a/docs/ru/query_language/functions/date_time_functions.md b/docs/ru/query_language/functions/date_time_functions.md
index 3f087505464..022b83e819b 100644
--- a/docs/ru/query_language/functions/date_time_functions.md
+++ b/docs/ru/query_language/functions/date_time_functions.md
@@ -166,4 +166,4 @@ SELECT
 |%Y|год, 4 цифры|2018|
 |%%|символ %|%|
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/date_time_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/date_time_functions/) 
diff --git a/docs/ru/query_language/functions/encoding_functions.md b/docs/ru/query_language/functions/encoding_functions.md
index 9c7737e5d14..a5a5b3b7148 100644
--- a/docs/ru/query_language/functions/encoding_functions.md
+++ b/docs/ru/query_language/functions/encoding_functions.md
@@ -20,4 +20,4 @@
 ## bitmaskToArray(num)
 Принимает целое число. Возвращает массив чисел типа UInt64, содержащий степени двойки, в сумме дающих исходное число; числа в массиве идут по возрастанию.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/encoding_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/encoding_functions/) 
diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md
index 8ccf9a31e91..29164aa6862 100644
--- a/docs/ru/query_language/functions/ext_dict_functions.md
+++ b/docs/ru/query_language/functions/ext_dict_functions.md
@@ -189,4 +189,4 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr)
 
 Если значение атрибута не удалось обработать или оно не соответствует типу данных атрибута, то ClickHouse генерирует исключение.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ext_dict_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ext_dict_functions/) 
diff --git a/docs/ru/query_language/functions/functions_for_nulls.md b/docs/ru/query_language/functions/functions_for_nulls.md
index 948b2d41f60..8558542a8f7 100644
--- a/docs/ru/query_language/functions/functions_for_nulls.md
+++ b/docs/ru/query_language/functions/functions_for_nulls.md
@@ -284,4 +284,4 @@ SELECT toTypeName(toNullable(10))
 └────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/functions_for_nulls/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/functions_for_nulls/) 
diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/query_language/functions/geo.md
index 151d98589ba..3994534ae26 100644
--- a/docs/ru/query_language/functions/geo.md
+++ b/docs/ru/query_language/functions/geo.md
@@ -381,4 +381,4 @@ SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index
 ```
 
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/geo/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/geo/) 
diff --git a/docs/ru/query_language/functions/hash_functions.md b/docs/ru/query_language/functions/hash_functions.md
index 47384e78565..fb460bf25f5 100644
--- a/docs/ru/query_language/functions/hash_functions.md
+++ b/docs/ru/query_language/functions/hash_functions.md
@@ -437,4 +437,4 @@ SELECT xxHash32('Hello, world!');
 
 - [xxHash](http://cyan4973.github.io/xxHash/).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/hash_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/hash_functions/) 
diff --git a/docs/ru/query_language/functions/higher_order_functions.md b/docs/ru/query_language/functions/higher_order_functions.md
index b64a561f58a..cd5d8945aaf 100644
--- a/docs/ru/query_language/functions/higher_order_functions.md
+++ b/docs/ru/query_language/functions/higher_order_functions.md
@@ -157,4 +157,4 @@ SELECT arrayReverseSort((x, y) -> y, ['hello', 'world'], [2, 1]) as res;
 
 Подробная информация о методе `arrayReverseSort` приведена в разделе [Функции по работе с массивами](array_functions.md#array_functions-reverse-sort).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/higher_order_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/higher_order_functions/) 
diff --git a/docs/ru/query_language/functions/in_functions.md b/docs/ru/query_language/functions/in_functions.md
index 8b4eccd0db6..b8b12ea3808 100644
--- a/docs/ru/query_language/functions/in_functions.md
+++ b/docs/ru/query_language/functions/in_functions.md
@@ -14,4 +14,4 @@
 N - индекс столбца начиная с 1. N должно быть константой. N должно быть целым строго положительным числом не большим размера кортежа.
 Выполнение функции ничего не стоит.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/in_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/in_functions/) 
diff --git a/docs/ru/query_language/functions/index.md b/docs/ru/query_language/functions/index.md
index c24e020097d..f978fb75096 100644
--- a/docs/ru/query_language/functions/index.md
+++ b/docs/ru/query_language/functions/index.md
@@ -62,4 +62,4 @@
 
 Если функция в запросе выполняется на сервере-инициаторе запроса, а вам нужно, чтобы она выполнялась на удалённых серверах, вы можете обернуть её в агрегатную функцию any или добавить в ключ в `GROUP BY`.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/) 
diff --git a/docs/ru/query_language/functions/ip_address_functions.md b/docs/ru/query_language/functions/ip_address_functions.md
index 5b1c5e32172..342e07e72b6 100644
--- a/docs/ru/query_language/functions/ip_address_functions.md
+++ b/docs/ru/query_language/functions/ip_address_functions.md
@@ -236,4 +236,4 @@ SELECT
 └───────────────────────────────────┴──────────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ip_address_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ip_address_functions/) 
diff --git a/docs/ru/query_language/functions/json_functions.md b/docs/ru/query_language/functions/json_functions.md
index 9269493473b..3e04698fba7 100644
--- a/docs/ru/query_language/functions/json_functions.md
+++ b/docs/ru/query_language/functions/json_functions.md
@@ -221,4 +221,4 @@ SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100,
 SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']'
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/json_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/json_functions/) 
diff --git a/docs/ru/query_language/functions/logical_functions.md b/docs/ru/query_language/functions/logical_functions.md
index 6fa2504f933..a5c6d1cdd00 100644
--- a/docs/ru/query_language/functions/logical_functions.md
+++ b/docs/ru/query_language/functions/logical_functions.md
@@ -12,4 +12,4 @@
 
 ## xor
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/logical_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/logical_functions/) 
diff --git a/docs/ru/query_language/functions/math_functions.md b/docs/ru/query_language/functions/math_functions.md
index 92feceb3a95..886a0be48a7 100644
--- a/docs/ru/query_language/functions/math_functions.md
+++ b/docs/ru/query_language/functions/math_functions.md
@@ -79,4 +79,4 @@ SELECT erf(3 / sqrt(2))
 ## pow(x, y)
 Принимает два числовых аргумента x и y. Возвращает число типа Float64, близкое к x в степени y.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/math_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/math_functions/) 
diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md
index bf3ee2a8420..ceed16923ac 100644
--- a/docs/ru/query_language/functions/other_functions.md
+++ b/docs/ru/query_language/functions/other_functions.md
@@ -920,4 +920,4 @@ SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers
 └────────┴────────────────────────────────┴──────────────────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/other_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/other_functions/) 
diff --git a/docs/ru/query_language/functions/random_functions.md b/docs/ru/query_language/functions/random_functions.md
index 1af271bc8de..886413b4f24 100644
--- a/docs/ru/query_language/functions/random_functions.md
+++ b/docs/ru/query_language/functions/random_functions.md
@@ -14,4 +14,4 @@
 Возвращает псевдослучайное число типа UInt64, равномерно распределённое среди всех чисел типа UInt64.
 Используется linear congruential generator.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/random_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/random_functions/) 
diff --git a/docs/ru/query_language/functions/rounding_functions.md b/docs/ru/query_language/functions/rounding_functions.md
index 8f5eeca5dc8..4b03475fe24 100644
--- a/docs/ru/query_language/functions/rounding_functions.md
+++ b/docs/ru/query_language/functions/rounding_functions.md
@@ -169,4 +169,4 @@ roundBankers(10.755, 2) = 11,76
 ## roundAge(num)
 Принимает число. Если число меньше 18 - возвращает 0. Иначе округляет число вниз до чисел из набора: 18, 25, 35, 45, 55. Эта функция специфична для Яндекс.Метрики и предназначена для реализации отчёта по возрасту посетителей.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/rounding_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/rounding_functions/) 
diff --git a/docs/ru/query_language/functions/splitting_merging_functions.md b/docs/ru/query_language/functions/splitting_merging_functions.md
index 8d9c6aef14f..a4dcec38d8b 100644
--- a/docs/ru/query_language/functions/splitting_merging_functions.md
+++ b/docs/ru/query_language/functions/splitting_merging_functions.md
@@ -28,4 +28,4 @@ SELECT alphaTokens('abca1abc')
 │ ['abca','abc']          │
 └─────────────────────────┘
 ```
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/splitting_merging_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/splitting_merging_functions/) 
diff --git a/docs/ru/query_language/functions/string_functions.md b/docs/ru/query_language/functions/string_functions.md
index 56886e83a3d..7e085b8838d 100644
--- a/docs/ru/query_language/functions/string_functions.md
+++ b/docs/ru/query_language/functions/string_functions.md
@@ -359,4 +359,4 @@ SELECT trimBoth('     Hello, world!     ')
 
 Тип результата - UInt64.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/string_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_functions/) 
diff --git a/docs/ru/query_language/functions/string_replace_functions.md b/docs/ru/query_language/functions/string_replace_functions.md
index 0c4cb9923f2..7e8a0682316 100644
--- a/docs/ru/query_language/functions/string_replace_functions.md
+++ b/docs/ru/query_language/functions/string_replace_functions.md
@@ -74,4 +74,4 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 └─────────────────────┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/string_replace_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_replace_functions/) 
diff --git a/docs/ru/query_language/functions/string_search_functions.md b/docs/ru/query_language/functions/string_search_functions.md
index 1a35947811b..050b99747a6 100644
--- a/docs/ru/query_language/functions/string_search_functions.md
+++ b/docs/ru/query_language/functions/string_search_functions.md
@@ -117,4 +117,4 @@
 !!! note "Примечание"
     Для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем (не)симметрическую разность между хэш таблицами -- могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` -- мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки, а также первый бит нулевого байта, если байтов больше 1 -- это работает для латиницы и почти для всех кириллических букв.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/string_search_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_search_functions/) 
diff --git a/docs/ru/query_language/functions/type_conversion_functions.md b/docs/ru/query_language/functions/type_conversion_functions.md
index 3fb431fa3b2..54913e26f5c 100644
--- a/docs/ru/query_language/functions/type_conversion_functions.md
+++ b/docs/ru/query_language/functions/type_conversion_functions.md
@@ -393,4 +393,4 @@ SELECT
 ```
 
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/type_conversion_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/type_conversion_functions/) 
diff --git a/docs/ru/query_language/functions/url_functions.md b/docs/ru/query_language/functions/url_functions.md
index a6e97b8a013..abf45003ab6 100644
--- a/docs/ru/query_language/functions/url_functions.md
+++ b/docs/ru/query_language/functions/url_functions.md
@@ -179,4 +179,4 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod
 ### cutURLParameter(URL, name)
 Удаляет параметр URL с именем name, если такой есть. Функция работает при допущении, что имя параметра закодировано в URL в точности таким же образом, что и в переданном аргументе.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/url_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/url_functions/) 
diff --git a/docs/ru/query_language/functions/uuid_functions.md b/docs/ru/query_language/functions/uuid_functions.md
index d933130d30c..aa8ae451d6b 100644
--- a/docs/ru/query_language/functions/uuid_functions.md
+++ b/docs/ru/query_language/functions/uuid_functions.md
@@ -111,5 +111,5 @@ SELECT
 - [dictGetUUID](ext_dict_functions.md)
 - [dictGetUUIDOrDefault](ext_dict_functions.md)
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/uuid_function/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) 
 
diff --git a/docs/ru/query_language/functions/ym_dict_functions.md b/docs/ru/query_language/functions/ym_dict_functions.md
index 2bf511438b1..4340d6c8e16 100644
--- a/docs/ru/query_language/functions/ym_dict_functions.md
+++ b/docs/ru/query_language/functions/ym_dict_functions.md
@@ -115,4 +115,4 @@ LIMIT 15
 
 `ua` и `uk` обозначают одно и то же - украинский язык.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ym_dict_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ym_dict_functions/) 
diff --git a/docs/ru/query_language/index.md b/docs/ru/query_language/index.md
index 881a6ef7cfc..20b8ecdabec 100644
--- a/docs/ru/query_language/index.md
+++ b/docs/ru/query_language/index.md
@@ -6,4 +6,4 @@
 * [ALTER](alter.md#query_language_queries_alter)
 * [Прочие виды запросов](misc.md)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) 
diff --git a/docs/ru/query_language/insert_into.md b/docs/ru/query_language/insert_into.md
index 4cd14e21871..9919984bffb 100644
--- a/docs/ru/query_language/insert_into.md
+++ b/docs/ru/query_language/insert_into.md
@@ -72,4 +72,4 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
 -   Данные поступают в режиме реального времени.
 -   Вы загружаете данные, которые как правило отсортированы по времени.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/insert_into/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/insert_into/) 
diff --git a/docs/ru/query_language/misc.md b/docs/ru/query_language/misc.md
index 5a89856a143..d75258c8e80 100644
--- a/docs/ru/query_language/misc.md
+++ b/docs/ru/query_language/misc.md
@@ -234,4 +234,4 @@ USE db
 Текущая база данных используется для поиска таблиц, если база данных не указана в запросе явно через точку перед именем таблицы.
 При использовании HTTP протокола запрос не может быть выполнен, так как понятия сессии не существует.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/misc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/misc/) 
diff --git a/docs/ru/query_language/operators.md b/docs/ru/query_language/operators.md
index 31378760fcb..1bc1fa62719 100644
--- a/docs/ru/query_language/operators.md
+++ b/docs/ru/query_language/operators.md
@@ -261,4 +261,4 @@ SELECT * FROM t_null WHERE y IS NOT NULL
 └───┴───┘
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/operators/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/operators/) 
diff --git a/docs/ru/query_language/select.md b/docs/ru/query_language/select.md
index 9bc5c7a4c04..76d695b5e2e 100644
--- a/docs/ru/query_language/select.md
+++ b/docs/ru/query_language/select.md
@@ -1394,4 +1394,4 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL
 
 В других случаях использование звёздочки является издевательством над системой, так как вместо преимуществ столбцовой СУБД вы получаете недостатки. То есть использовать звёздочку не рекомендуется.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/select/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/select/) 
diff --git a/docs/ru/query_language/syntax.md b/docs/ru/query_language/syntax.md
index 703394789b3..641224fded6 100644
--- a/docs/ru/query_language/syntax.md
+++ b/docs/ru/query_language/syntax.md
@@ -168,4 +168,4 @@ Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception
 Список выражений - одно выражение или несколько выражений через запятую.
 Функции и операторы, в свою очередь, в качестве аргументов, могут иметь произвольные выражения.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/syntax/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/syntax/) 
diff --git a/docs/ru/query_language/system.md b/docs/ru/query_language/system.md
index 31e0c3cf90b..2394c7481d7 100644
--- a/docs/ru/query_language/system.md
+++ b/docs/ru/query_language/system.md
@@ -103,5 +103,5 @@ SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
 SYSTEM START MERGES [[db.]merge_tree_family_table_name]
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/system/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/system/) 
 
diff --git a/docs/ru/query_language/table_functions/file.md b/docs/ru/query_language/table_functions/file.md
index d43f2773a42..0c51024dd33 100644
--- a/docs/ru/query_language/table_functions/file.md
+++ b/docs/ru/query_language/table_functions/file.md
@@ -100,4 +100,4 @@ FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, value UInt32')
 
 - [Виртуальные столбцы](index.md#table_engines-virtual_columns)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/file/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/file/) 
diff --git a/docs/ru/query_language/table_functions/hdfs.md b/docs/ru/query_language/table_functions/hdfs.md
index 768cf4e3b1d..9196f56b948 100644
--- a/docs/ru/query_language/table_functions/hdfs.md
+++ b/docs/ru/query_language/table_functions/hdfs.md
@@ -56,4 +56,4 @@ LIMIT 2
 
 - [Виртуальные столбцы](index.md#table_engines-virtual_columns)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/hdfs/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/hdfs/) 
diff --git a/docs/ru/query_language/table_functions/index.md b/docs/ru/query_language/table_functions/index.md
index 2883ae7c032..59e5df3a645 100644
--- a/docs/ru/query_language/table_functions/index.md
+++ b/docs/ru/query_language/table_functions/index.md
@@ -27,4 +27,4 @@
 [odbc](odbc.md) | Создаёт таблицу с движком [ODBC](../../operations/table_engines/odbc.md).
 [hdfs](hdfs.md) | Создаёт таблицу с движком [HDFS](../../operations/table_engines/hdfs.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/) 
diff --git a/docs/ru/query_language/table_functions/input.md b/docs/ru/query_language/table_functions/input.md
index 796ca38e0bc..e08fc934268 100644
--- a/docs/ru/query_language/table_functions/input.md
+++ b/docs/ru/query_language/table_functions/input.md
@@ -32,4 +32,4 @@ $ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV"
 $ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV"
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/input/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/input/) 
diff --git a/docs/ru/query_language/table_functions/jdbc.md b/docs/ru/query_language/table_functions/jdbc.md
index 50c3bf67292..807a054ffc0 100644
--- a/docs/ru/query_language/table_functions/jdbc.md
+++ b/docs/ru/query_language/table_functions/jdbc.md
@@ -21,4 +21,4 @@ SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema',
 SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table')
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/jdbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/jdbc/) 
diff --git a/docs/ru/query_language/table_functions/merge.md b/docs/ru/query_language/table_functions/merge.md
index 04a3ba98223..0cc7d7c9943 100644
--- a/docs/ru/query_language/table_functions/merge.md
+++ b/docs/ru/query_language/table_functions/merge.md
@@ -4,4 +4,4 @@
 
 Структура таблицы берётся из первой попавшейся таблицы, подходящей под регулярное выражение.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/merge/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/merge/) 
diff --git a/docs/ru/query_language/table_functions/mysql.md b/docs/ru/query_language/table_functions/mysql.md
index d698f5e69a2..2527b158d78 100644
--- a/docs/ru/query_language/table_functions/mysql.md
+++ b/docs/ru/query_language/table_functions/mysql.md
@@ -70,4 +70,4 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
 - [Движок таблиц 'MySQL'](../../operations/table_engines/mysql.md)
 - [Использование MySQL как источника данных для внешнего словаря](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/mysql/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/mysql/) 
diff --git a/docs/ru/query_language/table_functions/numbers.md b/docs/ru/query_language/table_functions/numbers.md
index 7406773c8c2..1f2710c3eae 100644
--- a/docs/ru/query_language/table_functions/numbers.md
+++ b/docs/ru/query_language/table_functions/numbers.md
@@ -18,4 +18,4 @@ SELECT * FROM system.numbers LIMIT 10;
 select toDate('2010-01-01') + number as d FROM numbers(365);
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/numbers/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/numbers/) 
diff --git a/docs/ru/query_language/table_functions/odbc.md b/docs/ru/query_language/table_functions/odbc.md
index a05e50b75a5..86a6a018b6a 100644
--- a/docs/ru/query_language/table_functions/odbc.md
+++ b/docs/ru/query_language/table_functions/odbc.md
@@ -97,4 +97,4 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test')
 - [Внешние словари ODBC](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-odbc)
 - [Движок таблиц ODBC](../../operations/table_engines/odbc.md).
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/jdbc/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/jdbc/) 
diff --git a/docs/ru/query_language/table_functions/remote.md b/docs/ru/query_language/table_functions/remote.md
index 6bf86502b47..89a423fa6f2 100644
--- a/docs/ru/query_language/table_functions/remote.md
+++ b/docs/ru/query_language/table_functions/remote.md
@@ -74,4 +74,4 @@ example01-{01..02}-{1|2}
 
 `remoteSecure` - аналогично функции `remote`, но с соединением по шифрованному каналу. Порт по умолчанию - `tcp_port_secure` из конфига или 9440.
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/remote/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/remote/) 
diff --git a/docs/ru/query_language/table_functions/url.md b/docs/ru/query_language/table_functions/url.md
index 79951209b3f..c34cd4aa53f 100644
--- a/docs/ru/query_language/table_functions/url.md
+++ b/docs/ru/query_language/table_functions/url.md
@@ -18,4 +18,4 @@ structure - структура таблицы в форме `'UserID UInt64, Nam
 SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3
 ```
 
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/url/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/url/) 
diff --git a/docs/ru/security_changelog.md b/docs/ru/security_changelog.md
index db742b5f990..6bb8f892b11 100644
--- a/docs/ru/security_changelog.md
+++ b/docs/ru/security_changelog.md
@@ -60,4 +60,4 @@ unixODBC позволял указать путь для подключения
 Некорректная конфигурация в deb пакете могла привести к неавторизованному доступу к базе данных.
 
 Обнаружено благодаря: the UK's National Cyber Security Centre (NCSC)
-[Оригинальная статья](https://clickhouse.yandex/docs/ru/security_changelog/) 
+[Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) 
diff --git a/docs/tools/README.md b/docs/tools/README.md
index 6332a22f164..57a2e5a99ca 100644
--- a/docs/tools/README.md
+++ b/docs/tools/README.md
@@ -2,7 +2,7 @@
 
 ClickHouse documentation is built using [build.py](build.py) script that uses [mkdocs](https://www.mkdocs.org) library and it's dependencies to separately build all version of documentations (all languages in either single and multi page mode) as static HTMLs. The results are then put in correct directory structure. It can also generate PDF version.
 
-[release.sh](release.sh) also pulls static files needed for [official ClickHouse website](https://clickhouse.yandex) from [../../website](../../website) folder, packs them alongside docs into Docker container and tries to deploy it (possible only from Yandex private network).
+[release.sh](release.sh) also pulls static files needed for [official ClickHouse website](https://clickhouse.tech) from [../../website](../../website) folder, packs them alongside docs into Docker container and tries to deploy it (possible only from Yandex private network).
 
 ## How to check if the documentation will look fine?
 
@@ -40,6 +40,6 @@ Then push to GitHub so you can use it's preview. It's better to use previous met
 At the moment there's no easy way to do just that, but you can consider:
 
 * Hit the "Watch" button on top of GitHub web interface to know as early as possible, even during pull request.
-* Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for .
+* Some search engines allow to subscribe on specific website changes via email and you can opt-in for that for .
 
 
diff --git a/docs/zh/data_types/domains/ipv4.md b/docs/zh/data_types/domains/ipv4.md
index 4adf13409fe..aab11f30c27 100644
--- a/docs/zh/data_types/domains/ipv4.md
+++ b/docs/zh/data_types/domains/ipv4.md
@@ -26,16 +26,16 @@ CREATE TABLE hits (url String, from IPv4) ENGINE = MergeTree() ORDER BY from;
 在写入与查询时，`IPv4`类型能够识别可读性更加友好的输入输出格式：
 
 ``` sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.yandex', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '116.253.40.133')('https://clickhouse.tech', '183.247.232.58')('https://clickhouse.yandex/docs/en/', '116.106.34.242');
 
 SELECT * FROM hits;
 ```
 
 ```
 ┌─url────────────────────────────────┬───────────from─┐
-│ https://clickhouse.yandex/docs/en/ │ 116.106.34.242 │
+│ https://clickhouse.tech/docs/en/ │ 116.106.34.242 │
 │ https://wikipedia.org              │ 116.253.40.133 │
-│ https://clickhouse.yandex          │ 183.247.232.58 │
+│ https://clickhouse.tech          │ 183.247.232.58 │
 └────────────────────────────────────┴────────────────┘
 ```
 
@@ -75,4 +75,4 @@ SELECT toTypeName(i), CAST(from as UInt32) as i FROM hits LIMIT 1;
 └──────────────────────────────────┴────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/domains/ipv4) 
+[来源文章](https://clickhouse.tech/docs/en/data_types/domains/ipv4) 
diff --git a/docs/zh/data_types/domains/ipv6.md b/docs/zh/data_types/domains/ipv6.md
index 1209350990f..1f89bd93047 100644
--- a/docs/zh/data_types/domains/ipv6.md
+++ b/docs/zh/data_types/domains/ipv6.md
@@ -26,15 +26,15 @@ CREATE TABLE hits (url String, from IPv6) ENGINE = MergeTree() ORDER BY from;
 在写入与查询时，`IPv6`类型能够识别可读性更加友好的输入输出格式：
 
 ``` sql
-INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.yandex', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
+INSERT INTO hits (url, from) VALUES ('https://wikipedia.org', '2a02:aa08:e000:3100::2')('https://clickhouse.tech', '2001:44c8:129:2632:33:0:252:2')('https://clickhouse.yandex/docs/en/', '2a02:e980:1e::1');
 
 SELECT * FROM hits;
 ```
 
 ```
 ┌─url────────────────────────────────┬─from──────────────────────────┐
-│ https://clickhouse.yandex          │ 2001:44c8:129:2632:33:0:252:2 │
-│ https://clickhouse.yandex/docs/en/ │ 2a02:e980:1e::1               │
+│ https://clickhouse.tech          │ 2001:44c8:129:2632:33:0:252:2 │
+│ https://clickhouse.tech/docs/en/ │ 2a02:e980:1e::1               │
 │ https://wikipedia.org              │ 2a02:aa08:e000:3100::2        │
 └────────────────────────────────────┴───────────────────────────────┘
 ```
@@ -75,4 +75,4 @@ SELECT toTypeName(i), CAST(from as FixedString(16)) as i FROM hits LIMIT 1;
 └───────────────────────────────────────────┴─────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/domains/ipv6) 
+[来源文章](https://clickhouse.tech/docs/en/data_types/domains/ipv6) 
diff --git a/docs/zh/data_types/domains/overview.md b/docs/zh/data_types/domains/overview.md
index b4db116e75b..2a56baa4fda 100644
--- a/docs/zh/data_types/domains/overview.md
+++ b/docs/zh/data_types/domains/overview.md
@@ -23,4 +23,4 @@ Domain类型是特定实现的类型，它总是与某个现存的基础类型
 * 当从其他列或表插入数据时，无法将string类型的值隐式地转换为Domain类型的值。
 * 无法对存储为Domain类型的值添加约束。
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/domains/overview) 
+[来源文章](https://clickhouse.tech/docs/en/data_types/domains/overview) 
diff --git a/docs/zh/data_types/fixedstring.md b/docs/zh/data_types/fixedstring.md
index 18ed3ed45bc..44f7bb775a6 100644
--- a/docs/zh/data_types/fixedstring.md
+++ b/docs/zh/data_types/fixedstring.md
@@ -52,4 +52,4 @@ WHERE a = 'b\0'
 
 请注意，`FixedString(N)`的长度是个常量。仅由空字符组成的字符串，函数[length](../query_language/functions/array_functions.md#array_functions-length)返回值为`N`,而函数[empty](../query_language/functions/string_functions.md#string_functions-empty)的返回值为`1`。
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/fixedstring/) 
+[来源文章](https://clickhouse.tech/docs/en/data_types/fixedstring/) 
diff --git a/docs/zh/data_types/nested_data_structures/aggregatefunction.md b/docs/zh/data_types/nested_data_structures/aggregatefunction.md
index ed3cc5960e2..4d4fadc5ffe 100644
--- a/docs/zh/data_types/nested_data_structures/aggregatefunction.md
+++ b/docs/zh/data_types/nested_data_structures/aggregatefunction.md
@@ -61,4 +61,4 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP
 请参阅 [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) 的说明
 
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/aggregatefunction/) 
+[来源文章](https://clickhouse.tech/docs/en/data_types/nested_data_structures/aggregatefunction/) 
diff --git a/docs/zh/data_types/nullable.md b/docs/zh/data_types/nullable.md
index 67d98fa9408..8bcbc115c20 100644
--- a/docs/zh/data_types/nullable.md
+++ b/docs/zh/data_types/nullable.md
@@ -36,4 +36,4 @@ SELECT x + y FROM t_null
 └────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/data_types/nullable/) 
\ No newline at end of file
+[来源文章](https://clickhouse.tech/docs/en/data_types/nullable/) 
\ No newline at end of file
diff --git a/docs/zh/database_engines/index.md b/docs/zh/database_engines/index.md
index f8ae05e2520..a3ea151d305 100644
--- a/docs/zh/database_engines/index.md
+++ b/docs/zh/database_engines/index.md
@@ -8,4 +8,4 @@
 
 - [MySQL](mysql.md)
 
-[来源文章](https://clickhouse.yandex/docs/en/database_engines/) 
+[来源文章](https://clickhouse.tech/docs/en/database_engines/) 
diff --git a/docs/zh/database_engines/mysql.md b/docs/zh/database_engines/mysql.md
index 38dfcb5ef64..40afdec5a7f 100644
--- a/docs/zh/database_engines/mysql.md
+++ b/docs/zh/database_engines/mysql.md
@@ -121,4 +121,4 @@ SELECT * FROM mysql_db.mysql_table
 └────────┴───────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/database_engines/mysql/) 
+[来源文章](https://clickhouse.tech/docs/en/database_engines/mysql/) 
diff --git a/docs/zh/development/architecture.md b/docs/zh/development/architecture.md
index dd6c96bf113..efb3d7c6716 100644
--- a/docs/zh/development/architecture.md
+++ b/docs/zh/development/architecture.md
@@ -191,4 +191,4 @@ ClickHouse 中的复制是基于表实现的。你可以在同一个服务器上
 
 > ClickHouse 集群由独立的分片组成，每一个分片由多个副本组成。集群不是弹性的，因此在添加新的分片后，数据不会自动在分片之间重新平衡。相反，集群负载将变得不均衡。该实现为你提供了更多控制，对于相对较小的集群，例如只有数十个节点的集群来说是很好的。但是对于我们在生产中使用的具有数百个节点的集群来说，这种方法成为一个重大缺陷。我们应该实现一个表引擎，使得该引擎能够跨集群扩展数据，同时具有动态复制的区域，这些区域能够在集群之间自动拆分和平衡。
 
-[来源文章](https://clickhouse.yandex/docs/en/development/architecture/) 
+[来源文章](https://clickhouse.tech/docs/en/development/architecture/) 
diff --git a/docs/zh/development/build.md b/docs/zh/development/build.md
index 5b3117ff39e..94db13689a9 100644
--- a/docs/zh/development/build.md
+++ b/docs/zh/development/build.md
@@ -84,4 +84,4 @@ cd ..
 这个命令会使得 `dbms/programs/clickhouse` 文件可执行，您可以使用 `client` or `server` 参数运行。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/development/build/) 
+[来源文章](https://clickhouse.tech/docs/en/development/build/) 
diff --git a/docs/zh/development/build_cross_osx.md b/docs/zh/development/build_cross_osx.md
index ef31386c9f6..31fee6ca31e 100644
--- a/docs/zh/development/build_cross_osx.md
+++ b/docs/zh/development/build_cross_osx.md
@@ -1,6 +1,6 @@
 # 如何在Linux中编译Mac OS X ClickHouse
 
-Linux机器也可以编译运行在OS X系统的`clickhouse`二进制包，这可以用于在Linux上跑持续集成测试。如果要在Mac OS X上直接构建ClickHouse，请参考另外一篇指南： https://clickhouse.yandex/docs/zh/development/build_osx/
+Linux机器也可以编译运行在OS X系统的`clickhouse`二进制包，这可以用于在Linux上跑持续集成测试。如果要在Mac OS X上直接构建ClickHouse，请参考另外一篇指南： https://clickhouse.tech/docs/zh/development/build_osx/
 
 Mac OS X的交叉编译基于以下构建说明，请首先遵循它们。
 
diff --git a/docs/zh/development/build_osx.md b/docs/zh/development/build_osx.md
index 50085fb9f78..c4918b4f145 100644
--- a/docs/zh/development/build_osx.md
+++ b/docs/zh/development/build_osx.md
@@ -80,4 +80,4 @@ $ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
 可以通过 `ulimit -n` 命令来检查是否生效。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/development/build_osx/) 
+[来源文章](https://clickhouse.tech/docs/en/development/build_osx/) 
diff --git a/docs/zh/development/developer_instruction.md b/docs/zh/development/developer_instruction.md
index f78a9f422b9..25762967a84 100644
--- a/docs/zh/development/developer_instruction.md
+++ b/docs/zh/development/developer_instruction.md
@@ -114,7 +114,7 @@ Yandex官方当前使用GCC构建ClickHouse，因为它生成的机器代码性
 
 在Ubuntu上安装GCC，请执行：`sudo apt install gcc g++`
 
-请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.yandex/docs/en/development/build/#install-gcc-9 。
+请使用`gcc --version`查看gcc的版本。如果gcc版本低于9，请参考此处的指示：https://clickhouse.tech/docs/en/development/build/#install-gcc-9 。
 
 在Mac OS X上安装GCC，请执行：`brew install gcc`
 
@@ -239,11 +239,11 @@ KDevelop和QTCreator是另外两款适合开发ClickHouse的替代IDE。尽管
 
 # 编写代码
 
-ClickHouse的架构描述可以在此处查看：https://clickhouse.yandex/docs/en/development/architecture/
+ClickHouse的架构描述可以在此处查看：https://clickhouse.tech/docs/en/development/architecture/
 
-代码风格指引：https://clickhouse.yandex/docs/en/development/style/
+代码风格指引：https://clickhouse.tech/docs/en/development/style/
 
-编写测试用例：https://clickhouse.yandex/docs/en/development/tests/
+编写测试用例：https://clickhouse.tech/docs/en/development/tests/
 
 任务列表：https://github.com/yandex/ClickHouse/blob/master/dbms/tests/instructions/easy_tasks_sorted_en.md
 
diff --git a/docs/zh/development/index.md b/docs/zh/development/index.md
index 7053cfcb7b4..d536d93441f 100644
--- a/docs/zh/development/index.md
+++ b/docs/zh/development/index.md
@@ -1,4 +1,4 @@
 # ClickHouse 开发
 
 
-[来源文章](https://clickhouse.yandex/docs/en/development/) 
+[来源文章](https://clickhouse.tech/docs/en/development/) 
diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md
index 06622a3afd5..c0489775fd6 100644
--- a/docs/zh/development/style.md
+++ b/docs/zh/development/style.md
@@ -835,4 +835,4 @@ function(
       size_t limit)
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/development/style/) 
+[来源文章](https://clickhouse.tech/docs/en/development/style/) 
diff --git a/docs/zh/development/tests.md b/docs/zh/development/tests.md
index 2861697fb0c..787314339c5 100644
--- a/docs/zh/development/tests.md
+++ b/docs/zh/development/tests.md
@@ -221,7 +221,7 @@ Yandex Cloud 部门的人员从安全角度对 ClickHouse 功能进行了一些
 
 ## 代码风格
 
-代码风格在[这里](https://clickhouse.yandex/docs/en/development/style/) 有说明。
+代码风格在[这里](https://clickhouse.tech/docs/en/development/style/) 有说明。
 
 要检查一些常见的样式冲突，您可以使用 `utils/check-style` 脚本。
 
@@ -254,4 +254,4 @@ Yandex Cloud 部门的人员从安全角度对 ClickHouse 功能进行了一些
 
 在 Jenkins，我们运行字典测试，指标B2B测试。 我们使用 Jenkins 来准备和发布版本。 Jenkins是一种传统的技术，所有的工作将被转移到沙箱中。
 
-[来源文章](https://clickhouse.yandex/docs/zh/development/tests/) 
+[来源文章](https://clickhouse.tech/docs/zh/development/tests/) 
diff --git a/docs/zh/faq/general.md b/docs/zh/faq/general.md
index a7568cc60b0..f8498cc3156 100644
--- a/docs/zh/faq/general.md
+++ b/docs/zh/faq/general.md
@@ -19,4 +19,4 @@ If you use Oracle through ODBC driver as a source of external dictionaries, you
 NLS_LANG=CHINESE_CHINA.ZHS16GBK
 ```
 
-[来源文章](https://clickhouse.yandex/docs/zh/faq/general/) 
+[来源文章](https://clickhouse.tech/docs/zh/faq/general/) 
diff --git a/docs/zh/getting_started/example_datasets/amplab_benchmark.md b/docs/zh/getting_started/example_datasets/amplab_benchmark.md
index 5afd7dfd705..d0c43a23862 100644
--- a/docs/zh/getting_started/example_datasets/amplab_benchmark.md
+++ b/docs/zh/getting_started/example_datasets/amplab_benchmark.md
@@ -120,4 +120,4 @@ LIMIT 1
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/amplab_benchmark/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/amplab_benchmark/) 
diff --git a/docs/zh/getting_started/example_datasets/criteo.md b/docs/zh/getting_started/example_datasets/criteo.md
index 3a86f630d7b..0c78b06ed75 100644
--- a/docs/zh/getting_started/example_datasets/criteo.md
+++ b/docs/zh/getting_started/example_datasets/criteo.md
@@ -72,4 +72,4 @@ DROP TABLE criteo_log;
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/criteo/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/criteo/) 
diff --git a/docs/zh/getting_started/example_datasets/nyc_taxi.md b/docs/zh/getting_started/example_datasets/nyc_taxi.md
index 338ac5ba0b7..9f7625391cc 100644
--- a/docs/zh/getting_started/example_datasets/nyc_taxi.md
+++ b/docs/zh/getting_started/example_datasets/nyc_taxi.md
@@ -382,4 +382,4 @@ Q4: 0.072 sec.
 | 140 | 0.028 | 0.043 | 0.051 | 0.072 |
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/nyc_taxi/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/nyc_taxi/) 
diff --git a/docs/zh/getting_started/example_datasets/star_schema.md b/docs/zh/getting_started/example_datasets/star_schema.md
index 865327b50ec..1bd83283163 100644
--- a/docs/zh/getting_started/example_datasets/star_schema.md
+++ b/docs/zh/getting_started/example_datasets/star_schema.md
@@ -165,4 +165,4 @@ Q4.3
 SELECT toYear(LO_ORDERDATE) AS year, S_CITY, P_BRAND, sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year, S_CITY, P_BRAND ORDER BY year, S_CITY, P_BRAND;
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/star_schema/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/star_schema/) 
diff --git a/docs/zh/getting_started/example_datasets/wikistat.md b/docs/zh/getting_started/example_datasets/wikistat.md
index ee3b800f47b..218f8fdb5b0 100644
--- a/docs/zh/getting_started/example_datasets/wikistat.md
+++ b/docs/zh/getting_started/example_datasets/wikistat.md
@@ -26,4 +26,4 @@ $ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikis
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/example_datasets/wikistat/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/wikistat/) 
diff --git a/docs/zh/getting_started/index.md b/docs/zh/getting_started/index.md
index c73181a6068..e8edf392cbd 100644
--- a/docs/zh/getting_started/index.md
+++ b/docs/zh/getting_started/index.md
@@ -7,4 +7,4 @@
 * [通过详细的教程](tutorial.md)
 * [试验示例数据集](example_datasets/ontime.md)
 
-[来源文章](https://clickhouse.yandex/docs/zh/getting_started/) 
+[来源文章](https://clickhouse.tech/docs/zh/getting_started/) 
diff --git a/docs/zh/getting_started/install.md b/docs/zh/getting_started/install.md
index aa3cb816218..34114ecd40c 100644
--- a/docs/zh/getting_started/install.md
+++ b/docs/zh/getting_started/install.md
@@ -149,4 +149,4 @@ SELECT 1
 为了继续进行实验，你可以尝试下载测试数据集。
 
 
-[Original article](https://clickhouse.yandex/docs/en/getting_started/install/) 
+[Original article](https://clickhouse.tech/docs/en/getting_started/install/) 
diff --git a/docs/zh/index.md b/docs/zh/index.md
index 6c16c93eafb..cc48a721a86 100644
--- a/docs/zh/index.md
+++ b/docs/zh/index.md
@@ -138,4 +138,4 @@ LIMIT 20
 
 请注意，为了提高CPU效率，查询语言必须是声明型的(SQL或MDX)， 或者至少一个向量(J，K)。 查询应该只包含隐式循环，允许进行优化。
 
-[来源文章](https://clickhouse.yandex/docs/zh/) 
+[来源文章](https://clickhouse.tech/docs/zh/) 
diff --git a/docs/zh/interfaces/cli.md b/docs/zh/interfaces/cli.md
index 8fd3ea7cf18..91a6f71ca5a 100644
--- a/docs/zh/interfaces/cli.md
+++ b/docs/zh/interfaces/cli.md
@@ -110,4 +110,4 @@ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMA
 ```
 
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/cli/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/cli/) 
diff --git a/docs/zh/interfaces/cpp.md b/docs/zh/interfaces/cpp.md
index ff791b38d13..7d7bf7c0cbf 100644
--- a/docs/zh/interfaces/cpp.md
+++ b/docs/zh/interfaces/cpp.md
@@ -2,4 +2,4 @@
 
 请参阅以下网站的自述文件[clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp)资料库。
 
-[Original article](https://clickhouse.yandex/docs/zh/interfaces/cpp/) 
+[Original article](https://clickhouse.tech/docs/zh/interfaces/cpp/) 
diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md
index e4663c2d418..aeba2df7f47 100644
--- a/docs/zh/interfaces/formats.md
+++ b/docs/zh/interfaces/formats.md
@@ -762,4 +762,4 @@ struct Message {
 
 Cap'n Proto 反序列化是很高效的，通常不会增加系统的负载。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/formats/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/formats/) 
diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md
index 5923359e763..a775d489e05 100644
--- a/docs/zh/interfaces/http.md
+++ b/docs/zh/interfaces/http.md
@@ -214,4 +214,4 @@ curl -sS 'http://localhost:8123/?max_result_bytes=4000000&buffer_size=3000000&wa
 
 查询请求响应状态码和 HTTP 头被发送到客户端后，若发生查询处理出错，使用缓冲区可以避免这种情况的发生。在这种情况下，响应主体的结尾会写入一条错误消息，而在客户端，只能在解析阶段检测到该错误。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/http_interface/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/http_interface/) 
diff --git a/docs/zh/interfaces/index.md b/docs/zh/interfaces/index.md
index 3336aa4d105..577cf707e57 100644
--- a/docs/zh/interfaces/index.md
+++ b/docs/zh/interfaces/index.md
@@ -16,4 +16,4 @@ ClickHouse提供了两个网络接口（两者都可以选择包装在TLS中以
 * [集成](third-party/integrations.md)
 * [可视界面](third-party/gui.md)
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/) 
diff --git a/docs/zh/interfaces/jdbc.md b/docs/zh/interfaces/jdbc.md
index 95640cdfaab..1a38d71c693 100644
--- a/docs/zh/interfaces/jdbc.md
+++ b/docs/zh/interfaces/jdbc.md
@@ -6,4 +6,4 @@
   * [clickhouse4j](https://github.com/blynkkk/clickhouse4j)
 
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/jdbc/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/jdbc/) 
diff --git a/docs/zh/interfaces/odbc.md b/docs/zh/interfaces/odbc.md
index c08fc101cb7..b3014e718c4 100644
--- a/docs/zh/interfaces/odbc.md
+++ b/docs/zh/interfaces/odbc.md
@@ -2,4 +2,4 @@
 
 - ClickHouse官方有 ODBC 的驱动。 见 [这里](https://github.com/ClickHouse/clickhouse-odbc)。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/odbc/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/odbc/) 
diff --git a/docs/zh/interfaces/tcp.md b/docs/zh/interfaces/tcp.md
index d07f5d14f1d..a5adda268c9 100644
--- a/docs/zh/interfaces/tcp.md
+++ b/docs/zh/interfaces/tcp.md
@@ -2,4 +2,4 @@
 
 本机协议用于 [命令行客户端](cli.md)，用于分布式查询处理期间的服务器间通信，以及其他C ++程序。 不幸的是，本机ClickHouse协议还没有正式的规范，但它可以从ClickHouse源代码进行逆向工程 [从这里开始](https://github.com/ClickHouse/ClickHouse/tree/master/dbms/src/Client)）和/或拦截和分析TCP流量。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/tcp/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/tcp/) 
diff --git a/docs/zh/interfaces/third-party/client_libraries.md b/docs/zh/interfaces/third-party/client_libraries.md
index 3a814f05237..54675acf4b1 100644
--- a/docs/zh/interfaces/third-party/client_libraries.md
+++ b/docs/zh/interfaces/third-party/client_libraries.md
@@ -46,4 +46,4 @@
     - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse)
 
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/third-party/client_libraries/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/client_libraries/) 
diff --git a/docs/zh/interfaces/third-party/gui.md b/docs/zh/interfaces/third-party/gui.md
index b8143792981..64234f4f8de 100644
--- a/docs/zh/interfaces/third-party/gui.md
+++ b/docs/zh/interfaces/third-party/gui.md
@@ -100,4 +100,4 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix).
  - 重构。
  - 搜索和导航。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/third-party/gui/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/gui/) 
diff --git a/docs/zh/interfaces/third-party/integrations.md b/docs/zh/interfaces/third-party/integrations.md
index 253939827df..a37e7610656 100644
--- a/docs/zh/interfaces/third-party/integrations.md
+++ b/docs/zh/interfaces/third-party/integrations.md
@@ -81,4 +81,4 @@
     - [Ecto](https://github.com/elixir-ecto/ecto)
         - [clickhouse_ecto](https://github.com/appodeal/clickhouse_ecto)
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/third-party/integrations/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/integrations/) 
diff --git a/docs/zh/interfaces/third-party/proxy.md b/docs/zh/interfaces/third-party/proxy.md
index ee5ff42bbb9..df92bd4dbe5 100644
--- a/docs/zh/interfaces/third-party/proxy.md
+++ b/docs/zh/interfaces/third-party/proxy.md
@@ -34,4 +34,4 @@
 
 在Go中实现。
 
-[来源文章](https://clickhouse.yandex/docs/zh/interfaces/third-party/proxy/) 
+[来源文章](https://clickhouse.tech/docs/zh/interfaces/third-party/proxy/) 
diff --git a/docs/zh/introduction/distinctive_features.md b/docs/zh/introduction/distinctive_features.md
index bf45b81ef2a..394627f1d87 100644
--- a/docs/zh/introduction/distinctive_features.md
+++ b/docs/zh/introduction/distinctive_features.md
@@ -61,4 +61,4 @@ ClickHouse使用异步的多主复制技术。当数据被写入任何一个可
 
 更多信息，参见 [数据复制](../operations/table_engines/replication.md)。
 
-[来源文章](https://clickhouse.yandex/docs/en/introduction/distinctive_features/) 
+[来源文章](https://clickhouse.tech/docs/en/introduction/distinctive_features/) 
diff --git a/docs/zh/introduction/features_considered_disadvantages.md b/docs/zh/introduction/features_considered_disadvantages.md
index 015f1481b80..9f916b2c20b 100644
--- a/docs/zh/introduction/features_considered_disadvantages.md
+++ b/docs/zh/introduction/features_considered_disadvantages.md
@@ -4,4 +4,4 @@
 2. 缺少高频率，低延迟的修改或删除已存在数据的能力。仅能用于批量删除或修改数据，但这符合 [GDPR](https://gdpr-info.eu)。
 3. 稀疏索引使得ClickHouse不适合通过其键检索单行的点查询。
 
-[来源文章](https://clickhouse.yandex/docs/zh/introduction/features_considered_disadvantages/) 
+[来源文章](https://clickhouse.tech/docs/zh/introduction/features_considered_disadvantages/) 
diff --git a/docs/zh/introduction/history.md b/docs/zh/introduction/history.md
index 86fe02f84d5..cd25af50942 100644
--- a/docs/zh/introduction/history.md
+++ b/docs/zh/introduction/history.md
@@ -47,4 +47,4 @@ OLAPServer可以很好的工作在非聚合数据上，但是它有诸多限制
 为了消除OLAPServer的这些局限性，解决所有报表使用非聚合数据的问题，我们开发了ClickHouse数据库管理系统。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/introduction/ya_metrika_task/) 
+[来源文章](https://clickhouse.tech/docs/en/introduction/ya_metrika_task/) 
diff --git a/docs/zh/introduction/performance.md b/docs/zh/introduction/performance.md
index a740db13995..21c3d0c562e 100644
--- a/docs/zh/introduction/performance.md
+++ b/docs/zh/introduction/performance.md
@@ -1,8 +1,8 @@
 # Performance
 
-根据Yandex的内部测试结果，ClickHouse表现出了比同类可比较产品更优的性能。你可以在 [这里](https://clickhouse.yandex/benchmark.html) 查看具体的测试结果。
+根据Yandex的内部测试结果，ClickHouse表现出了比同类可比较产品更优的性能。你可以在 [这里](/benchmark.html) 查看具体的测试结果。
 
-许多其他的测试也证实这一点。你可以使用互联网搜索到它们，或者你也可以从 [我们收集的部分相关连接](https://clickhouse.yandex/#independent-benchmarks) 中查看。
+许多其他的测试也证实这一点。你可以使用互联网搜索到它们，或者你也可以从 [我们收集的部分相关连接](/#independent-benchmarks) 中查看。
 
 ## 单个大查询的吞吐量
 
@@ -22,4 +22,4 @@
 
 我们建议每次写入不少于1000行的批量写入，或每秒不超过一个写入请求。当使用tab-separated格式将一份数据写入到MergeTree表中时，写入速度大约为50到200MB/s。如果您写入的数据每行为1Kb，那么写入的速度为50，000到200，000行每秒。如果您的行更小，那么写入速度将更高。为了提高写入性能，您可以使用多个INSERT进行并行写入，这将带来线性的性能提升。
 
-[来源文章](https://clickhouse.yandex/docs/en/introduction/performance/) 
+[来源文章](https://clickhouse.tech/docs/en/introduction/performance/) 
diff --git a/docs/zh/operations/access_rights.md b/docs/zh/operations/access_rights.md
index 621d1673d8f..ebe3a2e7c0f 100644
--- a/docs/zh/operations/access_rights.md
+++ b/docs/zh/operations/access_rights.md
@@ -99,4 +99,4 @@ The user can get a list of all databases and tables in them by using `SHOW` quer
 Database access is not related to the [readonly](settings/permissions_for_queries.md#settings_readonly) setting. You can't grant full access to one database and `readonly` access to another one.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/access_rights/) 
+[Original article](https://clickhouse.tech/docs/en/operations/access_rights/) 
diff --git a/docs/zh/operations/configuration_files.md b/docs/zh/operations/configuration_files.md
index eafa5d34235..c86e5720247 100644
--- a/docs/zh/operations/configuration_files.md
+++ b/docs/zh/operations/configuration_files.md
@@ -39,4 +39,4 @@ For each config file, the server also generates `file-preprocessed.xml` files wh
 
 The server tracks changes in config files, as well as files and ZooKeeper nodes that were used when performing substitutions and overrides, and reloads the settings for users and clusters on the fly. This means that you can modify the cluster, users, and their settings without restarting the server.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/configuration_files/) 
+[Original article](https://clickhouse.tech/docs/en/operations/configuration_files/) 
diff --git a/docs/zh/operations/index.md b/docs/zh/operations/index.md
index 63cb19bb639..1c5bb3198e3 100644
--- a/docs/zh/operations/index.md
+++ b/docs/zh/operations/index.md
@@ -1,4 +1,4 @@
 # Operations 
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/) 
+[Original article](https://clickhouse.tech/docs/en/operations/) 
diff --git a/docs/zh/operations/quotas.md b/docs/zh/operations/quotas.md
index 5379d242da3..85ba0577f03 100644
--- a/docs/zh/operations/quotas.md
+++ b/docs/zh/operations/quotas.md
@@ -103,4 +103,4 @@ For distributed query processing, the accumulated amounts are stored on the requ
 When the server is restarted, quotas are reset.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/quotas/) 
+[Original article](https://clickhouse.tech/docs/en/operations/quotas/) 
diff --git a/docs/zh/operations/server_settings/index.md b/docs/zh/operations/server_settings/index.md
index 3b721f95ef2..39f3a5bb6d4 100644
--- a/docs/zh/operations/server_settings/index.md
+++ b/docs/zh/operations/server_settings/index.md
@@ -9,4 +9,4 @@ Other settings are described in the "[Settings](../settings/index.md#settings)"
 Before studying the settings, read the [Configuration files](../configuration_files.md#configuration_files) section and note the use of substitutions (the `incl` and `optional` attributes).
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/server_settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/server_settings/) 
diff --git a/docs/zh/operations/settings/index.md b/docs/zh/operations/settings/index.md
index 73152183079..1d914cb2a24 100644
--- a/docs/zh/operations/settings/index.md
+++ b/docs/zh/operations/settings/index.md
@@ -21,4 +21,4 @@ Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you
 Settings that can only be made in the server config file are not covered in this section.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/) 
diff --git a/docs/zh/operations/settings/query_complexity.md b/docs/zh/operations/settings/query_complexity.md
index 1e3c94bc2e1..b1d5484ae4e 100644
--- a/docs/zh/operations/settings/query_complexity.md
+++ b/docs/zh/operations/settings/query_complexity.md
@@ -190,4 +190,4 @@ Maximum number of bytes (uncompressed data) that can be passed to a remote serve
 
 What to do when the amount of data exceeds one of the limits: 'throw' or 'break'. By default, throw.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/query_complexity/) 
+[Original article](https://clickhouse.tech/docs/en/operations/settings/query_complexity/) 
diff --git a/docs/zh/operations/table_engines/aggregatingmergetree.md b/docs/zh/operations/table_engines/aggregatingmergetree.md
index 4fd511ff9cf..f4e1319620a 100644
--- a/docs/zh/operations/table_engines/aggregatingmergetree.md
+++ b/docs/zh/operations/table_engines/aggregatingmergetree.md
@@ -90,4 +90,4 @@ GROUP BY StartDate
 ORDER BY StartDate;
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/aggregatingmergetree/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/aggregatingmergetree/) 
diff --git a/docs/zh/operations/table_engines/buffer.md b/docs/zh/operations/table_engines/buffer.md
index c326f619d70..aec20029933 100644
--- a/docs/zh/operations/table_engines/buffer.md
+++ b/docs/zh/operations/table_engines/buffer.md
@@ -53,4 +53,4 @@ PREWHERE，FINAL 和 SAMPLE 对缓冲表不起作用。这些条件将传递到
 请注意，一次插入一行数据是没有意义的，即使对于 Buffer 表也是如此。这将只产生每秒几千行的速度，而插入更大的数据块每秒可以产生超过一百万行（参见 "性能" 部分）。
 
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/buffer/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/buffer/) 
diff --git a/docs/zh/operations/table_engines/collapsingmergetree.md b/docs/zh/operations/table_engines/collapsingmergetree.md
index ce22aeadf29..fee3db919e2 100644
--- a/docs/zh/operations/table_engines/collapsingmergetree.md
+++ b/docs/zh/operations/table_engines/collapsingmergetree.md
@@ -214,4 +214,4 @@ SELECT * FROM UAct FINAL
 
 这种查询数据的方法是非常低效的。不要在大表中使用它。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/collapsingmergetree/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/collapsingmergetree/) 
diff --git a/docs/zh/operations/table_engines/custom_partitioning_key.md b/docs/zh/operations/table_engines/custom_partitioning_key.md
index 8f43f4068b8..0517e1ec7d5 100644
--- a/docs/zh/operations/table_engines/custom_partitioning_key.md
+++ b/docs/zh/operations/table_engines/custom_partitioning_key.md
@@ -117,4 +117,4 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb  1 16:48 detached
 
 ClickHouse 支持对分区执行这些操作：删除分区，从一个表复制到另一个表，或创建备份。了解分区的所有操作，请参阅 [分区和片段的操作](../../query_language/alter.md#alter_manipulations-with-partitions) 一节。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/custom_partitioning_key/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/custom_partitioning_key/) 
diff --git a/docs/zh/operations/table_engines/dictionary.md b/docs/zh/operations/table_engines/dictionary.md
index b5e49371101..d08155f2ca0 100644
--- a/docs/zh/operations/table_engines/dictionary.md
+++ b/docs/zh/operations/table_engines/dictionary.md
@@ -108,4 +108,4 @@ LIMIT 1
 ```
 
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/dictionary/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/dictionary/) 
diff --git a/docs/zh/operations/table_engines/distributed.md b/docs/zh/operations/table_engines/distributed.md
index f02b45e6d47..06b600b373c 100644
--- a/docs/zh/operations/table_engines/distributed.md
+++ b/docs/zh/operations/table_engines/distributed.md
@@ -122,4 +122,4 @@ SELECT 查询会被发送到所有分片，并且无论数据在分片中如何
 启用 max_parallel_replicas 选项后，会在分表的所有副本上并行查询处理。更多信息，请参阅“设置，max_parallel_replicas”部分。
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/distributed/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/distributed/) 
diff --git a/docs/zh/operations/table_engines/external_data.md b/docs/zh/operations/table_engines/external_data.md
index 60a9612da12..4c1d80183c4 100644
--- a/docs/zh/operations/table_engines/external_data.md
+++ b/docs/zh/operations/table_engines/external_data.md
@@ -59,4 +59,4 @@ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+count(
 对于分布式查询，将临时表发送到所有远程服务器。
 
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/external_data/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/external_data/) 
diff --git a/docs/zh/operations/table_engines/file.md b/docs/zh/operations/table_engines/file.md
index e7026177fb5..dcdc091a84b 100644
--- a/docs/zh/operations/table_engines/file.md
+++ b/docs/zh/operations/table_engines/file.md
@@ -74,4 +74,4 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64
   - 索引
   - 副本
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/file/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/file/) 
diff --git a/docs/zh/operations/table_engines/index.md b/docs/zh/operations/table_engines/index.md
index 3d618d01496..6ffc3149d39 100644
--- a/docs/zh/operations/table_engines/index.md
+++ b/docs/zh/operations/table_engines/index.md
@@ -12,51 +12,51 @@
 # 引擎类型
 ## MergeTree
 
-适用于高负载任务的最通用和功能最强大的表引擎。这些引擎的共同特点是可以快速插入数据并进行后续的后台数据处理。 MergeTree系列引擎支持数据复制（使用[Replicated*](https://clickhouse.yandex/docs/en/operations/table_engines/replication/) 的引擎版本），分区和一些其他引擎不支持的其他功能。
+适用于高负载任务的最通用和功能最强大的表引擎。这些引擎的共同特点是可以快速插入数据并进行后续的后台数据处理。 MergeTree系列引擎支持数据复制（使用[Replicated*](replication.md) 的引擎版本），分区和一些其他引擎不支持的其他功能。
 
 该类型的引擎：
-* [MergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/)
-* [ReplacingMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/replacingmergetree/)
-* [SummingMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/summingmergetree/)
-* [AggregatingMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/aggregatingmergetree/)
-* [CollapsingMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/collapsingmergetree/)
-* [VersionedCollapsingMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/versionedcollapsingmergetree/)
-* [GraphiteMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/graphitemergetree/)
+* [MergeTree](mergetree.md)
+* [ReplacingMergeTree](replacingmergetree.md)
+* [SummingMergeTree](summingmergetree.md)
+* [AggregatingMergeTree](aggregatingmergetree.md)
+* [CollapsingMergeTree](collapsingmergetree.md)
+* [VersionedCollapsingMergeTree](versionedcollapsingmergetree.md)
+* [GraphiteMergeTree](graphitemergetree.md)
 
 ## Log
-具有最小功能的[轻量级引擎](https://clickhouse.yandex/docs/en/operations/table_engines/log_family/)。当您需要快速写入许多小表（最多约100万行）并在以后整体读取它们时，该类型的引擎是最有效的。
+具有最小功能的[轻量级引擎](log_family.md)。当您需要快速写入许多小表（最多约100万行）并在以后整体读取它们时，该类型的引擎是最有效的。
 
 该类型的引擎：
 
-* [TinyLog](https://clickhouse.yandex/docs/en/operations/table_engines/tinylog/）
-* [StripeLog](https://clickhouse.yandex/docs/en/operations/table_engines/stripelog/）
-* [Log](https://clickhouse.yandex/docs/en/operations/table_engines/log/）
+* [TinyLog](tinylog/）
+* [StripeLog](stripelog/）
+* [Log](log/）
 
 ## Intergation engines
 用于与其他的数据存储与处理系统集成的引擎。
 该类型的引擎：
 
-* [Kafka](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/)
-* [MySQL](https://clickhouse.yandex/docs/en/operations/table_engines/mysql/)
-* [ODBC](https://clickhouse.yandex/docs/en/operations/table_engines/odbc/)
-* [JDBC](https://clickhouse.yandex/docs/en/operations/table_engines/jdbc/)
-* [HDFS](https://clickhouse.yandex/docs/en/operations/table_engines/hdfs/)
+* [Kafka](kafka.md)
+* [MySQL](mysql.md)
+* [ODBC](odbc.md)
+* [JDBC](jdbc.md)
+* [HDFS](hdfs.md)
 
 ## 用于其他特定功能的引擎
 该类型的引擎：
 
-* [Distributed](https://clickhouse.yandex/docs/en/operations/table_engines/distributed/)
-* [MaterializedView](https://clickhouse.yandex/docs/en/operations/table_engines/materializedview/)
-* [Dictionary](https://clickhouse.yandex/docs/en/operations/table_engines/dictionary/)
-* [Merge](https://clickhouse.yandex/docs/en/operations/table_engines/merge/)
-* [File](https://clickhouse.yandex/docs/en/operations/table_engines/file/)
-* [Null](https://clickhouse.yandex/docs/en/operations/table_engines/null/)
-* [Set](https://clickhouse.yandex/docs/en/operations/table_engines/set/)
-* [Join](https://clickhouse.yandex/docs/en/operations/table_engines/join/)
-* [URL](https://clickhouse.yandex/docs/en/operations/table_engines/url/)
-* [View](https://clickhouse.yandex/docs/en/operations/table_engines/view/)
-* [Memory](https://clickhouse.yandex/docs/en/operations/table_engines/memory/)
-* [Buffer](https://clickhouse.yandex/docs/en/operations/table_engines/buffer/)
+* [Distributed](distributed.md)
+* [MaterializedView](materializedview.md)
+* [Dictionary](dictionary.md)
+* [Merge](merge.md)
+* [File](file.md)
+* [Null](null.md)
+* [Set](set.md)
+* [Join](join.md)
+* [URL](url.md)
+* [View](view.md)
+* [Memory](memory.md)
+* [Buffer](buffer.md)
 
 #虚拟列
 虚拟列是表引擎组成的一部分，它在对应的表引擎的源代码中定义。
diff --git a/docs/zh/operations/table_engines/join.md b/docs/zh/operations/table_engines/join.md
index d037ddbed9a..397506daaec 100644
--- a/docs/zh/operations/table_engines/join.md
+++ b/docs/zh/operations/table_engines/join.md
@@ -27,4 +27,4 @@ When creating a table, the following settings are applied:
 
 The `Join`-engine tables can't be used in `GLOBAL JOIN` operations.
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/join/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/join/) 
diff --git a/docs/zh/operations/table_engines/kafka.md b/docs/zh/operations/table_engines/kafka.md
index cda367793bb..de70c4a6d35 100644
--- a/docs/zh/operations/table_engines/kafka.md
+++ b/docs/zh/operations/table_engines/kafka.md
@@ -138,4 +138,4 @@ Kafka SETTINGS
 
 有关详细配置选项列表，请参阅 [librdkafka configuration reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)。在 ClickHouse 配置中使用下划线 (`_`) ，并不是使用点 (`.`)。例如，`check.crcs=true` 将是 `true`。
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/kafka/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/kafka/) 
diff --git a/docs/zh/operations/table_engines/log.md b/docs/zh/operations/table_engines/log.md
index 96e4d59e637..47ad15f7ea8 100644
--- a/docs/zh/operations/table_engines/log.md
+++ b/docs/zh/operations/table_engines/log.md
@@ -3,5 +3,5 @@
 日志与 TinyLog 的不同之处在于，"标记" 的小文件与列文件存在一起。这些标记写在每个数据块上，并且包含偏移量，这些偏移量指示从哪里开始读取文件以便跳过指定的行数。这使得可以在多个线程中读取表数据。对于并发数据访问，可以同时执行读取操作，而写入操作则阻塞读取和其它写入。Log 引擎不支持索引。同样，如果写入表失败，则该表将被破坏，并且从该表读取将返回错误。Log 引擎适用于临时数据，write-once 表以及测试或演示目的。
 
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/log/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/log/) 
 
diff --git a/docs/zh/operations/table_engines/log_family.md b/docs/zh/operations/table_engines/log_family.md
index a1647cd473c..0726040ff07 100644
--- a/docs/zh/operations/table_engines/log_family.md
+++ b/docs/zh/operations/table_engines/log_family.md
@@ -39,4 +39,4 @@
 
 `TingLog` 引擎是该系列中最简单的引擎并且提供了最少的功能和最低的性能。`TingLog` 引擎不支持并行读取和并发数据访问，并将每一列存储在不同的文件中。它比其余两种支持并行读取的引擎的读取速度更慢，并且使用了和 `Log` 引擎同样多的描述符。你可以在简单的低负载的情景下使用它。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/log_family/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/log_family/) 
diff --git a/docs/zh/operations/table_engines/materializedview.md b/docs/zh/operations/table_engines/materializedview.md
index 146d3136ea9..c0c6aee88e5 100644
--- a/docs/zh/operations/table_engines/materializedview.md
+++ b/docs/zh/operations/table_engines/materializedview.md
@@ -3,4 +3,4 @@
 物化视图的使用（更多信息请参阅 [CREATE TABLE](../../query_language/create.md) ）。它需要使用一个不同的引擎来存储数据，这个引擎要在创建物化视图时指定。当从表中读取时，它就会使用该引擎。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/materializedview/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/materializedview/) 
diff --git a/docs/zh/operations/table_engines/memory.md b/docs/zh/operations/table_engines/memory.md
index 4e6c2d092c3..7d2d6dc0f60 100644
--- a/docs/zh/operations/table_engines/memory.md
+++ b/docs/zh/operations/table_engines/memory.md
@@ -5,5 +5,5 @@ Memory 引擎以未压缩的形式将数据存储在 RAM 中。数据完全以
 Memory 引擎是由系统用于临时表进行外部数据的查询（请参阅 "外部数据用于请求处理" 部分），以及用于实现 `GLOBAL IN`（请参见 "IN 运算符" 部分）。
 
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/memory/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/memory/) 
 
diff --git a/docs/zh/operations/table_engines/merge.md b/docs/zh/operations/table_engines/merge.md
index b0cf9fe097e..722f510d135 100644
--- a/docs/zh/operations/table_engines/merge.md
+++ b/docs/zh/operations/table_engines/merge.md
@@ -66,4 +66,4 @@ FROM WatchLog
 如果 `WHERE/PREWHERE` 子句包含了带 `_table` 的条件，并且没有依赖其他的列（如作为表达式谓词链接的一个子项或作为整个的表达式），这些条件的作用会像索引一样。这些条件会在那些可能被读数据的表的表名上执行，并且读操作只会在那些满足了该条件的表上去执行。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/merge/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/merge/) 
diff --git a/docs/zh/operations/table_engines/mergetree.md b/docs/zh/operations/table_engines/mergetree.md
index 2beeca1a9cb..984f0339aad 100644
--- a/docs/zh/operations/table_engines/mergetree.md
+++ b/docs/zh/operations/table_engines/mergetree.md
@@ -402,4 +402,4 @@ ALTER TABLE example_table
 
 ### Configuration {#table_engine-mergetree-multiple-volumes_configure}
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/mergetree/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/mergetree/) 
diff --git a/docs/zh/operations/table_engines/mysql.md b/docs/zh/operations/table_engines/mysql.md
index a7815f691e2..f4974b9a7d8 100644
--- a/docs/zh/operations/table_engines/mysql.md
+++ b/docs/zh/operations/table_engines/mysql.md
@@ -24,4 +24,4 @@ MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_
 
 `MySQL` 引擎不支持 [Nullable](../../data_types/nullable.md) 数据类型，因此，当从MySQL表中读取数据时，`NULL` 将转换为指定列类型的默认值（通常为0或空字符串）。
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/mysql/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/mysql/) 
diff --git a/docs/zh/operations/table_engines/null.md b/docs/zh/operations/table_engines/null.md
index d760f1efac2..dc2e777739a 100644
--- a/docs/zh/operations/table_engines/null.md
+++ b/docs/zh/operations/table_engines/null.md
@@ -5,4 +5,4 @@
 但是，可以在 Null 类型的表上创建物化视图。写入表的数据将转发到视图中。
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/null/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/null/) 
diff --git a/docs/zh/operations/table_engines/replacingmergetree.md b/docs/zh/operations/table_engines/replacingmergetree.md
index a9bb9d65b6d..b72838d9e21 100644
--- a/docs/zh/operations/table_engines/replacingmergetree.md
+++ b/docs/zh/operations/table_engines/replacingmergetree.md
@@ -55,4 +55,4 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
 - `ver` - 版本列。可选参数，有关说明，请参阅上文。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/replacingmergetree/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/replacingmergetree/) 
diff --git a/docs/zh/operations/table_engines/replication.md b/docs/zh/operations/table_engines/replication.md
index d699404452f..c79606b9d03 100644
--- a/docs/zh/operations/table_engines/replication.md
+++ b/docs/zh/operations/table_engines/replication.md
@@ -202,4 +202,4 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data
 
 如果 ZooKeeper 中的数据丢失或损坏，如上所述，你可以通过将数据转移到非复制表来保存数据。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/replication/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/replication/) 
diff --git a/docs/zh/operations/table_engines/set.md b/docs/zh/operations/table_engines/set.md
index 708ee890e6e..d47cc4721a7 100644
--- a/docs/zh/operations/table_engines/set.md
+++ b/docs/zh/operations/table_engines/set.md
@@ -9,4 +9,4 @@
 对于强制服务器重启，磁盘上的数据块可能会丢失或损坏。在数据块损坏的情况下，可能需要手动删除包含损坏数据的文件。
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/set/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/set/) 
diff --git a/docs/zh/operations/table_engines/stripelog.md b/docs/zh/operations/table_engines/stripelog.md
index 43378293f85..4def6d86a3b 100644
--- a/docs/zh/operations/table_engines/stripelog.md
+++ b/docs/zh/operations/table_engines/stripelog.md
@@ -83,4 +83,4 @@ SELECT * FROM stripe_log_table ORDER BY timestamp
 └─────────────────────┴──────────────┴────────────────────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/stripelog/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/stripelog/) 
diff --git a/docs/zh/operations/table_engines/summingmergetree.md b/docs/zh/operations/table_engines/summingmergetree.md
index ab3a50765e2..03cd0561b0e 100644
--- a/docs/zh/operations/table_engines/summingmergetree.md
+++ b/docs/zh/operations/table_engines/summingmergetree.md
@@ -130,4 +130,4 @@ ClickHouse 会按片段合并数据，以至于不同的数据片段中会包含
 
 对于嵌套数据结构，你无需在列的元组中指定列以进行汇总。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/summingmergetree/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/summingmergetree/) 
diff --git a/docs/zh/operations/table_engines/tinylog.md b/docs/zh/operations/table_engines/tinylog.md
index 2e9364c4cd8..620fb36c905 100644
--- a/docs/zh/operations/table_engines/tinylog.md
+++ b/docs/zh/operations/table_engines/tinylog.md
@@ -11,5 +11,5 @@
 在 Yandex.Metrica 中，TinyLog 表用于小批量处理的中间数据。
 
 
-[Original article](https://clickhouse.yandex/docs/zh/operations/table_engines/tinylog/) 
+[Original article](https://clickhouse.tech/docs/zh/operations/table_engines/tinylog/) 
 
diff --git a/docs/zh/operations/table_engines/url.md b/docs/zh/operations/table_engines/url.md
index be47e6601c7..8928b9232fd 100644
--- a/docs/zh/operations/table_engines/url.md
+++ b/docs/zh/operations/table_engines/url.md
@@ -70,4 +70,4 @@ SELECT * FROM url_engine_table
     - 索引。
     - 副本。
 
-[来源文章](https://clickhouse.yandex/docs/en/operations/table_engines/url/) 
+[来源文章](https://clickhouse.tech/docs/en/operations/table_engines/url/) 
diff --git a/docs/zh/operations/table_engines/view.md b/docs/zh/operations/table_engines/view.md
index 3a7909657a7..4191eae3974 100644
--- a/docs/zh/operations/table_engines/view.md
+++ b/docs/zh/operations/table_engines/view.md
@@ -3,4 +3,4 @@
 用于构建视图（有关更多信息，请参阅 `CREATE VIEW 查询`）。 它不存储数据，仅存储指定的 `SELECT` 查询。 从表中读取时，它会运行此查询（并从查询中删除所有不必要的列）。
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/view/) 
+[Original article](https://clickhouse.tech/docs/en/operations/table_engines/view/) 
diff --git a/docs/zh/operations/tips.md b/docs/zh/operations/tips.md
index 1dcbb74a2cc..4780d3a0b8c 100644
--- a/docs/zh/operations/tips.md
+++ b/docs/zh/operations/tips.md
@@ -256,4 +256,4 @@ end script
 ```
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/tips/) 
+[Original article](https://clickhouse.tech/docs/en/operations/tips/) 
diff --git a/docs/zh/operations/utils/clickhouse-copier.md b/docs/zh/operations/utils/clickhouse-copier.md
index cf5ef7cb7a5..6366c71d60e 100644
--- a/docs/zh/operations/utils/clickhouse-copier.md
+++ b/docs/zh/operations/utils/clickhouse-copier.md
@@ -165,4 +165,4 @@ Parameters:
 `clickhouse-copier` tracks the changes in `/task/path/description` and applies them on the fly. For instance, if you change the value of `max_workers`, the number of processes running tasks will also change.
 
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/clickhouse-copier/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/clickhouse-copier/) 
diff --git a/docs/zh/operations/utils/clickhouse-local.md b/docs/zh/operations/utils/clickhouse-local.md
index e91d1eda2d7..9a0880ac9f1 100644
--- a/docs/zh/operations/utils/clickhouse-local.md
+++ b/docs/zh/operations/utils/clickhouse-local.md
@@ -71,4 +71,4 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec.
 ...
 ```
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/clickhouse-local/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/clickhouse-local/) 
diff --git a/docs/zh/operations/utils/index.md b/docs/zh/operations/utils/index.md
index 3fcd66da8d8..71d46f3f7c8 100644
--- a/docs/zh/operations/utils/index.md
+++ b/docs/zh/operations/utils/index.md
@@ -4,4 +4,4 @@
 - [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
 - [clickhouse-benchmark](clickhouse-benchmark.md) — Loads server with the custom queries and settings.
 
-[Original article](https://clickhouse.yandex/docs/en/operations/utils/) 
+[Original article](https://clickhouse.tech/docs/en/operations/utils/) 
diff --git a/docs/zh/query_language/create.md b/docs/zh/query_language/create.md
index 54ef6ecb0b8..9dc0a8be560 100644
--- a/docs/zh/query_language/create.md
+++ b/docs/zh/query_language/create.md
@@ -246,7 +246,7 @@ SELECT a, b, c FROM (SELECT ...)
 
 没有单独的删除视图的语法。如果要删除视图，请使用`DROP TABLE`。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/create/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/create/) 
 
 ## CREATE DICTIONARY {#create-dictionary-query}
 
diff --git a/docs/zh/query_language/functions/arithmetic_functions.md b/docs/zh/query_language/functions/arithmetic_functions.md
index e75a2c9f5f0..014ec96ce03 100644
--- a/docs/zh/query_language/functions/arithmetic_functions.md
+++ b/docs/zh/query_language/functions/arithmetic_functions.md
@@ -76,4 +76,4 @@ Calculates a number with the reverse sign. The result is always signed.
 除以零或将最小负数除以-1时抛出异常。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/arithmetic_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/arithmetic_functions/) 
diff --git a/docs/zh/query_language/functions/array_functions.md b/docs/zh/query_language/functions/array_functions.md
index 9b760c7b6ae..6f87a62102f 100644
--- a/docs/zh/query_language/functions/array_functions.md
+++ b/docs/zh/query_language/functions/array_functions.md
@@ -718,4 +718,4 @@ arrayReduce（'agg_func'，arr1，...） - 将聚合函数`agg_func`应用于数
 
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) 
diff --git a/docs/zh/query_language/functions/array_join.md b/docs/zh/query_language/functions/array_join.md
index c5095bd5a96..400230d60a2 100644
--- a/docs/zh/query_language/functions/array_join.md
+++ b/docs/zh/query_language/functions/array_join.md
@@ -28,4 +28,4 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src
 ```
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/array_join/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/array_join/) 
diff --git a/docs/zh/query_language/functions/bit_functions.md b/docs/zh/query_language/functions/bit_functions.md
index 1960a5a0d71..0f8e8ad9e6c 100644
--- a/docs/zh/query_language/functions/bit_functions.md
+++ b/docs/zh/query_language/functions/bit_functions.md
@@ -28,4 +28,4 @@
 
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/bit_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/bit_functions/) 
diff --git a/docs/zh/query_language/functions/bitmap_functions.md b/docs/zh/query_language/functions/bitmap_functions.md
index 1fbfd3c8fe7..e8ad2645a1d 100644
--- a/docs/zh/query_language/functions/bitmap_functions.md
+++ b/docs/zh/query_language/functions/bitmap_functions.md
@@ -453,4 +453,4 @@ SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res
 ```
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/bitmap_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/bitmap_functions/) 
diff --git a/docs/zh/query_language/functions/comparison_functions.md b/docs/zh/query_language/functions/comparison_functions.md
index ae12b02a41e..68067a5586a 100644
--- a/docs/zh/query_language/functions/comparison_functions.md
+++ b/docs/zh/query_language/functions/comparison_functions.md
@@ -30,4 +30,4 @@
 ## greaterOrEquals, `>= operator`
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/comparison_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) 
diff --git a/docs/zh/query_language/functions/conditional_functions.md b/docs/zh/query_language/functions/conditional_functions.md
index 7737b865fa1..89babefd338 100644
--- a/docs/zh/query_language/functions/conditional_functions.md
+++ b/docs/zh/query_language/functions/conditional_functions.md
@@ -47,4 +47,4 @@ multiIf(cond_1, then_1, cond_2, then_2...else)
 └────────────────────────────────────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/conditional_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) 
diff --git a/docs/zh/query_language/functions/date_time_functions.md b/docs/zh/query_language/functions/date_time_functions.md
index bbf92dd9718..91f81757830 100644
--- a/docs/zh/query_language/functions/date_time_functions.md
+++ b/docs/zh/query_language/functions/date_time_functions.md
@@ -293,4 +293,4 @@ Supported unit values:
 |%Y|Year|2018|
 |%%|a % sign|%|
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/date_time_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) 
diff --git a/docs/zh/query_language/functions/encoding_functions.md b/docs/zh/query_language/functions/encoding_functions.md
index 7b97ece8fd1..4cc6afd39c7 100644
--- a/docs/zh/query_language/functions/encoding_functions.md
+++ b/docs/zh/query_language/functions/encoding_functions.md
@@ -26,4 +26,4 @@
 接受一个整数。返回一个UInt64类型数组，其中包含一组2的幂列表，其列表中的所有值相加等于这个整数。数组中的数字按升序排列。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/encoding_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/encoding_functions/) 
diff --git a/docs/zh/query_language/functions/ext_dict_functions.md b/docs/zh/query_language/functions/ext_dict_functions.md
index b9d02252e3e..fca1eeb6cc9 100644
--- a/docs/zh/query_language/functions/ext_dict_functions.md
+++ b/docs/zh/query_language/functions/ext_dict_functions.md
@@ -44,4 +44,4 @@
 - 检查字典是否存在指定的`id`。如果不存在，则返回0;如果存在，则返回1。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/ext_dict_functions/) 
diff --git a/docs/zh/query_language/functions/functions_for_nulls.md b/docs/zh/query_language/functions/functions_for_nulls.md
index ec1eda800ee..71bd7a2fe88 100644
--- a/docs/zh/query_language/functions/functions_for_nulls.md
+++ b/docs/zh/query_language/functions/functions_for_nulls.md
@@ -294,4 +294,4 @@ SELECT toTypeName(toNullable(10))
 ```
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/functions_for_nulls/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/functions_for_nulls/) 
diff --git a/docs/zh/query_language/functions/geo.md b/docs/zh/query_language/functions/geo.md
index 2182aaa2fcb..a9d30b91596 100644
--- a/docs/zh/query_language/functions/geo.md
+++ b/docs/zh/query_language/functions/geo.md
@@ -217,4 +217,4 @@ SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos
 └─────────────────────────────────────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/geo/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/geo/) 
diff --git a/docs/zh/query_language/functions/hash_functions.md b/docs/zh/query_language/functions/hash_functions.md
index 9fc6c79b0b3..98ed40d51dd 100644
--- a/docs/zh/query_language/functions/hash_functions.md
+++ b/docs/zh/query_language/functions/hash_functions.md
@@ -111,4 +111,4 @@ URL的层级与URLHierarchy中的层级相同。 此函数被用于Yandex.Metric
 接受一个String类型的参数。返回UInt64或UInt32。
 有关更多信息，请参见链接：[xxHash](http://cyan4973.github.io/xxHash/)
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/hash_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) 
diff --git a/docs/zh/query_language/functions/higher_order_functions.md b/docs/zh/query_language/functions/higher_order_functions.md
index 39c6770e5b8..fff702ab0ec 100644
--- a/docs/zh/query_language/functions/higher_order_functions.md
+++ b/docs/zh/query_language/functions/higher_order_functions.md
@@ -150,4 +150,4 @@ SELECT arrayReverseSort([1, nan, 2, NULL, 3, nan, 4, NULL])
 
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/higher_order_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/higher_order_functions/) 
diff --git a/docs/zh/query_language/functions/in_functions.md b/docs/zh/query_language/functions/in_functions.md
index 1cc90663a0e..3f26d0ce500 100644
--- a/docs/zh/query_language/functions/in_functions.md
+++ b/docs/zh/query_language/functions/in_functions.md
@@ -17,4 +17,4 @@
 执行该函数没有任何成本。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/in_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/in_functions/) 
diff --git a/docs/zh/query_language/functions/index.md b/docs/zh/query_language/functions/index.md
index 815725d7542..ad287b49062 100644
--- a/docs/zh/query_language/functions/index.md
+++ b/docs/zh/query_language/functions/index.md
@@ -64,4 +64,4 @@ ClickHouse中至少存在两种类型的函数 - 常规函数（它们称之为
 如果查询中的函数在请求服务器上执行，但您需要在远程服务器上执行它，则可以将其包装在“any”聚合函数中，或将其添加到“GROUP BY”中。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/) 
diff --git a/docs/zh/query_language/functions/ip_address_functions.md b/docs/zh/query_language/functions/ip_address_functions.md
index bf65d254ab8..ad4212b0e36 100644
--- a/docs/zh/query_language/functions/ip_address_functions.md
+++ b/docs/zh/query_language/functions/ip_address_functions.md
@@ -242,4 +242,4 @@ SELECT
 └───────────────────────────────────┴──────────────────────────────────┘
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/ip_address_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/ip_address_functions/) 
diff --git a/docs/zh/query_language/functions/json_functions.md b/docs/zh/query_language/functions/json_functions.md
index f918484eb53..7d205b1ea5f 100644
--- a/docs/zh/query_language/functions/json_functions.md
+++ b/docs/zh/query_language/functions/json_functions.md
@@ -193,4 +193,4 @@ SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8')
 select JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]'
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/json_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) 
diff --git a/docs/zh/query_language/functions/logical_functions.md b/docs/zh/query_language/functions/logical_functions.md
index 2cb703e9870..c71c8c5f2b8 100644
--- a/docs/zh/query_language/functions/logical_functions.md
+++ b/docs/zh/query_language/functions/logical_functions.md
@@ -13,4 +13,4 @@
 ## xor
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/logical_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) 
diff --git a/docs/zh/query_language/functions/math_functions.md b/docs/zh/query_language/functions/math_functions.md
index de73dbe136b..93b55728aee 100644
--- a/docs/zh/query_language/functions/math_functions.md
+++ b/docs/zh/query_language/functions/math_functions.md
@@ -106,4 +106,4 @@ SELECT erf(3 / sqrt(2))
 
 接受一个数值类型的参数并返回它的10的x次幂（UInt64）。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/math_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) 
diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md
index 329db222b48..d72cbe7ea38 100644
--- a/docs/zh/query_language/functions/other_functions.md
+++ b/docs/zh/query_language/functions/other_functions.md
@@ -660,4 +660,4 @@ SELECT replicate(1, ['a', 'b', 'c'])
 
 如果参数不为零则抛出异常。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/other_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) 
diff --git a/docs/zh/query_language/functions/random_functions.md b/docs/zh/query_language/functions/random_functions.md
index 27ea3482503..cd457257143 100644
--- a/docs/zh/query_language/functions/random_functions.md
+++ b/docs/zh/query_language/functions/random_functions.md
@@ -18,4 +18,4 @@
 
 返回一个UInt32类型的随机数字，该函数不同之处在于仅为每个数据块参数一个随机数。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/random_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) 
diff --git a/docs/zh/query_language/functions/rounding_functions.md b/docs/zh/query_language/functions/rounding_functions.md
index 11c8f1a51a2..c6e9de0ea15 100644
--- a/docs/zh/query_language/functions/rounding_functions.md
+++ b/docs/zh/query_language/functions/rounding_functions.md
@@ -90,4 +90,4 @@ round(3.65, 1) = 3.6
 
 接受一个数字，将其向下舍入到指定数组中的元素。如果该值小于数组中的最低边界，则返回最低边界。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/rounding_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/rounding_functions/) 
diff --git a/docs/zh/query_language/functions/splitting_merging_functions.md b/docs/zh/query_language/functions/splitting_merging_functions.md
index 6cc629f942e..6193ad5f6e7 100644
--- a/docs/zh/query_language/functions/splitting_merging_functions.md
+++ b/docs/zh/query_language/functions/splitting_merging_functions.md
@@ -27,4 +27,4 @@ SELECT alphaTokens('abca1abc')
 │ ['abca','abc']          │
 └─────────────────────────┘
 ```
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/splitting_merging_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/splitting_merging_functions/) 
diff --git a/docs/zh/query_language/functions/string_functions.md b/docs/zh/query_language/functions/string_functions.md
index 68c88ad7060..c2db80e8c51 100644
--- a/docs/zh/query_language/functions/string_functions.md
+++ b/docs/zh/query_language/functions/string_functions.md
@@ -164,4 +164,4 @@ SELECT format('{} {}', 'Hello', 'World')
 
 返回一个字符串，用于删除任一侧的空白字符。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/string_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) 
diff --git a/docs/zh/query_language/functions/string_replace_functions.md b/docs/zh/query_language/functions/string_replace_functions.md
index c50b740eaf8..78a2820358f 100644
--- a/docs/zh/query_language/functions/string_replace_functions.md
+++ b/docs/zh/query_language/functions/string_replace_functions.md
@@ -85,4 +85,4 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res
 这个实现与re2 :: RE2 :: QuoteMeta略有不同。它以\0而不是\x00转义零字节，它只转义所需的字符。
 有关详细信息，请参阅链接：[RE2]（https://github.com/google/re2/blob/master/re2/re2.cc#L473）
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/string_replace_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/string_replace_functions/) 
diff --git a/docs/zh/query_language/functions/string_search_functions.md b/docs/zh/query_language/functions/string_search_functions.md
index 99018670854..97edd104700 100644
--- a/docs/zh/query_language/functions/string_search_functions.md
+++ b/docs/zh/query_language/functions/string_search_functions.md
@@ -120,4 +120,4 @@
     对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/string_search_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) 
diff --git a/docs/zh/query_language/functions/type_conversion_functions.md b/docs/zh/query_language/functions/type_conversion_functions.md
index 3cabf07643b..5a53fddf0c8 100644
--- a/docs/zh/query_language/functions/type_conversion_functions.md
+++ b/docs/zh/query_language/functions/type_conversion_functions.md
@@ -180,4 +180,4 @@ SELECT
 
 与[parseDateTimeBestEffort](#type_conversion_functions-parsedatetimebesteffort)相同，但它遇到无法处理的日期格式时返回零Date或零DateTime。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/type_conversion_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) 
diff --git a/docs/zh/query_language/functions/url_functions.md b/docs/zh/query_language/functions/url_functions.md
index 7fcf2b3ff9d..7639bcf0580 100644
--- a/docs/zh/query_language/functions/url_functions.md
+++ b/docs/zh/query_language/functions/url_functions.md
@@ -120,4 +120,4 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod
 删除URL中名称为‘name’的参数。改函数假设参数名称以及参数值经过URL相同的编码。
 
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/url_functions/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/url_functions/) 
diff --git a/docs/zh/query_language/functions/uuid_functions.md b/docs/zh/query_language/functions/uuid_functions.md
index 1f9bd1f64c4..e398847c149 100644
--- a/docs/zh/query_language/functions/uuid_functions.md
+++ b/docs/zh/query_language/functions/uuid_functions.md
@@ -104,4 +104,4 @@ SELECT
 
 - [dictGetUUID](ext_dict_functions.md)
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/functions/uuid_function/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) 
diff --git a/docs/zh/query_language/functions/ym_dict_functions.md b/docs/zh/query_language/functions/ym_dict_functions.md
index 8039c77edea..130490b2c7a 100644
--- a/docs/zh/query_language/functions/ym_dict_functions.md
+++ b/docs/zh/query_language/functions/ym_dict_functions.md
@@ -124,4 +124,4 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. A string with
 `ua` and `uk` both mean Ukrainian.
 
 
-[Original article](https://clickhouse.yandex/docs/en/query_language/functions/ym_dict_functions/) 
+[Original article](https://clickhouse.tech/docs/en/query_language/functions/ym_dict_functions/) 
diff --git a/docs/zh/query_language/insert_into.md b/docs/zh/query_language/insert_into.md
index 8c3f8142651..7e91baa5afc 100644
--- a/docs/zh/query_language/insert_into.md
+++ b/docs/zh/query_language/insert_into.md
@@ -66,4 +66,4 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ...
 - 数据总是被实时的写入。
 - 写入的数据已经按照时间排序。
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/insert_into/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/insert_into/) 
diff --git a/docs/zh/query_language/operators.md b/docs/zh/query_language/operators.md
index 9dbddd681ed..8b53c0953eb 100644
--- a/docs/zh/query_language/operators.md
+++ b/docs/zh/query_language/operators.md
@@ -256,4 +256,4 @@ WHERE isNotNull(y)
 1 rows in set. Elapsed: 0.002 sec.
 ```
 
-[来源文章](https://clickhouse.yandex/docs/en/query_language/operators/) 
+[来源文章](https://clickhouse.tech/docs/en/query_language/operators/) 
diff --git a/docs/zh/query_language/select.md b/docs/zh/query_language/select.md
index a045799e9f3..8a77e6dfb78 100644
--- a/docs/zh/query_language/select.md
+++ b/docs/zh/query_language/select.md
@@ -975,4 +975,4 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL
 
 在所有的其他情况下，我们不建议使用星号，因为它是列式数据库的缺点而不是优点。
 
-[来源文章](https://clickhouse.yandex/docs/zh/query_language/select/) 
+[来源文章](https://clickhouse.tech/docs/zh/query_language/select/) 
diff --git a/docs/zh/roadmap.md b/docs/zh/roadmap.md
index 1f23d5f0ab4..7ace0ce36c8 100644
--- a/docs/zh/roadmap.md
+++ b/docs/zh/roadmap.md
@@ -6,4 +6,4 @@
 - 细粒度的授权管理
 - 与外部认证服务集成
 
-[来源文章](https://clickhouse.yandex/docs/en/roadmap/) 
+[来源文章](https://clickhouse.tech/docs/en/roadmap/) 
diff --git a/docs/zh/security_changelog.md b/docs/zh/security_changelog.md
index f4e5910c6d2..1b234f53a48 100644
--- a/docs/zh/security_changelog.md
+++ b/docs/zh/security_changelog.md
@@ -36,4 +36,4 @@ deb软件包中的错误配置可能导致使用未经授权的数据库。
 
 来源：英国国家网络安全中心（NCSC）
 
-[来源文章](https://clickhouse.yandex/docs/en/security_changelog/) 
+[来源文章](https://clickhouse.tech/docs/en/security_changelog/) 

From 4bdd374d9df5da6dd9e8a67260245b6c9de731e6 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 13:38:04 +0300
Subject: [PATCH 309/312] Few more static redirects

---
 docs/tools/build.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/tools/build.py b/docs/tools/build.py
index 581658a0162..6af292e4b1c 100755
--- a/docs/tools/build.py
+++ b/docs/tools/build.py
@@ -267,10 +267,15 @@ def build(args):
     if not args.skip_website:
         minify_website(args)
 
-    write_redirect_html(
-        os.path.join(args.output_dir, 'tutorial.html'),
-        '/docs/en/getting_started/tutorial/'
-    )
+    for static_redirect in [
+        ('tutorial.html', '/docs/en/getting_started/tutorial/',),
+        ('reference_en.html', '/docs/en/single/', ),
+        ('reference_ru.html', '/docs/ru/single/',),
+    ]:
+        write_redirect_html(
+            os.path.join(args.output_dir, static_redirect[0]),
+            static_redirect[1]
+        )
 
 
 if __name__ == '__main__':

From aba5f712fb869497d31c048c5bc8b499f0f07d05 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 13:39:37 +0300
Subject: [PATCH 310/312] One more static redirect

---
 docs/tools/build.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/tools/build.py b/docs/tools/build.py
index 6af292e4b1c..fa11baf85dd 100755
--- a/docs/tools/build.py
+++ b/docs/tools/build.py
@@ -271,6 +271,7 @@ def build(args):
         ('tutorial.html', '/docs/en/getting_started/tutorial/',),
         ('reference_en.html', '/docs/en/single/', ),
         ('reference_ru.html', '/docs/ru/single/',),
+        ('docs/index.html', '/docs/en/',),
     ]:
         write_redirect_html(
             os.path.join(args.output_dir, static_redirect[0]),

From 9cc403b6994a1f0a36089a7f500ed60f8632f13f Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 13:42:11 +0300
Subject: [PATCH 311/312] Final domain change

---
 website/benchmark_hardware.html |  2 +-
 website/index.html              | 14 +++++++-------
 website/nginx/default.conf      | 31 ++-----------------------------
 website/robots.txt              |  4 ++--
 website/sitemap.xml             | 12 ++++++------
 website/sitemap_static.xml      | 10 +++++-----
 6 files changed, 23 insertions(+), 50 deletions(-)

diff --git a/website/benchmark_hardware.html b/website/benchmark_hardware.html
index 10a1b6a26b8..e6ca7150f6e 100644
--- a/website/benchmark_hardware.html
+++ b/website/benchmark_hardware.html
@@ -2436,7 +2436,7 @@ try { var yaCounter18343495 = new Ya.Metrika({id:18343495,
 
 
 

-Submit your own results: https://clickhouse.yandex/docs/en/operations/performance_test/

+Submit your own results: https://clickhouse.yandex/docs/en/operations/performance_test/

 

 Results for Lenovo B580 Laptop are from Ragıp Ünal. 16GB RAM 1600 GHz, 240GB SSD, Intel(R) Core(TM) i5-3210M CPU @ 2.50GHz (2 Core / 4 HT)

 Results for Time4vps.eu are from Ragıp Ünal.

diff --git a/website/index.html b/website/index.html
index f376e55b84e..48786c37746 100644
--- a/website/index.html
+++ b/website/index.html
@@ -13,8 +13,8 @@
     
     
-    
-    
+    
+    
 
     
 
@@ -449,7 +449,7 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh
     
For other operating systems the easiest way to get started is using
         
-            official Docker images of ClickHouse, this is not the only option though.
+            official Docker images of ClickHouse, this is not the only option though.
         Alternatively, you can easily get a running ClickHouse instance or cluster at 
         
             Yandex Managed Service for ClickHouse.
@@ -488,13 +488,13 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh
     

 
     
Like ClickHouse?

-    
Help to spread the word about it via Facebook,
-        Twitter and
-        LinkedIn!

+    
Help to spread the word about it via Facebook,
+        Twitter and
+        LinkedIn!

 
     
Also hosting ClickHouse Meetups is very appreciated, fill this form
         if you are interested and we'll get in touch.
-        Short reports about previous meetups are published in official ClickHouse blog.

+        Short reports about previous meetups are published in official ClickHouse blog.

 
     

diff --git a/website/nginx/default.conf b/website/nginx/default.conf
index fc029323fe2..04b5783ee37 100644
--- a/website/nginx/default.conf
+++ b/website/nginx/default.conf
@@ -1,33 +1,6 @@
 server {
     listen 80 default;
     listen [::]:80 default ipv6only=on;
-    server_name localhost clickhouse.yandex test.clickhouse.yandex;
-
-    root /usr/share/nginx/html;
-    index index.html index.htm;
-
-    error_page 404 /404.html;
-    error_page 500 502 503 504 /50x.html;
-
-    location = /50x.html {
-        root /usr/share/nginx/www;
-    }
-
-    rewrite ^/docs/$ https://clickhouse.yandex/docs/en/ permanent;
-    rewrite ^/tutorial.html$ https://clickhouse.yandex/docs/en/getting_started/tutorial/ permanent;
-    rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent;
-    rewrite ^/reference_en.html$ https://clickhouse.yandex/docs/en/single/ permanent;
-    rewrite ^/reference_ru.html$ https://clickhouse.yandex/docs/ru/single/ permanent;
-    rewrite ^/presentations/(.*)$ https://clickhouse.github.io/clickhouse-presentations/$1 permanent;
-
-    include /usr/share/nginx/html/docs/redirects.conf;
-
-    if ( $uri !~ .*/index.html ){
-       rewrite ^/docs/(.*)/(.+)\.html$ /docs/$1/$2/ permanent;
-    }
-}
-
-server {
-    server_name clickhouse.yandex.ru clickhouse.yandex.com;
-    return 301 https://clickhouse.yandex$request_uri;
+    server_name localhost clickhouse.yandex test.clickhouse.yandex clickhouse.yandex.ru clickhouse.yandex.com;
+    return 301 https://clickhouse.tech$request_uri;
 }
diff --git a/website/robots.txt b/website/robots.txt
index 82708ceea95..300b2e7da6e 100644
--- a/website/robots.txt
+++ b/website/robots.txt
@@ -13,5 +13,5 @@ Disallow: /docs/ja/search.html
 Disallow: /docs/zh/search.html
 Disallow: /docs/fa/search.html
 Allow: /
-Host: https://clickhouse.yandex
-Sitemap: https://clickhouse.yandex/docs/sitemap.xml
+Host: https://clickhouse.tech
+Sitemap: https://clickhouse.tech/docs/sitemap.xml
diff --git a/website/sitemap.xml b/website/sitemap.xml
index e9319dc8701..cb1a6fddd9b 100644
--- a/website/sitemap.xml
+++ b/website/sitemap.xml
@@ -1,21 +1,21 @@
 
 
    
-      https://clickhouse.yandex/docs/en/sitemap.xml
+      https://clickhouse.tech/docs/en/sitemap.xml
    
    
-      https://clickhouse.yandex/docs/ru/sitemap.xml
+      https://clickhouse.tech/docs/ru/sitemap.xml
    
    
-      https://clickhouse.yandex/docs/zh/sitemap.xml
+      https://clickhouse.tech/docs/zh/sitemap.xml
    
    
-      https://clickhouse.yandex/docs/ja/sitemap.xml
+      https://clickhouse.tech/docs/ja/sitemap.xml
    
    
-      https://clickhouse.yandex/docs/fa/sitemap.xml
+      https://clickhouse.tech/docs/fa/sitemap.xml
    
    
-      https://clickhouse.yandex/docs/sitemap_static.xml
+      https://clickhouse.tech/docs/sitemap_static.xml
    
 
diff --git a/website/sitemap_static.xml b/website/sitemap_static.xml
index c1bc94bf00a..cfeaf1b8b79 100644
--- a/website/sitemap_static.xml
+++ b/website/sitemap_static.xml
@@ -1,23 +1,23 @@
 
 
 
-  https://clickhouse.yandex/ 
+  https://clickhouse.tech/ 
   daily
 
 
-  https://clickhouse.yandex/benchmark.html 
+  https://clickhouse.tech/benchmark.html 
   daily 
 
 
-  https://clickhouse.yandex/tutorial.html 
+  https://clickhouse.tech/tutorial.html 
   daily 
 
 
-  https://clickhouse.yandex/blog/en 
+  https://clickhouse.tech/blog/en 
   daily 
 
 
-  https://clickhouse.yandex/blog/ru 
+  https://clickhouse.tech/blog/ru 
   daily 
 
 
\ No newline at end of file

From 514f4d00816cc456a3b6d7e81b7d9b44147f0fa6 Mon Sep 17 00:00:00 2001
From: Ivan Blinkov 
Date: Thu, 30 Jan 2020 13:43:46 +0300
Subject: [PATCH 312/312] Separate redirect for test

---
 website/nginx/default.conf | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/website/nginx/default.conf b/website/nginx/default.conf
index 04b5783ee37..58245449e33 100644
--- a/website/nginx/default.conf
+++ b/website/nginx/default.conf
@@ -1,6 +1,10 @@
 server {
     listen 80 default;
     listen [::]:80 default ipv6only=on;
-    server_name localhost clickhouse.yandex test.clickhouse.yandex clickhouse.yandex.ru clickhouse.yandex.com;
+    server_name localhost clickhouse.yandex clickhouse.yandex.ru clickhouse.yandex.com;
     return 301 https://clickhouse.tech$request_uri;
 }
+server {
+    server_name localhost test.clickhouse.yandex;
+    return 301 https://test.clickhouse.tech$request_uri;
+}