Merge branch 'master' of github.com:ClickHouse/ClickHouse into disk-async-read

base/base/DateLUTImpl.cpp

@@ -177,8 +177,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
 }
 
 
-#if !defined(ARCADIA_BUILD) /// Arcadia's variant of CCTZ already has the same implementation.
-
 /// Prefer to load timezones from blobs linked to the binary.
 /// The blobs are provided by "tzdata" library.
 /// This allows to avoid dependency on system tzdata.
@@ -234,5 +232,3 @@ namespace cctz_extension
 
     ZoneInfoSourceFactory zone_info_source_factory = custom_factory;
 }
-
-#endif

base/base/LineReader.cpp

@@ -103,7 +103,6 @@ String LineReader::readLine(const String & first_prompt, const String & second_p
                 continue;
         }
 
-#if !defined(ARCADIA_BUILD) /// C++20
         const char * has_extender = nullptr;
         for (const auto * extender : extenders)
         {
@@ -133,7 +132,6 @@ String LineReader::readLine(const String & first_prompt, const String & second_p
             if (input.empty())
                 continue;
         }
-#endif
 
         line += (line.empty() ? "" : "\n") + input;
 

base/base/defines.h

@@ -83,10 +83,6 @@
 #    define BOOST_USE_UCONTEXT 1
 #endif
 
-#if defined(ARCADIA_BUILD) && defined(BOOST_USE_UCONTEXT)
-#    undef BOOST_USE_UCONTEXT
-#endif
-
 /// TODO: Strange enough, there is no way to detect UB sanitizer.
 
 /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute.

base/base/logger_useful.h

@@ -49,12 +49,3 @@ namespace
 #define LOG_WARNING(logger, ...) LOG_IMPL(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, __VA_ARGS__)
 #define LOG_ERROR(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, __VA_ARGS__)
 #define LOG_FATAL(logger, ...)   LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, __VA_ARGS__)
-
-
-/// Compatibility for external projects.
-#if defined(ARCADIA_BUILD)
-    using Poco::Logger;
-    using Poco::Message;
-    using DB::LogsLevel;
-    using DB::CurrentThread;
-#endif

base/base/sort.h

@@ -3,41 +3,24 @@
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wold-style-cast"
 
-#if !defined(ARCADIA_BUILD)
-#    include <miniselect/floyd_rivest_select.h>  // Y_IGNORE
-#else
-#    include <algorithm>
-#endif
+#include <miniselect/floyd_rivest_select.h>
 
 template <class RandomIt>
 void nth_element(RandomIt first, RandomIt nth, RandomIt last)
 {
-#if !defined(ARCADIA_BUILD)
     ::miniselect::floyd_rivest_select(first, nth, last);
-#else
-    ::std::nth_element(first, nth, last);
-#endif
 }
 
 template <class RandomIt>
 void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
 {
-#if !defined(ARCADIA_BUILD)
     ::miniselect::floyd_rivest_partial_sort(first, middle, last);
-#else
-    ::std::partial_sort(first, middle, last);
-#endif
 }
 
 template <class RandomIt, class Compare>
 void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
 {
-#if !defined(ARCADIA_BUILD)
     ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare);
-#else
-    ::std::partial_sort(first, middle, last, compare);
-#endif
+}
 
 #pragma GCC diagnostic pop
-
-}

base/daemon/BaseDaemon.cpp

@@ -63,9 +63,7 @@
 #include <Common/Elf.h>
 #include <filesystem>
 
-#if !defined(ARCADIA_BUILD)
-#   include <Common/config_version.h>
-#endif
+#include <Common/config_version.h>
 
 #if defined(OS_DARWIN)
 #   pragma GCC diagnostic ignored "-Wunused-macros"

base/daemon/SentryWriter.cpp

@@ -15,14 +15,12 @@
 #include <Core/ServerUUID.h>
 #include <Common/hex.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include "Common/config_version.h"
-#    include <Common/config.h>
-#endif
+#include "Common/config_version.h"
+#include <Common/config.h>
 
 #if USE_SENTRY
 
-#    include <sentry.h> // Y_IGNORE
+#    include <sentry.h>
 #    include <stdio.h>
 #    include <filesystem>
 

base/mysqlxx/Query.cpp

@@ -2,7 +2,7 @@
 #include <errmsg.h>
 #include <mysql.h>
 #else
-#include <mysql/errmsg.h> //Y_IGNORE
+#include <mysql/errmsg.h>
 #include <mysql/mysql.h>
 #endif
 

cmake/cpu_features.cmake

@@ -179,7 +179,7 @@ else ()
             set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi")
         endif ()
         if (HAVE_AVX512)
-            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw")
+            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mprefer-vector-width=256")
         endif ()
     endif ()
 endif ()

docs/en/interfaces/formats.md

@@ -164,8 +164,7 @@ This format is also available under the name `TSVRaw`.
 ## TabSeparatedWithNames {#tabseparatedwithnames}
 
 Differs from the `TabSeparated` format in that the column names are written in the first row.
-During parsing, the first row is completely ignored. You can’t use column names to determine their position or to check their correctness.
-(Support for parsing the header row may be added in the future.)
+During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness.
 
 This format is also available under the name `TSVWithNames`.
 

docs/en/introduction/adopters.md

@@ -24,7 +24,7 @@ toc_title: Adopters
 | <a href="https://arenadata.tech/" class="favicon">ArenaData</a> | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) |
 | <a href="https://www.argedor.com/en/clickhouse/" class="favicon">Argedor</a> | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) |
 | <a href="https://avito.ru/" class="favicon">Avito</a> | Classifieds | Monitoring | — | — | [Meetup, April 2020](https://www.youtube.com/watch?v=n1tm4j4W8ZQ) |
-| <a href="https://badoo.com" class="favicon">Badoo</a> | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.com/meetup38/forecast.pdf) |
+| <a href="https://badoo.com" class="favicon">Badoo</a> | Dating | Timeseries | — | 1.6 mln events/sec (2018) | [Slides in Russian, December 2019](https://presentations.clickhouse.com/meetup38/forecast.pdf) |
 | <a href="https://beeline.ru/" class="favicon">Beeline</a> | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) |
 | <a href="https://www.benocs.com/" class="favicon">Benocs</a> | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) |
 | <a href="https://www.bigo.sg/" class="favicon">BIGO</a> | Video | Computing Platform | — | — | [Blog Article, August 2020](https://www.programmersought.com/article/44544895251/) |

docs/en/operations/system-tables/replicas.md

@@ -8,43 +8,48 @@ Example:
 ``` sql
 SELECT *
 FROM system.replicas
-WHERE table = 'visits'
+WHERE table = 'test_table'
 FORMAT Vertical
 ```
 
 ``` text
+Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e
+
 Row 1:
-──────
-database:                   merge
-table:                      visits
-engine:                     ReplicatedCollapsingMergeTree
-is_leader:                  1
-can_become_leader:          1
-is_readonly:                0
-is_session_expired:         0
-future_parts:               1
-parts_to_check:             0
-zookeeper_path:             /clickhouse/tables/01-06/visits
-replica_name:               example01-06-1.yandex.ru
-replica_path:               /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru
-columns_version:            9
-queue_size:                 1
-inserts_in_queue:           0
-merges_in_queue:            1
-part_mutations_in_queue:    0
-queue_oldest_time:          2020-02-20 08:34:30
-inserts_oldest_time:        1970-01-01 00:00:00
-merges_oldest_time:         2020-02-20 08:34:30
-part_mutations_oldest_time: 1970-01-01 00:00:00
-oldest_part_to_get:
-oldest_part_to_merge_to:    20200220_20284_20840_7
-oldest_part_to_mutate_to:
-log_max_index:              596273
-log_pointer:                596274
-last_queue_update:          2020-02-20 08:34:32
-absolute_delay:             0
-total_replicas:             2
-active_replicas:            2
+───────
+database:                    db
+table:                       test_table
+engine:                      ReplicatedMergeTree
+is_leader:                   1
+can_become_leader:           1
+is_readonly:                 0
+is_session_expired:          0
+future_parts:                0
+parts_to_check:              0
+zookeeper_path:              /test/test_table
+replica_name:                r1
+replica_path:                /test/test_table/replicas/r1
+columns_version:             -1
+queue_size:                  27
+inserts_in_queue:            27
+merges_in_queue:             0
+part_mutations_in_queue:     0
+queue_oldest_time:           2021-10-12 14:48:48
+inserts_oldest_time:         2021-10-12 14:48:48
+merges_oldest_time:          1970-01-01 03:00:00
+part_mutations_oldest_time:  1970-01-01 03:00:00
+oldest_part_to_get:          1_17_17_0
+oldest_part_to_merge_to:     
+oldest_part_to_mutate_to:    
+log_max_index:               206
+log_pointer:                 207
+last_queue_update:           2021-10-12 14:50:08
+absolute_delay:              99
+total_replicas:              5
+active_replicas:             5
+last_queue_update_exception: 
+zookeeper_exception:         
+replica_is_active:           {'r1':1,'r2':1}
 ```
 
 Columns:
@@ -82,6 +87,8 @@ The next 4 columns have a non-zero value only where there is an active session w
 -   `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has.
 -   `total_replicas` (`UInt8`) - The total number of known replicas of this table.
 -   `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas).
+-   `last_queue_update_exception` (`String`) - When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions.
+-   `zookeeper_exception` (`String`) - The last exception message, got if the error happened when fetching the info from ZooKeeper. 
 -   `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active.
 
 If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row.

docs/en/sql-reference/functions/string-functions.md

@@ -810,6 +810,150 @@ Result:
 └─────┘
 ```
 
+## normalizeUTF8NFC {#normalizeutf8nfc}
+
+Converts a string to [NFC normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text.
+
+**Syntax**
+
+``` sql
+normalizeUTF8NFC(words)
+```
+
+**Arguments**
+
+-   `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+-   String transformed to NFC normalization form.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFC('â') AS nfc, length(nfc) AS nfc_len;
+```
+
+Result:
+
+``` text
+┌─length('â')─┬─nfc─┬─nfc_len─┐
+│           2 │ â   │       2 │
+└─────────────┴─────┴─────────┘
+```
+
+## normalizeUTF8NFD {#normalizeutf8nfd}
+
+Converts a string to [NFD normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text.
+
+**Syntax**
+
+``` sql
+normalizeUTF8NFD(words)
+```
+
+**Arguments**
+
+-   `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+-   String transformed to NFD normalization form.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFD('â') AS nfd, length(nfd) AS nfd_len;
+```
+
+Result:
+
+``` text
+┌─length('â')─┬─nfd─┬─nfd_len─┐
+│           2 │ â   │       3 │
+└─────────────┴─────┴─────────┘
+```
+
+## normalizeUTF8NFKC {#normalizeutf8nfkc}
+
+Converts a string to [NFKC normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text.
+
+**Syntax**
+
+``` sql
+normalizeUTF8NFKC(words)
+```
+
+**Arguments**
+
+-   `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+-   String transformed to NFKC normalization form.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFKC('â') AS nfkc, length(nfkc) AS nfkc_len;
+```
+
+Result:
+
+``` text
+┌─length('â')─┬─nfkc─┬─nfkc_len─┐
+│           2 │ â    │        2 │
+└─────────────┴──────┴──────────┘
+```
+
+## normalizeUTF8NFKD {#normalizeutf8nfkd}
+
+Converts a string to [NFKD normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text.
+
+**Syntax**
+
+``` sql
+normalizeUTF8NFKD(words)
+```
+
+**Arguments**
+
+-   `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+-   String transformed to NFKD normalization form.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFKD('â') AS nfkd, length(nfkd) AS nfkd_len;
+```
+
+Result:
+
+``` text
+┌─length('â')─┬─nfkd─┬─nfkd_len─┐
+│           2 │ â    │        3 │
+└─────────────┴──────┴──────────┘
+```
+
 ## encodeXMLComponent {#encode-xml-component}
 
 Escapes characters to place string into XML text node or attribute.

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -350,6 +350,45 @@ Result:
 │ ['eleven','11']  │
 │ ['twelve','6.0'] │
 └──────────────────┘
-```
+```  
+  
+## mapContainsKeyLike {#mapContainsKeyLike}  
+  
+**Syntax**
+
+```sql
+mapContainsKeyLike(map, pattern)
+```  
+  
+**Parameters**
+  
+-   `map` — Map. [Map](../../sql-reference/data-types/map.md).  
+-   `pattern`  - String pattern to match.  
+  
+**Returned value**
+
+-   `1` if `map` contains `key` like specified pattern, `0` if not.  
+  
+**Example**
+
+Query:
+
+```sql
+CREATE TABLE test (a Map(String,String)) ENGINE = Memory;
+
+INSERT INTO test VALUES ({'abc':'abc','def':'def'}), ({'hij':'hij','klm':'klm'});
+
+SELECT mapContainsKeyLike(a, 'a%') FROM test;
+```  
+  
+Result:  
+  
+```text
+┌─mapContainsKeyLike(a, 'a%')─┐
+│                           1 │
+│                           0 │
+└─────────────────────────────┘  
+```  
+  
 
 [Original article](https://clickhouse.com/docs/en/sql-reference/functions/tuple-map-functions/) <!--hide-->

docs/ru/operations/system-tables/replicas.md

@@ -8,43 +8,48 @@
 ``` sql
 SELECT *
 FROM system.replicas
-WHERE table = 'visits'
+WHERE table = 'test_table'
 FORMAT Vertical
 ```
 
 ``` text
+Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e
+
 Row 1:
-──────
-database:                   merge
-table:                      visits
-engine:                     ReplicatedCollapsingMergeTree
-is_leader:                  1
-can_become_leader:          1
-is_readonly:                0
-is_session_expired:         0
-future_parts:               1
-parts_to_check:             0
-zookeeper_path:             /clickhouse/tables/01-06/visits
-replica_name:               example01-06-1.yandex.ru
-replica_path:               /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru
-columns_version:            9
-queue_size:                 1
-inserts_in_queue:           0
-merges_in_queue:            1
-part_mutations_in_queue:    0
-queue_oldest_time:          2020-02-20 08:34:30
-inserts_oldest_time:        0000-00-00 00:00:00
-merges_oldest_time:         2020-02-20 08:34:30
-part_mutations_oldest_time: 0000-00-00 00:00:00
-oldest_part_to_get:
-oldest_part_to_merge_to:    20200220_20284_20840_7
-oldest_part_to_mutate_to:
-log_max_index:              596273
-log_pointer:                596274
-last_queue_update:          2020-02-20 08:34:32
-absolute_delay:             0
-total_replicas:             2
-active_replicas:            2
+───────
+database:                    db
+table:                       test_table
+engine:                      ReplicatedMergeTree
+is_leader:                   1
+can_become_leader:           1
+is_readonly:                 0
+is_session_expired:          0
+future_parts:                0
+parts_to_check:              0
+zookeeper_path:              /test/test_table
+replica_name:                r1
+replica_path:                /test/test_table/replicas/r1
+columns_version:             -1
+queue_size:                  27
+inserts_in_queue:            27
+merges_in_queue:             0
+part_mutations_in_queue:     0
+queue_oldest_time:           2021-10-12 14:48:48
+inserts_oldest_time:         2021-10-12 14:48:48
+merges_oldest_time:          1970-01-01 03:00:00
+part_mutations_oldest_time:  1970-01-01 03:00:00
+oldest_part_to_get:          1_17_17_0
+oldest_part_to_merge_to:     
+oldest_part_to_mutate_to:    
+log_max_index:               206
+log_pointer:                 207
+last_queue_update:           2021-10-12 14:50:08
+absolute_delay:              99
+total_replicas:              5
+active_replicas:             5
+last_queue_update_exception: 
+zookeeper_exception:         
+replica_is_active:           {'r1':1,'r2':1}
 ```
 
 Столбцы:
@@ -82,6 +87,8 @@ active_replicas:            2
 -   `absolute_delay` (`UInt64`) - задержка (в секундах) для текущей реплики.
 -   `total_replicas` (`UInt8`) - общее число известных реплик этой таблицы.
 -   `active_replicas` (`UInt8`) - число реплик этой таблицы, имеющих сессию в ZK; то есть, число работающих реплик.
+-   `last_queue_update_exception` (`String`) - если в очереди есть битые записи. Особенно важно, когда в ClickHouse нарушается обратная совместимость между версиями, а записи журнала, сделанные более новыми версиями, не могут быть проанализированы старыми версиями.
+-   `zookeeper_exception` (`String`) - последнее сообщение об исключении. Появляется, если ошибка произошла при получении информации из ZooKeeper.
 -   `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — соответствие между именем реплики и признаком активности реплики.
 
 Если запрашивать все столбцы, то таблица может работать слегка медленно, так как на каждую строчку делается несколько чтений из ZK.

docs/ru/sql-reference/functions/string-functions.md

@@ -781,6 +781,150 @@ SELECT normalizedQueryHash('SELECT 1 AS `xyz`') != normalizedQueryHash('SELECT 1
 └─────┘
 ```
 
+## normalizeUTF8NFC {#normalizeutf8nfc}
+
+Преобразует строку в нормализованную форму [NFC](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8.
+
+**Синтаксис**
+
+``` sql
+normalizeUTF8NFC(words)
+```
+
+**Аргументы**
+
+-   `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+-   Строка, преобразованная в нормализированную форму NFC.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFC('â') AS nfc, length(nfc) AS nfc_len;
+```
+
+Результат:
+
+``` text
+┌─length('â')─┬─nfc─┬─nfc_len─┐
+│           2 │ â   │       2 │
+└─────────────┴─────┴─────────┘
+```
+
+## normalizeUTF8NFD {#normalizeutf8nfd}
+
+Преобразует строку в нормализованную форму [NFD](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8.
+
+**Синтаксис**
+
+``` sql
+normalizeUTF8NFD(words)
+```
+
+**Аргументы**
+
+-   `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+-   Строка, преобразованная в нормализированную форму NFD.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFD('â') AS nfd, length(nfd) AS nfd_len;
+```
+
+Результат:
+
+``` text
+┌─length('â')─┬─nfd─┬─nfd_len─┐
+│           2 │ â   │       3 │
+└─────────────┴─────┴─────────┘
+```
+
+## normalizeUTF8NFKC {#normalizeutf8nfkc}
+
+Преобразует строку в нормализованную форму [NFKC](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8.
+
+**Синтаксис**
+
+``` sql
+normalizeUTF8NFKC(words)
+```
+
+**Аргументы**
+
+-   `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+-   Строка, преобразованная в нормализированную форму NFKC.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFKC('â') AS nfkc, length(nfkc) AS nfkc_len;
+```
+
+Результат:
+
+``` text
+┌─length('â')─┬─nfkc─┬─nfkc_len─┐
+│           2 │ â    │        2 │
+└─────────────┴──────┴──────────┘
+```
+
+## normalizeUTF8NFKD {#normalizeutf8nfkd}
+
+Преобразует строку в нормализованную форму [NFKD](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8.
+
+**Синтаксис**
+
+``` sql
+normalizeUTF8NFKD(words)
+```
+
+**Аргументы**
+
+-   `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+-   Строка, преобразованная в нормализированную форму NFKD.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Пример**
+
+Запрос:
+
+``` sql
+SELECT length('â'), normalizeUTF8NFKD('â') AS nfkd, length(nfkd) AS nfkd_len;
+```
+
+Результат:
+
+``` text
+┌─length('â')─┬─nfkd─┬─nfkd_len─┐
+│           2 │ â    │        3 │
+└─────────────┴──────┴──────────┘
+```
+
 ## encodeXMLComponent {#encode-xml-component}
 
 Экранирует символы для размещения строки в текстовом узле или атрибуте XML.

src/Access/GSSAcceptor.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#   include "config_core.h"
-#endif
+#include "config_core.h"
 
 #include <Access/Credentials.h>
 #include <base/types.h>

src/Access/LDAPClient.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#   include "config_core.h"
-#endif
+#include "config_core.h"
 
 #include <base/types.h>
 

src/AggregateFunctions/AggregateFunctionAvg.h

@@ -9,9 +9,7 @@
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <Core/DecimalFunctions.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionBitwise.h

@@ -9,9 +9,7 @@
 
 #include <AggregateFunctions/IAggregateFunction.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionCount.h

@@ -10,9 +10,7 @@
 #include <AggregateFunctions/IAggregateFunction.h>
 #include <Common/assert_cast.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionIf.h

@@ -5,9 +5,7 @@
 #include <Common/assert_cast.h>
 #include <AggregateFunctions/IAggregateFunction.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionMinMaxAny.h

@@ -14,9 +14,7 @@
 #include <DataTypes/DataTypeNullable.h>
 #include <AggregateFunctions/IAggregateFunction.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionNull.h

@@ -10,9 +10,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionSum.h

@@ -14,9 +14,7 @@
 
 #include <AggregateFunctions/IAggregateFunction.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_EMBEDDED_COMPILER
 #    include <llvm/IR/IRBuilder.h>

src/AggregateFunctions/AggregateFunctionSumMap.h

@@ -395,9 +395,7 @@ private:
     using Self = AggregateFunctionSumMapFiltered<T, overflow, tuple_argument>;
     using Base = AggregateFunctionMapBase<T, Self, FieldVisitorSum, overflow, tuple_argument, true>;
 
-    /// ARCADIA_BUILD disallow unordered_set for big ints for some reason
-    static constexpr const bool allow_hash = !is_over_big_int<T>;
-    using ContainerT = std::conditional_t<allow_hash, std::unordered_set<T>, std::set<T>>;
+    using ContainerT = std::unordered_set<T>;
 
     ContainerT keys_to_keep;
 
@@ -418,13 +416,10 @@ public:
                 "Aggregate function {} requires an Array as a parameter",
                 getName());
 
-        if constexpr (allow_hash)
-            keys_to_keep.reserve(keys_to_keep_.size());
+        keys_to_keep.reserve(keys_to_keep_.size());
 
         for (const Field & f : keys_to_keep_)
-        {
             keys_to_keep.emplace(f.safeGet<T>());
-        }
     }
 
     String getName() const override

src/AggregateFunctions/IAggregateFunction.h

@@ -9,9 +9,7 @@
 #include <Common/Exception.h>
 #include <base/types.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #include <cstddef>
 #include <memory>

src/AggregateFunctions/ThetaSketchData.h

@@ -1,15 +1,13 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_DATASKETCHES
 
 #include <boost/noncopyable.hpp>
 #include <memory>
-#include <theta_sketch.hpp> // Y_IGNORE
-#include <theta_union.hpp> // Y_IGNORE
+#include <theta_sketch.hpp>
+#include <theta_union.hpp>
 
 
 namespace DB

src/AggregateFunctions/registerAggregateFunctions.cpp

@@ -98,9 +98,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);
-#if !defined(ARCADIA_BUILD)
         registerAggregateFunctionsBitmap(factory);
-#endif
         registerAggregateFunctionsMaxIntersections(factory);
         registerAggregateFunctionHistogram(factory);
         registerAggregateFunctionRetention(factory);

src/Bridge/XDBCBridgeHelper.h

@@ -15,9 +15,7 @@
 #include <base/range.h>
 #include <Bridge/IBridgeHelper.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 
 namespace DB

src/CMakeLists.txt

@@ -295,6 +295,7 @@ set_source_files_properties(
         Columns/ColumnFixedString.cpp
         Columns/ColumnsCommon.cpp
         Columns/ColumnVector.cpp
+        Columns/ColumnDecimal.cpp
         PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}")
 
 if(RE2_LIBRARY)

src/Client/ClientBase.cpp

@@ -14,9 +14,7 @@
 #include "Core/Block.h"
 #include "Core/Protocol.h"
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config_version.h>
-#endif
+#include <Common/config_version.h>
 #include <Common/UTF8Helpers.h>
 #include <Common/TerminalSize.h>
 #include <Common/clearPasswordFromCommandLine.h>

src/Client/Connection.cpp

@@ -31,10 +31,8 @@
 #include <Processors/Executors/PipelineExecutor.h>
 #include <pcg_random.hpp>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config_version.h>
-#    include <Common/config.h>
-#endif
+#include <Common/config_version.h>
+#include <Common/config.h>
 
 #if USE_SSL
 #    include <Poco/Net/SecureStreamSocket.h>

src/Client/Connection.h

@@ -4,9 +4,7 @@
 
 #include <Poco/Net/StreamSocket.h>
 
-#if !defined(ARCADIA_BUILD)
-#   include <Common/config.h>
-#endif
+#include <Common/config.h>
 #include <Client/IServerConnection.h>
 #include <Core/Defines.h>
 

src/Client/ConnectionParameters.cpp

@@ -12,9 +12,7 @@
 #include <base/setTerminalEcho.h>
 #include <base/scope_guard.h>
 
-#if !defined(ARCADIA_BUILD)
-#include <readpassphrase/readpassphrase.h> // Y_IGNORE
-#endif
+#include <readpassphrase/readpassphrase.h>
 
 
 namespace DB
@@ -55,12 +53,10 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
     }
     if (password_prompt)
     {
-#if !defined(ARCADIA_BUILD)
         std::string prompt{"Password for user (" + user + "): "};
         char buf[1000] = {};
         if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0))
             password = result;
-#endif
     }
 
     /// By default compression is disabled if address looks like localhost.

src/Columns/Collator.cpp

@@ -1,8 +1,6 @@
 #include <Columns/Collator.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_ICU
 #    include <unicode/locid.h>

src/Columns/ColumnDecimal.cpp

@@ -237,25 +237,39 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
     const UInt8 * filt_end = filt_pos + size;
     const T * data_pos = data.data();
 
-#ifdef __SSE2__
-    static constexpr size_t SIMD_BYTES = 16;
-    const __m128i zero16 = _mm_setzero_si128();
-    const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
+    /** A slightly more optimized version.
+    * Based on the assumption that often pieces of consecutive values
+    *  completely pass or do not pass the filter.
+    * Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
+    */
+    static constexpr size_t SIMD_BYTES = 64;
+    const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
 
-    while (filt_pos < filt_end_sse)
+    while (filt_pos < filt_end_aligned)
     {
-        UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)), zero16));
-        mask = ~mask;
-        while (mask)
+        UInt64 mask = Bytes64MaskToBits64Mask(filt_pos);
+
+        if (0xffffffffffffffff == mask)
         {
-            size_t index = __builtin_ctz(mask);
-            res_data.push_back(*(data_pos + index));
-            mask = mask & (mask - 1);
+            res_data.insert(data_pos, data_pos + SIMD_BYTES);
         }
+        else
+        {
+            while (mask)
+            {
+                size_t index = __builtin_ctzll(mask);
+                res_data.push_back(data_pos[index]);
+            #ifdef __BMI__
+                mask = _blsr_u64(mask);
+            #else
+                mask = mask & (mask-1);
+            #endif
+            }
+        }
+
         filt_pos += SIMD_BYTES;
         data_pos += SIMD_BYTES;
     }
-#endif
 
     while (filt_pos < filt_end)
     {

src/Columns/ColumnFixedString.cpp

@@ -236,17 +236,15 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
         *  completely pass or do not pass the filter.
         * Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
         */
-#if defined(__AVX512F__) && defined(__AVX512BW__)
     static constexpr size_t SIMD_BYTES = 64;
-    const __m512i zero64 = _mm512_setzero_epi32();
-    const UInt8 * filt_end_avx512 = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES;
+    const UInt8 * filt_end_aligned = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES;
     const size_t chars_per_simd_elements = SIMD_BYTES * n;
 
-    while (filt_pos < filt_end_avx512)
+    while (filt_pos < filt_end_aligned)
     {
-        uint64_t mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i *>(filt_pos)), zero64, _MM_CMPINT_GT);
+        uint64_t mask = Bytes64MaskToBits64Mask(filt_pos);
 
-        if (0xFFFFFFFFFFFFFFFF == mask)
+        if (0xffffffffffffffff == mask)
         {
             res->chars.insert(data_pos, data_pos + chars_per_simd_elements);
         }
@@ -269,73 +267,6 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
         data_pos += chars_per_simd_elements;
         filt_pos += SIMD_BYTES;
     }
-#elif defined(__AVX__) && defined(__AVX2__)
-    static constexpr size_t SIMD_BYTES = 32;
-    const __m256i zero32 = _mm256_setzero_si256();
-    const UInt8 * filt_end_avx2 = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES;
-    const size_t chars_per_simd_elements = SIMD_BYTES * n;
-
-    while (filt_pos < filt_end_avx2)
-    {
-        uint32_t mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast<const __m256i *>(filt_pos)), zero32));
-
-        if (0xFFFFFFFF == mask)
-        {
-            res->chars.insert(data_pos, data_pos + chars_per_simd_elements);
-        }
-        else
-        {
-            size_t res_chars_size = res->chars.size();
-            while (mask)
-            {
-                size_t index = __builtin_ctz(mask);
-                res->chars.resize(res_chars_size + n);
-                memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n);
-                res_chars_size += n;
-            #ifdef __BMI__
-                mask = _blsr_u32(mask);
-            #else
-                mask = mask & (mask-1);
-            #endif
-            }
-        }
-        data_pos += chars_per_simd_elements;
-        filt_pos += SIMD_BYTES;
-    }
-
-#elif defined(__SSE2__)
-
-    static constexpr size_t SIMD_BYTES = 16;
-    const __m128i zero16 = _mm_setzero_si128();
-    const UInt8 * filt_end_sse = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES;
-    const size_t chars_per_simd_elements = SIMD_BYTES * n;
-
-    while (filt_pos < filt_end_sse)
-    {
-        UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)), zero16));
-        mask = ~mask;
-
-        if (0xFFFF == mask)
-        {
-            res->chars.insert(data_pos, data_pos + chars_per_simd_elements);
-        }
-        else
-        {
-            size_t res_chars_size = res->chars.size();
-            while (mask)
-            {
-                size_t index = __builtin_ctz(mask);
-                res->chars.resize(res_chars_size + n);
-                memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n);
-                res_chars_size += n;
-                mask = mask & (mask - 1);
-            }
-        }
-        data_pos += chars_per_simd_elements;
-        filt_pos += SIMD_BYTES;
-    }
-
-#endif
 
     size_t res_chars_size = res->chars.size();
     while (filt_pos < filt_end)

src/Columns/ColumnVector.cpp

@@ -310,21 +310,20 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
     const UInt8 * filt_pos = filt.data();
     const UInt8 * filt_end = filt_pos + size;
     const T * data_pos = data.data();
+
     /** A slightly more optimized version.
     * Based on the assumption that often pieces of consecutive values
     *  completely pass or do not pass the filter.
     * Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
     */
-#if defined(__AVX512F__) && defined(__AVX512BW__)
     static constexpr size_t SIMD_BYTES = 64;
-    const __m512i zero64 = _mm512_setzero_epi32();
-    const UInt8 * filt_end_avx512 = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
+    const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
 
-    while (filt_pos < filt_end_avx512)
+    while (filt_pos < filt_end_aligned)
     {
-        UInt64 mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i *>(filt_pos)), zero64, _MM_CMPINT_GT);
+        UInt64 mask = Bytes64MaskToBits64Mask(filt_pos);
 
-        if (0xFFFFFFFFFFFFFFFF == mask)
+        if (0xffffffffffffffff == mask)
         {
             res_data.insert(data_pos, data_pos + SIMD_BYTES);
         }
@@ -346,67 +345,6 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
         data_pos += SIMD_BYTES;
     }
 
-#elif defined(__AVX__) && defined(__AVX2__)
-    static constexpr size_t SIMD_BYTES = 32;
-    const __m256i zero32 = _mm256_setzero_si256();
-    const UInt8 * filt_end_avx2 = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
-
-    while (filt_pos < filt_end_avx2)
-    {
-        UInt32 mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast<const __m256i *>(filt_pos)), zero32));
-
-        if (0xFFFFFFFF == mask)
-        {
-            res_data.insert(data_pos, data_pos + SIMD_BYTES);
-        }
-        else
-        {
-            while (mask)
-            {
-                size_t index = __builtin_ctz(mask);
-                res_data.push_back(data_pos[index]);
-            #ifdef __BMI__
-                mask = _blsr_u32(mask);
-            #else
-                mask = mask & (mask-1);
-            #endif
-            }
-        }
-
-        filt_pos += SIMD_BYTES;
-        data_pos += SIMD_BYTES;
-    }
-
-#elif defined(__SSE2__)
-    static constexpr size_t SIMD_BYTES = 16;
-    const __m128i zero16 = _mm_setzero_si128();
-    const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
-
-    while (filt_pos < filt_end_sse)
-    {
-        UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)), zero16));
-        mask = ~mask;
-
-        if (0xFFFF == mask)
-        {
-            res_data.insert(data_pos, data_pos + SIMD_BYTES);
-        }
-        else
-        {
-            while (mask)
-            {
-                size_t index = __builtin_ctz(mask);
-                res_data.push_back(data_pos[index]);
-                mask = mask & (mask - 1);
-            }
-        }
-
-        filt_pos += SIMD_BYTES;
-        data_pos += SIMD_BYTES;
-    }
-
-#endif
-
     while (filt_pos < filt_end)
     {
         if (*filt_pos)

src/Columns/ColumnsCommon.cpp

@@ -1,7 +1,3 @@
-#ifdef __SSE2__
-    #include <emmintrin.h>
-#endif
-
 #include <Columns/IColumn.h>
 #include <Columns/ColumnVector.h>
 #include <Common/typeid_cast.h>
@@ -229,16 +225,19 @@ namespace
             memcpy(&res_elems[elems_size_old], &src_elems[arr_offset], arr_size * sizeof(T));
         };
 
-    #if defined(__AVX512F__) && defined(__AVX512BW__)
-        const __m512i zero_vec = _mm512_setzero_epi32();
+        /** A slightly more optimized version.
+        * Based on the assumption that often pieces of consecutive values
+        *  completely pass or do not pass the filter.
+        * Therefore, we will optimistically check the parts of `SIMD_BYTES` values.
+        */
         static constexpr size_t SIMD_BYTES = 64;
         const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
 
         while (filt_pos < filt_end_aligned)
         {
-            uint64_t mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i *>(filt_pos)), zero_vec, _MM_CMPINT_GT);
+            uint64_t mask = Bytes64MaskToBits64Mask(filt_pos);
 
-            if (mask == 0xffffffffffffffff)
+            if (0xffffffffffffffff == mask)
             {
                 /// SIMD_BYTES consecutive rows pass the filter
                 const auto first = offsets_pos == offsets_begin;
@@ -270,88 +269,6 @@ namespace
             filt_pos += SIMD_BYTES;
             offsets_pos += SIMD_BYTES;
         }
-    #elif defined(__AVX__) && defined(__AVX2__)
-        const __m256i zero_vec = _mm256_setzero_si256();
-        static constexpr size_t SIMD_BYTES = 32;
-        const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
-
-        while (filt_pos < filt_end_aligned)
-        {
-            uint32_t mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast<const __m256i *>(filt_pos)), zero_vec));
-
-            if (mask == 0xffffffff)
-            {
-                /// SIMD_BYTES consecutive rows pass the filter
-                const auto first = offsets_pos == offsets_begin;
-
-                const auto chunk_offset = first ? 0 : offsets_pos[-1];
-                const auto chunk_size = offsets_pos[SIMD_BYTES - 1] - chunk_offset;
-
-                result_offsets_builder.template insertChunk<SIMD_BYTES>(offsets_pos, first, chunk_offset, chunk_size);
-
-                /// copy elements for SIMD_BYTES arrays at once
-                const auto elems_size_old = res_elems.size();
-                res_elems.resize(elems_size_old + chunk_size);
-                memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
-            }
-            else
-            {
-                while (mask)
-                {
-                    size_t index = __builtin_ctz(mask);
-                    copy_array(offsets_pos + index);
-                #ifdef __BMI__
-                    mask = _blsr_u32(mask);
-                #else
-                    mask = mask & (mask-1);
-                #endif
-                }
-            }
-
-            filt_pos += SIMD_BYTES;
-            offsets_pos += SIMD_BYTES;
-        }
-    #elif defined(__SSE2__)
-        const __m128i zero_vec = _mm_setzero_si128();
-        static constexpr size_t SIMD_BYTES = 16;
-        const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
-
-        while (filt_pos < filt_end_aligned)
-        {
-            UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
-                _mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)),
-                zero_vec));
-            mask = ~mask;
-
-            if (mask == 0xffff)
-            {
-                /// SIMD_BYTES consecutive rows pass the filter
-                const auto first = offsets_pos == offsets_begin;
-
-                const auto chunk_offset = first ? 0 : offsets_pos[-1];
-                const auto chunk_size = offsets_pos[SIMD_BYTES - 1] - chunk_offset;
-
-                result_offsets_builder.template insertChunk<SIMD_BYTES>(offsets_pos, first, chunk_offset, chunk_size);
-
-                /// copy elements for SIMD_BYTES arrays at once
-                const auto elems_size_old = res_elems.size();
-                res_elems.resize(elems_size_old + chunk_size);
-                memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
-            }
-            else
-            {
-                while (mask)
-                {
-                    size_t index = __builtin_ctz(mask);
-                    copy_array(offsets_pos + index);
-                    mask = mask & (mask - 1);
-                }
-            }
-
-            filt_pos += SIMD_BYTES;
-            offsets_pos += SIMD_BYTES;
-        }
-    #endif
 
         while (filt_pos < filt_end)
         {

src/Columns/ColumnsCommon.h

@@ -2,7 +2,12 @@
 
 #include <Columns/IColumn.h>
 #include <Common/PODArray.h>
-
+#ifdef __SSE2__
+#include <emmintrin.h>
+#endif
+#if defined(__AVX512F__) || defined(__AVX512BW__) || defined(__AVX__) || defined(__AVX2__)
+#include <immintrin.h>
+#endif
 
 /// Common helper methods for implementation of different columns.
 
@@ -15,6 +20,40 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
+/// Transform 64-byte mask to 64-bit mask
+inline UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64)
+{
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+    static const __m512i zero64 = _mm512_setzero_epi32();
+    UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i *>(bytes64)), zero64, _MM_CMPINT_EQ);
+#elif defined(__AVX__) && defined(__AVX2__)
+    static const __m256i zero32 = _mm256_setzero_si256();
+    UInt64 res =
+        (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64)), zero32))) & 0xffffffff)
+        | (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64+32)), zero32))) << 32);
+#elif defined(__SSE2__) && defined(__POPCNT__)
+    static const __m128i zero16 = _mm_setzero_si128();
+    UInt64 res =
+        (static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes64)), zero16))) & 0xffff)
+        | ((static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes64 + 16)), zero16))) << 16) & 0xffff0000)
+        | ((static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes64 + 32)), zero16))) << 32) & 0xffff00000000)
+        | ((static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes64 + 48)), zero16))) << 48) & 0xffff000000000000);
+#else
+    UInt64 res = 0;
+    const UInt8 * pos = bytes64;
+    const UInt8 * end = pos + 64;
+    for (; pos < end; ++pos)
+        res |= ((*pos == 0)<<(pos-bytes64));
+#endif
+    return ~res;
+}
+
 /// Counts how many bytes of `filt` are greater than zero.
 size_t countBytesInFilter(const UInt8 * filt, size_t sz);
 size_t countBytesInFilter(const IColumn::Filter & filt);

src/Common/ClickHouseRevision.cpp

@@ -1,8 +1,5 @@
 #include <Common/ClickHouseRevision.h>
-
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config_version.h>
-#endif
+#include <Common/config_version.h>
 
 namespace ClickHouseRevision
 {

src/Common/CompactArray.h

@@ -55,28 +55,6 @@ public:
         return locus;
     }
 
-    /// Used only in arcadia/metrika
-    void readText(ReadBuffer & in)
-    {
-        for (size_t i = 0; i < BITSET_SIZE; ++i)
-        {
-            if (i != 0)
-                assertChar(',', in);
-            readIntText(bitset[i], in);
-        }
-    }
-
-    /// Used only in arcadia/metrika
-    void writeText(WriteBuffer & out) const
-    {
-        for (size_t i = 0; i < BITSET_SIZE; ++i)
-        {
-            if (i != 0)
-                writeCString(",", out);
-            writeIntText(bitset[i], out);
-        }
-    }
-
 private:
     /// number of bytes in bitset
     static constexpr size_t BITSET_SIZE = (static_cast<size_t>(bucket_count) * content_width + 7) / 8;

src/Common/Config/ConfigProcessor.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-    #include <Common/config.h>
-#endif
+#include <Common/config.h>
 #include "ConfigProcessor.h"
 #include "YAMLParser.h"
 

src/Common/Config/ConfigProcessor.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-    #include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #include <string>
 #include <unordered_set>

src/Common/Config/YAMLParser.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-    #include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_YAML_CPP
 #include "YAMLParser.h"
@@ -19,7 +17,7 @@
 #include <Poco/DOM/Text.h>
 #include <Common/Exception.h>
 
-#include <yaml-cpp/yaml.h> // Y_IGNORE
+#include <yaml-cpp/yaml.h>
 
 #include <base/logger_useful.h>
 

src/Common/Config/YAMLParser.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-    #include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #include <string>
 

src/Common/DNSResolver.cpp

@@ -114,11 +114,7 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host)
 
     try
     {
-#if defined(ARCADIA_BUILD)
-        addresses = Poco::Net::DNS::hostByName(host, &Poco::Net::DNS::DEFAULT_DNS_TIMEOUT, flags).addresses();
-#else
         addresses = Poco::Net::DNS::hostByName(host, flags).addresses();
-#endif
     }
     catch (const Poco::Net::DNSException & e)
     {

src/Common/ErrorCodes.cpp

@@ -577,7 +577,7 @@
     M(607, BACKUP_ELEMENT_DUPLICATE) \
     M(608, CANNOT_RESTORE_TABLE) \
     M(609, FUNCTION_ALREADY_EXISTS) \
-    M(610, CANNOT_DROP_SYSTEM_FUNCTION) \
+    M(610, CANNOT_DROP_FUNCTION) \
     M(611, CANNOT_CREATE_RECURSIVE_FUNCTION) \
     M(612, OBJECT_ALREADY_STORED_ON_DISK) \
     M(613, OBJECT_WAS_NOT_STORED_ON_DISK) \

src/Common/Exception.cpp

@@ -17,9 +17,7 @@
 #include <Common/ErrorCodes.h>
 #include <filesystem>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config_version.h>
-#endif
+#include <Common/config_version.h>
 
 namespace fs = std::filesystem;
 

src/Common/OpenSSLHelpers.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_SSL
 #include "OpenSSLHelpers.h"

src/Common/OpenSSLHelpers.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_SSL
 #    include <base/types.h>

src/Common/OptimizedRegularExpression.h

@@ -7,9 +7,7 @@
 #include <Common/StringSearcher.h>
 #include <re2/re2.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_RE2_ST
 #    include <re2_st/re2.h>

src/Common/QueryProfiler.h

@@ -4,9 +4,7 @@
 #include <signal.h>
 #include <time.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 
 namespace Poco

src/Common/SparseHashMap.h

@@ -1,24 +1,10 @@
 #pragma once
 
 /// SparseHashMap is a wrapper for google::sparse_hash_map.
-#if defined(ARCADIA_BUILD)
-#define HASH_FUN_H <unordered_map>
-template <typename T>
-struct THash;
-#endif
 
 #include <sparsehash/sparse_hash_map>
 
-#if !defined(ARCADIA_BUILD)
-    template <class Key, class T, class HashFcn = std::hash<Key>,
-        class EqualKey = std::equal_to<Key>,
-        class Alloc = google::libc_allocator_with_realloc<std::pair<const Key, T>>>
-    using SparseHashMap = google::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>;
-#else
-    template <class Key, class T, class HashFcn = std::hash<Key>,
-        class EqualKey = std::equal_to<Key>,
-        class Alloc = google::sparsehash::libc_allocator_with_realloc<std::pair<const Key, T>>>
-    using SparseHashMap = google::sparsehash::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>;
-
-    #undef THash
-#endif
+template <class Key, class T, class HashFcn = std::hash<Key>,
+    class EqualKey = std::equal_to<Key>,
+    class Alloc = google::libc_allocator_with_realloc<std::pair<const Key, T>>>
+using SparseHashMap = google::sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>;

src/Common/StackTrace.cpp

@@ -13,9 +13,7 @@
 #include <sstream>
 #include <unordered_map>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_UNWIND
 #    include <libunwind.h>
@@ -201,7 +199,7 @@ void StackTrace::symbolize(
     const StackTrace::FramePointers & frame_pointers, [[maybe_unused]] size_t offset,
     size_t size, StackTrace::Frames & frames)
 {
-#if defined(__ELF__) && !defined(__FreeBSD__) && !defined(ARCADIA_BUILD)
+#if defined(__ELF__) && !defined(__FreeBSD__)
 
     auto symbol_index_ptr = DB::SymbolIndex::instance();
     const DB::SymbolIndex & symbol_index = *symbol_index_ptr;

src/Common/ThreadStatus.cpp

@@ -26,7 +26,7 @@ namespace ErrorCodes
 thread_local ThreadStatus * current_thread = nullptr;
 thread_local ThreadStatus * main_thread = nullptr;
 
-#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
+#if !defined(SANITIZER)
 namespace
 {
 
@@ -88,7 +88,7 @@ ThreadStatus::ThreadStatus()
     /// Will set alternative signal stack to provide diagnostics for stack overflow errors.
     /// If not already installed for current thread.
     /// Sanitizer makes larger stack usage and also it's incompatible with alternative stack by default (it sets up and relies on its own).
-#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
+#if !defined(SANITIZER)
     if (!has_alt_stack)
     {
         /// Don't repeat tries even if not installed successfully.
@@ -147,11 +147,9 @@ ThreadStatus::~ThreadStatus()
         thread_group->threads.erase(this);
     }
 
-#if !defined(ARCADIA_BUILD)
     /// It may cause segfault if query_context was destroyed, but was not detached
     auto query_context_ptr = query_context.lock();
     assert((!query_context_ptr && query_id.empty()) || (query_context_ptr && query_id == query_context_ptr->getCurrentQueryId()));
-#endif
 
     if (deleter)
         deleter();

src/Common/ZooKeeper/ZooKeeperImpl.cpp

@@ -9,9 +9,7 @@
 #include <IO/WriteBufferFromString.h>
 #include <base/logger_useful.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 
 #if USE_SSL
 #    include <Poco/Net/SecureStreamSocket.h>

src/Compression/CompressionCodecEncrypted.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#endif
+#include <Common/config.h>
 #include <Common/Exception.h>
 #include <base/types.h>
 #include <IO/VarInt.h>
@@ -13,10 +11,10 @@
 // This depends on BoringSSL-specific API, notably <openssl/aead.h>.
 #if USE_SSL && USE_INTERNAL_SSL_LIBRARY
 #include <Parsers/ASTLiteral.h>
-#include <openssl/digest.h> // Y_IGNORE
+#include <openssl/digest.h>
 #include <openssl/err.h>
 #include <boost/algorithm/hex.hpp>
-#include <openssl/aead.h> // Y_IGNORE
+#include <openssl/aead.h>
 #endif
 
 // Common part for both parts (with SSL and without)

src/Compression/CompressionCodecEncrypted.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include <string_view>
 #include <unordered_map>
 #include <base/types.h>
@@ -134,5 +133,3 @@ private:
 };
 
 }
-
-#endif /* NOT Arcadia_build */

src/Compression/CompressionFactory.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#   include "config_core.h"
-#endif
+#include "config_core.h"
 
 #include <Compression/CompressionFactory.h>
 #include <Parsers/ASTFunction.h>

src/Coordination/Changelog.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <city.h>
 #include <optional>
 #include <IO/WriteBufferFromFile.h>

src/Coordination/InMemoryLogStore.h

@@ -4,7 +4,7 @@
 #include <map>
 #include <mutex>
 #include <Core/Types.h>
-#include <libnuraft/log_store.hxx> // Y_IGNORE
+#include <libnuraft/log_store.hxx>
 
 namespace DB
 {

src/Coordination/KeeperDispatcher.h

@@ -1,9 +1,7 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#    include "config_core.h"
-#endif
+#include <Common/config.h>
+#include "config_core.h"
 
 #if USE_NURAFT
 

src/Coordination/KeeperLogStore.h

@@ -1,5 +1,5 @@
 #pragma once
-#include <libnuraft/log_store.hxx> // Y_IGNORE
+#include <libnuraft/log_store.hxx>
 #include <map>
 #include <mutex>
 #include <Core/Types.h>

src/Coordination/KeeperServer.cpp

@@ -1,9 +1,7 @@
 #include <Coordination/KeeperServer.h>
 #include <Coordination/Defines.h>
 
-#if !defined(ARCADIA_BUILD)
-#   include "config_core.h"
-#endif
+#include "config_core.h"
 
 #include <Coordination/LoggerWrapper.h>
 #include <Coordination/KeeperStateMachine.h>

src/Coordination/KeeperServer.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <Coordination/InMemoryLogStore.h>
 #include <Coordination/KeeperStateManager.h>
 #include <Coordination/KeeperStateMachine.h>

src/Coordination/KeeperSnapshotManager.h

@@ -1,5 +1,5 @@
 #pragma once
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <Coordination/KeeperStorage.h>
 #include <IO/WriteBuffer.h>
 #include <IO/ReadBuffer.h>

src/Coordination/KeeperStateMachine.h

@@ -2,7 +2,7 @@
 
 #include <Common/ConcurrentBoundedQueue.h>
 #include <Coordination/KeeperStorage.h>
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <base/logger_useful.h>
 #include <Coordination/CoordinationSettings.h>
 #include <Coordination/KeeperSnapshotManager.h>

src/Coordination/KeeperStateManager.h

@@ -4,7 +4,7 @@
 #include <string>
 #include <Coordination/KeeperLogStore.h>
 #include <Coordination/CoordinationSettings.h>
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Coordination/KeeperSnapshotManager.h>
 

src/Coordination/LoggerWrapper.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <base/logger_useful.h>
 #include <Core/SettingsEnums.h>
 

src/Coordination/ReadBufferFromNuraftBuffer.h

@@ -1,7 +1,7 @@
 #pragma once
 #include <IO/ReadBufferFromMemory.h>
 
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 
 namespace DB
 {

src/Coordination/SummingStateMachine.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <Core/Types.h>
 #include <atomic>
 #include <map>

src/Coordination/WriteBufferFromNuraftBuffer.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <IO/WriteBuffer.h>
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 
 namespace DB
 {

src/Coordination/tests/gtest_coordination.cpp

@@ -1,9 +1,7 @@
 #include <gtest/gtest.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include <Common/config.h>
-#    include "config_core.h"
-#endif
+#include <Common/config.h>
+#include "config_core.h"
 
 #if USE_NURAFT
 #include <Poco/ConsoleChannel.h>
@@ -22,7 +20,7 @@
 #include <Common/ZooKeeper/ZooKeeperIO.h>
 #include <Common/Exception.h>
 #include <base/logger_useful.h>
-#include <libnuraft/nuraft.hxx> // Y_IGNORE
+#include <libnuraft/nuraft.hxx>
 #include <thread>
 #include <Coordination/KeeperLogStore.h>
 #include <Coordination/Changelog.h>

src/Core/MySQL/Authentication.h

@@ -4,9 +4,7 @@
 #include <Interpreters/Context.h>
 #include <Core/MySQL/PacketEndpoint.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_SSL
 #    include <openssl/pem.h>

src/Core/PostgreSQL/Connection.h

@@ -1,12 +1,10 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 
-#include <pqxx/pqxx> // Y_IGNORE
+#include <pqxx/pqxx>
 #include <Core/Types.h>
 #include <boost/noncopyable.hpp>
 

src/Core/PostgreSQL/ConnectionHolder.h

@@ -1,12 +1,10 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 
-#include <pqxx/pqxx> // Y_IGNORE
+#include <pqxx/pqxx>
 #include <Core/Types.h>
 #include <base/BorrowedObjectPool.h>
 

src/Core/PostgreSQL/PoolWithFailover.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 

src/Core/PostgreSQL/Utils.h

@@ -1,12 +1,10 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 
-#include <pqxx/pqxx> // Y_IGNORE
+#include <pqxx/pqxx>
 #include <Core/Types.h>
 #include "Connection.h"
 #include <Common/Exception.h>

src/Core/PostgreSQL/insertPostgreSQLValue.cpp

@@ -15,7 +15,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/ReadBufferFromString.h>
 #include <Common/assert_cast.h>
-#include <pqxx/pqxx> // Y_IGNORE
+#include <pqxx/pqxx>
 
 
 namespace DB

src/Core/PostgreSQL/insertPostgreSQLValue.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 

src/Core/Settings.h

@@ -263,7 +263,6 @@ class IColumn;
     M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
     M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
     M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
-    M(UInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
     M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
@@ -547,6 +546,7 @@ class IColumn;
     M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing.", 0) \
     M(UInt64, replication_alter_columns_timeout, 60, "Obsolete setting, does nothing.", 0) \
     M(Bool, optimize_fuse_sum_count_avg, false, "Obsolete, use optimize_syntax_fuse_functions", 0) \
+    M(UInt64, odbc_max_field_size, 0, "Obsolete setting, does nothing.", 0) \
     /** The section above is for obsolete settings. Do not add anything there. */
 
 

src/DataTypes/Native.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_EMBEDDED_COMPILER
 #    include <Common/Exception.h>

src/Databases/DatabaseFactory.cpp

@@ -16,9 +16,7 @@
 #include <Storages/ExternalDataSourceConfiguration.h>
 #include <filesystem>
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 #    include <Core/MySQL/MySQLClient.h>
@@ -36,7 +34,7 @@
 #endif
 
 #if USE_LIBPQXX
-#include <Databases/PostgreSQL/DatabasePostgreSQL.h> // Y_IGNORE
+#include <Databases/PostgreSQL/DatabasePostgreSQL.h>
 #include <Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h>
 #include <Storages/PostgreSQL/MaterializedPostgreSQLSettings.h>
 #include <Storages/StoragePostgreSQL.h>

src/Databases/MySQL/DatabaseMaterializedMySQL.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 

src/Databases/MySQL/DatabaseMySQL.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 #    include <string>

src/Databases/MySQL/FetchTablesColumnsList.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 #include <Core/Block.h>

src/Databases/MySQL/MaterializeMetadata.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 

src/Databases/MySQL/MaterializedMySQLSyncThread.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 

src/Databases/MySQL/MaterializedMySQLSyncThread.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 

src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 

src/Databases/PostgreSQL/DatabasePostgreSQL.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 

src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h

@@ -1,8 +1,6 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_LIBPQXX
 #include <Core/PostgreSQL/ConnectionHolder.h>

src/Databases/SQLite/DatabaseSQLite.h

@@ -1,15 +1,13 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_SQLITE
 #include <Core/Names.h>
 #include <Databases/DatabasesCommon.h>
 #include <Parsers/ASTCreateQuery.h>
 
-#include <sqlite3.h> // Y_IGNORE
+#include <sqlite3.h>
 
 
 namespace DB

src/Databases/SQLite/SQLiteUtils.h

@@ -1,13 +1,11 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_SQLITE
 #include <Core/Types.h>
 #include <Interpreters/Context.h>
-#include <sqlite3.h> // Y_IGNORE
+#include <sqlite3.h>
 
 
 namespace DB

src/Databases/SQLite/fetchSQLiteTableStructure.h

@@ -1,13 +1,11 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include "config_core.h"
-#endif
 
 #if USE_SQLITE
 
 #include <Storages/StorageSQLite.h>
-#include <sqlite3.h> // Y_IGNORE
+#include <sqlite3.h>
 
 
 namespace DB

src/Dictionaries/CassandraHelpers.h

@@ -1,11 +1,9 @@
 #pragma once
 
-#if !defined(ARCADIA_BUILD)
 #include <Common/config.h>
-#endif
 
 #if USE_CASSANDRA
-#include <cassandra.h> // Y_IGNORE
+#include <cassandra.h>
 #include <utility>
 #include <memory>
 

src/Dictionaries/CassandraSource.cpp

@@ -1,6 +1,4 @@
-#if !defined(ARCADIA_BUILD)
 #include <Common/config.h>
-#endif
 
 #if USE_CASSANDRA
 

src/Dictionaries/MySQLDictionarySource.h

@@ -2,9 +2,7 @@
 
 #include <Core/Block.h>
 
-#if !defined(ARCADIA_BUILD)
-#    include "config_core.h"
-#endif
+#include "config_core.h"
 
 #if USE_MYSQL
 #    include <base/LocalDateTime.h>