Merge remote-tracking branch 'master/master' into update_23

2024-09-21 09:10:48 +00:00 · 2018-12-03 19:20:35 +08:00 · 2018-12-03 19:20:35 +08:00 · fb06c5ab6b
commit fb06c5ab6b
parent 6f699e9f94 0090aaf8ee
366 changed files with 10191 additions and 13840 deletions
--- a/.clang-format
+++ b/.clang-format
@ -29,7 +29,9 @@ AllowShortFunctionsOnASingleLine: InlineOnly
 AlwaysBreakTemplateDeclarations: true
 IndentCaseLabels: true
 SpaceAfterTemplateKeyword: true
+SpaceBeforeCpp11BracedList: false
 SortIncludes: true
+IndentPPDirectives: AfterHash
 IncludeCategories:
  - Regex: '^<[a-z_]+>'
    Priority: 1
--- a/CHANGELOG_RU.md
+++ b/CHANGELOG_RU.md
@ -1,3 +1,12 @@
+## ClickHouse release 18.14.17, 2018-11-30
+
+### Исправления ошибок:
+* Исправлена ситуация, при которой ODBC Bridge продолжал работу после завершения работы сервера ClickHouse. Теперь ODBC Bridge всегда завершает работу вместе с сервером. [#3642](https://github.com/yandex/ClickHouse/pull/3642)
+* Исправлена синхронная вставка в `Distributed` таблицу в случае явного указания неполного списка столбцов или списка столбцов в измененном порядке. [#3673](https://github.com/yandex/ClickHouse/pull/3673)
+* Исправлен race condition в `BackgroundProcessingPoolTaskInfo` возникающий при попытке выполнения задачи одновременно с инвалидацией её итератора очереди. [#3680](https://github.com/yandex/ClickHouse/pull/3680)
+* Исправлен deadlock в `ParallelInputsProcessor` возникающий в случае исключения при создании потока. [#3643](https://github.com/yandex/ClickHouse/pull/3643)
+* Исправлена ошибка парсинга `Engine` при создании таблицы с синтаксисом `AS table` в случае, когда `AS table` указывался после `Engine`, что приводило к игнорированию указанного движка. [#3692](https://github.com/yandex/ClickHouse/pull/3692)
+
 ## ClickHouse release 18.14.15, 2018-11-21

 ### Исправления ошибок:
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -25,6 +25,18 @@ endif ()
 # Write compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)

+set(PARALLEL_COMPILE_JOBS "" CACHE STRING "Define the maximum number of concurrent compilation jobs")
+if (PARALLEL_COMPILE_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool="${PARALLEL_COMPILE_JOBS}")
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+endif ()
+
+set(PARALLEL_LINK_JOBS "" CACHE STRING "Define the maximum number of concurrent link jobs")
+if (LLVM_PARALLEL_LINK_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${PARALLEL_LINK_JOBS})
+    set(CMAKE_JOB_POOL_LINK link_job_pool)
+endif ()
+
 include (cmake/find_ccache.cmake)

 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@ -147,13 +159,6 @@ set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fn
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")

-if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD))
-    set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
-
-    # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained.
-    set (CMAKE_SHARED_LINKER_FLAGS       "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -static-libstdc++")
-endif ()
-
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package (Threads)

@ -169,7 +174,8 @@ if (OS_LINUX AND COMPILER_CLANG)
        set (CMAKE_CXX_FLAGS             "${CMAKE_CXX_FLAGS} -stdlib=libc++") # Ok for clang6, for older can cause 'not used option' warning
        set (CMAKE_CXX_FLAGS_DEBUG       "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
        if (MAKE_STATIC_LIBRARIES)
-            link_libraries (-Wl,-Bstatic -stdlib=libc++ c++ c++abi -Wl,-Bdynamic)
+            execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIB_PATH OUTPUT_STRIP_TRAILING_WHITESPACE)
+            link_libraries (-nodefaultlibs -Wl,-Bstatic -stdlib=libc++ c++ c++abi gcc_eh ${BUILTINS_LIB_PATH} rt -Wl,-Bdynamic dl pthread m c)
        else ()
            link_libraries (-stdlib=libc++ c++ c++abi)
        endif ()
@ -181,6 +187,19 @@ if (OS_LINUX AND COMPILER_CLANG)
    endif ()
 endif ()

+if (COMPILER_GCC)
+    set (STATIC_STDLIB_FLAGS "-static-libgcc -static-libstdc++")
+else ()
+    set (STATIC_STDLIB_FLAGS "")
+endif ()
+
+if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD))
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
+
+    # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained.
+    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${STATIC_STDLIB_FLAGS}")
+endif ()
+
 if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
    set (CMAKE_CXX_FLAGS                 "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}")
    set (CMAKE_C_FLAGS                   "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
--- a/cmake/generate_code.cmake
+++ b/cmake/generate_code.cmake
@ -0,0 +1,5 @@
+function(generate_code TEMPLATE_FILE)
+    foreach(NAME IN LISTS ARGN)
+       configure_file (${TEMPLATE_FILE}.cpp.in ${CMAKE_CURRENT_BINARY_DIR}/generated/${TEMPLATE_FILE}_${NAME}.cpp)
+    endforeach()
+endfunction()
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@ -52,5 +52,11 @@ target_include_directories(jemalloc PRIVATE
 target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)

 if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-    target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1)
+    target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
+
+    if (USE_UNWIND)
+        target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
+        target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR})
+        target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY})
+    endif ()
 endif ()
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@ -63,9 +63,6 @@ add_headers_and_sources(dbms src/Core)
 add_headers_and_sources(dbms src/DataStreams)
 add_headers_and_sources(dbms src/DataTypes)
 add_headers_and_sources(dbms src/Databases)
-add_headers_and_sources(dbms src/Dictionaries)
-add_headers_and_sources(dbms src/Dictionaries/Embedded)
-add_headers_and_sources(dbms src/Dictionaries/Embedded/GeodataProviders)
 add_headers_and_sources(dbms src/Interpreters)
 add_headers_and_sources(dbms src/Interpreters/ClusterProxy)
 add_headers_and_sources(dbms src/Columns)
@ -184,8 +181,11 @@ target_link_libraries (dbms
    clickhouse_common_config
        PUBLIC
    clickhouse_common_io
-    pocoext
+        PRIVATE
+    clickhouse_dictionaries
+    clickhouse_dictionaries_embedded
        PUBLIC
+    pocoext
    ${MYSQLXX_LIBRARY}
        PRIVATE
    ${BTRIE_LIBRARIES}
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@ -2,10 +2,10 @@
 set(VERSION_REVISION 54409 CACHE STRING "")
 set(VERSION_MAJOR 18 CACHE STRING "")
 set(VERSION_MINOR 14 CACHE STRING "")
-set(VERSION_PATCH 9 CACHE STRING "")
-set(VERSION_GITHASH 457f8fd495b2812940e69c15ab5b499cd863aae4 CACHE STRING "")
-set(VERSION_DESCRIBE v18.14.9-testing CACHE STRING "")
-set(VERSION_STRING 18.14.9 CACHE STRING "")
+set(VERSION_PATCH 17 CACHE STRING "")
+set(VERSION_GITHASH ac2895d769c3dcf070530dec7fcfdcf87bfa852a CACHE STRING "")
+set(VERSION_DESCRIBE v18.14.17-testing CACHE STRING "")
+set(VERSION_STRING 18.14.17 CACHE STRING "")
 # end of autochange

 set(VERSION_EXTRA "" CACHE STRING "")
--- a/dbms/programs/benchmark/Benchmark.cpp
+++ b/dbms/programs/benchmark/Benchmark.cpp
@ -31,6 +31,7 @@
 #include <Interpreters/Context.h>
 #include <Client/Connection.h>
 #include <Common/InterruptListener.h>
+#include <Common/Config/configReadClient.h>


 /** A tool for evaluating ClickHouse performance.
@ -46,17 +47,17 @@ namespace ErrorCodes
    extern const int EMPTY_DATA_PASSED;
 }

-class Benchmark
+class Benchmark : public Poco::Util::Application
 {
 public:
    Benchmark(unsigned concurrency_, double delay_,
-            const String & host_, UInt16 port_, const String & default_database_,
+            const String & host_, UInt16 port_, bool secure_, const String & default_database_,
            const String & user_, const String & password_, const String & stage,
            bool randomize_, size_t max_iterations_, double max_time_,
            const String & json_path_, const ConnectionTimeouts & timeouts, const Settings & settings_)
        :
        concurrency(concurrency_), delay(delay_), queue(concurrency),
-        connections(concurrency, host_, port_, default_database_, user_, password_, timeouts),
+        connections(concurrency, host_, port_, default_database_, user_, password_, timeouts, "benchmark", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable),
        randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_),
        json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency)
    {
@ -75,13 +76,26 @@ public:
        else
            throw Exception("Unknown query processing stage: " + stage, ErrorCodes::BAD_ARGUMENTS);

+    }
+
+    void initialize(Poco::Util::Application & self [[maybe_unused]])
+    {
+        std::string home_path;
+        const char * home_path_cstr = getenv("HOME");
+        if (home_path_cstr)
+            home_path = home_path_cstr;
+
+        configReadClient(config(), home_path);
+    }
+
+    int main(const std::vector<std::string> &)
+    {
        if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results
-        {
            Poco::File(json_path).remove();
-        }

        readQueries();
-        run();
+        runBenchmark();
+        return 0;
    }

 private:
@ -220,7 +234,7 @@ private:
        return true;
    }

-    void run()
+    void runBenchmark()
    {
        pcg64 generator(randomSeed());
        std::uniform_int_distribution<size_t> distribution(0, queries.size() - 1);
@ -432,6 +446,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
            ("json",          value<std::string>()->default_value(""),          "write final report to specified file in JSON format")
            ("host,h",        value<std::string>()->default_value("localhost"), "")
            ("port",          value<UInt16>()->default_value(9000),             "")
+            ("secure,s",                                                        "Use TLS connection")
            ("user",          value<std::string>()->default_value("default"),   "")
            ("password",      value<std::string>()->default_value(""),          "")
            ("database",      value<std::string>()->default_value("default"),   "")
@ -470,6 +485,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
            options["delay"].as<double>(),
            options["host"].as<std::string>(),
            options["port"].as<UInt16>(),
+            options.count("secure"),
            options["database"].as<std::string>(),
            options["user"].as<std::string>(),
            options["password"].as<std::string>(),
@ -480,6 +496,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
            options["json"].as<std::string>(),
            ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings),
            settings);
+        return benchmark.run();
    }
    catch (...)
    {
--- a/dbms/programs/benchmark/CMakeLists.txt
+++ b/dbms/programs/benchmark/CMakeLists.txt
@ -1,5 +1,5 @@
 add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
-target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse_aggregate_functions clickhouse-client-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 if (CLICKHOUSE_SPLIT_BINARY)
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@ -59,6 +59,7 @@
 #include <Common/InterruptListener.h>
 #include <Functions/registerFunctions.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
+#include <Common/Config/configReadClient.h>

 #if USE_READLINE
 #include "Suggest.h" // Y_IGNORE
@ -206,22 +207,7 @@ private:
        if (home_path_cstr)
            home_path = home_path_cstr;

-        std::string config_path;
-        if (config().has("config-file"))
-            config_path = config().getString("config-file");
-        else if (Poco::File("./clickhouse-client.xml").exists())
-            config_path = "./clickhouse-client.xml";
-        else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists())
-            config_path = home_path + "/.clickhouse-client/config.xml";
-        else if (Poco::File("/etc/clickhouse-client/config.xml").exists())
-            config_path = "/etc/clickhouse-client/config.xml";
-
-        if (!config_path.empty())
-        {
-            ConfigProcessor config_processor(config_path);
-            auto loaded_config = config_processor.loadConfig();
-            config().add(loaded_config.configuration);
-        }
+        configReadClient(config(), home_path);

        context.setApplicationType(Context::ApplicationType::CLIENT);

--- a/dbms/programs/local/LocalServer.cpp
+++ b/dbms/programs/local/LocalServer.cpp
@ -30,6 +30,7 @@
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
+#include <Dictionaries/registerDictionaries.h>
 #include <boost/program_options/options_description.hpp>
 #include <boost/program_options.hpp>

@ -142,6 +143,7 @@ try
    registerAggregateFunctions();
    registerTableFunctions();
    registerStorages();
+    registerDictionaries();

    /// Maybe useless
    if (config().has("macros"))
--- a/dbms/programs/performance-test/CMakeLists.txt
+++ b/dbms/programs/performance-test/CMakeLists.txt
@ -1,5 +1,5 @@
 add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
-target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 if (CLICKHOUSE_SPLIT_BINARY)
--- a/dbms/programs/performance-test/PerformanceTest.cpp
+++ b/dbms/programs/performance-test/PerformanceTest.cpp
@ -30,7 +30,9 @@
 #include <Poco/SAX/InputSource.h>
 #include <Poco/Util/XMLConfiguration.h>
 #include <Poco/XML/XMLStream.h>
+#include <Poco/Util/Application.h>
 #include <Common/InterruptListener.h>
+#include <Common/Config/configReadClient.h>

 #ifndef __clang__
 #pragma GCC optimize("-fno-var-tracking-assignments")
@ -487,17 +489,18 @@ struct Stats
 double Stats::avg_rows_speed_precision = 0.001;
 double Stats::avg_bytes_speed_precision = 0.001;

-class PerformanceTest
+class PerformanceTest : public Poco::Util::Application
 {
 public:
    using Strings = std::vector<String>;

    PerformanceTest(const String & host_,
        const UInt16 port_,
+        const bool secure_,
        const String & default_database_,
        const String & user_,
        const String & password_,
-        const bool & lite_output_,
+        const bool lite_output_,
        const String & profiles_file_,
        Strings && input_files_,
        Strings && tests_tags_,
@ -507,7 +510,7 @@ public:
        Strings && tests_names_regexp_,
        Strings && skip_names_regexp_,
        const ConnectionTimeouts & timeouts)
-        : connection(host_, port_, default_database_, user_, password_, timeouts),
+        : connection(host_, port_, default_database_, user_, password_, timeouts, "performance-test", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable),
          gotSIGINT(false),
          lite_output(lite_output_),
          profiles_file(profiles_file_),
@ -523,7 +526,19 @@ public:
        {
            throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
        }
+    }

+    void initialize(Poco::Util::Application & self [[maybe_unused]])
+    {
+        std::string home_path;
+        const char * home_path_cstr = getenv("HOME");
+        if (home_path_cstr)
+            home_path = home_path_cstr;
+        configReadClient(Poco::Util::Application::instance().config(), home_path);
+    }
+
+    int main(const std::vector < std::string > & /* args */)
+    {
        std::string name;
        UInt64 version_major;
        UInt64 version_minor;
@ -536,6 +551,8 @@ public:
        server_version = ss.str();

        processTestsConfigurations(input_files);
+
+        return 0;
    }

 private:
@ -1383,21 +1400,28 @@ try
    using Strings = std::vector<String>;

    boost::program_options::options_description desc("Allowed options");
-    desc.add_options()("help", "produce help message")("lite", "use lite version of output")(
-        "profiles-file", value<String>()->default_value(""), "Specify a file with global profiles")(
-        "host,h", value<String>()->default_value("localhost"), "")("port", value<UInt16>()->default_value(9000), "")(
-        "database", value<String>()->default_value("default"), "")("user", value<String>()->default_value("default"), "")(
-        "password", value<String>()->default_value(""), "")("tags", value<Strings>()->multitoken(), "Run only tests with tag")(
-        "skip-tags", value<Strings>()->multitoken(), "Do not run tests with tag")("names",
-        value<Strings>()->multitoken(),
-        "Run tests with specific name")("skip-names", value<Strings>()->multitoken(), "Do not run tests with name")(
-        "names-regexp", value<Strings>()->multitoken(), "Run tests with names matching regexp")("skip-names-regexp",
-        value<Strings>()->multitoken(),
-        "Do not run tests with names matching regexp")("recursive,r", "Recurse in directories to find all xml's");
+    desc.add_options()
+        ("help", "produce help message")
+        ("lite", "use lite version of output")
+        ("profiles-file", value<String>()->default_value(""), "Specify a file with global profiles")
+        ("host,h", value<String>()->default_value("localhost"), "")
+        ("port", value<UInt16>()->default_value(9000), "")
+        ("secure,s", "Use TLS connection")
+        ("database", value<String>()->default_value("default"), "")
+        ("user", value<String>()->default_value("default"), "")
+        ("password", value<String>()->default_value(""), "")
+        ("tags", value<Strings>()->multitoken(), "Run only tests with tag")
+        ("skip-tags", value<Strings>()->multitoken(), "Do not run tests with tag")
+        ("names", value<Strings>()->multitoken(), "Run tests with specific name")
+        ("skip-names", value<Strings>()->multitoken(), "Do not run tests with name")
+        ("names-regexp", value<Strings>()->multitoken(), "Run tests with names matching regexp")
+        ("skip-names-regexp", value<Strings>()->multitoken(), "Do not run tests with names matching regexp")
+        ("recursive,r", "Recurse in directories to find all xml's");

    /// These options will not be displayed in --help
    boost::program_options::options_description hidden("Hidden options");
-    hidden.add_options()("input-files", value<std::vector<String>>(), "");
+    hidden.add_options()
+        ("input-files", value<std::vector<String>>(), "");

    /// But they will be legit, though. And they must be given without name
    boost::program_options::positional_options_description positional;
@ -1474,8 +1498,10 @@ try

    DB::UseSSL use_ssl;

-    DB::PerformanceTest performanceTest(options["host"].as<String>(),
+    DB::PerformanceTest performance_test(
+        options["host"].as<String>(),
        options["port"].as<UInt16>(),
+        options.count("secure"),
        options["database"].as<String>(),
        options["user"].as<String>(),
        options["password"].as<String>(),
@ -1489,8 +1515,7 @@ try
        std::move(tests_names_regexp),
        std::move(skip_names_regexp),
        timeouts);
-
-    return 0;
+    return performance_test.run();
 }
 catch (...)
 {
--- a/dbms/programs/server/Server.cpp
+++ b/dbms/programs/server/Server.cpp
@ -36,6 +36,7 @@
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
+#include <Dictionaries/registerDictionaries.h>
 #include <Common/Config/ConfigReloader.h>
 #include "HTTPHandlerFactory.h"
 #include "MetricsTransmitter.h"
@ -109,6 +110,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
    registerAggregateFunctions();
    registerTableFunctions();
    registerStorages();
+    registerDictionaries();

    CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
    CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
@ -123,12 +125,13 @@ int Server::main(const std::vector<std::string> & /*args*/)
    bool has_zookeeper = config().has("zookeeper");

    zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); });
+    zkutil::EventPtr main_config_zk_changed_event = std::make_shared<Poco::Event>();
    if (loaded_config.has_zk_includes)
    {
        auto old_configuration = loaded_config.configuration;
        ConfigProcessor config_processor(config_path);
        loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
-            main_config_zk_node_cache, /* fallback_to_preprocessed = */ true);
+            main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true);
        config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
        config().removeConfiguration(old_configuration.get());
        config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
@ -303,6 +306,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
        include_from_path,
        config().getString("path", ""),
        std::move(main_config_zk_node_cache),
+        main_config_zk_changed_event,
        [&](ConfigurationPtr config)
        {
            buildLoggers(*config);
@ -325,6 +329,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
        include_from_path,
        config().getString("path", ""),
        zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
+        std::make_shared<Poco::Event>(),
        [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
        /* already_loaded = */ false);

--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@ -729,14 +729,6 @@ BlockStreamProfileInfo Connection::receiveProfileInfo()
    return profile_info;
 }

-void Connection::fillBlockExtraInfo(BlockExtraInfo & info) const
-{
-    info.is_valid = true;
-    info.host = host;
-    info.resolved_address = getResolvedAddress().toString();
-    info.port = port;
-    info.user = user;
-}

 void Connection::throwUnexpectedPacket(UInt64 packet_type, const char * expected) const
 {
--- a/dbms/src/Client/Connection.h
+++ b/dbms/src/Client/Connection.h
@ -159,11 +159,6 @@ public:
      */
    void disconnect();

-    /** Fill in the information that is needed when getting the block for some tasks
-      * (so far only for a DESCRIBE TABLE query with Distributed tables).
-      */
-    void fillBlockExtraInfo(BlockExtraInfo & info) const;
-
    size_t outBytesCount() const { return out ? out->count() : 0; }
    size_t inBytesCount() const { return in ? in->count() : 0; }

--- a/dbms/src/Client/MultiplexedConnections.cpp
+++ b/dbms/src/Client/MultiplexedConnections.cpp
@ -26,7 +26,7 @@ MultiplexedConnections::MultiplexedConnections(Connection & connection, const Se

 MultiplexedConnections::MultiplexedConnections(
        std::vector<IConnectionPool::Entry> && connections,
-        const Settings & settings_, const ThrottlerPtr & throttler, bool append_extra_info)
+        const Settings & settings_, const ThrottlerPtr & throttler)
    : settings(settings_)
 {
    /// If we didn't get any connections from pool and getMany() did not throw exceptions, this means that
@ -48,9 +48,6 @@ MultiplexedConnections::MultiplexedConnections(
    }

    active_connection_count = connections.size();
-
-    if (append_extra_info)
-        block_extra_info = std::make_unique<BlockExtraInfo>();
 }

 void MultiplexedConnections::sendExternalTablesData(std::vector<ExternalTablesData> & data)
@ -126,24 +123,9 @@ Connection::Packet MultiplexedConnections::receivePacket()
 {
    std::lock_guard<std::mutex> lock(cancel_mutex);
    Connection::Packet packet = receivePacketUnlocked();
-    if (block_extra_info)
-    {
-        if (packet.type == Protocol::Server::Data)
-            current_connection->fillBlockExtraInfo(*block_extra_info);
-        else
-            block_extra_info->is_valid = false;
-    }
    return packet;
 }

-BlockExtraInfo MultiplexedConnections::getBlockExtraInfo() const
-{
-    if (!block_extra_info)
-        throw Exception("MultiplexedConnections object not configured for block extra info support",
-            ErrorCodes::LOGICAL_ERROR);
-    return *block_extra_info;
-}
-
 void MultiplexedConnections::disconnect()
 {
    std::lock_guard<std::mutex> lock(cancel_mutex);
--- a/dbms/src/Client/MultiplexedConnections.h
+++ b/dbms/src/Client/MultiplexedConnections.h
@ -21,12 +21,10 @@ public:
    /// Accepts ready connection.
    MultiplexedConnections(Connection & connection, const Settings & settings_, const ThrottlerPtr & throttler_);

-    /** Accepts a vector of connections to replicas of one shard already taken from pool.
-      * If the append_extra_info flag is set, additional information appended to each received block.
-      */
+    /// Accepts a vector of connections to replicas of one shard already taken from pool.
    MultiplexedConnections(
-            std::vector<IConnectionPool::Entry> && connections,
-            const Settings & settings_, const ThrottlerPtr & throttler_, bool append_extra_info);
+        std::vector<IConnectionPool::Entry> && connections,
+        const Settings & settings_, const ThrottlerPtr & throttler_);

    /// Send all content of external tables to replicas.
    void sendExternalTablesData(std::vector<ExternalTablesData> & data);
@ -42,9 +40,6 @@ public:
    /// Get packet from any replica.
    Connection::Packet receivePacket();

-    /// Get information about the last received packet.
-    BlockExtraInfo getBlockExtraInfo() const;
-
    /// Break all active connections.
    void disconnect();

@ -99,11 +94,8 @@ private:

    /// Connection that received last block.
    Connection * current_connection = nullptr;
-    /// Information about the last received block, if supported.
-    std::unique_ptr<BlockExtraInfo> block_extra_info;

    bool sent_query = false;
-
    bool cancelled = false;

    /// A mutex for the sendCancel function to execute safely
--- a/dbms/src/Common/Config/ConfigProcessor.cpp
+++ b/dbms/src/Common/Config/ConfigProcessor.cpp
@ -231,6 +231,7 @@ void ConfigProcessor::doIncludesRecursive(
        XMLDocumentPtr include_from,
        Node * node,
        zkutil::ZooKeeperNodeCache * zk_node_cache,
+        const zkutil::EventPtr & zk_changed_event,
        std::unordered_set<std::string> & contributing_zk_paths)
 {
    if (node->nodeType() == Node::TEXT_NODE)
@ -349,12 +350,12 @@ void ConfigProcessor::doIncludesRecursive(
            XMLDocumentPtr zk_document;
            auto get_zk_node = [&](const std::string & name) -> const Node *
            {
-                std::optional<std::string> contents = zk_node_cache->get(name);
-                if (!contents)
+                zkutil::ZooKeeperNodeCache::ZNode znode = zk_node_cache->get(name, zk_changed_event);
+                if (!znode.exists)
                    return nullptr;

                /// Enclose contents into a fake <from_zk> tag to allow pure text substitutions.
-                zk_document = dom_parser.parseString("<from_zk>" + *contents + "</from_zk>");
+                zk_document = dom_parser.parseString("<from_zk>" + znode.contents + "</from_zk>");
                return getRootNode(zk_document.get());
            };

@ -380,13 +381,13 @@ void ConfigProcessor::doIncludesRecursive(
    }

    if (included_something)
-        doIncludesRecursive(config, include_from, node, zk_node_cache, contributing_zk_paths);
+        doIncludesRecursive(config, include_from, node, zk_node_cache, zk_changed_event, contributing_zk_paths);
    else
    {
        NodeListPtr children = node->childNodes();
        Node * child = nullptr;
        for (size_t i = 0; (child = children->item(i)); ++i)
-            doIncludesRecursive(config, include_from, child, zk_node_cache, contributing_zk_paths);
+            doIncludesRecursive(config, include_from, child, zk_node_cache, zk_changed_event, contributing_zk_paths);
    }
 }

@ -430,7 +431,8 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &

 XMLDocumentPtr ConfigProcessor::processConfig(
    bool * has_zk_includes,
-    zkutil::ZooKeeperNodeCache * zk_node_cache)
+    zkutil::ZooKeeperNodeCache * zk_node_cache,
+    const zkutil::EventPtr & zk_changed_event)
 {
    XMLDocumentPtr config = dom_parser.parse(path);

@ -460,7 +462,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
        if (node)
        {
            /// if we include_from env or zk.
-            doIncludesRecursive(config, nullptr, node, zk_node_cache, contributing_zk_paths);
+            doIncludesRecursive(config, nullptr, node, zk_node_cache, zk_changed_event, contributing_zk_paths);
            include_from_path = node->innerText();
        }
        else
@ -475,7 +477,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
            include_from = dom_parser.parse(include_from_path);
        }

-        doIncludesRecursive(config, include_from, getRootNode(config.get()), zk_node_cache, contributing_zk_paths);
+        doIncludesRecursive(config, include_from, getRootNode(config.get()), zk_node_cache, zk_changed_event, contributing_zk_paths);
    }
    catch (Poco::Exception & e)
    {
@ -524,6 +526,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes

 ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
        zkutil::ZooKeeperNodeCache & zk_node_cache,
+        const zkutil::EventPtr & zk_changed_event,
        bool fallback_to_preprocessed)
 {
    XMLDocumentPtr config_xml;
@ -531,7 +534,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
    bool processed_successfully = false;
    try
    {
-        config_xml = processConfig(&has_zk_includes, &zk_node_cache);
+        config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event);
        processed_successfully = true;
    }
    catch (const Poco::Exception & ex)
--- a/dbms/src/Common/Config/ConfigProcessor.h
+++ b/dbms/src/Common/Config/ConfigProcessor.h
@ -3,6 +3,7 @@
 #include <string>
 #include <unordered_set>
 #include <vector>
+#include <memory>

 #include <Poco/DOM/Document.h>
 #include <Poco/DOM/DOMParser.h>
@ -22,6 +23,7 @@
 namespace zkutil
 {
    class ZooKeeperNodeCache;
+    using EventPtr = std::shared_ptr<Poco::Event>;
 }

 namespace DB
@ -61,7 +63,8 @@ public:
    /// 5) (Yandex.Metrika-specific) Substitute "<layer/>" with "<layer>layer number from the hostname</layer>".
    XMLDocumentPtr processConfig(
        bool * has_zk_includes = nullptr,
-        zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr);
+        zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
+        const zkutil::EventPtr & zk_changed_event = nullptr);


    /// loadConfig* functions apply processConfig and create Poco::Util::XMLConfiguration.
@ -87,6 +90,7 @@ public:
    /// processing, load the configuration from the preprocessed file.
    LoadedConfig loadConfigWithZooKeeperIncludes(
        zkutil::ZooKeeperNodeCache & zk_node_cache,
+        const zkutil::EventPtr & zk_changed_event,
        bool fallback_to_preprocessed = false);

    /// Save preprocessed config to specified directory.
@ -134,6 +138,7 @@ private:
            XMLDocumentPtr include_from,
            Poco::XML::Node * node,
            zkutil::ZooKeeperNodeCache * zk_node_cache,
+            const zkutil::EventPtr & zk_changed_event,
            std::unordered_set<std::string> & contributing_zk_paths);
 };

--- a/dbms/src/Common/Config/ConfigReloader.cpp
+++ b/dbms/src/Common/Config/ConfigReloader.cpp
@ -17,11 +17,13 @@ ConfigReloader::ConfigReloader(
        const std::string & include_from_path_,
        const std::string & preprocessed_dir_,
        zkutil::ZooKeeperNodeCache && zk_node_cache_,
+        const zkutil::EventPtr & zk_changed_event_,
        Updater && updater_,
        bool already_loaded)
    : path(path_), include_from_path(include_from_path_)
    , preprocessed_dir(preprocessed_dir_)
    , zk_node_cache(std::move(zk_node_cache_))
+    , zk_changed_event(zk_changed_event_)
    , updater(std::move(updater_))
 {
    if (!already_loaded)
@ -40,7 +42,7 @@ ConfigReloader::~ConfigReloader()
    try
    {
        quit = true;
-        zk_node_cache.getChangedEvent().set();
+        zk_changed_event->set();

        if (thread.joinable())
            thread.join();
@ -60,7 +62,7 @@ void ConfigReloader::run()
    {
        try
        {
-            bool zk_changed = zk_node_cache.getChangedEvent().tryWait(std::chrono::milliseconds(reload_interval).count());
+            bool zk_changed = zk_changed_event->tryWait(std::chrono::milliseconds(reload_interval).count());
            if (quit)
                return;

@ -90,7 +92,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
            loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
            if (loaded_config.has_zk_includes)
                loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
-                        zk_node_cache, fallback_to_preprocessed);
+                    zk_node_cache, zk_changed_event, fallback_to_preprocessed);
        }
        catch (...)
        {
--- a/dbms/src/Common/Config/ConfigReloader.h
+++ b/dbms/src/Common/Config/ConfigReloader.h
@ -35,6 +35,7 @@ public:
            const std::string & include_from_path,
            const std::string & preprocessed_dir,
            zkutil::ZooKeeperNodeCache && zk_node_cache,
+            const zkutil::EventPtr & zk_changed_event,
            Updater && updater,
            bool already_loaded);

@ -74,6 +75,7 @@ private:
    std::string preprocessed_dir;
    FilesChangesTracker files;
    zkutil::ZooKeeperNodeCache zk_node_cache;
+    zkutil::EventPtr zk_changed_event = std::make_shared<Poco::Event>();

    Updater updater;

--- a/dbms/src/Common/Config/configReadClient.cpp
+++ b/dbms/src/Common/Config/configReadClient.cpp
@ -0,0 +1,31 @@
+#include "configReadClient.h"
+
+#include <Poco/Util/Application.h>
+#include <Poco/Util/LayeredConfiguration.h>
+#include <Poco/File.h>
+#include "ConfigProcessor.h"
+
+namespace DB
+{
+bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path)
+{
+    std::string config_path;
+    if (config.has("config-file"))
+        config_path = config.getString("config-file");
+    else if (Poco::File("./clickhouse-client.xml").exists())
+        config_path = "./clickhouse-client.xml";
+    else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists())
+        config_path = home_path + "/.clickhouse-client/config.xml";
+    else if (Poco::File("/etc/clickhouse-client/config.xml").exists())
+        config_path = "/etc/clickhouse-client/config.xml";
+
+    if (!config_path.empty())
+    {
+        ConfigProcessor config_processor(config_path);
+        auto loaded_config = config_processor.loadConfig();
+        config.add(loaded_config.configuration);
+        return true;
+    }
+    return false;
+}
+}
--- a/dbms/src/Common/Config/configReadClient.h
+++ b/dbms/src/Common/Config/configReadClient.h
@ -0,0 +1,10 @@
+#pragma once
+#include <string>
+
+namespace Poco { class Logger; namespace Util { class LayeredConfiguration; } }
+
+namespace DB
+{
+/// Read configuration files related to clickhouse-client like applications. Returns true if any configuration files were read.
+bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path);
+}
--- a/dbms/src/Common/ErrorCodes.cpp
+++ b/dbms/src/Common/ErrorCodes.cpp
@ -401,6 +401,7 @@ namespace ErrorCodes
    extern const int CANNOT_LINK = 424;
    extern const int SYSTEM_ERROR = 425;
    extern const int NULL_POINTER_DEREFERENCE = 426;
+    extern const int CANNOT_COMPILE_REGEXP = 427;

    extern const int KEEPER_EXCEPTION = 999;
    extern const int POCO_EXCEPTION = 1000;
--- a/dbms/src/Common/OptimizedRegularExpression.inl.h
+++ b/dbms/src/Common/OptimizedRegularExpression.inl.h
@ -1,7 +1,4 @@
-#include <iostream>
-
-#include <Poco/Exception.h>
-
+#include <Common/Exception.h>
 #include <Common/OptimizedRegularExpression.h>


@ -9,6 +6,15 @@
 #define MAX_SUBPATTERNS 5


+namespace DB
+{
+    namespace ErrorCodes
+    {
+        extern const int CANNOT_COMPILE_REGEXP;
+    }
+}
+
+
 template <bool thread_safe>
 void OptimizedRegularExpressionImpl<thread_safe>::analyze(
    const std::string & regexp,
@ -254,11 +260,11 @@ OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(cons

    /// Just three following options are supported
    if (options & (~(RE_CASELESS | RE_NO_CAPTURE | RE_DOT_NL)))
-        throw Poco::Exception("OptimizedRegularExpression: Unsupported option.");
+        throw DB::Exception("OptimizedRegularExpression: Unsupported option.", DB::ErrorCodes::CANNOT_COMPILE_REGEXP);

-    is_case_insensitive   = options & RE_CASELESS;
-    bool is_no_capture    = options & RE_NO_CAPTURE;
-    bool is_dot_nl        = options & RE_DOT_NL;
+    is_case_insensitive = options & RE_CASELESS;
+    bool is_no_capture = options & RE_NO_CAPTURE;
+    bool is_dot_nl = options & RE_DOT_NL;

    number_of_subpatterns = 0;
    if (!is_trivial)
@ -266,6 +272,9 @@ OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(cons
        /// Compile the re2 regular expression.
        typename RegexType::Options regexp_options;

+        /// Never write error messages to stderr. It's ignorant to do it from library code.
+        regexp_options.set_log_errors(false);
+
        if (is_case_insensitive)
            regexp_options.set_case_sensitive(false);

@ -274,13 +283,13 @@ OptimizedRegularExpressionImpl<thread_safe>::OptimizedRegularExpressionImpl(cons

        re2 = std::make_unique<RegexType>(regexp_, regexp_options);
        if (!re2->ok())
-            throw Poco::Exception("OptimizedRegularExpression: cannot compile re2: " + regexp_ + ", error: " + re2->error());
+            throw DB::Exception("OptimizedRegularExpression: cannot compile re2: " + regexp_ + ", error: " + re2->error() + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", DB::ErrorCodes::CANNOT_COMPILE_REGEXP);

        if (!is_no_capture)
        {
            number_of_subpatterns = re2->NumberOfCapturingGroups();
            if (number_of_subpatterns > MAX_SUBPATTERNS)
-                throw Poco::Exception("OptimizedRegularExpression: too many subpatterns in regexp: " + regexp_);
+                throw DB::Exception("OptimizedRegularExpression: too many subpatterns in regexp: " + regexp_, DB::ErrorCodes::CANNOT_COMPILE_REGEXP);
        }
    }
 }
@ -432,6 +441,5 @@ unsigned OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject
    }
 }

-#undef MIN_LENGTH_FOR_STRSTR
-#undef MAX_SUBPATTERNS
-
+template class OptimizedRegularExpressionImpl<true>;
+template class OptimizedRegularExpressionImpl<false>;
--- a/dbms/src/Common/OptimizedRegularExpression.h
+++ b/dbms/src/Common/OptimizedRegularExpression.h
@ -45,9 +45,9 @@ class OptimizedRegularExpressionImpl
 public:
    enum Options
    {
-        RE_CASELESS        = 0x00000001,
-        RE_NO_CAPTURE    = 0x00000010,
-        RE_DOT_NL        = 0x00000100
+        RE_CASELESS   = 0x00000001,
+        RE_NO_CAPTURE = 0x00000010,
+        RE_DOT_NL     = 0x00000100
    };

    using Match = OptimizedRegularExpressionDetails::Match;
@ -106,5 +106,3 @@ private:
 };

 using OptimizedRegularExpression = OptimizedRegularExpressionImpl<true>;
-
-#include "OptimizedRegularExpression.inl.h"
--- a/dbms/src/Common/RWLockFIFO.cpp
+++ b/dbms/src/Common/RWLockFIFO.cpp
@ -1,4 +1,4 @@
-#include "RWLockFIFO.h"
+#include "RWLock.h"
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
 #include <Poco/Ext/ThreadNumber.h>
@ -33,15 +33,15 @@ namespace ErrorCodes
 }


-class RWLockFIFO::LockHandlerImpl
+class RWLockImpl::LockHandlerImpl
 {
-    RWLockFIFOPtr parent;
+    RWLock parent;
    GroupsContainer::iterator it_group;
    ClientsContainer::iterator it_client;
    ThreadToHandler::iterator it_handler;
    CurrentMetrics::Increment active_client_increment;

-    LockHandlerImpl(RWLockFIFOPtr && parent, GroupsContainer::iterator it_group, ClientsContainer::iterator it_client);
+    LockHandlerImpl(RWLock && parent, GroupsContainer::iterator it_group, ClientsContainer::iterator it_client);

 public:

@ -49,11 +49,11 @@ public:

    ~LockHandlerImpl();

-    friend class RWLockFIFO;
+    friend class RWLockImpl;
 };


-RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::Client client)
+RWLockImpl::LockHandler RWLockImpl::getLock(RWLockImpl::Type type)
 {
    Stopwatch watch(CLOCK_MONOTONIC_COARSE);
    CurrentMetrics::Increment waiting_client_increment((type == Read) ? CurrentMetrics::RWLockWaitingReaders
@ -78,15 +78,16 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
    {
        auto handler_ptr = it_handler->second.lock();

-        if (!handler_ptr)
-            throw Exception("Lock handler cannot be nullptr. This is a bug", ErrorCodes::LOGICAL_ERROR);
+        /// Lock may be released in another thread, but not yet deleted inside |~LogHandlerImpl()|

-        if (type != Read || handler_ptr->it_group->type != Read)
-            throw Exception("Attempt to acquire exclusive lock recursively", ErrorCodes::LOGICAL_ERROR);
+        if (handler_ptr)
+        {
+            /// XXX: it means we can't upgrade lock from read to write - with proper waiting!
+            if (type != Read || handler_ptr->it_group->type != Read)
+                throw Exception("Attempt to acquire exclusive lock recursively", ErrorCodes::LOGICAL_ERROR);

-        handler_ptr->it_client->info += "; " + client.info;
-
-        return handler_ptr;
+            return handler_ptr;
+        }
    }

    if (type == Type::Write || queue.empty() || queue.back().type == Type::Write)
@ -104,7 +105,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
    auto & clients = it_group->clients;
    try
    {
-        it_client = clients.emplace(clients.end(), std::move(client));
+        it_client = clients.emplace(clients.end(), type);
    }
    catch (...)
    {
@ -114,10 +115,6 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
        throw;
    }

-    it_client->thread_number = Poco::ThreadNumber::get();
-    it_client->enqueue_time = time(nullptr);
-    it_client->type = type;
-
    LockHandler res(new LockHandlerImpl(shared_from_this(), it_group, it_client));

    /// Insert myself (weak_ptr to the handler) to threads set to implement recursive lock
@ -128,7 +125,6 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
    /// If we are not the first client in the group, a notification could be already sent
    if (it_group == queue.begin())
    {
-        it_client->start_time = it_client->enqueue_time;
        finalize_metrics();
        return res;
    }
@ -136,30 +132,12 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
    /// Wait a notification
    it_group->cv.wait(lock, [&] () { return it_group == queue.begin(); });

-    it_client->start_time = time(nullptr);
    finalize_metrics();
    return res;
 }


-RWLockFIFO::Clients RWLockFIFO::getClientsInTheQueue() const
-{
-    std::unique_lock<std::mutex> lock(mutex);
-
-    Clients res;
-    for (const auto & group : queue)
-    {
-        for (const auto & client : group.clients)
-        {
-            res.emplace_back(client);
-        }
-    }
-
-    return res;
-}
-
-
-RWLockFIFO::LockHandlerImpl::~LockHandlerImpl()
+RWLockImpl::LockHandlerImpl::~LockHandlerImpl()
 {
    std::unique_lock<std::mutex> lock(parent->mutex);

@ -183,11 +161,11 @@ RWLockFIFO::LockHandlerImpl::~LockHandlerImpl()
 }


-RWLockFIFO::LockHandlerImpl::LockHandlerImpl(RWLockFIFOPtr && parent, RWLockFIFO::GroupsContainer::iterator it_group,
-                                             RWLockFIFO::ClientsContainer::iterator it_client)
+RWLockImpl::LockHandlerImpl::LockHandlerImpl(RWLock && parent, RWLockImpl::GroupsContainer::iterator it_group,
+                                             RWLockImpl::ClientsContainer::iterator it_client)
    : parent{std::move(parent)}, it_group{it_group}, it_client{it_client},
-      active_client_increment{(it_client->type == RWLockFIFO::Read) ? CurrentMetrics::RWLockActiveReaders
-                                                                    : CurrentMetrics::RWLockActiveWriters}
+      active_client_increment{(*it_client == RWLockImpl::Read) ? CurrentMetrics::RWLockActiveReaders
+                                                               : CurrentMetrics::RWLockActiveWriters}
 {}

 }
--- a/dbms/src/Common/RWLock.h
+++ b/dbms/src/Common/RWLock.h
@ -0,0 +1,68 @@
+#pragma once
+#include <boost/core/noncopyable.hpp>
+#include <list>
+#include <vector>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+#include <map>
+#include <string>
+
+
+namespace DB
+{
+
+class RWLockImpl;
+using RWLock = std::shared_ptr<RWLockImpl>;
+
+
+/// Implements shared lock with FIFO service
+/// Can be acquired recursively (several calls from the same thread) in Read mode
+class RWLockImpl : public std::enable_shared_from_this<RWLockImpl>
+{
+public:
+    enum Type
+    {
+        Read,
+        Write,
+    };
+
+    static RWLock create() { return RWLock(new RWLockImpl); }
+
+    /// Just use LockHandler::reset() to release the lock
+    class LockHandlerImpl;
+    friend class LockHandlerImpl;
+    using LockHandler = std::shared_ptr<LockHandlerImpl>;
+
+
+    /// Waits in the queue and returns appropriate lock
+    LockHandler getLock(Type type);
+
+private:
+    RWLockImpl() = default;
+
+    struct Group;
+    using GroupsContainer = std::list<Group>;
+    using ClientsContainer = std::list<Type>;
+    using ThreadToHandler = std::map<std::thread::id, std::weak_ptr<LockHandlerImpl>>;
+
+    /// Group of clients that should be executed concurrently
+    /// i.e. a group could contain several readers, but only one writer
+    struct Group
+    {
+        // FIXME: there is only redundant |type| information inside |clients|.
+        const Type type;
+        ClientsContainer clients;
+
+        std::condition_variable cv; /// all clients of the group wait group condvar
+
+        explicit Group(Type type) : type{type} {}
+    };
+
+    mutable std::mutex mutex;
+    GroupsContainer queue;
+    ThreadToHandler thread_to_handler;
+};
+
+
+}
--- a/dbms/src/Common/RWLockFIFO.h
+++ b/dbms/src/Common/RWLockFIFO.h
@ -1,98 +0,0 @@
-#pragma once
-#include <boost/core/noncopyable.hpp>
-#include <list>
-#include <vector>
-#include <mutex>
-#include <condition_variable>
-#include <thread>
-#include <map>
-#include <string>
-
-
-namespace DB
-{
-
-class RWLockFIFO;
-using RWLockFIFOPtr = std::shared_ptr<RWLockFIFO>;
-
-
-/// Implements shared lock with FIFO service
-/// You could call it recursively (several calls from the same thread) in Read mode
-class RWLockFIFO : public std::enable_shared_from_this<RWLockFIFO>
-{
-public:
-    enum Type
-    {
-        Read,
-        Write
-    };
-
-private:
-    /// Client is that who wants to acquire the lock.
-    struct Client
-    {
-        explicit Client(const std::string & info = {}) : info{info} {}
-
-        bool isStarted() { return start_time != 0; }
-
-        /// TODO: delete extra info below if there is no need fot it already.
-        std::string info;
-        int thread_number = 0;
-        std::time_t enqueue_time = 0;
-        std::time_t start_time = 0;
-        Type type = Read;
-    };
-
-public:
-    static RWLockFIFOPtr create()
-    {
-        return RWLockFIFOPtr(new RWLockFIFO);
-    }
-
-
-    /// Just use LockHandler::reset() to release the lock
-    class LockHandlerImpl;
-    friend class LockHandlerImpl;
-    using LockHandler = std::shared_ptr<LockHandlerImpl>;
-
-
-    /// Waits in the queue and returns appropriate lock
-    LockHandler getLock(Type type, Client client = Client{});
-
-    LockHandler getLock(Type type, const std::string & who)
-    {
-        return getLock(type, Client(who));
-    }
-
-    using Clients = std::vector<Client>;
-
-    /// Returns list of executing and waiting clients
-    Clients getClientsInTheQueue() const;
-
-private:
-    RWLockFIFO() = default;
-
-    struct Group;
-    using GroupsContainer = std::list<Group>;
-    using ClientsContainer = std::list<Client>;
-    using ThreadToHandler = std::map<std::thread::id, std::weak_ptr<LockHandlerImpl>>;
-
-    /// Group of clients that should be executed concurrently
-    /// i.e. a group could contain several readers, but only one writer
-    struct Group
-    {
-        const Type type;
-        ClientsContainer clients;
-
-        std::condition_variable cv; /// all clients of the group wait group condvar
-
-        explicit Group(Type type) : type{type} {}
-    };
-
-    mutable std::mutex mutex;
-    GroupsContainer queue;
-    ThreadToHandler thread_to_handler;
-};
-
-
-}
--- a/dbms/src/Common/ShellCommand.cpp
+++ b/dbms/src/Common/ShellCommand.cpp
@ -5,7 +5,6 @@
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
 #include <common/logger_useful.h>
-#include <IO/WriteBufferFromVector.h>
 #include <IO/WriteHelpers.h>
 #include <port/unistd.h>
 #include <csignal>
--- a/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp
+++ b/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.cpp
@ -9,7 +9,16 @@ ZooKeeperNodeCache::ZooKeeperNodeCache(GetZooKeeper get_zookeeper_)
 {
 }

-std::optional<std::string> ZooKeeperNodeCache::get(const std::string & path)
+ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, EventPtr watch_event)
+{
+    Coordination::WatchCallback watch_callback;
+    if (watch_event)
+        watch_callback = [watch_event](const Coordination::WatchResponse &) { watch_event->set(); };
+
+    return get(path, watch_callback);
+}
+
+ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coordination::WatchCallback caller_watch_callback)
 {
    zkutil::ZooKeeperPtr zookeeper;
    std::unordered_set<std::string> invalidated_paths;
@ -19,8 +28,7 @@ std::optional<std::string> ZooKeeperNodeCache::get(const std::string & path)
        if (!context->zookeeper)
        {
            /// Possibly, there was a previous session and it has expired. Clear the cache.
-            nonexistent_nodes.clear();
-            node_cache.clear();
+            path_to_cached_znode.clear();

            context->zookeeper = get_zookeeper();
        }
@ -33,65 +41,62 @@ std::optional<std::string> ZooKeeperNodeCache::get(const std::string & path)
        throw DB::Exception("Could not get znode: `" + path + "'. ZooKeeper not configured.", DB::ErrorCodes::NO_ZOOKEEPER);

    for (const auto & invalidated_path : invalidated_paths)
-    {
-        nonexistent_nodes.erase(invalidated_path);
-        node_cache.erase(invalidated_path);
-    }
+        path_to_cached_znode.erase(invalidated_path);

-    if (nonexistent_nodes.count(path))
-        return std::nullopt;
+    auto cache_it = path_to_cached_znode.find(path);
+    if (cache_it != path_to_cached_znode.end())
+        return cache_it->second;

-    auto watch_callback = [context=context](const Coordination::WatchResponse & response)
+    std::weak_ptr<Context> weak_context(context);
+    auto watch_callback = [weak_context, caller_watch_callback](const Coordination::WatchResponse & response)
    {
        if (!(response.type != Coordination::SESSION || response.state == Coordination::EXPIRED_SESSION))
            return;

+        auto owned_context = weak_context.lock();
+        if (!owned_context)
+            return;
+
        bool changed = false;
        {
-            std::lock_guard<std::mutex> lock(context->mutex);
+            std::lock_guard<std::mutex> lock(owned_context->mutex);

            if (response.type != Coordination::SESSION)
-                changed = context->invalidated_paths.emplace(response.path).second;
+                changed = owned_context->invalidated_paths.emplace(response.path).second;
            else if (response.state == Coordination::EXPIRED_SESSION)
            {
-                context->zookeeper = nullptr;
-                context->invalidated_paths.clear();
+                owned_context->zookeeper = nullptr;
+                owned_context->invalidated_paths.clear();
                changed = true;
            }
        }
-        if (changed)
-            context->changed_event.set();
+        if (changed && caller_watch_callback)
+            caller_watch_callback(response);
    };

-    std::string contents;
+    ZNode result;

-    auto cache_it = node_cache.find(path);
-    if (cache_it != node_cache.end())
+    result.exists = zookeeper->tryGetWatch(path, result.contents, &result.stat, watch_callback);
+    if (result.exists)
    {
-        return cache_it->second;
+        path_to_cached_znode.emplace(path, result);
+        return result;
    }

-    if (zookeeper->tryGetWatch(path, contents, /* stat = */nullptr, watch_callback))
+    /// Node doesn't exist. We must set a watch on node creation (because it wasn't set by tryGetWatch).
+
+    result.exists = zookeeper->existsWatch(path, &result.stat, watch_callback);
+    if (!result.exists)
    {
-        node_cache.emplace(path, contents);
-        return contents;
+        path_to_cached_znode.emplace(path, result);
+        return result;
    }

-    /// Node doesn't exist. Create a watch on node creation.
-    nonexistent_nodes.insert(path);
-
-    if (!zookeeper->existsWatch(path, /* stat = */nullptr, watch_callback))
-        return std::nullopt;
-
    /// Node was created between the two previous calls, try again. Watch is already set.
-    if (zookeeper->tryGet(path, contents))
-    {
-        nonexistent_nodes.erase(path);
-        node_cache.emplace(path, contents);
-        return contents;
-    }

-    return std::nullopt;
+    result.exists = zookeeper->tryGet(path, result.contents, &result.stat);
+    path_to_cached_znode.emplace(path, result);
+    return result;
 }

 }
--- a/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.h
+++ b/dbms/src/Common/ZooKeeper/ZooKeeperNodeCache.h
@ -22,8 +22,13 @@ namespace zkutil

 /// This class allows querying the contents of ZooKeeper nodes and caching the results.
 /// Watches are set for cached nodes and for nodes that were nonexistent at the time of query.
-/// After a watch fires, a notification is generated for the change event.
+/// After a watch fires, the callback or event that was passed by the user is notified.
+///
 /// NOTE: methods of this class are not thread-safe.
+///
+/// Intended use case: if you need one thread to watch changes in several nodes.
+/// If instead you use simple a watch event for this, watches will accumulate for nodes that do not change
+/// or change rarely.
 class ZooKeeperNodeCache
 {
 public:
@ -32,17 +37,21 @@ public:
    ZooKeeperNodeCache(const ZooKeeperNodeCache &) = delete;
    ZooKeeperNodeCache(ZooKeeperNodeCache &&) = default;

-    std::optional<std::string> get(const std::string & path);
+    struct ZNode
+    {
+        bool exists = false;
+        std::string contents;
+        Coordination::Stat stat;
+    };

-    Poco::Event & getChangedEvent() { return context->changed_event; }
+    ZNode get(const std::string & path, EventPtr watch_event);
+    ZNode get(const std::string & path, Coordination::WatchCallback watch_callback);

 private:
    GetZooKeeper get_zookeeper;

    struct Context
    {
-        Poco::Event changed_event;
-
        std::mutex mutex;
        zkutil::ZooKeeperPtr zookeeper;
        std::unordered_set<std::string> invalidated_paths;
@ -50,8 +59,7 @@ private:

    std::shared_ptr<Context> context;

-    std::unordered_set<std::string> nonexistent_nodes;
-    std::unordered_map<std::string, std::string> node_cache;
+    std::unordered_map<std::string, ZNode> path_to_cached_znode;
 };

 }
--- a/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp
+++ b/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp
@ -4,7 +4,7 @@
 #endif
 #include <gtest/gtest.h>

-#include <Common/RWLockFIFO.h>
+#include <Common/RWLock.h>
 #include <Common/Stopwatch.h>
 #include <common/Types.h>
 #include <common/ThreadPool.h>
@ -18,7 +18,7 @@
 using namespace DB;


-TEST(Common, RWLockFIFO_1)
+TEST(Common, RWLock_1)
 {
    constexpr int cycles = 1000;
    const std::vector<size_t> pool_sizes{1, 2, 4, 8};
@ -26,7 +26,7 @@ TEST(Common, RWLockFIFO_1)
    static std::atomic<int> readers{0};
    static std::atomic<int> writers{0};

-    static auto fifo_lock = RWLockFIFO::create();
+    static auto fifo_lock = RWLockImpl::create();

    static thread_local std::random_device rd;
    static thread_local pcg64 gen(rd());
@ -35,12 +35,12 @@ TEST(Common, RWLockFIFO_1)
    {
        for (int i = 0; i < cycles; ++i)
        {
-            auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockFIFO::Read : RWLockFIFO::Write;
+            auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockImpl::Read : RWLockImpl::Write;
            auto sleep_for = std::chrono::duration<int, std::micro>(std::uniform_int_distribution<>(1, 100)(gen));

-            auto lock = fifo_lock->getLock(type, "RW");
+            auto lock = fifo_lock->getLock(type);

-            if (type == RWLockFIFO::Write)
+            if (type == RWLockImpl::Write)
            {
                ++writers;

@ -85,11 +85,11 @@ TEST(Common, RWLockFIFO_1)
    }
 }

-TEST(Common, RWLockFIFO_Recursive)
+TEST(Common, RWLock_Recursive)
 {
    constexpr auto cycles = 10000;

-    static auto fifo_lock = RWLockFIFO::create();
+    static auto fifo_lock = RWLockImpl::create();

    static thread_local std::random_device rd;
    static thread_local pcg64 gen(rd());
@ -98,7 +98,7 @@ TEST(Common, RWLockFIFO_Recursive)
    {
        for (int i = 0; i < 2 * cycles; ++i)
        {
-            auto lock = fifo_lock->getLock(RWLockFIFO::Write);
+            auto lock = fifo_lock->getLock(RWLockImpl::Write);

            auto sleep_for = std::chrono::duration<int, std::micro>(std::uniform_int_distribution<>(1, 100)(gen));
            std::this_thread::sleep_for(sleep_for);
@ -109,17 +109,17 @@ TEST(Common, RWLockFIFO_Recursive)
    {
        for (int i = 0; i < cycles; ++i)
        {
-            auto lock1 = fifo_lock->getLock(RWLockFIFO::Read);
+            auto lock1 = fifo_lock->getLock(RWLockImpl::Read);

            auto sleep_for = std::chrono::duration<int, std::micro>(std::uniform_int_distribution<>(1, 100)(gen));
            std::this_thread::sleep_for(sleep_for);

-            auto lock2 = fifo_lock->getLock(RWLockFIFO::Read);
+            auto lock2 = fifo_lock->getLock(RWLockImpl::Read);

-            EXPECT_ANY_THROW({fifo_lock->getLock(RWLockFIFO::Write);});
+            EXPECT_ANY_THROW({fifo_lock->getLock(RWLockImpl::Write);});
        }

-        fifo_lock->getLock(RWLockFIFO::Write);
+        fifo_lock->getLock(RWLockImpl::Write);
    });

    t1.join();
@ -127,12 +127,12 @@ TEST(Common, RWLockFIFO_Recursive)
 }


-TEST(Common, RWLockFIFO_PerfTest_Readers)
+TEST(Common, RWLock_PerfTest_Readers)
 {
    constexpr int cycles = 100000; // 100k
    const std::vector<size_t> pool_sizes{1, 2, 4, 8};

-    static auto fifo_lock = RWLockFIFO::create();
+    static auto fifo_lock = RWLockImpl::create();

    for (auto pool_size : pool_sizes)
    {
@ -142,7 +142,7 @@ TEST(Common, RWLockFIFO_PerfTest_Readers)
            {
                for (auto i = 0; i < cycles; ++i)
                {
-                    auto lock = fifo_lock->getLock(RWLockFIFO::Read);
+                    auto lock = fifo_lock->getLock(RWLockImpl::Read);
                }
            };

--- a/dbms/src/Core/Block.h
+++ b/dbms/src/Core/Block.h
@ -151,22 +151,4 @@ void assertBlocksHaveEqualStructure(const Block & lhs, const Block & rhs, const
 /// Calculate difference in structure of blocks and write description into output strings. NOTE It doesn't compare values of constant columns.
 void getBlocksDifference(const Block & lhs, const Block & rhs, std::string & out_lhs_diff, std::string & out_rhs_diff);

-
-/** Additional data to the blocks. They are only needed for a query
-  * DESCRIBE TABLE with Distributed tables.
-  */
-struct BlockExtraInfo
-{
-    BlockExtraInfo() {}
-    operator bool() const { return is_valid; }
-    bool operator!() const { return !is_valid; }
-
-    std::string host;
-    std::string resolved_address;
-    std::string user;
-    UInt16 port = 0;
-
-    bool is_valid = false;
-};
-
 }
--- a/dbms/src/Core/NamesAndTypes.cpp
+++ b/dbms/src/Core/NamesAndTypes.cpp
@ -84,6 +84,19 @@ size_t NamesAndTypesList::sizeOfDifference(const NamesAndTypesList & rhs) const
    return (std::unique(vector.begin(), vector.end()) - vector.begin()) * 2 - size() - rhs.size();
 }

+void NamesAndTypesList::getDifference(const NamesAndTypesList & rhs, NamesAndTypesList & deleted, NamesAndTypesList & added) const
+{
+    NamesAndTypes lhs_vector(begin(), end());
+    std::sort(lhs_vector.begin(), lhs_vector.end());
+    NamesAndTypes rhs_vector(rhs.begin(), rhs.end());
+    std::sort(rhs_vector.begin(), rhs_vector.end());
+
+    std::set_difference(lhs_vector.begin(), lhs_vector.end(), rhs_vector.begin(), rhs_vector.end(),
+        std::back_inserter(deleted));
+    std::set_difference(rhs_vector.begin(), rhs_vector.end(), lhs_vector.begin(), lhs_vector.end(),
+        std::back_inserter(added));
+}
+
 Names NamesAndTypesList::getNames() const
 {
    Names res;
--- a/dbms/src/Core/NamesAndTypes.h
+++ b/dbms/src/Core/NamesAndTypes.h
@ -58,6 +58,9 @@ public:
    ///  (in other words, the added and deleted columns are counted once, the columns that changed the type - twice).
    size_t sizeOfDifference(const NamesAndTypesList & rhs) const;

+    /// If an element changes type, it is present both in deleted (with the old type) and in added (with the new type).
+    void getDifference(const NamesAndTypesList & rhs, NamesAndTypesList & deleted, NamesAndTypesList & added) const;
+
    Names getNames() const;
    DataTypes getTypes() const;

--- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp
+++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp
@ -27,7 +27,7 @@ Block ExpressionBlockInputStream::getTotals()
 Block ExpressionBlockInputStream::getHeader() const
 {
    Block res = children.back()->getHeader();
-    expression->execute(res);
+    expression->execute(res, true);
    return res;
 }

--- a/dbms/src/DataStreams/IBlockInputStream.h
+++ b/dbms/src/DataStreams/IBlockInputStream.h
@ -63,13 +63,6 @@ public:
      */
    virtual Block read() = 0;

-    /** Get information about the last block received.
-      */
-    virtual BlockExtraInfo getBlockExtraInfo() const
-    {
-        throw Exception("Method getBlockExtraInfo is not supported by the data stream " + getName(), ErrorCodes::NOT_IMPLEMENTED);
-    }
-
    /** Read something before starting all data or after the end of all data.
      * In the `readSuffix` function, you can implement a finalization that can lead to an exception.
      * readPrefix() must be called before the first call to read().
--- a/dbms/src/DataStreams/ParallelInputsProcessor.h
+++ b/dbms/src/DataStreams/ParallelInputsProcessor.h
@ -39,23 +39,12 @@ namespace CurrentMetrics
 namespace DB
 {

-/** Union mode.
-  */
-enum class StreamUnionMode
-{
-    Basic = 0, /// take out blocks
-    ExtraInfo  /// take out blocks + additional information
-};
-
 /// Example of the handler.
 struct ParallelInputsHandler
 {
    /// Processing the data block.
    void onBlock(Block & /*block*/, size_t /*thread_num*/) {}

-    /// Processing the data block + additional information.
-    void onBlock(Block & /*block*/, BlockExtraInfo & /*extra_info*/, size_t /*thread_num*/) {}
-
    /// Called for each thread, when the thread has nothing else to do.
    /// Due to the fact that part of the sources has run out, and now there are fewer sources left than streams.
    /// Called if the `onException` method does not throw an exception; is called before the `onFinish` method.
@ -70,7 +59,7 @@ struct ParallelInputsHandler
 };


-template <typename Handler, StreamUnionMode mode = StreamUnionMode::Basic>
+template <typename Handler>
 class ParallelInputsProcessor
 {
 public:
@ -183,15 +172,9 @@ private:
        InputData(const BlockInputStreamPtr & in_, size_t i_) : in(in_), i(i_) {}
    };

-    void publishPayload(BlockInputStreamPtr & stream, Block & block, size_t thread_num)
+    void publishPayload(Block & block, size_t thread_num)
    {
-        if constexpr (mode == StreamUnionMode::Basic)
-            handler.onBlock(block, thread_num);
-        else
-        {
-            BlockExtraInfo extra_info = stream->getBlockExtraInfo();
-            handler.onBlock(block, extra_info, thread_num);
-        }
+        handler.onBlock(block, thread_num);
    }

    void thread(ThreadGroupStatusPtr thread_group, size_t thread_num)
@ -249,7 +232,7 @@ private:
                {
                    additional_input_at_end->readPrefix();
                    while (Block block = additional_input_at_end->read())
-                        publishPayload(additional_input_at_end, block, thread_num);
+                        publishPayload(block, thread_num);
                }
                catch (...)
                {
@ -312,7 +295,7 @@ private:
                    break;

                if (block)
-                    publishPayload(input.in, block, thread_num);
+                    publishPayload(block, thread_num);
            }
        }
    }
--- a/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
@ -20,7 +20,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
      * Although now any insertion into the table is done via PushingToViewsBlockOutputStream,
      *  but it's clear that here is not the best place for this functionality.
      */
-    addTableLock(storage->lockStructure(true, __PRETTY_FUNCTION__));
+    addTableLock(storage->lockStructure(true));

    /// If the "root" table deduplactes blocks, there are no need to make deduplication for children
    /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks
@ -45,7 +45,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
            auto & materialized_view = dynamic_cast<const StorageMaterializedView &>(*dependent_table);

            if (StoragePtr inner_table = materialized_view.tryGetTargetTable())
-                addTableLock(inner_table->lockStructure(true, __PRETTY_FUNCTION__));
+                addTableLock(inner_table->lockStructure(true));

            auto query = materialized_view.getInnerQuery();
            BlockOutputStreamPtr out = std::make_shared<PushingToViewsBlockOutputStream>(
--- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp
+++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp
@ -45,7 +45,7 @@ RemoteBlockInputStream::RemoteBlockInputStream(
    create_multiplexed_connections = [this, connections, throttler]() mutable
    {
        return std::make_unique<MultiplexedConnections>(
-                std::move(connections), context.getSettingsRef(), throttler, append_extra_info);
+            std::move(connections), context.getSettingsRef(), throttler);
    };
 }

@ -74,7 +74,7 @@ RemoteBlockInputStream::RemoteBlockInputStream(
            connections = pool->getMany(&current_settings, pool_mode);

        return std::make_unique<MultiplexedConnections>(
-                std::move(connections), current_settings, throttler, append_extra_info);
+            std::move(connections), current_settings, throttler);
    };
 }

@ -88,11 +88,6 @@ RemoteBlockInputStream::~RemoteBlockInputStream()
        multiplexed_connections->disconnect();
 }

-void RemoteBlockInputStream::appendExtraInfo()
-{
-    append_extra_info = true;
-}
-
 void RemoteBlockInputStream::readPrefix()
 {
    if (!sent_query)
--- a/dbms/src/DataStreams/RemoteBlockInputStream.h
+++ b/dbms/src/DataStreams/RemoteBlockInputStream.h
@ -51,9 +51,6 @@ public:

    void setMainTable(QualifiedTableName main_table_) { main_table = std::move(main_table_); }

-    /// Besides blocks themself, get blocks' extra info
-    void appendExtraInfo();
-
    /// Sends query (initiates calculation) before read()
    void readPrefix() override;

@ -66,11 +63,6 @@ public:

    String getName() const override { return "Remote"; }

-    BlockExtraInfo getBlockExtraInfo() const override
-    {
-        return multiplexed_connections->getBlockExtraInfo();
-    }
-
    Block getHeader() const override { return header; }

 protected:
@ -143,7 +135,6 @@ private:
      */
    std::atomic<bool> got_unknown_packet_from_replica { false };

-    bool append_extra_info = false;
    PoolMode pool_mode = PoolMode::GET_MANY;
    std::optional<QualifiedTableName> main_table;

--- a/dbms/src/DataStreams/UnionBlockInputStream.h
+++ b/dbms/src/DataStreams/UnionBlockInputStream.h
@ -16,39 +16,6 @@ namespace ErrorCodes
 }


-namespace
-{
-
-template <StreamUnionMode mode>
-struct OutputData;
-
-/// A block or an exception.
-template <>
-struct OutputData<StreamUnionMode::Basic>
-{
-    Block block;
-    std::exception_ptr exception;
-
-    OutputData() {}
-    OutputData(Block & block_) : block(block_) {}
-    OutputData(std::exception_ptr & exception_) : exception(exception_) {}
-};
-
-/// Block + additional information or an exception.
-template <>
-struct OutputData<StreamUnionMode::ExtraInfo>
-{
-    Block block;
-    BlockExtraInfo extra_info;
-    std::exception_ptr exception;
-
-    OutputData() {}
-    OutputData(Block & block_, BlockExtraInfo & extra_info_) : block(block_), extra_info(extra_info_) {}
-    OutputData(std::exception_ptr & exception_) : exception(exception_) {}
-};
-
-}
-
 /** Merges several sources into one.
  * Blocks from different sources are interleaved with each other in an arbitrary way.
  * You can specify the number of threads (max_threads),
@ -58,20 +25,24 @@ struct OutputData<StreamUnionMode::ExtraInfo>
  * - with the help of ParallelInputsProcessor in several threads it takes out blocks from the sources;
  * - the completed blocks are added to a limited queue of finished blocks;
  * - the main thread takes out completed blocks from the queue of finished blocks;
-  * - if the StreamUnionMode::ExtraInfo mode is specified, in addition to the UnionBlockInputStream
-  *   extracts blocks information; In this case all sources should support such mode.
  */
-
-template <StreamUnionMode mode = StreamUnionMode::Basic>
 class UnionBlockInputStream final : public IProfilingBlockInputStream
 {
+private:
+    /// A block or an exception.
+    struct OutputData
+    {
+        Block block;
+        std::exception_ptr exception;
+
+        OutputData() {}
+        OutputData(Block & block_) : block(block_) {}
+        OutputData(std::exception_ptr & exception_) : exception(exception_) {}
+    };
+
 public:
    using ExceptionCallback = std::function<void()>;

-private:
-    using Self = UnionBlockInputStream;
-
-public:
    UnionBlockInputStream(BlockInputStreams inputs, BlockInputStreamPtr additional_input_at_end, size_t max_threads,
        ExceptionCallback exception_callback_ = ExceptionCallback()) :
        output_queue(std::min(inputs.size(), max_threads)),
@ -125,11 +96,6 @@ public:
        processor.cancel(kill);
    }

-    BlockExtraInfo getBlockExtraInfo() const override
-    {
-        return doGetBlockExtraInfo();
-    }
-
    Block getHeader() const override { return children.at(0)->getHeader(); }

 protected:
@ -146,7 +112,7 @@ protected:
            /** Let's read everything up to the end, so that ParallelInputsProcessor is not blocked when trying to insert into the queue.
              * Maybe there is an exception in the queue.
              */
-            OutputData<mode> res;
+            OutputData res;
            while (true)
            {
                //std::cerr << "popping\n";
@ -230,20 +196,9 @@ protected:
    }

 private:
-    BlockExtraInfo doGetBlockExtraInfo() const
-    {
-        if constexpr (mode == StreamUnionMode::ExtraInfo)
-            return received_payload.extra_info;
-        else
-            throw Exception("Method getBlockExtraInfo is not supported for mode StreamUnionMode::Basic",
-                ErrorCodes::NOT_IMPLEMENTED);
-    }
-
-private:
-    using Payload = OutputData<mode>;
+    using Payload = OutputData;
    using OutputQueue = ConcurrentBoundedQueue<Payload>;

-private:
    /** The queue of the finished blocks. Also, you can put an exception instead of a block.
      * When data is run out, an empty block is inserted into the queue.
      * Sooner or later, an empty block is always inserted into the queue (even after exception or query cancellation).
@ -254,18 +209,13 @@ private:

    struct Handler
    {
-        Handler(Self & parent_) : parent(parent_) {}
+        Handler(UnionBlockInputStream & parent_) : parent(parent_) {}

        void onBlock(Block & block, size_t /*thread_num*/)
        {
            parent.output_queue.push(Payload(block));
        }

-        void onBlock(Block & block, BlockExtraInfo & extra_info, size_t /*thread_num*/)
-        {
-            parent.output_queue.push(Payload(block, extra_info));
-        }
-
        void onFinish()
        {
            parent.output_queue.push(Payload());
@ -287,11 +237,11 @@ private:
            parent.cancel(false);    /// Does not throw exceptions.
        }

-        Self & parent;
+        UnionBlockInputStream & parent;
    };

    Handler handler;
-    ParallelInputsProcessor<Handler, mode> processor;
+    ParallelInputsProcessor<Handler> processor;

    ExceptionCallback exception_callback;

--- a/dbms/src/DataStreams/tests/union_stream2.cpp
+++ b/dbms/src/DataStreams/tests/union_stream2.cpp
@ -40,7 +40,7 @@ try
    for (size_t i = 0, size = streams.size(); i < size; ++i)
        streams[i] = std::make_shared<AsynchronousBlockInputStream>(streams[i]);

-    BlockInputStreamPtr stream = std::make_shared<UnionBlockInputStream<>>(streams, nullptr, settings.max_threads);
+    BlockInputStreamPtr stream = std::make_shared<UnionBlockInputStream>(streams, nullptr, settings.max_threads);
    stream = std::make_shared<LimitBlockInputStream>(stream, 10, 0);

    WriteBufferFromFileDescriptor wb(STDERR_FILENO);
--- a/dbms/src/Databases/IDatabase.h
+++ b/dbms/src/Databases/IDatabase.h
@ -91,7 +91,7 @@ public:
        const StoragePtr & table,
        const ASTPtr & query) = 0;

-    /// Delete the table from the database and return it. Delete the metadata.
+    /// Delete the table from the database. Delete the metadata.
    virtual void removeTable(
        const Context & context,
        const String & name) = 0;
--- a/dbms/src/Dictionaries/CMakeLists.txt
+++ b/dbms/src/Dictionaries/CMakeLists.txt
@ -0,0 +1,43 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+include(${ClickHouse_SOURCE_DIR}/cmake/generate_code.cmake)
+
+add_headers_and_sources(clickhouse_dictionaries .)
+
+generate_code(ComplexKeyCacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(ComplexKeyCacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(ComplexKeyCacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/generated/)
+
+add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources})
+target_link_libraries(clickhouse_dictionaries PRIVATE clickhouse_common_io pocoext ${MYSQLXX_LIBRARY} ${BTRIE_LIBRARIES})
+
+if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
+endif()
+
+if(USE_POCO_SQLODBC)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
+    if (NOT USE_INTERNAL_POCO_LIBRARY)
+        target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} ${Poco_SQL_INCLUDE_DIR})
+    endif()
+endif()
+
+if(Poco_Data_FOUND)
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
+endif()
+
+if(USE_POCO_DATAODBC)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_DataODBC_LIBRARY} ${Poco_Data_LIBRARY})
+    if (NOT USE_INTERNAL_POCO_LIBRARY)
+        target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+    endif()
+endif()
+
+if(USE_POCO_MONGODB)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_MongoDB_LIBRARY})
+endif()
+
+add_subdirectory(Embedded)
--- a/dbms/src/Dictionaries/CacheDictionary.cpp
+++ b/dbms/src/Dictionaries/CacheDictionary.cpp
@ -1,3 +1,5 @@
+#include "CacheDictionary.h"
+
 #include <functional>
 #include <sstream>
 #include <memory>
@ -11,12 +13,12 @@
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/typeid_cast.h>
-#include <Dictionaries/CacheDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "DictionaryBlockInputStream.h"
 #include <ext/size.h>
 #include <ext/range.h>
 #include <ext/map.h>
-
+#include "DictionaryFactory.h"
+#include "CacheDictionary.inc.h"

 namespace ProfileEvents
 {
@ -47,6 +49,7 @@ namespace ErrorCodes
    extern const int BAD_ARGUMENTS;
    extern const int UNSUPPORTED_METHOD;
    extern const int LOGICAL_ERROR;
+    extern const int TOO_SMALL_BUFFER_SIZE;
 }


@ -206,34 +209,6 @@ void CacheDictionary::isInConstantVector(
        out[i] = std::find(ancestors.begin(), ancestors.end(), ancestor_ids[i]) != ancestors.end();
 }

-
-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    const auto null_value = std::get<TYPE>(attribute.null_values);\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t) { return null_value; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
 {
    auto & attribute = getAttribute(attribute_name);
@ -245,33 +220,6 @@ void CacheDictionary::getString(const std::string & attribute_name, const Padded
    getItemsString(attribute, ids, out, [&] (const size_t) { return null_value; });
 }

-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(\
-    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
-    ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t row) { return def[row]; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(
    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def,
    ColumnString * const out) const
@ -283,32 +231,6 @@ void CacheDictionary::getString(
    getItemsString(attribute, ids, out, [&] (const size_t row) { return def->getDataAt(row); });
 }

-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(\
-    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t) { return def; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(
    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def,
    ColumnString * const out) const
@ -487,374 +409,6 @@ CacheDictionary::Attribute CacheDictionary::createAttributeWithType(const Attrib
    return attr;
 }

-
-template <typename OutputType, typename DefaultGetter>
-void CacheDictionary::getItemsNumber(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ResultArrayType<OutputType> & out,
-    DefaultGetter && get_default) const
-{
-    if (false) {}
-#define DISPATCH(TYPE) \
-    else if (attribute.type == AttributeUnderlyingType::TYPE) \
-        getItemsNumberImpl<TYPE, OutputType>(attribute, ids, out, std::forward<DefaultGetter>(get_default));
-    DISPATCH(UInt8)
-    DISPATCH(UInt16)
-    DISPATCH(UInt32)
-    DISPATCH(UInt64)
-    DISPATCH(UInt128)
-    DISPATCH(Int8)
-    DISPATCH(Int16)
-    DISPATCH(Int32)
-    DISPATCH(Int64)
-    DISPATCH(Float32)
-    DISPATCH(Float64)
-    DISPATCH(Decimal32)
-    DISPATCH(Decimal64)
-    DISPATCH(Decimal128)
-#undef DISPATCH
-    else
-        throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
-}
-
-template <typename AttributeType, typename OutputType, typename DefaultGetter>
-void CacheDictionary::getItemsNumberImpl(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ResultArrayType<OutputType> & out,
-    DefaultGetter && get_default) const
-{
-    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
-    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
-    auto & attribute_array = std::get<ContainerPtrType<AttributeType>>(attribute.arrays);
-    const auto rows = ext::size(ids);
-
-    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
-
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        /// fetch up-to-date values, decide which ones require update
-        for (const auto row : ext::range(0, rows))
-        {
-            const auto id = ids[row];
-
-            /** cell should be updated if either:
-                *    1. ids do not match,
-                *    2. cell has expired,
-                *    3. explicit defaults were specified and cell was set default. */
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                outdated_ids[id].push_back(row);
-                if (find_result.outdated)
-                    ++cache_expired;
-                else
-                    ++cache_not_found;
-            }
-            else
-            {
-                ++cache_hit;
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                out[row] = cell.isDefault() ? get_default(row) : static_cast<OutputType>(attribute_array[cell_idx]);
-            }
-        }
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
-
-    query_count.fetch_add(rows, std::memory_order_relaxed);
-    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
-
-    if (outdated_ids.empty())
-        return;
-
-    std::vector<Key> required_ids(outdated_ids.size());
-    std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
-        [] (auto & pair) { return pair.first; });
-
-    /// request new values
-    update(required_ids,
-    [&] (const auto id, const auto cell_idx)
-    {
-        const auto attribute_value = attribute_array[cell_idx];
-
-        for (const size_t row : outdated_ids[id])
-            out[row] = static_cast<OutputType>(attribute_value);
-    },
-    [&] (const auto id, const auto)
-    {
-        for (const size_t row : outdated_ids[id])
-            out[row] = get_default(row);
-    });
-}
-
-template <typename DefaultGetter>
-void CacheDictionary::getItemsString(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ColumnString * out,
-    DefaultGetter && get_default) const
-{
-    const auto rows = ext::size(ids);
-
-    /// save on some allocations
-    out->getOffsets().reserve(rows);
-
-    auto & attribute_array = std::get<ContainerPtrType<StringRef>>(attribute.arrays);
-
-    auto found_outdated_values = false;
-
-    /// perform optimistic version, fallback to pessimistic if failed
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        /// fetch up-to-date values, discard on fail
-        for (const auto row : ext::range(0, rows))
-        {
-            const auto id = ids[row];
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                found_outdated_values = true;
-                break;
-            }
-            else
-            {
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
-                out->insertData(string_ref.data, string_ref.size);
-            }
-        }
-    }
-
-    /// optimistic code completed successfully
-    if (!found_outdated_values)
-    {
-        query_count.fetch_add(rows, std::memory_order_relaxed);
-        hit_count.fetch_add(rows, std::memory_order_release);
-        return;
-    }
-
-    /// now onto the pessimistic one, discard possible partial results from the optimistic path
-    out->getChars().resize_assume_reserved(0);
-    out->getOffsets().resize_assume_reserved(0);
-
-    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
-    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
-    /// we are going to store every string separately
-    std::unordered_map<Key, String> map;
-
-    size_t total_length = 0;
-    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        for (const auto row : ext::range(0, ids.size()))
-        {
-            const auto id = ids[row];
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                outdated_ids[id].push_back(row);
-                if (find_result.outdated)
-                    ++cache_expired;
-                else
-                    ++cache_not_found;
-            }
-            else
-            {
-                ++cache_hit;
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
-
-                if (!cell.isDefault())
-                    map[id] = String{string_ref};
-
-                total_length += string_ref.size + 1;
-            }
-        }
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
-
-    query_count.fetch_add(rows, std::memory_order_relaxed);
-    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
-
-    /// request new values
-    if (!outdated_ids.empty())
-    {
-        std::vector<Key> required_ids(outdated_ids.size());
-        std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
-            [] (auto & pair) { return pair.first; });
-
-        update(required_ids,
-        [&] (const auto id, const auto cell_idx)
-        {
-            const auto attribute_value = attribute_array[cell_idx];
-
-            map[id] = String{attribute_value};
-            total_length += (attribute_value.size + 1) * outdated_ids[id].size();
-        },
-        [&] (const auto id, const auto)
-        {
-            for (const auto row : outdated_ids[id])
-                total_length += get_default(row).size + 1;
-        });
-    }
-
-    out->getChars().reserve(total_length);
-
-    for (const auto row : ext::range(0, ext::size(ids)))
-    {
-        const auto id = ids[row];
-        const auto it = map.find(id);
-
-        const auto string_ref = it != std::end(map) ? StringRef{it->second} : get_default(row);
-        out->insertData(string_ref.data, string_ref.size);
-    }
-}
-
-template <typename PresentIdHandler, typename AbsentIdHandler>
-void CacheDictionary::update(
-    const std::vector<Key> & requested_ids,
-    PresentIdHandler && on_cell_updated,
-    AbsentIdHandler && on_id_not_found) const
-{
-    std::unordered_map<Key, UInt8> remaining_ids{requested_ids.size()};
-    for (const auto id : requested_ids)
-        remaining_ids.insert({ id, 0 });
-
-    std::uniform_int_distribution<UInt64> distribution
-    {
-        dict_lifetime.min_sec,
-        dict_lifetime.max_sec
-    };
-
-    const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
-
-    {
-        CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests};
-        Stopwatch watch;
-        auto stream = source_ptr->loadIds(requested_ids);
-        stream->readPrefix();
-
-        const auto now = std::chrono::system_clock::now();
-
-        while (const auto block = stream->read())
-        {
-            const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
-            if (!id_column)
-                throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
-
-            const auto & ids = id_column->getData();
-
-            /// cache column pointers
-            const auto column_ptrs = ext::map<std::vector>(ext::range(0, attributes.size()), [&block] (size_t i)
-            {
-                return block.safeGetByPosition(i + 1).column.get();
-            });
-
-            for (const auto i : ext::range(0, ids.size()))
-            {
-                const auto id = ids[i];
-
-                const auto find_result = findCellIdx(id, now);
-                const auto & cell_idx = find_result.cell_idx;
-
-                auto & cell = cells[cell_idx];
-
-                for (const auto attribute_idx : ext::range(0, attributes.size()))
-                {
-                    const auto & attribute_column = *column_ptrs[attribute_idx];
-                    auto & attribute = attributes[attribute_idx];
-
-                    setAttributeValue(attribute, cell_idx, attribute_column[i]);
-                }
-
-                /// if cell id is zero and zero does not map to this cell, then the cell is unused
-                if (cell.id == 0 && cell_idx != zero_cell_idx)
-                    element_count.fetch_add(1, std::memory_order_relaxed);
-
-                cell.id = id;
-                if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
-                    cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
-                else
-                    cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
-
-                /// inform caller
-                on_cell_updated(id, cell_idx);
-                /// mark corresponding id as found
-                remaining_ids[id] = 1;
-            }
-        }
-
-        stream->readSuffix();
-
-        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size());
-        ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed());
-    }
-
-    size_t not_found_num = 0, found_num = 0;
-
-    const auto now = std::chrono::system_clock::now();
-    /// Check which ids have not been found and require setting null_value
-    for (const auto & id_found_pair : remaining_ids)
-    {
-        if (id_found_pair.second)
-        {
-            ++found_num;
-            continue;
-        }
-        ++not_found_num;
-
-        const auto id = id_found_pair.first;
-
-        const auto find_result = findCellIdx(id, now);
-        const auto & cell_idx = find_result.cell_idx;
-
-        auto & cell = cells[cell_idx];
-
-        /// Set null_value for each attribute
-        for (auto & attribute : attributes)
-            setDefaultAttributeValue(attribute, cell_idx);
-
-        /// Check if cell had not been occupied before and increment element counter if it hadn't
-        if (cell.id == 0 && cell_idx != zero_cell_idx)
-            element_count.fetch_add(1, std::memory_order_relaxed);
-
-        cell.id = id;
-        if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
-            cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
-        else
-            cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
-
-        cell.setDefault();
-
-        /// inform caller that the cell has not been found
-        on_id_not_found(id, cell_idx);
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedFound, found_num);
-    ProfileEvents::increment(ProfileEvents::DictCacheRequests);
-}
-
-
 void CacheDictionary::setDefaultAttributeValue(Attribute & attribute, const Key idx) const
 {
    switch (attribute.type)
@ -981,5 +535,41 @@ BlockInputStreamPtr CacheDictionary::getBlockInputStream(const Names & column_na
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getCachedIds(), column_names);
 }

+void registerDictionaryCache(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+
+        if (dict_struct.key)
+            throw Exception {"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
+
+        if (dict_struct.range_min || dict_struct.range_max)
+            throw Exception {name
+                                 + ": elements .structure.range_min and .structure.range_max should be defined only "
+                                   "for a dictionary of layout 'range_hashed'",
+                             ErrorCodes::BAD_ARGUMENTS};
+        const auto & layout_prefix = config_prefix + ".layout";
+        const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
+        if (size == 0)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
+
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        if (require_nonempty)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
+                             ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
+
+
+    };
+    factory.registerLayout("cache", create_layout);
+}
+

 }
--- a/dbms/src/Dictionaries/CacheDictionary.h
+++ b/dbms/src/Dictionaries/CacheDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <Common/ArenaWithFreeLists.h>
 #include <Common/CurrentMetrics.h>
 #include <Columns/ColumnDecimal.h>
--- a/dbms/src/Dictionaries/CacheDictionary.inc.h
+++ b/dbms/src/Dictionaries/CacheDictionary.inc.h
@ -0,0 +1,403 @@
+#include "CacheDictionary.h"
+
+#include <ext/size.h>
+#include <ext/map.h>
+#include <ext/range.h>
+#include <Common/ProfilingScopedRWLock.h>
+#include <Common/typeid_cast.h>
+#include <Columns/ColumnsNumber.h>
+
+namespace ProfileEvents
+{
+    extern const Event DictCacheKeysRequested;
+    extern const Event DictCacheKeysRequestedMiss;
+    extern const Event DictCacheKeysRequestedFound;
+    extern const Event DictCacheKeysExpired;
+    extern const Event DictCacheKeysNotFound;
+    extern const Event DictCacheKeysHit;
+    extern const Event DictCacheRequestTimeNs;
+    extern const Event DictCacheRequests;
+    extern const Event DictCacheLockWriteNs;
+    extern const Event DictCacheLockReadNs;
+}
+
+namespace CurrentMetrics
+{
+    extern const Metric DictCacheRequests;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+template <typename OutputType, typename DefaultGetter>
+void CacheDictionary::getItemsNumber(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ResultArrayType<OutputType> & out,
+    DefaultGetter && get_default) const
+{
+    if (false) {}
+#define DISPATCH(TYPE) \
+    else if (attribute.type == AttributeUnderlyingType::TYPE) \
+        getItemsNumberImpl<TYPE, OutputType>(attribute, ids, out, std::forward<DefaultGetter>(get_default));
+    DISPATCH(UInt8)
+    DISPATCH(UInt16)
+    DISPATCH(UInt32)
+    DISPATCH(UInt64)
+    DISPATCH(UInt128)
+    DISPATCH(Int8)
+    DISPATCH(Int16)
+    DISPATCH(Int32)
+    DISPATCH(Int64)
+    DISPATCH(Float32)
+    DISPATCH(Float64)
+    DISPATCH(Decimal32)
+    DISPATCH(Decimal64)
+    DISPATCH(Decimal128)
+#undef DISPATCH
+    else
+        throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
+}
+
+template <typename AttributeType, typename OutputType, typename DefaultGetter>
+void CacheDictionary::getItemsNumberImpl(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ResultArrayType<OutputType> & out,
+    DefaultGetter && get_default) const
+{
+    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
+    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
+    auto & attribute_array = std::get<ContainerPtrType<AttributeType>>(attribute.arrays);
+    const auto rows = ext::size(ids);
+
+    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
+
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        /// fetch up-to-date values, decide which ones require update
+        for (const auto row : ext::range(0, rows))
+        {
+            const auto id = ids[row];
+
+            /** cell should be updated if either:
+                *    1. ids do not match,
+                *    2. cell has expired,
+                *    3. explicit defaults were specified and cell was set default. */
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                outdated_ids[id].push_back(row);
+                if (find_result.outdated)
+                    ++cache_expired;
+                else
+                    ++cache_not_found;
+            }
+            else
+            {
+                ++cache_hit;
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                out[row] = cell.isDefault() ? get_default(row) : static_cast<OutputType>(attribute_array[cell_idx]);
+            }
+        }
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
+
+    query_count.fetch_add(rows, std::memory_order_relaxed);
+    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
+
+    if (outdated_ids.empty())
+        return;
+
+    std::vector<Key> required_ids(outdated_ids.size());
+    std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
+        [] (auto & pair) { return pair.first; });
+
+    /// request new values
+    update(required_ids,
+    [&] (const auto id, const auto cell_idx)
+    {
+        const auto attribute_value = attribute_array[cell_idx];
+
+        for (const size_t row : outdated_ids[id])
+            out[row] = static_cast<OutputType>(attribute_value);
+    },
+    [&] (const auto id, const auto)
+    {
+        for (const size_t row : outdated_ids[id])
+            out[row] = get_default(row);
+    });
+}
+
+template <typename DefaultGetter>
+void CacheDictionary::getItemsString(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ColumnString * out,
+    DefaultGetter && get_default) const
+{
+    const auto rows = ext::size(ids);
+
+    /// save on some allocations
+    out->getOffsets().reserve(rows);
+
+    auto & attribute_array = std::get<ContainerPtrType<StringRef>>(attribute.arrays);
+
+    auto found_outdated_values = false;
+
+    /// perform optimistic version, fallback to pessimistic if failed
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        /// fetch up-to-date values, discard on fail
+        for (const auto row : ext::range(0, rows))
+        {
+            const auto id = ids[row];
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                found_outdated_values = true;
+                break;
+            }
+            else
+            {
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
+                out->insertData(string_ref.data, string_ref.size);
+            }
+        }
+    }
+
+    /// optimistic code completed successfully
+    if (!found_outdated_values)
+    {
+        query_count.fetch_add(rows, std::memory_order_relaxed);
+        hit_count.fetch_add(rows, std::memory_order_release);
+        return;
+    }
+
+    /// now onto the pessimistic one, discard possible partial results from the optimistic path
+    out->getChars().resize_assume_reserved(0);
+    out->getOffsets().resize_assume_reserved(0);
+
+    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
+    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
+    /// we are going to store every string separately
+    std::unordered_map<Key, String> map;
+
+    size_t total_length = 0;
+    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        for (const auto row : ext::range(0, ids.size()))
+        {
+            const auto id = ids[row];
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                outdated_ids[id].push_back(row);
+                if (find_result.outdated)
+                    ++cache_expired;
+                else
+                    ++cache_not_found;
+            }
+            else
+            {
+                ++cache_hit;
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
+
+                if (!cell.isDefault())
+                    map[id] = String{string_ref};
+
+                total_length += string_ref.size + 1;
+            }
+        }
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
+
+    query_count.fetch_add(rows, std::memory_order_relaxed);
+    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
+
+    /// request new values
+    if (!outdated_ids.empty())
+    {
+        std::vector<Key> required_ids(outdated_ids.size());
+        std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
+            [] (auto & pair) { return pair.first; });
+
+        update(required_ids,
+        [&] (const auto id, const auto cell_idx)
+        {
+            const auto attribute_value = attribute_array[cell_idx];
+
+            map[id] = String{attribute_value};
+            total_length += (attribute_value.size + 1) * outdated_ids[id].size();
+        },
+        [&] (const auto id, const auto)
+        {
+            for (const auto row : outdated_ids[id])
+                total_length += get_default(row).size + 1;
+        });
+    }
+
+    out->getChars().reserve(total_length);
+
+    for (const auto row : ext::range(0, ext::size(ids)))
+    {
+        const auto id = ids[row];
+        const auto it = map.find(id);
+
+        const auto string_ref = it != std::end(map) ? StringRef{it->second} : get_default(row);
+        out->insertData(string_ref.data, string_ref.size);
+    }
+}
+
+template <typename PresentIdHandler, typename AbsentIdHandler>
+void CacheDictionary::update(
+    const std::vector<Key> & requested_ids,
+    PresentIdHandler && on_cell_updated,
+    AbsentIdHandler && on_id_not_found) const
+{
+    std::unordered_map<Key, UInt8> remaining_ids{requested_ids.size()};
+    for (const auto id : requested_ids)
+        remaining_ids.insert({ id, 0 });
+
+    std::uniform_int_distribution<UInt64> distribution
+    {
+        dict_lifetime.min_sec,
+        dict_lifetime.max_sec
+    };
+
+    const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
+
+    {
+        CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests};
+        Stopwatch watch;
+        auto stream = source_ptr->loadIds(requested_ids);
+        stream->readPrefix();
+
+        const auto now = std::chrono::system_clock::now();
+
+        while (const auto block = stream->read())
+        {
+            const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
+            if (!id_column)
+                throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
+
+            const auto & ids = id_column->getData();
+
+            /// cache column pointers
+            const auto column_ptrs = ext::map<std::vector>(ext::range(0, attributes.size()), [&block] (size_t i)
+            {
+                return block.safeGetByPosition(i + 1).column.get();
+            });
+
+            for (const auto i : ext::range(0, ids.size()))
+            {
+                const auto id = ids[i];
+
+                const auto find_result = findCellIdx(id, now);
+                const auto & cell_idx = find_result.cell_idx;
+
+                auto & cell = cells[cell_idx];
+
+                for (const auto attribute_idx : ext::range(0, attributes.size()))
+                {
+                    const auto & attribute_column = *column_ptrs[attribute_idx];
+                    auto & attribute = attributes[attribute_idx];
+
+                    setAttributeValue(attribute, cell_idx, attribute_column[i]);
+                }
+
+                /// if cell id is zero and zero does not map to this cell, then the cell is unused
+                if (cell.id == 0 && cell_idx != zero_cell_idx)
+                    element_count.fetch_add(1, std::memory_order_relaxed);
+
+                cell.id = id;
+                if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
+                    cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
+                else
+                    cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
+
+                /// inform caller
+                on_cell_updated(id, cell_idx);
+                /// mark corresponding id as found
+                remaining_ids[id] = 1;
+            }
+        }
+
+        stream->readSuffix();
+
+        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size());
+        ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed());
+    }
+
+    size_t not_found_num = 0, found_num = 0;
+
+    const auto now = std::chrono::system_clock::now();
+    /// Check which ids have not been found and require setting null_value
+    for (const auto & id_found_pair : remaining_ids)
+    {
+        if (id_found_pair.second)
+        {
+            ++found_num;
+            continue;
+        }
+        ++not_found_num;
+
+        const auto id = id_found_pair.first;
+
+        const auto find_result = findCellIdx(id, now);
+        const auto & cell_idx = find_result.cell_idx;
+
+        auto & cell = cells[cell_idx];
+
+        /// Set null_value for each attribute
+        for (auto & attribute : attributes)
+            setDefaultAttributeValue(attribute, cell_idx);
+
+        /// Check if cell had not been occupied before and increment element counter if it hadn't
+        if (cell.id == 0 && cell_idx != zero_cell_idx)
+            element_count.fetch_add(1, std::memory_order_relaxed);
+
+        cell.id = id;
+        if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
+            cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
+        else
+            cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
+
+        cell.setDefault();
+
+        /// inform caller that the cell has not been found
+        on_id_not_found(id, cell_idx);
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedFound, found_num);
+    ProfileEvents::increment(ProfileEvents::DictCacheRequests);
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate1.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate1.cpp.in
@ -0,0 +1,24 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    const auto null_value = std::get<TYPE>(attribute.null_values);
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return null_value; });
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate2.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate2.cpp.in
@ -0,0 +1,25 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name,
+      const PaddedPODArray<Key> & ids,
+      const PaddedPODArray<TYPE> & def,
+      ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t row) { return def[row]; });
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate3.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate3.cpp.in
@ -0,0 +1,22 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return def; });
+}
+
+}
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
@ -1,14 +1,17 @@
-#include <Dictionaries/ClickHouseDictionarySource.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
-#include <Dictionaries/writeParenthesisedString.h>
+#include "ClickHouseDictionarySource.h"
+#include "ExternalQueryBuilder.h"
+#include "writeParenthesisedString.h"
 #include <Client/ConnectionPool.h>
 #include <DataStreams/RemoteBlockInputStream.h>
-#include <Dictionaries/readInvalidateQuery.h>
+#include "readInvalidateQuery.h"
 #include <Interpreters/executeQuery.h>
 #include <Common/isLocalAddress.h>
 #include <memory>
 #include <ext/range.h>
 #include <IO/ConnectionTimeouts.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+

 namespace DB
 {
@ -175,4 +178,17 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re
    }
 }

+
+void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 Context & context) -> DictionarySourcePtr {
+        return std::make_unique<ClickHouseDictionarySource>(dict_struct, config, config_prefix + ".clickhouse", sample_block, context);
+    };
+    factory.registerSource("clickhouse", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.h
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.h
@ -1,10 +1,9 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
+#include "ExternalQueryBuilder.h"
 #include <Client/ConnectionPoolWithFailover.h>
-#include <Poco/Util/AbstractConfiguration.h>
 #include <memory>


--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
@ -1,5 +1,5 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "ComplexKeyCacheDictionary.h"
+#include "DictionaryBlockInputStream.h"
 #include <Common/Arena.h>
 #include <Common/BitHelpers.h>
 #include <Common/randomSeed.h>
@ -9,6 +9,7 @@
 #include <Common/CurrentMetrics.h>
 #include <ext/range.h>
 #include <ext/map.h>
+#include "DictionaryFactory.h"


 namespace ProfileEvents
@ -39,6 +40,7 @@ namespace ErrorCodes
    extern const int TYPE_MISMATCH;
    extern const int BAD_ARGUMENTS;
    extern const int UNSUPPORTED_METHOD;
+    extern const int TOO_SMALL_BUFFER_SIZE;
 }


@ -378,4 +380,32 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, keys, column_names);
 }

+void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+        if (!dict_struct.key)
+            throw Exception {"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
+        const auto & layout_prefix = config_prefix + ".layout";
+        const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
+        if (size == 0)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
+
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        if (require_nonempty)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
+                             ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
+    };
+    factory.registerLayout("complex_key_cache", create_layout);
+}
+
+
 }
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
@ -12,9 +12,9 @@
 #include <Common/HashTable/HashMap.h>
 #include <Common/ProfilingScopedRWLock.h>
 #include <Common/SmallObjectPool.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
+#include "DictionaryStructure.h"
+#include "IDictionary.h"
+#include "IDictionarySource.h"
 #include <common/StringRef.h>
 #include <ext/bit_cast.h>
 #include <ext/map.h>
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_createAttributeWithType.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_createAttributeWithType.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp
@ -1,40 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                                   \
-    void ComplexKeyCacheDictionary::get##TYPE(                                                                                          \
-        const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const \
-    {                                                                                                                                   \
-        dict_struct.validateKeyTypes(key_types);                                                                                        \
-                                                                                                                                        \
-        auto & attribute = getAttribute(attribute_name);                                                                                \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                               \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),            \
-                ErrorCodes::TYPE_MISMATCH};                                                                                             \
-                                                                                                                                        \
-        const auto null_value = std::get<TYPE>(attribute.null_values);                                                                  \
-                                                                                                                                        \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });                                    \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in
@ -0,0 +1,24 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    const auto null_value = std::get<TYPE>(attribute.null_values);
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp
@ -1,41 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                        \
-    void ComplexKeyCacheDictionary::get##TYPE(const std::string & attribute_name,                                            \
-        const Columns & key_columns,                                                                                         \
-        const DataTypes & key_types,                                                                                         \
-        const PaddedPODArray<TYPE> & def,                                                                                    \
-        ResultArrayType<TYPE> & out) const                                                                                    \
-    {                                                                                                                        \
-        dict_struct.validateKeyTypes(key_types);                                                                             \
-                                                                                                                             \
-        auto & attribute = getAttribute(attribute_name);                                                                     \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                    \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
-                ErrorCodes::TYPE_MISMATCH};                                                                                  \
-                                                                                                                             \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });                       \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in
@ -0,0 +1,27 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
+      const Columns & key_columns,
+      const DataTypes & key_types,
+      const PaddedPODArray<TYPE> & def,
+      ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp
@ -1,41 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                        \
-    void ComplexKeyCacheDictionary::get##TYPE(const std::string & attribute_name,                                            \
-        const Columns & key_columns,                                                                                         \
-        const DataTypes & key_types,                                                                                         \
-        const TYPE def,                                                                                                      \
-        ResultArrayType<TYPE> & out) const                                                                                    \
-    {                                                                                                                        \
-        dict_struct.validateKeyTypes(key_types);                                                                             \
-                                                                                                                             \
-        auto & attribute = getAttribute(attribute_name);                                                                     \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                    \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
-                ErrorCodes::TYPE_MISMATCH};                                                                                  \
-                                                                                                                             \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });                                \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in
@ -0,0 +1,27 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
+      const Columns & key_columns,
+      const DataTypes & key_types,
+      const TYPE def,
+      ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
@ -1,8 +1,8 @@
 #include <ext/map.h>
 #include <ext/range.h>
-#include <Dictionaries/ComplexKeyHashedDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
-
+#include "ComplexKeyHashedDictionary.h"
+#include "DictionaryBlockInputStream.h"
+#include "DictionaryFactory.h"

 namespace DB
 {
@ -661,5 +661,24 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getKeys(), column_names);
 }

+void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+        if (!dict_struct.key)
+            throw Exception {"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
+    };
+    factory.registerLayout("complex_key_hashed", create_layout);
+}
+

 }
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <common/StringRef.h>
 #include <Common/HashTable/HashMap.h>
 #include <Columns/ColumnDecimal.h>
@ -243,5 +243,4 @@ private:
    BlockPtr saved_block;
 };

-
 }
--- a/dbms/src/Dictionaries/DictionaryBlockInputStream.h
+++ b/dbms/src/Dictionaries/DictionaryBlockInputStream.h
@ -6,9 +6,9 @@
 #include <Columns/IColumn.h>
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <DataTypes/DataTypesNumber.h>
-#include <Dictionaries/DictionaryBlockInputStreamBase.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/IDictionary.h>
+#include "DictionaryBlockInputStreamBase.h"
+#include "DictionaryStructure.h"
+#include "IDictionary.h"
 #include <ext/range.h>
 #include <common/logger_useful.h>
 #include <Core/Names.h>
--- a/dbms/src/Dictionaries/DictionaryBlockInputStreamBase.cpp
+++ b/dbms/src/Dictionaries/DictionaryBlockInputStreamBase.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/DictionaryBlockInputStreamBase.h>
+#include "DictionaryBlockInputStreamBase.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/DictionaryFactory.cpp
+++ b/dbms/src/Dictionaries/DictionaryFactory.cpp
@ -0,0 +1,51 @@
+#include "DictionaryFactory.h"
+
+#include <memory>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
+    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
+}
+
+void DictionaryFactory::registerLayout(const std::string & layout_type, Creator create_layout)
+{
+    //LOG_DEBUG(log, "Register dictionary layout type `" + layout_type + "`");
+    if (!registered_layouts.emplace(layout_type, std::move(create_layout)).second)
+        throw Exception("DictionaryFactory: the layout name '" + layout_type + "' is not unique", ErrorCodes::LOGICAL_ERROR);
+}
+
+
+DictionaryPtr DictionaryFactory::create(
+    const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) const
+{
+    Poco::Util::AbstractConfiguration::Keys keys;
+    const auto & layout_prefix = config_prefix + ".layout";
+    config.keys(layout_prefix, keys);
+    if (keys.size() != 1)
+        throw Exception {name + ": element dictionary.layout should have exactly one child element",
+                         ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
+
+    const DictionaryStructure dict_struct {config, config_prefix + ".structure"};
+
+    auto source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context);
+
+    const auto & layout_type = keys.front();
+
+    {
+        const auto found = registered_layouts.find(layout_type);
+        if (found != registered_layouts.end())
+        {
+            const auto & create_layout = found->second;
+            return create_layout(name, dict_struct, config, config_prefix, std::move(source_ptr));
+        }
+    }
+
+    throw Exception {name + ": unknown dictionary layout type: " + layout_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
+}
+
+}
--- a/dbms/src/Dictionaries/DictionaryFactory.h
+++ b/dbms/src/Dictionaries/DictionaryFactory.h
@ -1,20 +1,41 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Poco/Util/AbstractConfiguration.h>
 #include <ext/singleton.h>
+#include "IDictionary.h"

+namespace Poco
+{
+namespace Util
+{
+    class AbstractConfiguration;
+}
+
+class Logger;
+}

 namespace DB
 {
-
 class Context;

 class DictionaryFactory : public ext::singleton<DictionaryFactory>
 {
 public:
-    DictionaryPtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix, Context & context) const;
+    DictionaryPtr
+    create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context)
+        const;
+
+    using Creator = std::function<DictionaryPtr(
+        const std::string & name,
+        const DictionaryStructure & dict_struct,
+        const Poco::Util::AbstractConfiguration & config,
+        const std::string & config_prefix,
+        DictionarySourcePtr source_ptr)>;
+
+    void registerLayout(const std::string & layout_type, Creator create_layout);
+
+private:
+    using LayoutRegistry = std::unordered_map<std::string, Creator>;
+    LayoutRegistry registered_layouts;
 };

 }
--- a/dbms/src/Dictionaries/DictionarySourceFactory.cpp
+++ b/dbms/src/Dictionaries/DictionarySourceFactory.cpp
@ -1,41 +1,16 @@
-#include <Dictionaries/DictionarySourceFactory.h>
+#include "DictionarySourceFactory.h"

-#include <Core/Block.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/FileDictionarySource.h>
-#include <Dictionaries/ClickHouseDictionarySource.h>
-#include <Dictionaries/ExecutableDictionarySource.h>
-#include <Dictionaries/HTTPDictionarySource.h>
-#include <Dictionaries/LibraryDictionarySource.h>
-#include <Dictionaries/XDBCDictionarySource.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeDate.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <Common/FieldVisitors.h>
-#include <Common/XDBCBridgeHelper.h>
 #include <Columns/ColumnsNumber.h>
-#include <IO/HTTPCommon.h>
-#include <memory>
-#include <mutex>
-
-#include <Common/config.h>
-#if USE_POCO_MONGODB
-    #include <Dictionaries/MongoDBDictionarySource.h>
-#endif
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-    #include <Poco/Data/ODBC/Connector.h>
-#endif
-#if USE_MYSQL
-    #include <Dictionaries/MySQLDictionarySource.h>
-#endif
-
+#include <Core/Block.h>
+#include <Core/ColumnWithTypeAndName.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypesNumber.h>
 #include <Poco/Logger.h>
-
 #include <common/logger_useful.h>
+#include "DictionaryStructure.h"

 namespace DB
 {
-
 namespace ErrorCodes
 {
    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
@ -46,149 +21,78 @@ namespace ErrorCodes

 namespace
 {
-
-Block createSampleBlock(const DictionaryStructure & dict_struct)
-{
-    Block block;
-
-    if (dict_struct.id)
-        block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
-
-    if (dict_struct.key)
+    Block createSampleBlock(const DictionaryStructure & dict_struct)
    {
-        for (const auto & attribute : *dict_struct.key)
+        Block block;
+
+        if (dict_struct.id)
+            block.insert(ColumnWithTypeAndName {ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
+
+        if (dict_struct.key)
+        {
+            for (const auto & attribute : *dict_struct.key)
+            {
+                auto column = attribute.type->createColumn();
+                column->insertDefault();
+
+                block.insert(ColumnWithTypeAndName {std::move(column), attribute.type, attribute.name});
+            }
+        }
+
+        if (dict_struct.range_min)
+        {
+            for (const auto & attribute : {dict_struct.range_min, dict_struct.range_max})
+            {
+                const auto & type = std::make_shared<DataTypeNullable>(attribute->type);
+                auto column = type->createColumn();
+                column->insertDefault();
+
+                block.insert(ColumnWithTypeAndName {std::move(column), type, attribute->name});
+            }
+        }
+
+        for (const auto & attribute : dict_struct.attributes)
        {
            auto column = attribute.type->createColumn();
-            column->insertDefault();
+            column->insert(attribute.null_value);

-            block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
+            block.insert(ColumnWithTypeAndName {std::move(column), attribute.type, attribute.name});
        }
+
+        return block;
    }

-    if (dict_struct.range_min)
-    {
-        for (const auto & attribute : { dict_struct.range_min, dict_struct.range_max })
-        {
-            const auto & type = std::make_shared<DataTypeNullable>(attribute->type);
-            auto column = type->createColumn();
-            column->insertDefault();
-
-            block.insert(ColumnWithTypeAndName{std::move(column), type, attribute->name});
-        }
-    }
-
-    for (const auto & attribute : dict_struct.attributes)
-    {
-        auto column = attribute.type->createColumn();
-        column->insert(attribute.null_value);
-
-        block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
-    }
-
-    return block;
-}
-
 }


-DictionarySourceFactory::DictionarySourceFactory()
-    : log(&Poco::Logger::get("DictionarySourceFactory"))
+DictionarySourceFactory::DictionarySourceFactory() : log(&Poco::Logger::get("DictionarySourceFactory"))
 {
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-    Poco::Data::ODBC::Connector::registerConnector();
-#endif
 }

 void DictionarySourceFactory::registerSource(const std::string & source_type, Creator create_source)
 {
    LOG_DEBUG(log, "Register dictionary source type `" + source_type + "`");
    if (!registered_sources.emplace(source_type, std::move(create_source)).second)
-        throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique",
-            ErrorCodes::LOGICAL_ERROR);
+        throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique", ErrorCodes::LOGICAL_ERROR);
 }

 DictionarySourcePtr DictionarySourceFactory::create(
-    const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
-    const DictionaryStructure & dict_struct, Context & context) const
+    const std::string & name,
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const DictionaryStructure & dict_struct,
+    Context & context) const
 {
    Poco::Util::AbstractConfiguration::Keys keys;
    config.keys(config_prefix, keys);
    if (keys.size() != 1)
-        throw Exception{name +": element dictionary.source should have exactly one child element", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
+        throw Exception {name + ": element dictionary.source should have exactly one child element",
+                         ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};

    auto sample_block = createSampleBlock(dict_struct);

    const auto & source_type = keys.front();

-    if ("file" == source_type)
-    {
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `file` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        const auto filename = config.getString(config_prefix + ".file.path");
-        const auto format = config.getString(config_prefix + ".file.format");
-        return std::make_unique<FileDictionarySource>(filename, format, sample_block, context);
-    }
-    else if ("mysql" == source_type)
-    {
-#if USE_MYSQL
-        return std::make_unique<MySQLDictionarySource>(dict_struct, config, config_prefix + ".mysql", sample_block);
-#else
-        throw Exception{"Dictionary source of type `mysql` is disabled because ClickHouse was built without mysql support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("clickhouse" == source_type)
-    {
-        return std::make_unique<ClickHouseDictionarySource>(dict_struct, config, config_prefix + ".clickhouse",
-            sample_block, context);
-    }
-    else if ("mongodb" == source_type)
-    {
-#if USE_POCO_MONGODB
-        return std::make_unique<MongoDBDictionarySource>(dict_struct, config, config_prefix + ".mongodb", sample_block);
-#else
-        throw Exception{"Dictionary source of type `mongodb` is disabled because poco library was built without mongodb support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("odbc" == source_type)
-    {
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-        BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(context, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string"));
-        return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge);
-#else
-        throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("jdbc" == source_type)
-    {
-        throw Exception{"Dictionary source of type `jdbc` is disabled until consistent support for nullable fields.",
-                        ErrorCodes::SUPPORT_IS_DISABLED};
-//        BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<JDBCBridgeMixin>>(config, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".connection_string"));
-//        return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".jdbc", sample_block, context, bridge);
-    }
-    else if ("executable" == source_type)
-    {
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
-    }
-    else if ("http" == source_type)
-    {
-
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        return std::make_unique<HTTPDictionarySource>(dict_struct, config, config_prefix + ".http", sample_block, context);
-    }
-    else if ("library" == source_type)
-    {
-        return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
-    }
-    else
    {
        const auto found = registered_sources.find(source_type);
        if (found != registered_sources.end())
@ -198,7 +102,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
        }
    }

-    throw Exception{name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
+    throw Exception {name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
 }

 }
--- a/dbms/src/Dictionaries/DictionarySourceFactory.h
+++ b/dbms/src/Dictionaries/DictionarySourceFactory.h
@ -1,23 +1,22 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <ext/singleton.h>
+#include "IDictionarySource.h"

 #include <unordered_map>
+#include <ext/singleton.h>

 namespace Poco
 {
-    namespace Util
-    {
-        class AbstractConfiguration;
-    }
+namespace Util
+{
+    class AbstractConfiguration;
+}

-    class Logger;
+class Logger;
 }

 namespace DB
 {
-
 class Context;
 struct DictionaryStructure;

@ -30,15 +29,18 @@ public:
        const Poco::Util::AbstractConfiguration & config,
        const std::string & config_prefix,
        Block & sample_block,
-        const Context & context)>;
+        Context & context)>;

    DictionarySourceFactory();

    void registerSource(const std::string & source_type, Creator create_source);

    DictionarySourcePtr create(
-        const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
-        const DictionaryStructure & dict_struct, Context & context) const;
+        const std::string & name,
+        const Poco::Util::AbstractConfiguration & config,
+        const std::string & config_prefix,
+        const DictionaryStructure & dict_struct,
+        Context & context) const;

 private:
    using SourceRegistry = std::unordered_map<std::string, Creator>;
--- a/dbms/src/Dictionaries/DictionarySourceHelpers.cpp
+++ b/dbms/src/Dictionaries/DictionarySourceHelpers.cpp
@ -1,5 +1,5 @@
-#include <Dictionaries/DictionarySourceHelpers.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "DictionarySourceHelpers.h"
+#include "DictionaryStructure.h"
 #include <Core/ColumnWithTypeAndName.h>
 #include <Core/Block.h>
 #include <Columns/ColumnsNumber.h>
--- a/dbms/src/Dictionaries/DictionaryStructure.cpp
+++ b/dbms/src/Dictionaries/DictionaryStructure.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/DictionaryStructure.h>
+#include "DictionaryStructure.h"
 #include <Formats/FormatSettings.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeNullable.h>
--- a/dbms/src/Dictionaries/Embedded/CMakeLists.txt
+++ b/dbms/src/Dictionaries/Embedded/CMakeLists.txt
@ -0,0 +1,5 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+add_headers_and_sources(clickhouse_dictionaries_embedded .)
+add_headers_and_sources(clickhouse_dictionaries_embedded GeodataProviders)
+add_library(clickhouse_dictionaries_embedded ${LINK_MODE} ${clickhouse_dictionaries_embedded_sources})
+target_link_libraries(clickhouse_dictionaries_embedded PRIVATE clickhouse_common_io ${MYSQLXX_LIBRARY})
--- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp
@ -1,8 +1,8 @@
-#include <Dictionaries/Embedded/GeoDictionariesLoader.h>
-
-#include <Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/NamesProvider.h>
+#include "GeoDictionariesLoader.h"

+#include <Poco/Util/AbstractConfiguration.h>
+#include "GeodataProviders/HierarchiesProvider.h"
+#include "GeodataProviders/NamesProvider.h"

 std::unique_ptr<RegionsHierarchies> GeoDictionariesLoader::reloadRegionsHierarchies(
    const Poco::Util::AbstractConfiguration & config)
--- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h
+++ b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h
@ -1,6 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/IGeoDictionariesLoader.h>
+#include "IGeoDictionariesLoader.h"


 // Default implementation of geo dictionaries loader used by native server application
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
@ -1,6 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Types.h>
+#include "Types.h"
 #include <string>

 struct RegionEntry
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp
@ -1,8 +1,7 @@
-#include <Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h>
+#include "HierarchiesProvider.h"

+#include "HierarchyFormatReader.h"
 #include <IO/ReadBufferFromFile.h>
-
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
 #include <Poco/DirectoryIterator.h>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h
@ -1,9 +1,8 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
+#include "IHierarchiesProvider.h"

 #include <Common/FileUpdatesTracker.h>
-
 #include <unordered_map>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h>
+#include "HierarchyFormatReader.h"

 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "IHierarchiesProvider.h"
 #include <IO/ReadBuffer.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Entries.h>
-
+#include "Entries.h"
 #include <memory>
 #include <string>
 #include <vector>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Entries.h>
-
+#include "Entries.h"
 #include <memory>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h>
+#include "NamesFormatReader.h"

 #include <IO/ReadHelpers.h>

--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "INamesProvider.h"
 #include <IO/ReadBuffer.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp
@ -1,6 +1,6 @@
-#include <Dictionaries/Embedded/GeodataProviders/NamesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h>
+#include "NamesProvider.h"

+#include "NamesFormatReader.h"
 #include <IO/ReadBufferFromFile.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "INamesProvider.h"
 #include <Common/FileUpdatesTracker.h>


--- a/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h
+++ b/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h
@ -1,12 +1,19 @@
 #pragma once

-#include <Dictionaries/Embedded/RegionsHierarchies.h>
-#include <Dictionaries/Embedded/RegionsNames.h>
-
-#include <Poco/Util/AbstractConfiguration.h>
-
+#include "RegionsHierarchies.h"
+#include "RegionsNames.h"
 #include <memory>

+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+
+    class Logger;
+}
+

 // Provides actual versions of geo dictionaries (regions hierarchies, regions names)
 // Bind data structures (RegionsHierarchies, RegionsNames) with data providers
--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp
@ -1,7 +1,6 @@
-#include <Dictionaries/Embedded/RegionsHierarchies.h>
+#include "RegionsHierarchies.h"

 #include <common/logger_useful.h>
-
 #include <Poco/DirectoryIterator.h>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h
@ -1,10 +1,8 @@
 #pragma once

-#include <Dictionaries/Embedded/RegionsHierarchy.h>
-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "RegionsHierarchy.h"
+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <Poco/Exception.h>
-
 #include <unordered_map>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp
@ -1,12 +1,10 @@
-#include <Dictionaries/Embedded/RegionsHierarchy.h>
-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
+#include "RegionsHierarchy.h"

+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
-
 #include <common/logger_useful.h>
 #include <ext/singleton.h>
-
 #include <IO/WriteHelpers.h>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <vector>
 #include <boost/noncopyable.hpp>
 #include <common/Types.h>
--- a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp
@ -1,11 +1,9 @@
-#include <Dictionaries/Embedded/RegionsNames.h>
-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
+#include "RegionsNames.h"

+#include "GeodataProviders/INamesProvider.h"
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
-
 #include <common/logger_useful.h>
-
 #include <IO/WriteHelpers.h>

 namespace DB
--- a/dbms/src/Dictionaries/Embedded/RegionsNames.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsNames.h
@ -1,12 +1,9 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "GeodataProviders/INamesProvider.h"
 #include <Poco/Exception.h>
-
 #include <common/Types.h>
 #include <common/StringRef.h>
-
 #include <string>
 #include <vector>

--- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp
+++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp
@ -1,7 +1,7 @@
 #include <Common/config.h>
 #if USE_MYSQL

-#include <Dictionaries/Embedded/TechDataHierarchy.h>
+#include "TechDataHierarchy.h"

 #include <common/logger_useful.h>
 #include <mysqlxx/PoolWithFailover.h>
--- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h
+++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h
@ -1,12 +1,18 @@
 #pragma once

-#include <Poco/Util/AbstractConfiguration.h>
-#include <Poco/Exception.h>
-
 #include <common/Types.h>
-
 #include <ext/singleton.h>

+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+
+    class Logger;
+}
+

 /** @brief Class that lets you know if a search engine or operating system belongs
  * another search engine or operating system, respectively.
--- a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp
+++ b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp
@ -1,12 +1,15 @@
+#include "ExecutableDictionarySource.h"
+
 #include <thread>
 #include <future>
-#include <Dictionaries/ExecutableDictionarySource.h>
 #include <Common/ShellCommand.h>
 #include <Interpreters/Context.h>
 #include <DataStreams/OwningBlockInputStream.h>
-#include <Dictionaries/DictionarySourceHelpers.h>
+#include "DictionarySourceHelpers.h"
 #include <DataStreams/IBlockOutputStream.h>
 #include <common/logger_useful.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"


 namespace DB
@ -229,4 +232,19 @@ std::string ExecutableDictionarySource::toString() const
    return "Executable: " + command;
 }

+void registerDictionarySourceExecutable(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & context) -> DictionarySourcePtr {
+        if (dict_struct.has_expressions)
+            throw Exception {"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
+
+        return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
+    };
+    factory.registerSource("executable", createTableSource);
+}
+
 }
--- a/Show More
+++ b/Show More