Merge branch 'master' into alter-order-by

Conflicts: dbms/programs/server/Server.cpp
2024-09-20 08:40:50 +00:00 · 2018-11-28 16:47:07 +03:00 · 2018-11-28 16:47:07 +03:00 · 1b27fb046d
commit 1b27fb046d
parent f3519bb297 dd2371e071
257 changed files with 5736 additions and 3963 deletions
--- a/CHANGELOG.draft.md
+++ b/CHANGELOG.draft.md
@ -1 +1,4 @@
 * Настройка `enable_optimize_predicate_expression` выключена по-умолчанию.
+
+### Улучшения:
+* Файлы *-preprocessed.xml записываются в директорию с данными (/var/lib/clickhouse/preprocessed_configs). Для /etc/clickhouse-server больше не нужен +w для пользователя clickhouse. Для удобства создан симлинк /var/lib/clickhouse/preprocessed_configs -> /etc/clickhouse-server/preprocessed
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -236,7 +236,7 @@ include(GNUInstallDirs)

 include (cmake/find_ssl.cmake)
 include (cmake/lib_name.cmake)
-include (cmake/find_icu4c.cmake)
+include (cmake/find_icu.cmake)
 include (cmake/find_boost.cmake)
 include (cmake/find_zlib.cmake)
 include (cmake/find_zstd.cmake)
--- a/README.md
+++ b/README.md
@ -9,3 +9,4 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
 * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
+* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
--- a/cmake/Modules/FindICU.cmake
+++ b/cmake/Modules/FindICU.cmake
@ -0,0 +1,394 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#.rst:
+# FindICU
+# -------
+#
+# Find the International Components for Unicode (ICU) libraries and
+# programs.
+#
+# This module supports multiple components.
+# Components can include any of: ``data``, ``i18n``, ``io``, ``le``,
+# ``lx``, ``test``, ``tu`` and ``uc``.
+#
+# Note that on Windows ``data`` is named ``dt`` and ``i18n`` is named
+# ``in``; any of the names may be used, and the appropriate
+# platform-specific library name will be automatically selected.
+#
+# This module reports information about the ICU installation in
+# several variables.  General variables::
+#
+#   ICU_VERSION - ICU release version
+#   ICU_FOUND - true if the main programs and libraries were found
+#   ICU_LIBRARIES - component libraries to be linked
+#   ICU_INCLUDE_DIRS - the directories containing the ICU headers
+#
+# Imported targets::
+#
+#   ICU::<C>
+#
+# Where ``<C>`` is the name of an ICU component, for example
+# ``ICU::i18n``.
+#
+# ICU programs are reported in::
+#
+#   ICU_GENCNVAL_EXECUTABLE - path to gencnval executable
+#   ICU_ICUINFO_EXECUTABLE - path to icuinfo executable
+#   ICU_GENBRK_EXECUTABLE - path to genbrk executable
+#   ICU_ICU-CONFIG_EXECUTABLE - path to icu-config executable
+#   ICU_GENRB_EXECUTABLE - path to genrb executable
+#   ICU_GENDICT_EXECUTABLE - path to gendict executable
+#   ICU_DERB_EXECUTABLE - path to derb executable
+#   ICU_PKGDATA_EXECUTABLE - path to pkgdata executable
+#   ICU_UCONV_EXECUTABLE - path to uconv executable
+#   ICU_GENCFU_EXECUTABLE - path to gencfu executable
+#   ICU_MAKECONV_EXECUTABLE - path to makeconv executable
+#   ICU_GENNORM2_EXECUTABLE - path to gennorm2 executable
+#   ICU_GENCCODE_EXECUTABLE - path to genccode executable
+#   ICU_GENSPREP_EXECUTABLE - path to gensprep executable
+#   ICU_ICUPKG_EXECUTABLE - path to icupkg executable
+#   ICU_GENCMN_EXECUTABLE - path to gencmn executable
+#
+# ICU component libraries are reported in::
+#
+#   ICU_<C>_FOUND - ON if component was found
+#   ICU_<C>_LIBRARIES - libraries for component
+#
+# ICU datafiles are reported in::
+#
+#   ICU_MAKEFILE_INC - Makefile.inc
+#   ICU_PKGDATA_INC - pkgdata.inc
+#
+# Note that ``<C>`` is the uppercased name of the component.
+#
+# This module reads hints about search results from::
+#
+#   ICU_ROOT - the root of the ICU installation
+#
+# The environment variable ``ICU_ROOT`` may also be used; the
+# ICU_ROOT variable takes precedence.
+#
+# The following cache variables may also be set::
+#
+#   ICU_<P>_EXECUTABLE - the path to executable <P>
+#   ICU_INCLUDE_DIR - the directory containing the ICU headers
+#   ICU_<C>_LIBRARY - the library for component <C>
+#
+# .. note::
+#
+#   In most cases none of the above variables will require setting,
+#   unless multiple ICU versions are available and a specific version
+#   is required.
+#
+# Other variables one may set to control this module are::
+#
+#   ICU_DEBUG - Set to ON to enable debug output from FindICU.
+
+# Written by Roger Leigh <rleigh@codelibre.net>
+
+set(icu_programs
+  gencnval
+  icuinfo
+  genbrk
+  icu-config
+  genrb
+  gendict
+  derb
+  pkgdata
+  uconv
+  gencfu
+  makeconv
+  gennorm2
+  genccode
+  gensprep
+  icupkg
+  gencmn)
+
+set(icu_data
+  Makefile.inc
+  pkgdata.inc)
+
+# The ICU checks are contained in a function due to the large number
+# of temporary variables needed.
+function(_ICU_FIND)
+  # Set up search paths, taking compiler into account.  Search ICU_ROOT,
+  # with ICU_ROOT in the environment as a fallback if unset.
+  if(ICU_ROOT)
+    list(APPEND icu_roots "${ICU_ROOT}")
+  else()
+    if(NOT "$ENV{ICU_ROOT}" STREQUAL "")
+      file(TO_CMAKE_PATH "$ENV{ICU_ROOT}" NATIVE_PATH)
+      list(APPEND icu_roots "${NATIVE_PATH}")
+      set(ICU_ROOT "${NATIVE_PATH}"
+          CACHE PATH "Location of the ICU installation" FORCE)
+    endif()
+  endif()
+
+  # Find include directory
+  list(APPEND icu_include_suffixes "include")
+  find_path(ICU_INCLUDE_DIR
+            NAMES "unicode/utypes.h"
+            HINTS ${icu_roots}
+            PATH_SUFFIXES ${icu_include_suffixes}
+            DOC "ICU include directory")
+  set(ICU_INCLUDE_DIR "${ICU_INCLUDE_DIR}" PARENT_SCOPE)
+
+  # Get version
+  if(ICU_INCLUDE_DIR AND EXISTS "${ICU_INCLUDE_DIR}/unicode/uvernum.h")
+    file(STRINGS "${ICU_INCLUDE_DIR}/unicode/uvernum.h" icu_header_str
+      REGEX "^#define[\t ]+U_ICU_VERSION[\t ]+\".*\".*")
+
+    string(REGEX REPLACE "^#define[\t ]+U_ICU_VERSION[\t ]+\"([^ \\n]*)\".*"
+      "\\1" icu_version_string "${icu_header_str}")
+    set(ICU_VERSION "${icu_version_string}")
+    set(ICU_VERSION "${icu_version_string}" PARENT_SCOPE)
+    unset(icu_header_str)
+    unset(icu_version_string)
+  endif()
+
+  if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+    # 64-bit binary directory
+    set(_bin64 "bin64")
+    # 64-bit library directory
+    set(_lib64 "lib64")
+  endif()
+
+
+  # Find all ICU programs
+  list(APPEND icu_binary_suffixes "${_bin64}" "bin")
+  foreach(program ${icu_programs})
+    string(TOUPPER "${program}" program_upcase)
+    set(cache_var "ICU_${program_upcase}_EXECUTABLE")
+    set(program_var "ICU_${program_upcase}_EXECUTABLE")
+    find_program("${cache_var}" "${program}"
+      HINTS ${icu_roots}
+      PATH_SUFFIXES ${icu_binary_suffixes}
+      DOC "ICU ${program} executable")
+    mark_as_advanced(cache_var)
+    set("${program_var}" "${${cache_var}}" PARENT_SCOPE)
+  endforeach()
+
+  # Find all ICU libraries
+  list(APPEND icu_library_suffixes "${_lib64}" "lib")
+  set(ICU_REQUIRED_LIBS_FOUND ON)
+  foreach(component ${ICU_FIND_COMPONENTS})
+    string(TOUPPER "${component}" component_upcase)
+    set(component_cache "ICU_${component_upcase}_LIBRARY")
+    set(component_cache_release "${component_cache}_RELEASE")
+    set(component_cache_debug "${component_cache}_DEBUG")
+    set(component_found "${component_upcase}_FOUND")
+    set(component_libnames "icu${component}")
+    set(component_debug_libnames "icu${component}d")
+
+    # Special case deliberate library naming mismatches between Unix
+    # and Windows builds
+    unset(component_libnames)
+    unset(component_debug_libnames)
+    list(APPEND component_libnames "icu${component}")
+    list(APPEND component_debug_libnames "icu${component}d")
+    if(component STREQUAL "data")
+      list(APPEND component_libnames "icudt")
+      # Note there is no debug variant at present
+      list(APPEND component_debug_libnames "icudtd")
+    endif()
+    if(component STREQUAL "dt")
+      list(APPEND component_libnames "icudata")
+      # Note there is no debug variant at present
+      list(APPEND component_debug_libnames "icudatad")
+    endif()
+    if(component STREQUAL "i18n")
+      list(APPEND component_libnames "icuin")
+      list(APPEND component_debug_libnames "icuind")
+    endif()
+    if(component STREQUAL "in")
+      list(APPEND component_libnames "icui18n")
+      list(APPEND component_debug_libnames "icui18nd")
+    endif()
+
+    find_library("${component_cache_release}" ${component_libnames}
+      HINTS ${icu_roots}
+      PATH_SUFFIXES ${icu_library_suffixes}
+      DOC "ICU ${component} library (release)")
+    find_library("${component_cache_debug}" ${component_debug_libnames}
+      HINTS ${icu_roots}
+      PATH_SUFFIXES ${icu_library_suffixes}
+      DOC "ICU ${component} library (debug)")
+    include(SelectLibraryConfigurations)
+    select_library_configurations(ICU_${component_upcase})
+    mark_as_advanced("${component_cache_release}" "${component_cache_debug}")
+    if(${component_cache})
+      set("${component_found}" ON)
+      list(APPEND ICU_LIBRARY "${${component_cache}}")
+    endif()
+    mark_as_advanced("${component_found}")
+    set("${component_cache}" "${${component_cache}}" PARENT_SCOPE)
+    set("${component_found}" "${${component_found}}" PARENT_SCOPE)
+    if(${component_found})
+      if (ICU_FIND_REQUIRED_${component})
+        list(APPEND ICU_LIBS_FOUND "${component} (required)")
+      else()
+        list(APPEND ICU_LIBS_FOUND "${component} (optional)")
+      endif()
+    else()
+      if (ICU_FIND_REQUIRED_${component})
+        set(ICU_REQUIRED_LIBS_FOUND OFF)
+        list(APPEND ICU_LIBS_NOTFOUND "${component} (required)")
+      else()
+        list(APPEND ICU_LIBS_NOTFOUND "${component} (optional)")
+      endif()
+    endif()
+  endforeach()
+  set(_ICU_REQUIRED_LIBS_FOUND "${ICU_REQUIRED_LIBS_FOUND}" PARENT_SCOPE)
+  set(ICU_LIBRARY "${ICU_LIBRARY}" PARENT_SCOPE)
+
+  # Find all ICU data files
+  if(CMAKE_LIBRARY_ARCHITECTURE)
+    list(APPEND icu_data_suffixes
+      "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}"
+      "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}"
+      "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu"
+      "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu")
+  endif()
+  list(APPEND icu_data_suffixes
+    "${_lib64}/icu/${ICU_VERSION}"
+    "lib/icu/${ICU_VERSION}"
+    "${_lib64}/icu"
+    "lib/icu")
+  foreach(data ${icu_data})
+    string(TOUPPER "${data}" data_upcase)
+    string(REPLACE "." "_" data_upcase "${data_upcase}")
+    set(cache_var "ICU_${data_upcase}")
+    set(data_var "ICU_${data_upcase}")
+    find_file("${cache_var}" "${data}"
+      HINTS ${icu_roots}
+      PATH_SUFFIXES ${icu_data_suffixes}
+      DOC "ICU ${data} data file")
+    mark_as_advanced(cache_var)
+    set("${data_var}" "${${cache_var}}" PARENT_SCOPE)
+  endforeach()
+
+  if(NOT ICU_FIND_QUIETLY)
+    if(ICU_LIBS_FOUND)
+      message(STATUS "Found the following ICU libraries:")
+      foreach(found ${ICU_LIBS_FOUND})
+        message(STATUS "  ${found}")
+      endforeach()
+    endif()
+    if(ICU_LIBS_NOTFOUND)
+      message(STATUS "The following ICU libraries were not found:")
+      foreach(notfound ${ICU_LIBS_NOTFOUND})
+        message(STATUS "  ${notfound}")
+      endforeach()
+    endif()
+  endif()
+
+  if(ICU_DEBUG)
+    message(STATUS "--------FindICU.cmake search debug--------")
+    message(STATUS "ICU binary path search order: ${icu_roots}")
+    message(STATUS "ICU include path search order: ${icu_roots}")
+    message(STATUS "ICU library path search order: ${icu_roots}")
+    message(STATUS "----------------")
+  endif()
+endfunction()
+
+_ICU_FIND()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(ICU
+                                  FOUND_VAR ICU_FOUND
+                                  REQUIRED_VARS ICU_INCLUDE_DIR
+                                                ICU_LIBRARY
+                                                _ICU_REQUIRED_LIBS_FOUND
+                                  VERSION_VAR ICU_VERSION
+                                  FAIL_MESSAGE "Failed to find all ICU components")
+
+unset(_ICU_REQUIRED_LIBS_FOUND)
+
+if(ICU_FOUND)
+  set(ICU_INCLUDE_DIRS "${ICU_INCLUDE_DIR}")
+  set(ICU_LIBRARIES "${ICU_LIBRARY}")
+  foreach(_ICU_component ${ICU_FIND_COMPONENTS})
+    string(TOUPPER "${_ICU_component}" _ICU_component_upcase)
+    set(_ICU_component_cache "ICU_${_ICU_component_upcase}_LIBRARY")
+    set(_ICU_component_cache_release "ICU_${_ICU_component_upcase}_LIBRARY_RELEASE")
+    set(_ICU_component_cache_debug "ICU_${_ICU_component_upcase}_LIBRARY_DEBUG")
+    set(_ICU_component_lib "ICU_${_ICU_component_upcase}_LIBRARIES")
+    set(_ICU_component_found "${_ICU_component_upcase}_FOUND")
+    set(_ICU_imported_target "ICU::${_ICU_component}")
+    if(${_ICU_component_found})
+      set("${_ICU_component_lib}" "${${_ICU_component_cache}}")
+      if(NOT TARGET ${_ICU_imported_target})
+        add_library(${_ICU_imported_target} UNKNOWN IMPORTED)
+        if(ICU_INCLUDE_DIR)
+          set_target_properties(${_ICU_imported_target} PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}")
+        endif()
+        if(EXISTS "${${_ICU_component_cache}}")
+          set_target_properties(${_ICU_imported_target} PROPERTIES
+            IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+            IMPORTED_LOCATION "${${_ICU_component_cache}}")
+        endif()
+        if(EXISTS "${${_ICU_component_cache_release}}")
+          set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY
+            IMPORTED_CONFIGURATIONS RELEASE)
+          set_target_properties(${_ICU_imported_target} PROPERTIES
+            IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX"
+            IMPORTED_LOCATION_RELEASE "${${_ICU_component_cache_release}}")
+        endif()
+        if(EXISTS "${${_ICU_component_cache_debug}}")
+          set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY
+            IMPORTED_CONFIGURATIONS DEBUG)
+          set_target_properties(${_ICU_imported_target} PROPERTIES
+            IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "CXX"
+            IMPORTED_LOCATION_DEBUG "${${_ICU_component_cache_debug}}")
+        endif()
+      endif()
+    endif()
+    unset(_ICU_component_upcase)
+    unset(_ICU_component_cache)
+    unset(_ICU_component_lib)
+    unset(_ICU_component_found)
+    unset(_ICU_imported_target)
+  endforeach()
+endif()
+
+if(ICU_DEBUG)
+  message(STATUS "--------FindICU.cmake results debug--------")
+  message(STATUS "ICU found: ${ICU_FOUND}")
+  message(STATUS "ICU_VERSION number: ${ICU_VERSION}")
+  message(STATUS "ICU_ROOT directory: ${ICU_ROOT}")
+  message(STATUS "ICU_INCLUDE_DIR directory: ${ICU_INCLUDE_DIR}")
+  message(STATUS "ICU_LIBRARIES: ${ICU_LIBRARIES}")
+
+  foreach(program IN LISTS icu_programs)
+    string(TOUPPER "${program}" program_upcase)
+    set(program_lib "ICU_${program_upcase}_EXECUTABLE")
+    message(STATUS "${program} program: ${${program_lib}}")
+    unset(program_upcase)
+    unset(program_lib)
+  endforeach()
+
+  foreach(data IN LISTS icu_data)
+    string(TOUPPER "${data}" data_upcase)
+    string(REPLACE "." "_" data_upcase "${data_upcase}")
+    set(data_lib "ICU_${data_upcase}")
+    message(STATUS "${data} data: ${${data_lib}}")
+    unset(data_upcase)
+    unset(data_lib)
+  endforeach()
+
+  foreach(component IN LISTS ICU_FIND_COMPONENTS)
+    string(TOUPPER "${component}" component_upcase)
+    set(component_lib "ICU_${component_upcase}_LIBRARIES")
+    set(component_found "${component_upcase}_FOUND")
+    message(STATUS "${component} library found: ${${component_found}}")
+    message(STATUS "${component} library: ${${component_lib}}")
+    unset(component_upcase)
+    unset(component_lib)
+    unset(component_found)
+  endforeach()
+  message(STATUS "----------------")
+endif()
+
+unset(icu_programs)
--- a/cmake/dbms_generate_function.cmake
+++ b/cmake/dbms_generate_function.cmake
@ -1,10 +0,0 @@
-function(generate_function_register FUNCTION_AREA)
-    foreach(FUNCTION IN LISTS ARGN)
-        configure_file (registerFunction.h.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.h)
-        configure_file (registerFunction.cpp.in ${FUNCTIONS_GENERATED_DIR}register${FUNCTION}.cpp)
-        set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n")
-        set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS}    register${FUNCTION}(factory);\n")
-  endforeach()
-
-  configure_file (registerFunctions_area.cpp.in ${FUNCTIONS_GENERATED_DIR}registerFunctions${FUNCTION_AREA}.cpp)
-endfunction()
--- a/cmake/find_icu.cmake
+++ b/cmake/find_icu.cmake
@ -0,0 +1,16 @@
+option (ENABLE_ICU "Enable ICU" ON)
+
+if (ENABLE_ICU)
+    find_package(ICU COMPONENTS data i18n uc) # TODO: remove Modules/FindICU.cmake after cmake 3.7
+    #set (ICU_LIBRARIES ${ICU_I18N_LIBRARY} ${ICU_UC_LIBRARY} ${ICU_DATA_LIBRARY} CACHE STRING "")
+    set (ICU_LIBRARIES ICU::i18n ICU::uc ICU::data CACHE STRING "")
+    if (ICU_FOUND)
+        set(USE_ICU 1)
+    endif ()
+endif ()
+
+if (USE_ICU)
+    message (STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}")
+else ()
+    message (STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)")
+endif ()
--- a/cmake/find_icu4c.cmake
+++ b/cmake/find_icu4c.cmake
@ -1,21 +0,0 @@
-option (ENABLE_ICU "Enable ICU" ON)
-
-if (ENABLE_ICU)
-    set (ICU_PATHS "/usr/local/opt/icu4c/lib")
-    set (ICU_INCLUDE_PATHS "/usr/local/opt/icu4c/include")
-    find_library (ICUI18N icui18n PATHS ${ICU_PATHS})
-    find_library (ICUUC icuuc PATHS ${ICU_PATHS})
-    find_library (ICUDATA icudata PATHS ${ICU_PATHS})
-    set (ICU_LIBS ${ICUI18N} ${ICUUC} ${ICUDATA})
-
-    find_path (ICU_INCLUDE_DIR NAMES unicode/unistr.h PATHS ${ICU_INCLUDE_PATHS})
-    if (ICU_INCLUDE_DIR AND ICU_LIBS)
-        set(USE_ICU 1)
-    endif ()
-endif ()
-
-if (USE_ICU)
-    message (STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBS}")
-else ()
-    message (STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)")
-endif ()
--- a/cmake/generate_code.cmake
+++ b/cmake/generate_code.cmake
@ -0,0 +1,5 @@
+function(generate_code TEMPLATE_FILE)
+    foreach(NAME IN LISTS ARGN)
+       configure_file (${TEMPLATE_FILE}.cpp.in ${CMAKE_CURRENT_BINARY_DIR}/generated/${TEMPLATE_FILE}_${NAME}.cpp)
+    endforeach()
+endfunction()
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@ -63,9 +63,6 @@ add_headers_and_sources(dbms src/Core)
 add_headers_and_sources(dbms src/DataStreams)
 add_headers_and_sources(dbms src/DataTypes)
 add_headers_and_sources(dbms src/Databases)
-add_headers_and_sources(dbms src/Dictionaries)
-add_headers_and_sources(dbms src/Dictionaries/Embedded)
-add_headers_and_sources(dbms src/Dictionaries/Embedded/GeodataProviders)
 add_headers_and_sources(dbms src/Interpreters)
 add_headers_and_sources(dbms src/Interpreters/ClusterProxy)
 add_headers_and_sources(dbms src/Columns)
@ -184,8 +181,11 @@ target_link_libraries (dbms
    clickhouse_common_config
        PUBLIC
    clickhouse_common_io
-    pocoext
+        PRIVATE
+    clickhouse_dictionaries
+    clickhouse_dictionaries_embedded
        PUBLIC
+    pocoext
    ${MYSQLXX_LIBRARY}
        PRIVATE
    ${BTRIE_LIBRARIES}
@ -242,8 +242,7 @@ endif()
 target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})

 if (USE_ICU)
-    target_link_libraries (dbms PRIVATE ${ICU_LIBS})
-    target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
+    target_link_libraries (dbms PRIVATE ${ICU_LIBRARIES})
 endif ()

 if (USE_CAPNP)
--- a/dbms/programs/CMakeLists.txt
+++ b/dbms/programs/CMakeLists.txt
@ -48,7 +48,7 @@ else ()
        link_directories (${LLVM_LIBRARY_DIRS})
    endif ()
    add_executable (clickhouse main.cpp)
-    target_link_libraries (clickhouse PRIVATE clickhouse_common_io)
+    target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
    target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
    target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

--- a/dbms/programs/extract-from-config/ExtractFromConfig.cpp
+++ b/dbms/programs/extract-from-config/ExtractFromConfig.cpp
@ -26,18 +26,18 @@ static void setupLogging(const std::string & log_level)
 static std::string extractFromConfig(
        const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false)
 {
-    ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
+    DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
    bool has_zk_includes;
-    XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
+    DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
    if (has_zk_includes && process_zk_includes)
    {
-        ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
+        DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
        zkutil::ZooKeeperPtr zookeeper = std::make_shared<zkutil::ZooKeeper>(
                *bootstrap_configuration, "zookeeper");
        zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; });
        config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
    }
-    ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
+    DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
    // do not throw exception if not found
    if (try_get)
        return configuration->getString(key, "");
--- a/dbms/programs/local/LocalServer.cpp
+++ b/dbms/programs/local/LocalServer.cpp
@ -30,6 +30,7 @@
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
+#include <Dictionaries/registerDictionaries.h>
 #include <boost/program_options/options_description.hpp>
 #include <boost/program_options.hpp>

@ -115,9 +116,11 @@ try
    /// Load config files if exists
    if (config().has("config-file") || Poco::File("config.xml").exists())
    {
-        ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true);
+        const auto config_path = config().getString("config-file", "config.xml");
+        ConfigProcessor config_processor(config_path, false, true);
+        config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
        auto loaded_config = config_processor.loadConfig();
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH));
        config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
    }

@ -140,6 +143,7 @@ try
    registerAggregateFunctions();
    registerTableFunctions();
    registerStorages();
+    registerDictionaries();

    /// Maybe useless
    if (config().has("macros"))
@ -348,7 +352,7 @@ void LocalServer::setupUsers()
        const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
        ConfigProcessor config_processor(users_config_path);
        const auto loaded_config = config_processor.loadConfig();
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
        users_config = loaded_config.configuration;
    }
    else
--- a/dbms/programs/server/Server.cpp
+++ b/dbms/programs/server/Server.cpp
@ -36,6 +36,7 @@
 #include <Functions/registerFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
 #include <Storages/registerStorages.h>
+#include <Dictionaries/registerDictionaries.h>
 #include <Common/Config/ConfigReloader.h>
 #include "HTTPHandlerFactory.h"
 #include "MetricsTransmitter.h"
@ -96,7 +97,7 @@ void Server::initialize(Poco::Util::Application & self)

 std::string Server::getDefaultCorePath() const
 {
-    return getCanonicalPath(config().getString("path")) + "cores";
+    return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores";
 }

 int Server::main(const std::vector<std::string> & /*args*/)
@ -109,6 +110,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
    registerAggregateFunctions();
    registerTableFunctions();
    registerStorages();
+    registerDictionaries();

    CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
    CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
@ -130,7 +132,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
        ConfigProcessor config_processor(config_path);
        loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
            main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true);
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
        config().removeConfiguration(old_configuration.get());
        config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
    }
@ -161,7 +163,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
    }
 #endif

-    std::string path = getCanonicalPath(config().getString("path"));
+    std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
    std::string default_database = config().getString("default_database", "default");

    global_context->setPath(path);
@ -302,6 +304,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
    std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
    auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
        include_from_path,
+        config().getString("path", ""),
        std::move(main_config_zk_node_cache),
        main_config_zk_changed_event,
        [&](ConfigurationPtr config)
@ -324,6 +327,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
    }
    auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
        include_from_path,
+        config().getString("path", ""),
        zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
        std::make_shared<Poco::Event>(),
        [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
--- a/dbms/src/Client/CMakeLists.txt
+++ b/dbms/src/Client/CMakeLists.txt
@ -4,3 +4,5 @@
 #add_library(clickhouse_client ${LINK_MODE} ${clickhouse_client_headers} ${clickhouse_client_sources})
 #target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY})
 #target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR})
+
+add_subdirectory(tests)
--- a/dbms/src/Client/Connection.cpp
+++ b/dbms/src/Client/Connection.cpp
@ -27,7 +27,6 @@
 #include <Poco/Net/SecureStreamSocket.h>
 #endif

-
 namespace CurrentMetrics
 {
    extern const Metric SendExternalTables;
--- a/dbms/src/Client/tests/CMakeLists.txt
+++ b/dbms/src/Client/tests/CMakeLists.txt
@ -0,0 +1,2 @@
+add_executable(test-connect test_connect.cpp)
+target_link_libraries (test-connect dbms)
--- a/dbms/src/Client/tests/test_connect.cpp
+++ b/dbms/src/Client/tests/test_connect.cpp
@ -0,0 +1,59 @@
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <iostream>
+#include <Poco/Net/StreamSocket.h>
+#include <Common/Exception.h>
+#include <IO/ReadHelpers.h>
+
+
+/** In a loop it connects to the server and immediately breaks the connection.
+  * Using the SO_LINGER option, we ensure that the connection is terminated by sending a RST packet (not FIN).
+  * This behavior causes a bug in the TCPServer implementation in the Poco library.
+  */
+int main(int argc, char ** argv)
+try
+{
+    for (size_t i = 0, num_iters = argc >= 2 ? DB::parse<size_t>(argv[1]) : 1; i < num_iters; ++i)
+    {
+        std::cerr << ".";
+
+        Poco::Net::SocketAddress address("localhost", 9000);
+
+        int fd = socket(PF_INET, SOCK_STREAM, IPPROTO_IP);
+
+        if (fd < 0)
+            DB::throwFromErrno("Cannot create socket", 0);
+
+        linger linger_value;
+        linger_value.l_onoff = 1;
+        linger_value.l_linger = 0;
+
+        if (0 != setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger_value, sizeof(linger_value)))
+            DB::throwFromErrno("Cannot set linger", 0);
+
+        try
+        {
+            int res = connect(fd, address.addr(), address.length());
+
+            if (res != 0 && errno != EINPROGRESS && errno != EWOULDBLOCK)
+            {
+                close(fd);
+                DB::throwFromErrno("Cannot connect", 0);
+            }
+
+            close(fd);
+        }
+        catch (const Poco::Exception & e)
+        {
+            std::cerr << e.displayText() << "\n";
+        }
+    }
+
+    std::cerr << "\n";
+}
+catch (const Poco::Exception & e)
+{
+    std::cerr << e.displayText() << "\n";
+}
--- a/dbms/src/Columns/ColumnLowCardinality.cpp
+++ b/dbms/src/Columns/ColumnLowCardinality.cpp
@ -259,7 +259,7 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di
    if (limit == 0)
        limit = size();

-    size_t unique_limit = std::min(limit, getDictionary().size());
+    size_t unique_limit = getDictionary().size();
    Permutation unique_perm;
    getDictionary().getNestedColumn()->getPermutation(reverse, unique_limit, nan_direction_hint, unique_perm);

--- a/dbms/src/Common/Config/ConfigProcessor.cpp
+++ b/dbms/src/Common/Config/ConfigProcessor.cpp
@ -20,6 +20,11 @@

 using namespace Poco::XML;

+namespace DB
+{
+
+/// For cutting prerpocessed path to this base
+std::string main_config_path;

 /// Extracts from a string the first encountered number consisting of at least two digits.
 static std::string numberFromHost(const std::string & s)
@ -40,13 +45,6 @@ static std::string numberFromHost(const std::string & s)
    return "";
 }

-static std::string preprocessedConfigPath(const std::string & path)
-{
-    Poco::Path preprocessed_path(path);
-    preprocessed_path.setBaseName(preprocessed_path.getBaseName() + PREPROCESSED_SUFFIX);
-    return preprocessed_path.toString();
-}
-
 bool ConfigProcessor::isPreprocessedFile(const std::string & path)
 {
    return endsWith(Poco::Path(path).getBaseName(), PREPROCESSED_SUFFIX);
@ -59,7 +57,6 @@ ConfigProcessor::ConfigProcessor(
    bool log_to_console,
    const Substitutions & substitutions_)
    : path(path_)
-    , preprocessed_path(preprocessedConfigPath(path))
    , throw_on_bad_incl(throw_on_bad_incl_)
    , substitutions(substitutions_)
    /// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse.
@ -524,7 +521,7 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes

    ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));

-    return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml};
+    return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml, path};
 }

 ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
@ -559,11 +556,44 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(

    ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));

-    return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml};
+    return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml, path};
 }

-void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
+void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir)
 {
+    if (preprocessed_path.empty())
+    {
+        auto new_path = loaded_config.config_path;
+        if (new_path.substr(0, main_config_path.size()) == main_config_path)
+            new_path.replace(0, main_config_path.size(), "");
+        std::replace(new_path.begin(), new_path.end(), '/', '_');
+
+        if (preprocessed_dir.empty())
+        {
+            if (!loaded_config.configuration->has("path"))
+            {
+                // Will use current directory
+                auto parent_path = Poco::Path(loaded_config.config_path).makeParent();
+                preprocessed_dir = parent_path.toString();
+                Poco::Path poco_new_path(new_path);
+                poco_new_path.setBaseName(poco_new_path.getBaseName() + PREPROCESSED_SUFFIX);
+                new_path = poco_new_path.toString();
+            }
+            else
+            {
+                preprocessed_dir = loaded_config.configuration->getString("path") + "/preprocessed_configs/";
+            }
+        }
+        else
+        {
+            preprocessed_dir += "/preprocessed_configs/";
+        }
+
+        preprocessed_path = preprocessed_dir + new_path;
+        auto path = Poco::Path(preprocessed_path).makeParent();
+        if (!path.toString().empty())
+            Poco::File(path).createDirectories();
+    }
    try
    {
        DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
@ -573,3 +603,10 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
        LOG_WARNING(log, "Couldn't save preprocessed config to " << preprocessed_path << ": " << e.displayText());
    }
 }
+
+void ConfigProcessor::setConfigPath(const std::string & config_path)
+{
+    main_config_path = config_path;
+}
+
+}
--- a/dbms/src/Common/Config/ConfigProcessor.h
+++ b/dbms/src/Common/Config/ConfigProcessor.h
@ -25,6 +25,9 @@ namespace zkutil
    using EventPtr = std::shared_ptr<Poco::Event>;
 }

+namespace DB
+{
+
 using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
 using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;

@ -74,6 +77,7 @@ public:
        bool has_zk_includes;
        bool loaded_from_preprocessed;
        XMLDocumentPtr preprocessed_xml;
+        std::string config_path;
    };

    /// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
@ -88,7 +92,12 @@ public:
        const zkutil::EventPtr & zk_changed_event,
        bool fallback_to_preprocessed = false);

-    void savePreprocessedConfig(const LoadedConfig & loaded_config);
+    /// Save preprocessed config to specified directory.
+    /// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/
+    void savePreprocessedConfig(const LoadedConfig & loaded_config, std::string preprocessed_dir);
+
+    /// Set path of main config.xml . It will be cutted from all configs placed to preprocessed_configs/
+    void setConfigPath(const std::string & config_path);

 public:
    using Files = std::vector<std::string>;
@ -102,7 +111,7 @@ public:

 private:
    const std::string path;
-    const std::string preprocessed_path;
+    std::string preprocessed_path;

    bool throw_on_bad_incl;

@ -131,3 +140,5 @@ private:
            const zkutil::EventPtr & zk_changed_event,
            std::unordered_set<std::string> & contributing_zk_paths);
 };
+
+}
--- a/dbms/src/Common/Config/ConfigReloader.cpp
+++ b/dbms/src/Common/Config/ConfigReloader.cpp
@ -15,11 +15,13 @@ constexpr decltype(ConfigReloader::reload_interval) ConfigReloader::reload_inter
 ConfigReloader::ConfigReloader(
        const std::string & path_,
        const std::string & include_from_path_,
+        const std::string & preprocessed_dir_,
        zkutil::ZooKeeperNodeCache && zk_node_cache_,
        const zkutil::EventPtr & zk_changed_event_,
        Updater && updater_,
        bool already_loaded)
    : path(path_), include_from_path(include_from_path_)
+    , preprocessed_dir(preprocessed_dir_)
    , zk_node_cache(std::move(zk_node_cache_))
    , zk_changed_event(zk_changed_event_)
    , updater(std::move(updater_))
@ -100,7 +102,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
            tryLogCurrentException(log, "Error loading config from `" + path + "'");
            return;
        }
-        config_processor.savePreprocessedConfig(loaded_config);
+        config_processor.savePreprocessedConfig(loaded_config, preprocessed_dir);

        /** We should remember last modification time if and only if config was sucessfully loaded
         * Otherwise a race condition could occur during config files update:
--- a/dbms/src/Common/Config/ConfigReloader.h
+++ b/dbms/src/Common/Config/ConfigReloader.h
@ -33,6 +33,7 @@ public:
    ConfigReloader(
            const std::string & path,
            const std::string & include_from_path,
+            const std::string & preprocessed_dir,
            zkutil::ZooKeeperNodeCache && zk_node_cache,
            const zkutil::EventPtr & zk_changed_event,
            Updater && updater,
@ -71,6 +72,7 @@ private:

    std::string path;
    std::string include_from_path;
+    std::string preprocessed_dir;
    FilesChangesTracker files;
    zkutil::ZooKeeperNodeCache zk_node_cache;
    zkutil::EventPtr zk_changed_event = std::make_shared<Poco::Event>();
--- a/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp
+++ b/dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp
@ -23,7 +23,7 @@ int main(int argc, char ** argv)
            return 3;
        }

-        ConfigProcessor processor(argv[1], false, true);
+        DB::ConfigProcessor processor(argv[1], false, true);
        auto config = processor.loadConfig().configuration;
        zkutil::ZooKeeper zk(*config, "zookeeper");
        zkutil::EventPtr watch = std::make_shared<Poco::Event>();
--- a/dbms/src/Core/Defines.h
+++ b/dbms/src/Core/Defines.h
@ -66,6 +66,8 @@
 /// the number is unmotivated
 #define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15

+#define DBMS_DEFAULT_PATH "/var/lib/clickhouse/"
+
 // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html

 #if defined(_MSC_VER)
--- a/dbms/src/DataStreams/AddingDefaultBlockInputStream.cpp
+++ b/dbms/src/DataStreams/AddingDefaultBlockInputStream.cpp
@ -0,0 +1,28 @@
+#include <DataStreams/AddingDefaultBlockInputStream.h>
+#include <Interpreters/addMissingDefaults.h>
+
+
+namespace DB
+{
+
+AddingDefaultBlockInputStream::AddingDefaultBlockInputStream(
+    const BlockInputStreamPtr & input_,
+    const Block & header_,
+    const ColumnDefaults & column_defaults_,
+    const Context & context_)
+    : input(input_), header(header_),
+      column_defaults(column_defaults_), context(context_)
+{
+    children.emplace_back(input);
+}
+
+Block AddingDefaultBlockInputStream::readImpl()
+{
+    Block src = children.back()->read();
+    if (!src)
+        return src;
+
+    return addMissingDefaults(src, header.getNamesAndTypesList(), column_defaults, context);
+}
+
+}
--- a/dbms/src/DataStreams/AddingDefaultBlockInputStream.h
+++ b/dbms/src/DataStreams/AddingDefaultBlockInputStream.h
@ -0,0 +1,40 @@
+#pragma once
+
+#include <DataStreams/IProfilingBlockInputStream.h>
+#include <Storages/ColumnDefault.h>
+
+
+namespace DB
+{
+
+
+/** This stream adds three types of columns into block
+  * 1. Columns, that are missed inside request, but present in table without defaults (missed columns)
+  * 2. Columns, that are missed inside request, but present in table with defaults (columns with default values)
+  * 3. Columns that materialized from other columns (materialized columns)
+  * All three types of columns are materialized (not constants).
+  */
+class AddingDefaultBlockInputStream : public IProfilingBlockInputStream
+{
+public:
+    AddingDefaultBlockInputStream(
+        const BlockInputStreamPtr & input_,
+        const Block & header_,
+        const ColumnDefaults & column_defaults_,
+        const Context & context_);
+
+    String getName() const override { return "AddingDefault"; }
+    Block getHeader() const override { return header; }
+
+private:
+    Block readImpl() override;
+
+    BlockInputStreamPtr input;
+    /// Blocks after this stream should have this structure
+    const Block header;
+    const ColumnDefaults column_defaults;
+    const Context & context;
+};
+
+
+}
--- a/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp
+++ b/dbms/src/DataStreams/AddingDefaultBlockOutputStream.cpp
@ -1,11 +1,5 @@
 #include <DataStreams/AddingDefaultBlockOutputStream.h>
-
-#include <Common/typeid_cast.h>
-#include <DataTypes/NestedUtils.h>
-#include <DataTypes/DataTypeArray.h>
-#include <Columns/ColumnArray.h>
-#include <Interpreters/evaluateMissingDefaults.h>
-#include <Core/Block.h>
+#include <Interpreters/addMissingDefaults.h>


 namespace DB
@ -13,68 +7,7 @@ namespace DB

 void AddingDefaultBlockOutputStream::write(const Block & block)
 {
-    Block res;
-    /// We take given columns from input block
-    /// and missed columns without default value (default and meterialized will be computed later)
-    for (const auto & column : output_block)
-    {
-        if (block.has(column.name))
-            res.insert(block.getByName(column.name));
-        else if (!column_defaults.count(column.name))
-            res.insert(column);
-    }
-
-    /// Adds not specified default values.
-    size_t rows = block.rows();
-
-    /// For missing columns of nested structure, you need to create not a column of empty arrays, but a column of arrays of correct lengths.
-    /// First, remember the offset columns for all arrays in the block.
-    std::map<String, ColumnPtr> offset_columns;
-
-    for (size_t i = 0, size = block.columns(); i < size; ++i)
-    {
-        const auto & elem = block.getByPosition(i);
-
-        if (const ColumnArray * array = typeid_cast<const ColumnArray *>(&*elem.column))
-        {
-            String offsets_name = Nested::extractTableName(elem.name);
-            auto & offsets_column = offset_columns[offsets_name];
-
-            /// If for some reason there are different offset columns for one nested structure, then we take nonempty.
-            if (!offsets_column || offsets_column->empty())
-                offsets_column = array->getOffsetsPtr();
-        }
-    }
-
-    /// In this loop we fill missed columns
-    for (auto & column : res)
-    {
-        if (block.has(column.name))
-            continue;
-
-        String offsets_name = Nested::extractTableName(column.name);
-        if (offset_columns.count(offsets_name))
-        {
-            ColumnPtr offsets_column = offset_columns[offsets_name];
-            DataTypePtr nested_type = typeid_cast<const DataTypeArray &>(*column.type).getNestedType();
-            UInt64 nested_rows = rows ? get<UInt64>((*offsets_column)[rows - 1]) : 0;
-
-            ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst();
-            column.column = ColumnArray::create(nested_column, offsets_column);
-        }
-        else
-        {
-            /** It is necessary to turn a constant column into a full column, since in part of blocks (from other parts),
-            *  it can be full (or the interpreter may decide that it is constant everywhere).
-            */
-            column.column = column.type->createColumnConstWithDefaultValue(rows)->convertToFullColumnIfConst();
-        }
-    }
-
-    /// Computes explicitly specified values (in column_defaults) by default and materialized columns.
-    evaluateMissingDefaults(res, output_block.getNamesAndTypesList(), column_defaults, context);
-
-    output->write(res);
+    output->write(addMissingDefaults(block, output_block.getNamesAndTypesList(), column_defaults, context));
 }

 void AddingDefaultBlockOutputStream::flush()
--- a/dbms/src/Dictionaries/CMakeLists.txt
+++ b/dbms/src/Dictionaries/CMakeLists.txt
@ -0,0 +1,43 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+include(${ClickHouse_SOURCE_DIR}/cmake/generate_code.cmake)
+
+add_headers_and_sources(clickhouse_dictionaries .)
+
+generate_code(ComplexKeyCacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(ComplexKeyCacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(ComplexKeyCacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate1 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate2 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+generate_code(CacheDictionary_generate3 UInt8 UInt16 UInt32 UInt64 UInt128 Int8 Int16 Int32 Int64 Float32 Float64 Decimal32 Decimal64 Decimal128)
+add_headers_and_sources(clickhouse_dictionaries ${CMAKE_CURRENT_BINARY_DIR}/generated/)
+
+add_library(clickhouse_dictionaries ${LINK_MODE} ${clickhouse_dictionaries_sources})
+target_link_libraries(clickhouse_dictionaries PRIVATE clickhouse_common_io pocoext ${MYSQLXX_LIBRARY} ${BTRIE_LIBRARIES})
+
+if(Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
+endif()
+
+if(USE_POCO_SQLODBC)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
+    if (NOT USE_INTERNAL_POCO_LIBRARY)
+        target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} ${Poco_SQL_INCLUDE_DIR})
+    endif()
+endif()
+
+if(Poco_Data_FOUND)
+    target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
+endif()
+
+if(USE_POCO_DATAODBC)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_DataODBC_LIBRARY} ${Poco_Data_LIBRARY})
+    if (NOT USE_INTERNAL_POCO_LIBRARY)
+        target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+    endif()
+endif()
+
+if(USE_POCO_MONGODB)
+    target_link_libraries(clickhouse_dictionaries PRIVATE ${Poco_MongoDB_LIBRARY})
+endif()
+
+add_subdirectory(Embedded)
--- a/dbms/src/Dictionaries/CacheDictionary.cpp
+++ b/dbms/src/Dictionaries/CacheDictionary.cpp
@ -1,3 +1,5 @@
+#include "CacheDictionary.h"
+
 #include <functional>
 #include <sstream>
 #include <memory>
@ -11,12 +13,12 @@
 #include <Common/ProfileEvents.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/typeid_cast.h>
-#include <Dictionaries/CacheDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "DictionaryBlockInputStream.h"
 #include <ext/size.h>
 #include <ext/range.h>
 #include <ext/map.h>
-
+#include "DictionaryFactory.h"
+#include "CacheDictionary.inc.h"

 namespace ProfileEvents
 {
@ -47,6 +49,7 @@ namespace ErrorCodes
    extern const int BAD_ARGUMENTS;
    extern const int UNSUPPORTED_METHOD;
    extern const int LOGICAL_ERROR;
+    extern const int TOO_SMALL_BUFFER_SIZE;
 }


@ -206,34 +209,6 @@ void CacheDictionary::isInConstantVector(
        out[i] = std::find(ancestors.begin(), ancestors.end(), ancestor_ids[i]) != ancestors.end();
 }

-
-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    const auto null_value = std::get<TYPE>(attribute.null_values);\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t) { return null_value; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ColumnString * out) const
 {
    auto & attribute = getAttribute(attribute_name);
@ -245,33 +220,6 @@ void CacheDictionary::getString(const std::string & attribute_name, const Padded
    getItemsString(attribute, ids, out, [&] (const size_t) { return null_value; });
 }

-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(\
-    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const PaddedPODArray<TYPE> & def,\
-    ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t row) { return def[row]; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(
    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const ColumnString * const def,
    ColumnString * const out) const
@ -283,32 +231,6 @@ void CacheDictionary::getString(
    getItemsString(attribute, ids, out, [&] (const size_t row) { return def->getDataAt(row); });
 }

-#define DECLARE(TYPE)\
-void CacheDictionary::get##TYPE(\
-    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const\
-{\
-    auto & attribute = getAttribute(attribute_name);\
-    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
-        throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
-    \
-    getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t) { return def; });\
-}
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-
 void CacheDictionary::getString(
    const std::string & attribute_name, const PaddedPODArray<Key> & ids, const String & def,
    ColumnString * const out) const
@ -487,374 +409,6 @@ CacheDictionary::Attribute CacheDictionary::createAttributeWithType(const Attrib
    return attr;
 }

-
-template <typename OutputType, typename DefaultGetter>
-void CacheDictionary::getItemsNumber(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ResultArrayType<OutputType> & out,
-    DefaultGetter && get_default) const
-{
-    if (false) {}
-#define DISPATCH(TYPE) \
-    else if (attribute.type == AttributeUnderlyingType::TYPE) \
-        getItemsNumberImpl<TYPE, OutputType>(attribute, ids, out, std::forward<DefaultGetter>(get_default));
-    DISPATCH(UInt8)
-    DISPATCH(UInt16)
-    DISPATCH(UInt32)
-    DISPATCH(UInt64)
-    DISPATCH(UInt128)
-    DISPATCH(Int8)
-    DISPATCH(Int16)
-    DISPATCH(Int32)
-    DISPATCH(Int64)
-    DISPATCH(Float32)
-    DISPATCH(Float64)
-    DISPATCH(Decimal32)
-    DISPATCH(Decimal64)
-    DISPATCH(Decimal128)
-#undef DISPATCH
-    else
-        throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
-}
-
-template <typename AttributeType, typename OutputType, typename DefaultGetter>
-void CacheDictionary::getItemsNumberImpl(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ResultArrayType<OutputType> & out,
-    DefaultGetter && get_default) const
-{
-    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
-    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
-    auto & attribute_array = std::get<ContainerPtrType<AttributeType>>(attribute.arrays);
-    const auto rows = ext::size(ids);
-
-    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
-
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        /// fetch up-to-date values, decide which ones require update
-        for (const auto row : ext::range(0, rows))
-        {
-            const auto id = ids[row];
-
-            /** cell should be updated if either:
-                *    1. ids do not match,
-                *    2. cell has expired,
-                *    3. explicit defaults were specified and cell was set default. */
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                outdated_ids[id].push_back(row);
-                if (find_result.outdated)
-                    ++cache_expired;
-                else
-                    ++cache_not_found;
-            }
-            else
-            {
-                ++cache_hit;
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                out[row] = cell.isDefault() ? get_default(row) : static_cast<OutputType>(attribute_array[cell_idx]);
-            }
-        }
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
-
-    query_count.fetch_add(rows, std::memory_order_relaxed);
-    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
-
-    if (outdated_ids.empty())
-        return;
-
-    std::vector<Key> required_ids(outdated_ids.size());
-    std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
-        [] (auto & pair) { return pair.first; });
-
-    /// request new values
-    update(required_ids,
-    [&] (const auto id, const auto cell_idx)
-    {
-        const auto attribute_value = attribute_array[cell_idx];
-
-        for (const size_t row : outdated_ids[id])
-            out[row] = static_cast<OutputType>(attribute_value);
-    },
-    [&] (const auto id, const auto)
-    {
-        for (const size_t row : outdated_ids[id])
-            out[row] = get_default(row);
-    });
-}
-
-template <typename DefaultGetter>
-void CacheDictionary::getItemsString(
-    Attribute & attribute,
-    const PaddedPODArray<Key> & ids,
-    ColumnString * out,
-    DefaultGetter && get_default) const
-{
-    const auto rows = ext::size(ids);
-
-    /// save on some allocations
-    out->getOffsets().reserve(rows);
-
-    auto & attribute_array = std::get<ContainerPtrType<StringRef>>(attribute.arrays);
-
-    auto found_outdated_values = false;
-
-    /// perform optimistic version, fallback to pessimistic if failed
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        /// fetch up-to-date values, discard on fail
-        for (const auto row : ext::range(0, rows))
-        {
-            const auto id = ids[row];
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                found_outdated_values = true;
-                break;
-            }
-            else
-            {
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
-                out->insertData(string_ref.data, string_ref.size);
-            }
-        }
-    }
-
-    /// optimistic code completed successfully
-    if (!found_outdated_values)
-    {
-        query_count.fetch_add(rows, std::memory_order_relaxed);
-        hit_count.fetch_add(rows, std::memory_order_release);
-        return;
-    }
-
-    /// now onto the pessimistic one, discard possible partial results from the optimistic path
-    out->getChars().resize_assume_reserved(0);
-    out->getOffsets().resize_assume_reserved(0);
-
-    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
-    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
-    /// we are going to store every string separately
-    std::unordered_map<Key, String> map;
-
-    size_t total_length = 0;
-    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
-    {
-        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
-
-        const auto now = std::chrono::system_clock::now();
-        for (const auto row : ext::range(0, ids.size()))
-        {
-            const auto id = ids[row];
-
-            const auto find_result = findCellIdx(id, now);
-            if (!find_result.valid)
-            {
-                outdated_ids[id].push_back(row);
-                if (find_result.outdated)
-                    ++cache_expired;
-                else
-                    ++cache_not_found;
-            }
-            else
-            {
-                ++cache_hit;
-                const auto & cell_idx = find_result.cell_idx;
-                const auto & cell = cells[cell_idx];
-                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
-
-                if (!cell.isDefault())
-                    map[id] = String{string_ref};
-
-                total_length += string_ref.size + 1;
-            }
-        }
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
-
-    query_count.fetch_add(rows, std::memory_order_relaxed);
-    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
-
-    /// request new values
-    if (!outdated_ids.empty())
-    {
-        std::vector<Key> required_ids(outdated_ids.size());
-        std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
-            [] (auto & pair) { return pair.first; });
-
-        update(required_ids,
-        [&] (const auto id, const auto cell_idx)
-        {
-            const auto attribute_value = attribute_array[cell_idx];
-
-            map[id] = String{attribute_value};
-            total_length += (attribute_value.size + 1) * outdated_ids[id].size();
-        },
-        [&] (const auto id, const auto)
-        {
-            for (const auto row : outdated_ids[id])
-                total_length += get_default(row).size + 1;
-        });
-    }
-
-    out->getChars().reserve(total_length);
-
-    for (const auto row : ext::range(0, ext::size(ids)))
-    {
-        const auto id = ids[row];
-        const auto it = map.find(id);
-
-        const auto string_ref = it != std::end(map) ? StringRef{it->second} : get_default(row);
-        out->insertData(string_ref.data, string_ref.size);
-    }
-}
-
-template <typename PresentIdHandler, typename AbsentIdHandler>
-void CacheDictionary::update(
-    const std::vector<Key> & requested_ids,
-    PresentIdHandler && on_cell_updated,
-    AbsentIdHandler && on_id_not_found) const
-{
-    std::unordered_map<Key, UInt8> remaining_ids{requested_ids.size()};
-    for (const auto id : requested_ids)
-        remaining_ids.insert({ id, 0 });
-
-    std::uniform_int_distribution<UInt64> distribution
-    {
-        dict_lifetime.min_sec,
-        dict_lifetime.max_sec
-    };
-
-    const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
-
-    {
-        CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests};
-        Stopwatch watch;
-        auto stream = source_ptr->loadIds(requested_ids);
-        stream->readPrefix();
-
-        const auto now = std::chrono::system_clock::now();
-
-        while (const auto block = stream->read())
-        {
-            const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
-            if (!id_column)
-                throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
-
-            const auto & ids = id_column->getData();
-
-            /// cache column pointers
-            const auto column_ptrs = ext::map<std::vector>(ext::range(0, attributes.size()), [&block] (size_t i)
-            {
-                return block.safeGetByPosition(i + 1).column.get();
-            });
-
-            for (const auto i : ext::range(0, ids.size()))
-            {
-                const auto id = ids[i];
-
-                const auto find_result = findCellIdx(id, now);
-                const auto & cell_idx = find_result.cell_idx;
-
-                auto & cell = cells[cell_idx];
-
-                for (const auto attribute_idx : ext::range(0, attributes.size()))
-                {
-                    const auto & attribute_column = *column_ptrs[attribute_idx];
-                    auto & attribute = attributes[attribute_idx];
-
-                    setAttributeValue(attribute, cell_idx, attribute_column[i]);
-                }
-
-                /// if cell id is zero and zero does not map to this cell, then the cell is unused
-                if (cell.id == 0 && cell_idx != zero_cell_idx)
-                    element_count.fetch_add(1, std::memory_order_relaxed);
-
-                cell.id = id;
-                if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
-                    cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
-                else
-                    cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
-
-                /// inform caller
-                on_cell_updated(id, cell_idx);
-                /// mark corresponding id as found
-                remaining_ids[id] = 1;
-            }
-        }
-
-        stream->readSuffix();
-
-        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size());
-        ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed());
-    }
-
-    size_t not_found_num = 0, found_num = 0;
-
-    const auto now = std::chrono::system_clock::now();
-    /// Check which ids have not been found and require setting null_value
-    for (const auto & id_found_pair : remaining_ids)
-    {
-        if (id_found_pair.second)
-        {
-            ++found_num;
-            continue;
-        }
-        ++not_found_num;
-
-        const auto id = id_found_pair.first;
-
-        const auto find_result = findCellIdx(id, now);
-        const auto & cell_idx = find_result.cell_idx;
-
-        auto & cell = cells[cell_idx];
-
-        /// Set null_value for each attribute
-        for (auto & attribute : attributes)
-            setDefaultAttributeValue(attribute, cell_idx);
-
-        /// Check if cell had not been occupied before and increment element counter if it hadn't
-        if (cell.id == 0 && cell_idx != zero_cell_idx)
-            element_count.fetch_add(1, std::memory_order_relaxed);
-
-        cell.id = id;
-        if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
-            cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
-        else
-            cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
-
-        cell.setDefault();
-
-        /// inform caller that the cell has not been found
-        on_id_not_found(id, cell_idx);
-    }
-
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);
-    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedFound, found_num);
-    ProfileEvents::increment(ProfileEvents::DictCacheRequests);
-}
-
-
 void CacheDictionary::setDefaultAttributeValue(Attribute & attribute, const Key idx) const
 {
    switch (attribute.type)
@ -981,5 +535,41 @@ BlockInputStreamPtr CacheDictionary::getBlockInputStream(const Names & column_na
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getCachedIds(), column_names);
 }

+void registerDictionaryCache(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+
+        if (dict_struct.key)
+            throw Exception {"'key' is not supported for dictionary of layout 'cache'", ErrorCodes::UNSUPPORTED_METHOD};
+
+        if (dict_struct.range_min || dict_struct.range_max)
+            throw Exception {name
+                                 + ": elements .structure.range_min and .structure.range_max should be defined only "
+                                   "for a dictionary of layout 'range_hashed'",
+                             ErrorCodes::BAD_ARGUMENTS};
+        const auto & layout_prefix = config_prefix + ".layout";
+        const auto size = config.getInt(layout_prefix + ".cache.size_in_cells");
+        if (size == 0)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
+
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        if (require_nonempty)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
+                             ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        return std::make_unique<CacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
+
+
+    };
+    factory.registerLayout("cache", create_layout);
+}
+

 }
--- a/dbms/src/Dictionaries/CacheDictionary.h
+++ b/dbms/src/Dictionaries/CacheDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <Common/ArenaWithFreeLists.h>
 #include <Common/CurrentMetrics.h>
 #include <Columns/ColumnDecimal.h>
--- a/dbms/src/Dictionaries/CacheDictionary.inc.h
+++ b/dbms/src/Dictionaries/CacheDictionary.inc.h
@ -0,0 +1,403 @@
+#include "CacheDictionary.h"
+
+#include <ext/size.h>
+#include <ext/map.h>
+#include <ext/range.h>
+#include <Common/ProfilingScopedRWLock.h>
+#include <Common/typeid_cast.h>
+#include <Columns/ColumnsNumber.h>
+
+namespace ProfileEvents
+{
+    extern const Event DictCacheKeysRequested;
+    extern const Event DictCacheKeysRequestedMiss;
+    extern const Event DictCacheKeysRequestedFound;
+    extern const Event DictCacheKeysExpired;
+    extern const Event DictCacheKeysNotFound;
+    extern const Event DictCacheKeysHit;
+    extern const Event DictCacheRequestTimeNs;
+    extern const Event DictCacheRequests;
+    extern const Event DictCacheLockWriteNs;
+    extern const Event DictCacheLockReadNs;
+}
+
+namespace CurrentMetrics
+{
+    extern const Metric DictCacheRequests;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+template <typename OutputType, typename DefaultGetter>
+void CacheDictionary::getItemsNumber(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ResultArrayType<OutputType> & out,
+    DefaultGetter && get_default) const
+{
+    if (false) {}
+#define DISPATCH(TYPE) \
+    else if (attribute.type == AttributeUnderlyingType::TYPE) \
+        getItemsNumberImpl<TYPE, OutputType>(attribute, ids, out, std::forward<DefaultGetter>(get_default));
+    DISPATCH(UInt8)
+    DISPATCH(UInt16)
+    DISPATCH(UInt32)
+    DISPATCH(UInt64)
+    DISPATCH(UInt128)
+    DISPATCH(Int8)
+    DISPATCH(Int16)
+    DISPATCH(Int32)
+    DISPATCH(Int64)
+    DISPATCH(Float32)
+    DISPATCH(Float64)
+    DISPATCH(Decimal32)
+    DISPATCH(Decimal64)
+    DISPATCH(Decimal128)
+#undef DISPATCH
+    else
+        throw Exception("Unexpected type of attribute: " + toString(attribute.type), ErrorCodes::LOGICAL_ERROR);
+}
+
+template <typename AttributeType, typename OutputType, typename DefaultGetter>
+void CacheDictionary::getItemsNumberImpl(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ResultArrayType<OutputType> & out,
+    DefaultGetter && get_default) const
+{
+    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
+    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
+    auto & attribute_array = std::get<ContainerPtrType<AttributeType>>(attribute.arrays);
+    const auto rows = ext::size(ids);
+
+    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
+
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        /// fetch up-to-date values, decide which ones require update
+        for (const auto row : ext::range(0, rows))
+        {
+            const auto id = ids[row];
+
+            /** cell should be updated if either:
+                *    1. ids do not match,
+                *    2. cell has expired,
+                *    3. explicit defaults were specified and cell was set default. */
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                outdated_ids[id].push_back(row);
+                if (find_result.outdated)
+                    ++cache_expired;
+                else
+                    ++cache_not_found;
+            }
+            else
+            {
+                ++cache_hit;
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                out[row] = cell.isDefault() ? get_default(row) : static_cast<OutputType>(attribute_array[cell_idx]);
+            }
+        }
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
+
+    query_count.fetch_add(rows, std::memory_order_relaxed);
+    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
+
+    if (outdated_ids.empty())
+        return;
+
+    std::vector<Key> required_ids(outdated_ids.size());
+    std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
+        [] (auto & pair) { return pair.first; });
+
+    /// request new values
+    update(required_ids,
+    [&] (const auto id, const auto cell_idx)
+    {
+        const auto attribute_value = attribute_array[cell_idx];
+
+        for (const size_t row : outdated_ids[id])
+            out[row] = static_cast<OutputType>(attribute_value);
+    },
+    [&] (const auto id, const auto)
+    {
+        for (const size_t row : outdated_ids[id])
+            out[row] = get_default(row);
+    });
+}
+
+template <typename DefaultGetter>
+void CacheDictionary::getItemsString(
+    Attribute & attribute,
+    const PaddedPODArray<Key> & ids,
+    ColumnString * out,
+    DefaultGetter && get_default) const
+{
+    const auto rows = ext::size(ids);
+
+    /// save on some allocations
+    out->getOffsets().reserve(rows);
+
+    auto & attribute_array = std::get<ContainerPtrType<StringRef>>(attribute.arrays);
+
+    auto found_outdated_values = false;
+
+    /// perform optimistic version, fallback to pessimistic if failed
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        /// fetch up-to-date values, discard on fail
+        for (const auto row : ext::range(0, rows))
+        {
+            const auto id = ids[row];
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                found_outdated_values = true;
+                break;
+            }
+            else
+            {
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
+                out->insertData(string_ref.data, string_ref.size);
+            }
+        }
+    }
+
+    /// optimistic code completed successfully
+    if (!found_outdated_values)
+    {
+        query_count.fetch_add(rows, std::memory_order_relaxed);
+        hit_count.fetch_add(rows, std::memory_order_release);
+        return;
+    }
+
+    /// now onto the pessimistic one, discard possible partial results from the optimistic path
+    out->getChars().resize_assume_reserved(0);
+    out->getOffsets().resize_assume_reserved(0);
+
+    /// Mapping: <id> -> { all indices `i` of `ids` such that `ids[i]` = <id> }
+    std::unordered_map<Key, std::vector<size_t>> outdated_ids;
+    /// we are going to store every string separately
+    std::unordered_map<Key, String> map;
+
+    size_t total_length = 0;
+    size_t cache_expired = 0, cache_not_found = 0, cache_hit = 0;
+    {
+        const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
+
+        const auto now = std::chrono::system_clock::now();
+        for (const auto row : ext::range(0, ids.size()))
+        {
+            const auto id = ids[row];
+
+            const auto find_result = findCellIdx(id, now);
+            if (!find_result.valid)
+            {
+                outdated_ids[id].push_back(row);
+                if (find_result.outdated)
+                    ++cache_expired;
+                else
+                    ++cache_not_found;
+            }
+            else
+            {
+                ++cache_hit;
+                const auto & cell_idx = find_result.cell_idx;
+                const auto & cell = cells[cell_idx];
+                const auto string_ref = cell.isDefault() ? get_default(row) : attribute_array[cell_idx];
+
+                if (!cell.isDefault())
+                    map[id] = String{string_ref};
+
+                total_length += string_ref.size + 1;
+            }
+        }
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysExpired, cache_expired);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysNotFound, cache_not_found);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysHit, cache_hit);
+
+    query_count.fetch_add(rows, std::memory_order_relaxed);
+    hit_count.fetch_add(rows - outdated_ids.size(), std::memory_order_release);
+
+    /// request new values
+    if (!outdated_ids.empty())
+    {
+        std::vector<Key> required_ids(outdated_ids.size());
+        std::transform(std::begin(outdated_ids), std::end(outdated_ids), std::begin(required_ids),
+            [] (auto & pair) { return pair.first; });
+
+        update(required_ids,
+        [&] (const auto id, const auto cell_idx)
+        {
+            const auto attribute_value = attribute_array[cell_idx];
+
+            map[id] = String{attribute_value};
+            total_length += (attribute_value.size + 1) * outdated_ids[id].size();
+        },
+        [&] (const auto id, const auto)
+        {
+            for (const auto row : outdated_ids[id])
+                total_length += get_default(row).size + 1;
+        });
+    }
+
+    out->getChars().reserve(total_length);
+
+    for (const auto row : ext::range(0, ext::size(ids)))
+    {
+        const auto id = ids[row];
+        const auto it = map.find(id);
+
+        const auto string_ref = it != std::end(map) ? StringRef{it->second} : get_default(row);
+        out->insertData(string_ref.data, string_ref.size);
+    }
+}
+
+template <typename PresentIdHandler, typename AbsentIdHandler>
+void CacheDictionary::update(
+    const std::vector<Key> & requested_ids,
+    PresentIdHandler && on_cell_updated,
+    AbsentIdHandler && on_id_not_found) const
+{
+    std::unordered_map<Key, UInt8> remaining_ids{requested_ids.size()};
+    for (const auto id : requested_ids)
+        remaining_ids.insert({ id, 0 });
+
+    std::uniform_int_distribution<UInt64> distribution
+    {
+        dict_lifetime.min_sec,
+        dict_lifetime.max_sec
+    };
+
+    const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
+
+    {
+        CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests};
+        Stopwatch watch;
+        auto stream = source_ptr->loadIds(requested_ids);
+        stream->readPrefix();
+
+        const auto now = std::chrono::system_clock::now();
+
+        while (const auto block = stream->read())
+        {
+            const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
+            if (!id_column)
+                throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
+
+            const auto & ids = id_column->getData();
+
+            /// cache column pointers
+            const auto column_ptrs = ext::map<std::vector>(ext::range(0, attributes.size()), [&block] (size_t i)
+            {
+                return block.safeGetByPosition(i + 1).column.get();
+            });
+
+            for (const auto i : ext::range(0, ids.size()))
+            {
+                const auto id = ids[i];
+
+                const auto find_result = findCellIdx(id, now);
+                const auto & cell_idx = find_result.cell_idx;
+
+                auto & cell = cells[cell_idx];
+
+                for (const auto attribute_idx : ext::range(0, attributes.size()))
+                {
+                    const auto & attribute_column = *column_ptrs[attribute_idx];
+                    auto & attribute = attributes[attribute_idx];
+
+                    setAttributeValue(attribute, cell_idx, attribute_column[i]);
+                }
+
+                /// if cell id is zero and zero does not map to this cell, then the cell is unused
+                if (cell.id == 0 && cell_idx != zero_cell_idx)
+                    element_count.fetch_add(1, std::memory_order_relaxed);
+
+                cell.id = id;
+                if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
+                    cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
+                else
+                    cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
+
+                /// inform caller
+                on_cell_updated(id, cell_idx);
+                /// mark corresponding id as found
+                remaining_ids[id] = 1;
+            }
+        }
+
+        stream->readSuffix();
+
+        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, requested_ids.size());
+        ProfileEvents::increment(ProfileEvents::DictCacheRequestTimeNs, watch.elapsed());
+    }
+
+    size_t not_found_num = 0, found_num = 0;
+
+    const auto now = std::chrono::system_clock::now();
+    /// Check which ids have not been found and require setting null_value
+    for (const auto & id_found_pair : remaining_ids)
+    {
+        if (id_found_pair.second)
+        {
+            ++found_num;
+            continue;
+        }
+        ++not_found_num;
+
+        const auto id = id_found_pair.first;
+
+        const auto find_result = findCellIdx(id, now);
+        const auto & cell_idx = find_result.cell_idx;
+
+        auto & cell = cells[cell_idx];
+
+        /// Set null_value for each attribute
+        for (auto & attribute : attributes)
+            setDefaultAttributeValue(attribute, cell_idx);
+
+        /// Check if cell had not been occupied before and increment element counter if it hadn't
+        if (cell.id == 0 && cell_idx != zero_cell_idx)
+            element_count.fetch_add(1, std::memory_order_relaxed);
+
+        cell.id = id;
+        if (dict_lifetime.min_sec != 0 && dict_lifetime.max_sec != 0)
+            cell.setExpiresAt(std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)});
+        else
+            cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
+
+        cell.setDefault();
+
+        /// inform caller that the cell has not been found
+        on_id_not_found(id, cell_idx);
+    }
+
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);
+    ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedFound, found_num);
+    ProfileEvents::increment(ProfileEvents::DictCacheRequests);
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate1.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate1.cpp.in
@ -0,0 +1,24 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    const auto null_value = std::get<TYPE>(attribute.null_values);
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return null_value; });
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate2.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate2.cpp.in
@ -0,0 +1,25 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name,
+      const PaddedPODArray<Key> & ids,
+      const PaddedPODArray<TYPE> & def,
+      ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t row) { return def[row]; });
+}
+
+}
--- a/dbms/src/Dictionaries/CacheDictionary_generate3.cpp.in
+++ b/dbms/src/Dictionaries/CacheDictionary_generate3.cpp.in
@ -0,0 +1,22 @@
+#include <Dictionaries/CacheDictionary.h>
+#include <Dictionaries/CacheDictionary.inc.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void CacheDictionary::get@NAME@(const std::string & attribute_name, const PaddedPODArray<Key> & ids, const TYPE def, ResultArrayType<TYPE> & out) const
+{
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, ids, out, [&](const size_t) { return def; });
+}
+
+}
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.cpp
@ -1,14 +1,17 @@
-#include <Dictionaries/ClickHouseDictionarySource.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
-#include <Dictionaries/writeParenthesisedString.h>
+#include "ClickHouseDictionarySource.h"
+#include "ExternalQueryBuilder.h"
+#include "writeParenthesisedString.h"
 #include <Client/ConnectionPool.h>
 #include <DataStreams/RemoteBlockInputStream.h>
-#include <Dictionaries/readInvalidateQuery.h>
+#include "readInvalidateQuery.h"
 #include <Interpreters/executeQuery.h>
 #include <Common/isLocalAddress.h>
 #include <memory>
 #include <ext/range.h>
 #include <IO/ConnectionTimeouts.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+

 namespace DB
 {
@ -175,4 +178,17 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re
    }
 }

+
+void registerDictionarySourceClickHouse(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 Context & context) -> DictionarySourcePtr {
+        return std::make_unique<ClickHouseDictionarySource>(dict_struct, config, config_prefix + ".clickhouse", sample_block, context);
+    };
+    factory.registerSource("clickhouse", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/ClickHouseDictionarySource.h
+++ b/dbms/src/Dictionaries/ClickHouseDictionarySource.h
@ -1,10 +1,9 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
+#include "ExternalQueryBuilder.h"
 #include <Client/ConnectionPoolWithFailover.h>
-#include <Poco/Util/AbstractConfiguration.h>
 #include <memory>


--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
@ -1,5 +1,5 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "ComplexKeyCacheDictionary.h"
+#include "DictionaryBlockInputStream.h"
 #include <Common/Arena.h>
 #include <Common/BitHelpers.h>
 #include <Common/randomSeed.h>
@ -9,6 +9,7 @@
 #include <Common/CurrentMetrics.h>
 #include <ext/range.h>
 #include <ext/map.h>
+#include "DictionaryFactory.h"


 namespace ProfileEvents
@ -39,6 +40,7 @@ namespace ErrorCodes
    extern const int TYPE_MISMATCH;
    extern const int BAD_ARGUMENTS;
    extern const int UNSUPPORTED_METHOD;
+    extern const int TOO_SMALL_BUFFER_SIZE;
 }


@ -378,4 +380,32 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, keys, column_names);
 }

+void registerDictionaryComplexKeyCache(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+        if (!dict_struct.key)
+            throw Exception {"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
+        const auto & layout_prefix = config_prefix + ".layout";
+        const auto size = config.getInt(layout_prefix + ".complex_key_cache.size_in_cells");
+        if (size == 0)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 0 cells", ErrorCodes::TOO_SMALL_BUFFER_SIZE};
+
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        if (require_nonempty)
+            throw Exception {name + ": dictionary of layout 'cache' cannot have 'require_nonempty' attribute set",
+                             ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        return std::make_unique<ComplexKeyCacheDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, size);
+    };
+    factory.registerLayout("complex_key_cache", create_layout);
+}
+
+
 }
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
@ -12,9 +12,9 @@
 #include <Common/HashTable/HashMap.h>
 #include <Common/ProfilingScopedRWLock.h>
 #include <Common/SmallObjectPool.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
+#include "DictionaryStructure.h"
+#include "IDictionary.h"
+#include "IDictionarySource.h"
 #include <common/StringRef.h>
 #include <ext/bit_cast.h>
 #include <ext/map.h>
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_createAttributeWithType.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_createAttributeWithType.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp
@ -1,40 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                                   \
-    void ComplexKeyCacheDictionary::get##TYPE(                                                                                          \
-        const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const \
-    {                                                                                                                                   \
-        dict_struct.validateKeyTypes(key_types);                                                                                        \
-                                                                                                                                        \
-        auto & attribute = getAttribute(attribute_name);                                                                                \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                               \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),            \
-                ErrorCodes::TYPE_MISMATCH};                                                                                             \
-                                                                                                                                        \
-        const auto null_value = std::get<TYPE>(attribute.null_values);                                                                  \
-                                                                                                                                        \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });                                    \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate1.cpp.in
@ -0,0 +1,24 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    const auto null_value = std::get<TYPE>(attribute.null_values);
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return null_value; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp
@ -1,41 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                        \
-    void ComplexKeyCacheDictionary::get##TYPE(const std::string & attribute_name,                                            \
-        const Columns & key_columns,                                                                                         \
-        const DataTypes & key_types,                                                                                         \
-        const PaddedPODArray<TYPE> & def,                                                                                    \
-        ResultArrayType<TYPE> & out) const                                                                                    \
-    {                                                                                                                        \
-        dict_struct.validateKeyTypes(key_types);                                                                             \
-                                                                                                                             \
-        auto & attribute = getAttribute(attribute_name);                                                                     \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                    \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
-                ErrorCodes::TYPE_MISMATCH};                                                                                  \
-                                                                                                                             \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });                       \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate2.cpp.in
@ -0,0 +1,27 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
+      const Columns & key_columns,
+      const DataTypes & key_types,
+      const PaddedPODArray<TYPE> & def,
+      ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t row) { return def[row]; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp
@ -1,41 +0,0 @@
-#include "ComplexKeyCacheDictionary.h"
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int TYPE_MISMATCH;
-}
-
-#define DECLARE(TYPE)                                                                                                        \
-    void ComplexKeyCacheDictionary::get##TYPE(const std::string & attribute_name,                                            \
-        const Columns & key_columns,                                                                                         \
-        const DataTypes & key_types,                                                                                         \
-        const TYPE def,                                                                                                      \
-        ResultArrayType<TYPE> & out) const                                                                                    \
-    {                                                                                                                        \
-        dict_struct.validateKeyTypes(key_types);                                                                             \
-                                                                                                                             \
-        auto & attribute = getAttribute(attribute_name);                                                                     \
-        if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))                                    \
-            throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), \
-                ErrorCodes::TYPE_MISMATCH};                                                                                  \
-                                                                                                                             \
-        getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });                                \
-    }
-DECLARE(UInt8)
-DECLARE(UInt16)
-DECLARE(UInt32)
-DECLARE(UInt64)
-DECLARE(UInt128)
-DECLARE(Int8)
-DECLARE(Int16)
-DECLARE(Int32)
-DECLARE(Int64)
-DECLARE(Float32)
-DECLARE(Float64)
-DECLARE(Decimal32)
-DECLARE(Decimal64)
-DECLARE(Decimal128)
-#undef DECLARE
-}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_generate3.cpp.in
@ -0,0 +1,27 @@
+#include <Dictionaries/ComplexKeyCacheDictionary.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int TYPE_MISMATCH;
+}
+
+using TYPE = @NAME@;
+
+void ComplexKeyCacheDictionary::get@NAME@(const std::string & attribute_name,
+      const Columns & key_columns,
+      const DataTypes & key_types,
+      const TYPE def,
+      ResultArrayType<TYPE> & out) const
+{
+    dict_struct.validateKeyTypes(key_types);
+
+    auto & attribute = getAttribute(attribute_name);
+    if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::@NAME@))
+        throw Exception {name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
+                         ErrorCodes::TYPE_MISMATCH};
+
+    getItemsNumber<TYPE>(attribute, key_columns, out, [&](const size_t) { return def; });
+}
+}
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setAttributeValue.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary_setDefaultAttributeValue.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/ComplexKeyCacheDictionary.h>
+#include "ComplexKeyCacheDictionary.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
@ -1,8 +1,8 @@
 #include <ext/map.h>
 #include <ext/range.h>
-#include <Dictionaries/ComplexKeyHashedDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
-
+#include "ComplexKeyHashedDictionary.h"
+#include "DictionaryBlockInputStream.h"
+#include "DictionaryFactory.h"

 namespace DB
 {
@ -661,5 +661,24 @@ BlockInputStreamPtr ComplexKeyHashedDictionary::getBlockInputStream(const Names
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getKeys(), column_names);
 }

+void registerDictionaryComplexKeyHashed(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+        if (!dict_struct.key)
+            throw Exception {"'key' is required for dictionary of layout 'complex_key_hashed'", ErrorCodes::BAD_ARGUMENTS};
+
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        return std::make_unique<ComplexKeyHashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
+    };
+    factory.registerLayout("complex_key_hashed", create_layout);
+}
+

 }
--- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
+++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <common/StringRef.h>
 #include <Common/HashTable/HashMap.h>
 #include <Columns/ColumnDecimal.h>
@ -243,5 +243,4 @@ private:
    BlockPtr saved_block;
 };

-
 }
--- a/dbms/src/Dictionaries/DictionaryBlockInputStream.h
+++ b/dbms/src/Dictionaries/DictionaryBlockInputStream.h
@ -6,9 +6,9 @@
 #include <Columns/IColumn.h>
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <DataTypes/DataTypesNumber.h>
-#include <Dictionaries/DictionaryBlockInputStreamBase.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/IDictionary.h>
+#include "DictionaryBlockInputStreamBase.h"
+#include "DictionaryStructure.h"
+#include "IDictionary.h"
 #include <ext/range.h>
 #include <common/logger_useful.h>
 #include <Core/Names.h>
--- a/dbms/src/Dictionaries/DictionaryBlockInputStreamBase.cpp
+++ b/dbms/src/Dictionaries/DictionaryBlockInputStreamBase.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/DictionaryBlockInputStreamBase.h>
+#include "DictionaryBlockInputStreamBase.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/DictionaryFactory.cpp
+++ b/dbms/src/Dictionaries/DictionaryFactory.cpp
@ -0,0 +1,51 @@
+#include "DictionaryFactory.h"
+
+#include <memory>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
+    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
+}
+
+void DictionaryFactory::registerLayout(const std::string & layout_type, Creator create_layout)
+{
+    //LOG_DEBUG(log, "Register dictionary layout type `" + layout_type + "`");
+    if (!registered_layouts.emplace(layout_type, std::move(create_layout)).second)
+        throw Exception("DictionaryFactory: the layout name '" + layout_type + "' is not unique", ErrorCodes::LOGICAL_ERROR);
+}
+
+
+DictionaryPtr DictionaryFactory::create(
+    const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context) const
+{
+    Poco::Util::AbstractConfiguration::Keys keys;
+    const auto & layout_prefix = config_prefix + ".layout";
+    config.keys(layout_prefix, keys);
+    if (keys.size() != 1)
+        throw Exception {name + ": element dictionary.layout should have exactly one child element",
+                         ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
+
+    const DictionaryStructure dict_struct {config, config_prefix + ".structure"};
+
+    auto source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context);
+
+    const auto & layout_type = keys.front();
+
+    {
+        const auto found = registered_layouts.find(layout_type);
+        if (found != registered_layouts.end())
+        {
+            const auto & create_layout = found->second;
+            return create_layout(name, dict_struct, config, config_prefix, std::move(source_ptr));
+        }
+    }
+
+    throw Exception {name + ": unknown dictionary layout type: " + layout_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
+}
+
+}
--- a/dbms/src/Dictionaries/DictionaryFactory.h
+++ b/dbms/src/Dictionaries/DictionaryFactory.h
@ -1,20 +1,41 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Poco/Util/AbstractConfiguration.h>
 #include <ext/singleton.h>
+#include "IDictionary.h"

+namespace Poco
+{
+namespace Util
+{
+    class AbstractConfiguration;
+}
+
+class Logger;
+}

 namespace DB
 {
-
 class Context;

 class DictionaryFactory : public ext::singleton<DictionaryFactory>
 {
 public:
-    DictionaryPtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config,
-        const std::string & config_prefix, Context & context) const;
+    DictionaryPtr
+    create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Context & context)
+        const;
+
+    using Creator = std::function<DictionaryPtr(
+        const std::string & name,
+        const DictionaryStructure & dict_struct,
+        const Poco::Util::AbstractConfiguration & config,
+        const std::string & config_prefix,
+        DictionarySourcePtr source_ptr)>;
+
+    void registerLayout(const std::string & layout_type, Creator create_layout);
+
+private:
+    using LayoutRegistry = std::unordered_map<std::string, Creator>;
+    LayoutRegistry registered_layouts;
 };

 }
--- a/dbms/src/Dictionaries/DictionarySourceFactory.cpp
+++ b/dbms/src/Dictionaries/DictionarySourceFactory.cpp
@ -1,41 +1,16 @@
-#include <Dictionaries/DictionarySourceFactory.h>
+#include "DictionarySourceFactory.h"

-#include <Core/Block.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/FileDictionarySource.h>
-#include <Dictionaries/ClickHouseDictionarySource.h>
-#include <Dictionaries/ExecutableDictionarySource.h>
-#include <Dictionaries/HTTPDictionarySource.h>
-#include <Dictionaries/LibraryDictionarySource.h>
-#include <Dictionaries/XDBCDictionarySource.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeDate.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <Common/FieldVisitors.h>
-#include <Common/XDBCBridgeHelper.h>
 #include <Columns/ColumnsNumber.h>
-#include <IO/HTTPCommon.h>
-#include <memory>
-#include <mutex>
-
-#include <Common/config.h>
-#if USE_POCO_MONGODB
-    #include <Dictionaries/MongoDBDictionarySource.h>
-#endif
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-    #include <Poco/Data/ODBC/Connector.h>
-#endif
-#if USE_MYSQL
-    #include <Dictionaries/MySQLDictionarySource.h>
-#endif
-
+#include <Core/Block.h>
+#include <Core/ColumnWithTypeAndName.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypesNumber.h>
 #include <Poco/Logger.h>
-
 #include <common/logger_useful.h>
+#include "DictionaryStructure.h"

 namespace DB
 {
-
 namespace ErrorCodes
 {
    extern const int UNKNOWN_ELEMENT_IN_CONFIG;
@ -46,149 +21,78 @@ namespace ErrorCodes

 namespace
 {
-
-Block createSampleBlock(const DictionaryStructure & dict_struct)
-{
-    Block block;
-
-    if (dict_struct.id)
-        block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
-
-    if (dict_struct.key)
+    Block createSampleBlock(const DictionaryStructure & dict_struct)
    {
-        for (const auto & attribute : *dict_struct.key)
+        Block block;
+
+        if (dict_struct.id)
+            block.insert(ColumnWithTypeAndName {ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
+
+        if (dict_struct.key)
+        {
+            for (const auto & attribute : *dict_struct.key)
+            {
+                auto column = attribute.type->createColumn();
+                column->insertDefault();
+
+                block.insert(ColumnWithTypeAndName {std::move(column), attribute.type, attribute.name});
+            }
+        }
+
+        if (dict_struct.range_min)
+        {
+            for (const auto & attribute : {dict_struct.range_min, dict_struct.range_max})
+            {
+                const auto & type = std::make_shared<DataTypeNullable>(attribute->type);
+                auto column = type->createColumn();
+                column->insertDefault();
+
+                block.insert(ColumnWithTypeAndName {std::move(column), type, attribute->name});
+            }
+        }
+
+        for (const auto & attribute : dict_struct.attributes)
        {
            auto column = attribute.type->createColumn();
-            column->insertDefault();
+            column->insert(attribute.null_value);

-            block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
+            block.insert(ColumnWithTypeAndName {std::move(column), attribute.type, attribute.name});
        }
+
+        return block;
    }

-    if (dict_struct.range_min)
-    {
-        for (const auto & attribute : { dict_struct.range_min, dict_struct.range_max })
-        {
-            const auto & type = std::make_shared<DataTypeNullable>(attribute->type);
-            auto column = type->createColumn();
-            column->insertDefault();
-
-            block.insert(ColumnWithTypeAndName{std::move(column), type, attribute->name});
-        }
-    }
-
-    for (const auto & attribute : dict_struct.attributes)
-    {
-        auto column = attribute.type->createColumn();
-        column->insert(attribute.null_value);
-
-        block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
-    }
-
-    return block;
-}
-
 }


-DictionarySourceFactory::DictionarySourceFactory()
-    : log(&Poco::Logger::get("DictionarySourceFactory"))
+DictionarySourceFactory::DictionarySourceFactory() : log(&Poco::Logger::get("DictionarySourceFactory"))
 {
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-    Poco::Data::ODBC::Connector::registerConnector();
-#endif
 }

 void DictionarySourceFactory::registerSource(const std::string & source_type, Creator create_source)
 {
    LOG_DEBUG(log, "Register dictionary source type `" + source_type + "`");
    if (!registered_sources.emplace(source_type, std::move(create_source)).second)
-        throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique",
-            ErrorCodes::LOGICAL_ERROR);
+        throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique", ErrorCodes::LOGICAL_ERROR);
 }

 DictionarySourcePtr DictionarySourceFactory::create(
-    const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
-    const DictionaryStructure & dict_struct, Context & context) const
+    const std::string & name,
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const DictionaryStructure & dict_struct,
+    Context & context) const
 {
    Poco::Util::AbstractConfiguration::Keys keys;
    config.keys(config_prefix, keys);
    if (keys.size() != 1)
-        throw Exception{name +": element dictionary.source should have exactly one child element", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
+        throw Exception {name + ": element dictionary.source should have exactly one child element",
+                         ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};

    auto sample_block = createSampleBlock(dict_struct);

    const auto & source_type = keys.front();

-    if ("file" == source_type)
-    {
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `file` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        const auto filename = config.getString(config_prefix + ".file.path");
-        const auto format = config.getString(config_prefix + ".file.format");
-        return std::make_unique<FileDictionarySource>(filename, format, sample_block, context);
-    }
-    else if ("mysql" == source_type)
-    {
-#if USE_MYSQL
-        return std::make_unique<MySQLDictionarySource>(dict_struct, config, config_prefix + ".mysql", sample_block);
-#else
-        throw Exception{"Dictionary source of type `mysql` is disabled because ClickHouse was built without mysql support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("clickhouse" == source_type)
-    {
-        return std::make_unique<ClickHouseDictionarySource>(dict_struct, config, config_prefix + ".clickhouse",
-            sample_block, context);
-    }
-    else if ("mongodb" == source_type)
-    {
-#if USE_POCO_MONGODB
-        return std::make_unique<MongoDBDictionarySource>(dict_struct, config, config_prefix + ".mongodb", sample_block);
-#else
-        throw Exception{"Dictionary source of type `mongodb` is disabled because poco library was built without mongodb support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("odbc" == source_type)
-    {
-#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
-        BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(context, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string"));
-        return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge);
-#else
-        throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.",
-            ErrorCodes::SUPPORT_IS_DISABLED};
-#endif
-    }
-    else if ("jdbc" == source_type)
-    {
-        throw Exception{"Dictionary source of type `jdbc` is disabled until consistent support for nullable fields.",
-                        ErrorCodes::SUPPORT_IS_DISABLED};
-//        BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<JDBCBridgeMixin>>(config, context.getSettings().http_receive_timeout, config.getString(config_prefix + ".connection_string"));
-//        return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".jdbc", sample_block, context, bridge);
-    }
-    else if ("executable" == source_type)
-    {
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
-    }
-    else if ("http" == source_type)
-    {
-
-        if (dict_struct.has_expressions)
-            throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
-
-        return std::make_unique<HTTPDictionarySource>(dict_struct, config, config_prefix + ".http", sample_block, context);
-    }
-    else if ("library" == source_type)
-    {
-        return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
-    }
-    else
    {
        const auto found = registered_sources.find(source_type);
        if (found != registered_sources.end())
@ -198,7 +102,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
        }
    }

-    throw Exception{name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
+    throw Exception {name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
 }

 }
--- a/dbms/src/Dictionaries/DictionarySourceFactory.h
+++ b/dbms/src/Dictionaries/DictionarySourceFactory.h
@ -1,23 +1,22 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <ext/singleton.h>
+#include "IDictionarySource.h"

 #include <unordered_map>
+#include <ext/singleton.h>

 namespace Poco
 {
-    namespace Util
-    {
-        class AbstractConfiguration;
-    }
+namespace Util
+{
+    class AbstractConfiguration;
+}

-    class Logger;
+class Logger;
 }

 namespace DB
 {
-
 class Context;
 struct DictionaryStructure;

@ -30,15 +29,18 @@ public:
        const Poco::Util::AbstractConfiguration & config,
        const std::string & config_prefix,
        Block & sample_block,
-        const Context & context)>;
+        Context & context)>;

    DictionarySourceFactory();

    void registerSource(const std::string & source_type, Creator create_source);

    DictionarySourcePtr create(
-        const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
-        const DictionaryStructure & dict_struct, Context & context) const;
+        const std::string & name,
+        const Poco::Util::AbstractConfiguration & config,
+        const std::string & config_prefix,
+        const DictionaryStructure & dict_struct,
+        Context & context) const;

 private:
    using SourceRegistry = std::unordered_map<std::string, Creator>;
--- a/dbms/src/Dictionaries/DictionarySourceHelpers.cpp
+++ b/dbms/src/Dictionaries/DictionarySourceHelpers.cpp
@ -1,5 +1,5 @@
-#include <Dictionaries/DictionarySourceHelpers.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "DictionarySourceHelpers.h"
+#include "DictionaryStructure.h"
 #include <Core/ColumnWithTypeAndName.h>
 #include <Core/Block.h>
 #include <Columns/ColumnsNumber.h>
--- a/dbms/src/Dictionaries/DictionaryStructure.cpp
+++ b/dbms/src/Dictionaries/DictionaryStructure.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/DictionaryStructure.h>
+#include "DictionaryStructure.h"
 #include <Formats/FormatSettings.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeNullable.h>
--- a/dbms/src/Dictionaries/Embedded/CMakeLists.txt
+++ b/dbms/src/Dictionaries/Embedded/CMakeLists.txt
@ -0,0 +1,5 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+add_headers_and_sources(clickhouse_dictionaries_embedded .)
+add_headers_and_sources(clickhouse_dictionaries_embedded GeodataProviders)
+add_library(clickhouse_dictionaries_embedded ${LINK_MODE} ${clickhouse_dictionaries_embedded_sources})
+target_link_libraries(clickhouse_dictionaries_embedded PRIVATE clickhouse_common_io ${MYSQLXX_LIBRARY})
--- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.cpp
@ -1,8 +1,8 @@
-#include <Dictionaries/Embedded/GeoDictionariesLoader.h>
-
-#include <Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/NamesProvider.h>
+#include "GeoDictionariesLoader.h"

+#include <Poco/Util/AbstractConfiguration.h>
+#include "GeodataProviders/HierarchiesProvider.h"
+#include "GeodataProviders/NamesProvider.h"

 std::unique_ptr<RegionsHierarchies> GeoDictionariesLoader::reloadRegionsHierarchies(
    const Poco::Util::AbstractConfiguration & config)
--- a/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h
+++ b/dbms/src/Dictionaries/Embedded/GeoDictionariesLoader.h
@ -1,6 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/IGeoDictionariesLoader.h>
+#include "IGeoDictionariesLoader.h"


 // Default implementation of geo dictionaries loader used by native server application
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/Entries.h
@ -1,6 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Types.h>
+#include "Types.h"
 #include <string>

 struct RegionEntry
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.cpp
@ -1,8 +1,7 @@
-#include <Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h>
+#include "HierarchiesProvider.h"

+#include "HierarchyFormatReader.h"
 #include <IO/ReadBufferFromFile.h>
-
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
 #include <Poco/DirectoryIterator.h>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchiesProvider.h
@ -1,9 +1,8 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
+#include "IHierarchiesProvider.h"

 #include <Common/FileUpdatesTracker.h>
-
 #include <unordered_map>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h>
+#include "HierarchyFormatReader.h"

 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/HierarchyFormatReader.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "IHierarchiesProvider.h"
 #include <IO/ReadBuffer.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Entries.h>
-
+#include "Entries.h"
 #include <memory>
 #include <string>
 #include <vector>
--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/INamesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/Entries.h>
-
+#include "Entries.h"
 #include <memory>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h>
+#include "NamesFormatReader.h"

 #include <IO/ReadHelpers.h>

--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "INamesProvider.h"
 #include <IO/ReadBuffer.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.cpp
@ -1,6 +1,6 @@
-#include <Dictionaries/Embedded/GeodataProviders/NamesProvider.h>
-#include <Dictionaries/Embedded/GeodataProviders/NamesFormatReader.h>
+#include "NamesProvider.h"

+#include "NamesFormatReader.h"
 #include <IO/ReadBufferFromFile.h>


--- a/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h
+++ b/dbms/src/Dictionaries/Embedded/GeodataProviders/NamesProvider.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "INamesProvider.h"
 #include <Common/FileUpdatesTracker.h>


--- a/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h
+++ b/dbms/src/Dictionaries/Embedded/IGeoDictionariesLoader.h
@ -1,12 +1,19 @@
 #pragma once

-#include <Dictionaries/Embedded/RegionsHierarchies.h>
-#include <Dictionaries/Embedded/RegionsNames.h>
-
-#include <Poco/Util/AbstractConfiguration.h>
-
+#include "RegionsHierarchies.h"
+#include "RegionsNames.h"
 #include <memory>

+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+
+    class Logger;
+}
+

 // Provides actual versions of geo dictionaries (regions hierarchies, regions names)
 // Bind data structures (RegionsHierarchies, RegionsNames) with data providers
--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp
@ -1,7 +1,6 @@
-#include <Dictionaries/Embedded/RegionsHierarchies.h>
+#include "RegionsHierarchies.h"

 #include <common/logger_useful.h>
-
 #include <Poco/DirectoryIterator.h>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h
@ -1,10 +1,8 @@
 #pragma once

-#include <Dictionaries/Embedded/RegionsHierarchy.h>
-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "RegionsHierarchy.h"
+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <Poco/Exception.h>
-
 #include <unordered_map>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp
@ -1,12 +1,10 @@
-#include <Dictionaries/Embedded/RegionsHierarchy.h>
-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
+#include "RegionsHierarchy.h"

+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
-
 #include <common/logger_useful.h>
 #include <ext/singleton.h>
-
 #include <IO/WriteHelpers.h>


--- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h
@ -1,7 +1,6 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/IHierarchiesProvider.h>
-
+#include "GeodataProviders/IHierarchiesProvider.h"
 #include <vector>
 #include <boost/noncopyable.hpp>
 #include <common/Types.h>
--- a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp
+++ b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp
@ -1,11 +1,9 @@
-#include <Dictionaries/Embedded/RegionsNames.h>
-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
+#include "RegionsNames.h"

+#include "GeodataProviders/INamesProvider.h"
 #include <Poco/Util/Application.h>
 #include <Poco/Exception.h>
-
 #include <common/logger_useful.h>
-
 #include <IO/WriteHelpers.h>

 namespace DB
--- a/dbms/src/Dictionaries/Embedded/RegionsNames.h
+++ b/dbms/src/Dictionaries/Embedded/RegionsNames.h
@ -1,12 +1,9 @@
 #pragma once

-#include <Dictionaries/Embedded/GeodataProviders/INamesProvider.h>
-
+#include "GeodataProviders/INamesProvider.h"
 #include <Poco/Exception.h>
-
 #include <common/Types.h>
 #include <common/StringRef.h>
-
 #include <string>
 #include <vector>

--- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp
+++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp
@ -1,7 +1,7 @@
 #include <Common/config.h>
 #if USE_MYSQL

-#include <Dictionaries/Embedded/TechDataHierarchy.h>
+#include "TechDataHierarchy.h"

 #include <common/logger_useful.h>
 #include <mysqlxx/PoolWithFailover.h>
--- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h
+++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h
@ -1,12 +1,18 @@
 #pragma once

-#include <Poco/Util/AbstractConfiguration.h>
-#include <Poco/Exception.h>
-
 #include <common/Types.h>
-
 #include <ext/singleton.h>

+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+
+    class Logger;
+}
+

 /** @brief Class that lets you know if a search engine or operating system belongs
  * another search engine or operating system, respectively.
--- a/dbms/src/Dictionaries/ExecutableDictionarySource.cpp
+++ b/dbms/src/Dictionaries/ExecutableDictionarySource.cpp
@ -1,12 +1,15 @@
+#include "ExecutableDictionarySource.h"
+
 #include <thread>
 #include <future>
-#include <Dictionaries/ExecutableDictionarySource.h>
 #include <Common/ShellCommand.h>
 #include <Interpreters/Context.h>
 #include <DataStreams/OwningBlockInputStream.h>
-#include <Dictionaries/DictionarySourceHelpers.h>
+#include "DictionarySourceHelpers.h"
 #include <DataStreams/IBlockOutputStream.h>
 #include <common/logger_useful.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"


 namespace DB
@ -229,4 +232,19 @@ std::string ExecutableDictionarySource::toString() const
    return "Executable: " + command;
 }

+void registerDictionarySourceExecutable(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & context) -> DictionarySourcePtr {
+        if (dict_struct.has_expressions)
+            throw Exception {"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
+
+        return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
+    };
+    factory.registerSource("executable", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/ExecutableDictionarySource.h
+++ b/dbms/src/Dictionaries/ExecutableDictionarySource.h
@ -1,7 +1,7 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"


 namespace Poco { class Logger; }
--- a/dbms/src/Dictionaries/ExternalQueryBuilder.cpp
+++ b/dbms/src/Dictionaries/ExternalQueryBuilder.cpp
@ -3,9 +3,9 @@
 #include <IO/WriteBuffer.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
-#include <Dictionaries/writeParenthesisedString.h>
-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/ExternalQueryBuilder.h>
+#include "writeParenthesisedString.h"
+#include "DictionaryStructure.h"
+#include "ExternalQueryBuilder.h"


 namespace DB
--- a/dbms/src/Dictionaries/ExternalResultDescription.cpp
+++ b/dbms/src/Dictionaries/ExternalResultDescription.cpp
@ -1,5 +1,5 @@
 #include <ext/range.h>
-#include <Dictionaries/ExternalResultDescription.h>
+#include "ExternalResultDescription.h"
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeDate.h>
--- a/dbms/src/Dictionaries/FileDictionarySource.cpp
+++ b/dbms/src/Dictionaries/FileDictionarySource.cpp
@ -1,9 +1,11 @@
+#include "FileDictionarySource.h"
+
 #include <Interpreters/Context.h>
 #include <DataStreams/OwningBlockInputStream.h>
-#include <Dictionaries/FileDictionarySource.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Poco/File.h>
-
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"

 namespace DB
 {
@ -46,4 +48,23 @@ Poco::Timestamp FileDictionarySource::getLastModification() const
    return Poco::File{filename}.getLastModified();
 }

+void registerDictionarySourceFile(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & context) -> DictionarySourcePtr {
+        if (dict_struct.has_expressions)
+            throw Exception {"Dictionary source of type `file` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
+
+        const auto filename = config.getString(config_prefix + ".file.path");
+        const auto format = config.getString(config_prefix + ".file.format");
+
+        return std::make_unique<FileDictionarySource>(filename, format, sample_block, context);
+    };
+
+    factory.registerSource("file", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/FileDictionarySource.h
+++ b/dbms/src/Dictionaries/FileDictionarySource.h
@ -1,6 +1,6 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
+#include "IDictionarySource.h"
 #include <Poco/Timestamp.h>


@ -9,7 +9,6 @@ namespace DB

 class Context;

-
 /// Allows loading dictionaries from a file with given format, does not support "random access"
 class FileDictionarySource final : public IDictionarySource
 {
--- a/dbms/src/Dictionaries/FlatDictionary.cpp
+++ b/dbms/src/Dictionaries/FlatDictionary.cpp
@ -1,7 +1,7 @@
-#include <Dictionaries/FlatDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "FlatDictionary.h"
+#include "DictionaryBlockInputStream.h"
 #include <IO/WriteHelpers.h>
-
+#include "DictionaryFactory.h"

 namespace DB
 {
@ -14,9 +14,9 @@ namespace ErrorCodes
    extern const int DICTIONARY_IS_EMPTY;
    extern const int LOGICAL_ERROR;
    extern const int UNKNOWN_TYPE;
+    extern const int UNSUPPORTED_METHOD;
 }

-
 static const auto initial_array_size = 1024;
 static const auto max_array_size = 500000;

@ -634,5 +634,33 @@ BlockInputStreamPtr FlatDictionary::getBlockInputStream(const Names & column_nam
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getIds() ,column_names);
 }

+void registerDictionaryFlat(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+
+        if (dict_struct.key)
+            throw Exception {"'key' is not supported for dictionary of layout 'flat'", ErrorCodes::UNSUPPORTED_METHOD};
+
+        if (dict_struct.range_min || dict_struct.range_max)
+            throw Exception {name
+                                 + ": elements .structure.range_min and .structure.range_max should be defined only "
+                                   "for a dictionary of layout 'range_hashed'",
+                             ErrorCodes::BAD_ARGUMENTS};
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        return std::make_unique<FlatDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
+
+
+    };
+    factory.registerLayout("flat", create_layout);
+}
+
+

 }
--- a/dbms/src/Dictionaries/FlatDictionary.h
+++ b/dbms/src/Dictionaries/FlatDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <Columns/ColumnDecimal.h>
 #include <Columns/ColumnString.h>
 #include <Common/Arena.h>
--- a/dbms/src/Dictionaries/HTTPDictionarySource.cpp
+++ b/dbms/src/Dictionaries/HTTPDictionarySource.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/HTTPDictionarySource.h>
+#include "HTTPDictionarySource.h"

 #include <Poco/Net/HTTPRequest.h>
 #include <Interpreters/Context.h>
@ -6,9 +6,12 @@
 #include <IO/ReadWriteBufferFromHTTP.h>
 #include <DataStreams/IBlockOutputStream.h>
 #include <IO/WriteBufferFromOStream.h>
-#include <Dictionaries/DictionarySourceHelpers.h>
+#include "DictionarySourceHelpers.h"
 #include <common/logger_useful.h>
 #include <IO/ConnectionTimeouts.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+

 namespace DB
 {
@ -149,4 +152,19 @@ std::string HTTPDictionarySource::toString() const
    return uri.toString();
 }

+void registerDictionarySourceHTTP(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & context) -> DictionarySourcePtr {
+        if (dict_struct.has_expressions)
+            throw Exception {"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
+
+        return std::make_unique<HTTPDictionarySource>(dict_struct, config, config_prefix + ".http", sample_block, context);
+    };
+    factory.registerSource("http", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/HTTPDictionarySource.h
+++ b/dbms/src/Dictionaries/HTTPDictionarySource.h
@ -1,8 +1,8 @@
 #pragma once

 #include <Poco/URI.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <common/LocalDateTime.h>
 #include <IO/ConnectionTimeouts.h>

--- a/dbms/src/Dictionaries/HashedDictionary.cpp
+++ b/dbms/src/Dictionaries/HashedDictionary.cpp
@ -1,6 +1,7 @@
 #include <ext/size.h>
-#include <Dictionaries/HashedDictionary.h>
-#include <Dictionaries/DictionaryBlockInputStream.h>
+#include "HashedDictionary.h"
+#include "DictionaryBlockInputStream.h"
+#include "DictionaryFactory.h"

 namespace DB
 {
@ -11,6 +12,7 @@ namespace ErrorCodes
    extern const int ARGUMENT_OUT_OF_BOUND;
    extern const int BAD_ARGUMENTS;
    extern const int DICTIONARY_IS_EMPTY;
+    extern const int UNSUPPORTED_METHOD;
 }


@ -623,4 +625,29 @@ BlockInputStreamPtr HashedDictionary::getBlockInputStream(const Names & column_n
    return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, getIds(), column_names);
 }

+void registerDictionaryHashed(DictionaryFactory & factory)
+{
+    auto create_layout = [=](
+                                 const std::string & name,
+                                 const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 DictionarySourcePtr source_ptr
+                                 ) -> DictionaryPtr {
+        if (dict_struct.key)
+            throw Exception {"'key' is not supported for dictionary of layout 'hashed'", ErrorCodes::UNSUPPORTED_METHOD};
+
+        if (dict_struct.range_min || dict_struct.range_max)
+            throw Exception {name
+                                 + ": elements .structure.range_min and .structure.range_max should be defined only "
+                                   "for a dictionary of layout 'range_hashed'",
+                             ErrorCodes::BAD_ARGUMENTS};
+        const DictionaryLifetime dict_lifetime {config, config_prefix + ".lifetime"};
+        const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false);
+        return std::make_unique<HashedDictionary>(name, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty);
+
+    };
+    factory.registerLayout("hashed", create_layout);
+}
+
 }
--- a/dbms/src/Dictionaries/HashedDictionary.h
+++ b/dbms/src/Dictionaries/HashedDictionary.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/IDictionary.h>
-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include "IDictionary.h"
+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"
 #include <Common/HashTable/HashMap.h>
 #include <Columns/ColumnDecimal.h>
 #include <Columns/ColumnString.h>
--- a/dbms/src/Dictionaries/IDictionary.h
+++ b/dbms/src/Dictionaries/IDictionary.h
@ -8,7 +8,7 @@
 #include <Common/PODArray.h>
 #include <memory>
 #include <chrono>
-#include <Dictionaries/IDictionarySource.h>
+#include "IDictionarySource.h"

 namespace DB
 {
--- a/dbms/src/Dictionaries/LibraryDictionarySource.cpp
+++ b/dbms/src/Dictionaries/LibraryDictionarySource.cpp
@ -1,12 +1,15 @@
 #include <DataStreams/OneBlockInputStream.h>
-#include <Dictionaries/LibraryDictionarySource.h>
-#include <Dictionaries/LibraryDictionarySourceExternal.h>
+#include "LibraryDictionarySource.h"
+#include "LibraryDictionarySourceExternal.h"
 #include <Interpreters/Context.h>
 #include <Poco/File.h>
 #include <common/logger_useful.h>
 #include <ext/bit_cast.h>
 #include <ext/range.h>
 #include <ext/scope_guard.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+

 namespace DB
 {
@ -269,4 +272,17 @@ std::string LibraryDictionarySource::toString() const
 {
    return path;
 }
+
+void registerDictionarySourceLibrary(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & context) -> DictionarySourcePtr {
+        return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
+    };
+    factory.registerSource("library", createTableSource);
+}
+
 }
--- a/dbms/src/Dictionaries/LibraryDictionarySource.h
+++ b/dbms/src/Dictionaries/LibraryDictionarySource.h
@ -1,8 +1,8 @@
 #pragma once

-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/ExternalResultDescription.h>
-#include <Dictionaries/IDictionarySource.h>
+#include "DictionaryStructure.h"
+#include "ExternalResultDescription.h"
+#include "IDictionarySource.h"
 #include <Common/SharedLibrary.h>
 #include <common/LocalDateTime.h>

@ -78,4 +78,5 @@ private:
    std::shared_ptr<CStringsHolder> settings;
    void * lib_data = nullptr;
 };
+
 }
--- a/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp
+++ b/dbms/src/Dictionaries/LibraryDictionarySourceExternal.cpp
@ -1,4 +1,4 @@
-#include <Dictionaries/LibraryDictionarySourceExternal.h>
+#include "LibraryDictionarySourceExternal.h"
 #include <common/logger_useful.h>

 namespace
--- a/dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
+++ b/dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
@ -10,8 +10,8 @@
 #include <Poco/MongoDB/Element.h>
 #include <Poco/MongoDB/ObjectId.h>

-#include <Dictionaries/DictionaryStructure.h>
-#include <Dictionaries/MongoDBBlockInputStream.h>
+#include "DictionaryStructure.h"
+#include "MongoDBBlockInputStream.h"
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnNullable.h>
--- a/dbms/src/Dictionaries/MongoDBBlockInputStream.h
+++ b/dbms/src/Dictionaries/MongoDBBlockInputStream.h
@ -2,7 +2,7 @@

 #include <Core/Block.h>
 #include <DataStreams/IProfilingBlockInputStream.h>
-#include <Dictionaries/ExternalResultDescription.h>
+#include "ExternalResultDescription.h"


 namespace Poco
--- a/dbms/src/Dictionaries/MongoDBDictionarySource.cpp
+++ b/dbms/src/Dictionaries/MongoDBDictionarySource.cpp
@ -1,21 +1,54 @@
-#include <Common/config.h>
-#if USE_POCO_MONGODB
-#include <Poco/Util/AbstractConfiguration.h>
+#include "DictionarySourceFactory.h"
+#include "DictionaryStructure.h"
+#include "MongoDBDictionarySource.h"

+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int SUPPORT_IS_DISABLED;
+}
+
+void registerDictionarySourceMongoDB(DictionarySourceFactory & factory)
+{
+    auto createTableSource = [=](const DictionaryStructure & dict_struct,
+                                 const Poco::Util::AbstractConfiguration & config,
+                                 const std::string & config_prefix,
+                                 Block & sample_block,
+                                 const Context & /* context */) -> DictionarySourcePtr {
+#if USE_POCO_MONGODB
+        return std::make_unique<MongoDBDictionarySource>(dict_struct, config, config_prefix + ".mongodb", sample_block);
+#else
+        (void)dict_struct;
+        (void)config;
+        (void)config_prefix;
+        (void)sample_block;
+        throw Exception {"Dictionary source of type `mongodb` is disabled because poco library was built without mongodb support.",
+                         ErrorCodes::SUPPORT_IS_DISABLED};
+#endif
+    };
+    factory.registerSource("mongodb", createTableSource);
+}
+
+}
+
+
+#if USE_POCO_MONGODB
+
+#include <Poco/Util/AbstractConfiguration.h>
 #include <Poco/MongoDB/Connection.h>
 #include <Poco/MongoDB/Database.h>
 #include <Poco/MongoDB/Cursor.h>
 #include <Poco/MongoDB/Array.h>
 #include <Poco/MongoDB/ObjectId.h>
-
 #include <Poco/Version.h>

 // only after poco
 // naming conflict:
 // Poco/MongoDB/BSONWriter.h:54: void writeCString(const std::string & value);
 // dbms/src/IO/WriteHelpers.h:146 #define writeCString(s, buf)
-#include <Dictionaries/MongoDBDictionarySource.h>
-#include <Dictionaries/MongoDBBlockInputStream.h>
+#include "MongoDBBlockInputStream.h"
 #include <Common/FieldVisitors.h>
 #include <IO/WriteHelpers.h>
 #include <ext/enumerate.h>
--- a/dbms/src/Dictionaries/MongoDBDictionarySource.h
+++ b/dbms/src/Dictionaries/MongoDBDictionarySource.h
@ -1,8 +1,10 @@
 #pragma once

-#include <Dictionaries/IDictionarySource.h>
-#include <Dictionaries/DictionaryStructure.h>
+#include <Common/config.h>
+#if USE_POCO_MONGODB

+#include "IDictionarySource.h"
+#include "DictionaryStructure.h"

 namespace Poco
 {
@ -79,3 +81,10 @@ private:
 };

 }
+#endif
+
+/*namespace DB
+{
+class DictionarySourceFactory;
+void registerDictionarySourceMongoDB(DictionarySourceFactory & factory);
+}*/
--- a/Show More
+++ b/Show More