Merge remote-tracking branch 'upstream/master' into updating-docs

CHANGELOG.draft.md

@@ -1 +1,31 @@
 ## RU
+
+## ClickHouse release 18.10.3, 2018-08-13
+
+### Новые возможности:
+* поддержка межсерверной репликации по HTTPS
+* MurmurHash
+* ODBCDriver2 с поддержкой NULL-ов
+* поддержка UUID в ключевых колонках (экспериментально)
+
+### Улучшения:
+* добавлена поддержка SETTINGS для движка Kafka
+* поддежка пустых кусков после мержей в движках Summing, Collapsing and VersionedCollapsing
+* удаление старых записей о полностью выполнившихся мутациях
+* исправлена логика REPLACE PARTITION для движка RplicatedMergeTree
+* добавлена системная таблица system.merge_tree_settings
+* в системную таблицу system.tables добавлены столбцы зависимостей: dependencies_database и dependencies_table
+* заменен аллокатор, теперь используется jemalloc вместо tcmalloc
+* улучшена валидация connection string ODBC
+* удалена поддержка CHECK TABLE для распределенных таблиц
+* добавлены stateful тесты (пока без данных)
+* добавлена опция конфига max_partition_size_to_drop
+* добавлена настройка output_format_json_escape_slashes
+* добавлена настройка max_fetch_partition_retries_count
+* добавлена настройка prefer_localhost_replica
+* добавлены libressl, unixodbc и mariadb-connector-c как сабмодули
+
+### Исправление ошибок:
+* #2786
+* #2777
+* #2795

CMakeLists.txt

@@ -1,5 +1,5 @@
 project (ClickHouse)
-cmake_minimum_required (VERSION 2.8)
+cmake_minimum_required (VERSION 3.3)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/")
 
@@ -218,7 +218,7 @@ else ()
     set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc")
 endif ()
 
-option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF)
+option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
 if (UNBUNDLED)
     set(NOT_UNBUNDLED 0)
 else ()

cmake/arch.cmake

@@ -7,9 +7,9 @@ endif ()
 if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
     set (ARCH_I386 1)
 endif ()
-if ( ( ARCH_ARM AND NOT ARCH_AARCH64 ) OR ARCH_I386)
+if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
     set (ARCH_32 1)
-    message (WARNING "Support for 32bit platforms is highly experimental")
+    message (FATAL_ERROR "32bit platforms are not supported")
 endif ()
 
 if (CMAKE_SYSTEM MATCHES "Linux")
@@ -24,7 +24,3 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     set (COMPILER_CLANG 1)
 endif ()
-
-if (OS_LINUX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
-    set (OS_LINUX_X86_64 1)
-endif ()

cmake/find_rdkafka.cmake

@@ -2,7 +2,7 @@ option (ENABLE_RDKAFKA "Enable kafka" ON)
 
 if (ENABLE_RDKAFKA)
 
-if (OS_LINUX_X86_64)
+if (OS_LINUX)
     option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED})
 endif ()
 

cmake/find_zlib.cmake

@@ -1,4 +1,4 @@
-if (NOT OS_FREEBSD)
+if (NOT OS_FREEBSD AND NOT APPLE)
     option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
 endif ()
 

contrib/CMakeLists.txt

@@ -1,10 +1,10 @@
 # Third-party libraries may have substandard code.
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z")
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
 endif ()
 
@@ -151,7 +151,7 @@ endif ()
 
 if (USE_INTERNAL_LLVM_LIBRARY)
     # ld: unknown option: --color-diagnostics
-    if (APPLE AND COMPILER_GCC)
+    if (APPLE)
         set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
     endif ()
     add_subdirectory (llvm/llvm)

dbms/CMakeLists.txt

@@ -157,6 +157,7 @@ target_link_libraries (dbms
     ${RE2_LIBRARY}
     ${RE2_ST_LIBRARY}
     ${BTRIE_LIBRARIES}
+    ${Boost_PROGRAM_OPTIONS_LIBRARY}
 )
 
 if (NOT USE_INTERNAL_RE2_LIBRARY)

dbms/cmake/version.cmake

@@ -1,11 +1,11 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54405 CACHE STRING "")
+set(VERSION_REVISION 54406 CACHE STRING "")
 set(VERSION_MAJOR 18 CACHE STRING "")
-set(VERSION_MINOR 10 CACHE STRING "")
-set(VERSION_PATCH 3 CACHE STRING "")
-set(VERSION_GITHASH 1fa1b34f1ab01ea2e1a833eebd36a4806e529f52 CACHE STRING "")
-set(VERSION_DESCRIBE v18.10.3-testing CACHE STRING "")
-set(VERSION_STRING 18.10.3 CACHE STRING "")
+set(VERSION_MINOR 11 CACHE STRING "")
+set(VERSION_PATCH 0 CACHE STRING "")
+set(VERSION_GITHASH 76af46ed5d223b3a7af92e31eae291174da16355 CACHE STRING "")
+set(VERSION_DESCRIBE v18.11.0-testing CACHE STRING "")
+set(VERSION_STRING 18.11.0 CACHE STRING "")
 # end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")

dbms/programs/CMakeLists.txt

@@ -13,6 +13,7 @@ option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLI
 option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL})
+option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL})
 
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
 
@@ -27,10 +28,11 @@ add_subdirectory (copier)
 add_subdirectory (format)
 add_subdirectory (clang)
 add_subdirectory (obfuscator)
+add_subdirectory (odbc-bridge)
 
 if (CLICKHOUSE_SPLIT_BINARY)
     set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
-            clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier)
+            clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier clickhouse-odbc-bridge)
 
     if (USE_EMBEDDED_COMPILER)
         list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld)
@@ -83,6 +85,9 @@ else ()
     if (USE_EMBEDDED_COMPILER)
         target_link_libraries (clickhouse clickhouse-compiler-lib)
     endif ()
+    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+        target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
+    endif()
 
     set (CLICKHOUSE_BUNDLE)
     if (ENABLE_CLICKHOUSE_SERVER)
@@ -135,6 +140,12 @@ else ()
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
     endif ()
+    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+        add_custom_target (clickhouse-odbc-bridge ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-odbc-bridge DEPENDS clickhouse)
+        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-odbc-bridge DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+        list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge)
+    endif ()
+
 
     # install always because depian package want this files:
     add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse)

dbms/programs/client/Client.cpp

@@ -1,3 +1,5 @@
+#include "TestHint.h"
+
 #include <port/unistd.h>
 #include <stdlib.h>
 #include <fcntl.h>
@@ -20,7 +22,6 @@
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
 #include <Common/ShellCommand.h>
-#include <Common/ExternalTable.h>
 #include <Common/UnicodeBar.h>
 #include <Common/formatReadable.h>
 #include <Common/NetException.h>
@@ -31,6 +32,7 @@
 #include <Common/config_version.h>
 #include <Core/Types.h>
 #include <Core/QueryProcessingStage.h>
+#include <Core/ExternalTable.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
@@ -39,6 +41,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <DataStreams/AsynchronousBlockInputStream.h>
+#include <DataStreams/InternalTextLogsRowOutputStream.h>
 #include <Parsers/ParserQuery.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ASTUseQuery.h>
@@ -87,105 +90,11 @@ namespace ErrorCodes
     extern const int UNEXPECTED_PACKET_FROM_SERVER;
     extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED;
     extern const int LOGICAL_ERROR;
+    extern const int CANNOT_SET_SIGNAL_HANDLER;
+    extern const int CANNOT_READLINE;
 }
 
 
-/// Checks expected server and client error codes in testmode.
-/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }".
-class TestHint
-{
-public:
-    TestHint(bool enabled_, const String & query)
-    :   enabled(enabled_),
-        server_error(0),
-        client_error(0)
-    {
-        if (!enabled_)
-            return;
-
-        size_t pos = query.find("--");
-        if (pos != String::npos && query.find("--", pos + 2) != String::npos)
-            return; /// It's not last comment. Hint belongs to commented query.
-
-        if (pos != String::npos)
-        {
-            pos = query.find('{', pos + 2);
-            if (pos != String::npos)
-            {
-                String hint = query.substr(pos + 1);
-                pos = hint.find('}');
-                hint.resize(pos);
-                parse(hint);
-            }
-        }
-    }
-
-    /// @returns true if it's possible to continue without reconnect
-    bool checkActual(int & actual_server_error, int & actual_client_error,
-                     bool & got_exception, std::unique_ptr<Exception> & last_exception) const
-    {
-        if (!enabled)
-            return true;
-
-        if (allErrorsExpected(actual_server_error, actual_client_error))
-        {
-            got_exception = false;
-            last_exception.reset();
-            actual_server_error = 0;
-            actual_client_error = 0;
-            return false;
-        }
-
-        if (lostExpectedError(actual_server_error, actual_client_error))
-        {
-            std::cerr << "Success when error expected. It expects server error "
-                << server_error << ", client error " << client_error << "." << std::endl;
-            got_exception = true;
-            last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS
-            return false;
-        }
-
-        return true;
-    }
-
-    int serverError() const { return server_error; }
-    int clientError() const { return client_error; }
-
-private:
-    bool enabled;
-    int server_error;
-    int client_error;
-
-    void parse(const String & hint)
-    {
-        std::stringstream ss;
-        ss << hint;
-        while (!ss.eof())
-        {
-            String item;
-            ss >> item;
-            if (item.empty())
-                break;
-
-            if (item == "serverError")
-                ss >> server_error;
-            else if (item == "clientError")
-                ss >> client_error;
-        }
-    }
-
-    bool allErrorsExpected(int actual_server_error, int actual_client_error) const
-    {
-        return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error);
-    }
-
-    bool lostExpectedError(int actual_server_error, int actual_client_error) const
-    {
-        return (server_error && !actual_server_error) || (client_error && !actual_client_error);
-    }
-};
-
-
 class Client : public Poco::Util::Application
 {
 public:
@@ -235,6 +144,11 @@ private:
     std::optional<WriteBufferFromFile> out_file_buf;
     BlockOutputStreamPtr block_out_stream;
 
+    /// The user could specify special file for server logs (stderr by default)
+    std::unique_ptr<WriteBuffer> out_logs_buf;
+    String server_logs_file;
+    BlockOutputStreamPtr logs_out_stream;
+
     String home_path;
 
     String current_profile;
@@ -408,20 +322,10 @@ private:
             /// If exception code isn't zero, we should return non-zero return code anyway.
             return e.code() ? e.code() : -1;
         }
-        catch (const Poco::Exception & e)
-        {
-            std::cerr << "Poco::Exception: " << e.displayText() << std::endl;
-            return ErrorCodes::POCO_EXCEPTION;
-        }
-        catch (const std::exception & e)
-        {
-            std::cerr << "std::exception: " << e.what() << std::endl;
-            return ErrorCodes::STD_EXCEPTION;
-        }
         catch (...)
         {
-            std::cerr << "Unknown exception" << std::endl;
-            return ErrorCodes::UNKNOWN_EXCEPTION;
+            std::cerr << getCurrentExceptionMessage(false) << std::endl;
+            return getCurrentExceptionCode();
         }
     }
 
@@ -469,7 +373,12 @@ private:
         format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size);
 
         insert_format = "Values";
-        insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size);
+
+        /// Setting value from cmd arg overrides one from config
+        if (context.getSettingsRef().max_insert_block_size.changed)
+            insert_format_max_block_size = context.getSettingsRef().max_insert_block_size;
+        else
+            insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size);
 
         if (!is_interactive)
         {
@@ -782,6 +691,7 @@ private:
             {
                 const char * pos = begin;
                 ASTPtr ast = parseQuery(pos, end, true);
+
                 if (!ast)
                 {
                     if (ignore_error)
@@ -797,7 +707,7 @@ private:
                     return true;
                 }
 
-                ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(&*ast);
+                ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(ast.get());
 
                 if (insert && insert->data)
                 {
@@ -990,7 +900,7 @@ private:
             /// If structure was received (thus, server has not thrown an exception),
             /// send our data with that structure.
             sendData(sample);
-            receivePacket();
+            receiveEndOfQuery();
         }
     }
 
@@ -1072,6 +982,11 @@ private:
             connection->sendData(block);
             processed_rows += block.rows();
 
+            /// Check if server send Log packet
+            auto packet_type = connection->checkPacket();
+            if (packet_type && *packet_type == Protocol::Server::Log)
+                receiveAndProcessPacket();
+
             if (!block)
                 break;
         }
@@ -1083,18 +998,28 @@ private:
     /// Flush all buffers.
     void resetOutput()
     {
-        block_out_stream = nullptr;
+        block_out_stream.reset();
+        logs_out_stream.reset();
+
         if (pager_cmd)
         {
             pager_cmd->in.close();
             pager_cmd->wait();
         }
         pager_cmd = nullptr;
+
         if (out_file_buf)
         {
             out_file_buf->next();
             out_file_buf.reset();
         }
+
+        if (out_logs_buf)
+        {
+            out_logs_buf->next();
+            out_logs_buf.reset();
+        }
+
         std_out.next();
     }
 
@@ -1127,7 +1052,7 @@ private:
                     continue;    /// If there is no new data, continue checking whether the query was cancelled after a timeout.
             }
 
-            if (!receivePacket())
+            if (!receiveAndProcessPacket())
                 break;
         }
 
@@ -1138,7 +1063,7 @@ private:
 
     /// Receive a part of the result, or progress info or an exception and process it.
     /// Returns true if one should continue receiving packets.
-    bool receivePacket()
+    bool receiveAndProcessPacket()
     {
         Connection::Packet packet = connection->receivePacket();
 
@@ -1169,6 +1094,10 @@ private:
                 last_exception = std::move(packet.exception);
                 return false;
 
+            case Protocol::Server::Log:
+                onLogData(packet.block);
+                return true;
+
             case Protocol::Server::EndOfStream:
                 onEndOfStream();
                 return false;
@@ -1182,22 +1111,59 @@ private:
     /// Receive the block that serves as an example of the structure of table where data will be inserted.
     bool receiveSampleBlock(Block & out)
     {
-        Connection::Packet packet = connection->receivePacket();
-
-        switch (packet.type)
+        while (true)
         {
-            case Protocol::Server::Data:
-                out = packet.block;
-                return true;
+            Connection::Packet packet = connection->receivePacket();
 
-            case Protocol::Server::Exception:
-                onException(*packet.exception);
-                last_exception = std::move(packet.exception);
-                return false;
+            switch (packet.type)
+            {
+                case Protocol::Server::Data:
+                    out = packet.block;
+                    return true;
 
-            default:
-                throw NetException("Unexpected packet from server (expected Data, got "
-                    + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
+                case Protocol::Server::Exception:
+                    onException(*packet.exception);
+                    last_exception = std::move(packet.exception);
+                    return false;
+
+                case Protocol::Server::Log:
+                    onLogData(packet.block);
+                    break;
+
+                default:
+                    throw NetException("Unexpected packet from server (expected Data, Exception or Log, got "
+                        + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
+            }
+        }
+    }
+
+
+    /// Process Log packets, exit when recieve Exception or EndOfStream
+    bool receiveEndOfQuery()
+    {
+        while (true)
+        {
+            Connection::Packet packet = connection->receivePacket();
+
+            switch (packet.type)
+            {
+                case Protocol::Server::EndOfStream:
+                    onEndOfStream();
+                    return true;
+
+                case Protocol::Server::Exception:
+                    onException(*packet.exception);
+                    last_exception = std::move(packet.exception);
+                    return false;
+
+                case Protocol::Server::Log:
+                    onLogData(packet.block);
+                    break;
+
+                default:
+                    throw NetException("Unexpected packet from server (expected Exception, EndOfStream or Log, got "
+                        + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
+            }
         }
     }
 
@@ -1253,6 +1219,38 @@ private:
     }
 
 
+    void initLogsOutputStream()
+    {
+        if (!logs_out_stream)
+        {
+            WriteBuffer * wb = out_logs_buf.get();
+
+            if (!out_logs_buf)
+            {
+                if (server_logs_file.empty())
+                {
+                    /// Use stderr by default
+                    out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
+                    wb = out_logs_buf.get();
+                }
+                else if (server_logs_file == "-")
+                {
+                    /// Use stdout if --server_logs_file=- specified
+                    wb = &std_out;
+                }
+                else
+                {
+                    out_logs_buf = std::make_unique<WriteBufferFromFile>(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT);
+                    wb = out_logs_buf.get();
+                }
+            }
+
+            logs_out_stream = std::make_shared<InternalTextLogsRowOutputStream>(*wb);
+            logs_out_stream->writePrefix();
+        }
+    }
+
+
     void onData(Block & block)
     {
         if (written_progress_chars)
@@ -1276,6 +1274,14 @@ private:
     }
 
 
+    void onLogData(Block & block)
+    {
+        initLogsOutputStream();
+        logs_out_stream->write(block);
+        logs_out_stream->flush();
+    }
+
+
     void onTotals(Block & block)
     {
         initBlockOutputStream(block);
@@ -1436,6 +1442,9 @@ private:
         if (block_out_stream)
             block_out_stream->writeSuffix();
 
+        if (logs_out_stream)
+            logs_out_stream->writeSuffix();
+
         resetOutput();
 
         if (is_interactive && !written_first_block)
@@ -1509,9 +1518,38 @@ public:
             }
         }
 
+#if USE_READLINE
+        if (rl_initialize())
+            throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE);
+
+        auto clear_prompt_or_exit = [](int)
+        {
+            /// This is signal safe.
+            ssize_t res = write(STDOUT_FILENO, "\n", 1);
+
+            if (res == 1 && rl_line_buffer[0])
+            {
+                rl_replace_line("", 0);
+                if (rl_forced_update_display())
+                    _exit(0);
+            }
+            else
+            {
+                /// A little dirty, but we struggle to find better way to correctly
+                /// force readline to exit after returning from the signal handler.
+                _exit(0);
+            }
+        };
+
+        if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
+            throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER);
+#endif
+
         ioctl(0, TIOCGWINSZ, &terminal_size);
 
-        unsigned line_length = boost::program_options::options_description::m_default_line_length;
+        namespace po = boost::program_options;
+
+        unsigned line_length = po::options_description::m_default_line_length;
         unsigned min_description_length = line_length / 2;
         if (!stdin_is_not_tty)
         {
@@ -1519,28 +1557,28 @@ public:
             min_description_length = std::min(min_description_length, line_length - 2);
         }
 
-#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value<std::string> (), DESCRIPTION)
+#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string> (), DESCRIPTION)
 
         /// Main commandline options related to client functionality and all parameters from Settings.
-        boost::program_options::options_description main_description("Main options", line_length, min_description_length);
+        po::options_description main_description("Main options", line_length, min_description_length);
         main_description.add_options()
             ("help", "produce help message")
-            ("config-file,c", boost::program_options::value<std::string>(), "config-file path")
-            ("host,h", boost::program_options::value<std::string>()->default_value("localhost"), "server host")
-            ("port", boost::program_options::value<int>()->default_value(9000), "server port")
+            ("config-file,c", po::value<std::string>(), "config-file path")
+            ("host,h", po::value<std::string>()->default_value("localhost"), "server host")
+            ("port", po::value<int>()->default_value(9000), "server port")
             ("secure,s", "secure")
-            ("user,u", boost::program_options::value<std::string>()->default_value("default"), "user")
-            ("password", boost::program_options::value<std::string>(), "password")
+            ("user,u", po::value<std::string>()->default_value("default"), "user")
+            ("password", po::value<std::string>(), "password")
             ("ask-password", "ask-password")
-            ("query_id", boost::program_options::value<std::string>(), "query_id")
-            ("query,q", boost::program_options::value<std::string>(), "query")
-            ("database,d", boost::program_options::value<std::string>(), "database")
-            ("pager", boost::program_options::value<std::string>(), "pager")
+            ("query_id", po::value<std::string>(), "query_id")
+            ("query,q", po::value<std::string>(), "query")
+            ("database,d", po::value<std::string>(), "database")
+            ("pager", po::value<std::string>(), "pager")
             ("multiline,m", "multiline")
             ("multiquery,n", "multiquery")
+            ("format,f", po::value<std::string>(), "default output format")
             ("testmode,T", "enable test hints in comments")
             ("ignore-error", "do not stop processing in multiquery mode")
-            ("format,f", boost::program_options::value<std::string>(), "default output format")
             ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
             ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
             ("stacktrace", "print stack traces of exceptions")
@@ -1548,27 +1586,29 @@ public:
             ("version,V", "print version information and exit")
             ("version-clean", "print version in machine-readable format and exit")
             ("echo", "in batch mode, print query before execution")
-            ("max_client_network_bandwidth", boost::program_options::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
-            ("compression", boost::program_options::value<bool>(), "enable or disable compression")
+            ("max_client_network_bandwidth", po::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
+            ("compression", po::value<bool>(), "enable or disable compression")
+            ("log-level", po::value<std::string>(), "client log level")
+            ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
             APPLY_FOR_SETTINGS(DECLARE_SETTING)
         ;
 #undef DECLARE_SETTING
 
         /// Commandline options related to external tables.
-        boost::program_options::options_description external_description("External tables options");
+        po::options_description external_description("External tables options");
         external_description.add_options()
-            ("file", boost::program_options::value<std::string>(), "data file or - for stdin")
-            ("name", boost::program_options::value<std::string>()->default_value("_data"), "name of the table")
-            ("format", boost::program_options::value<std::string>()->default_value("TabSeparated"), "data format")
-            ("structure", boost::program_options::value<std::string>(), "structure")
-            ("types", boost::program_options::value<std::string>(), "types")
+            ("file", po::value<std::string>(), "data file or - for stdin")
+            ("name", po::value<std::string>()->default_value("_data"), "name of the table")
+            ("format", po::value<std::string>()->default_value("TabSeparated"), "data format")
+            ("structure", po::value<std::string>(), "structure")
+            ("types", po::value<std::string>(), "types")
         ;
 
         /// Parse main commandline options.
-        boost::program_options::parsed_options parsed = boost::program_options::command_line_parser(
+        po::parsed_options parsed = po::command_line_parser(
             common_arguments.size(), common_arguments.data()).options(main_description).run();
-        boost::program_options::variables_map options;
-        boost::program_options::store(parsed, options);
+        po::variables_map options;
+        po::store(parsed, options);
 
         if (options.count("version") || options.count("V"))
         {
@@ -1591,14 +1631,17 @@ public:
             exit(0);
         }
 
+        if (options.count("log-level"))
+            Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
+
         size_t number_of_external_tables_with_stdin_source = 0;
         for (size_t i = 0; i < external_tables_arguments.size(); ++i)
         {
             /// Parse commandline options related to external tables.
-            boost::program_options::parsed_options parsed = boost::program_options::command_line_parser(
+            po::parsed_options parsed = po::command_line_parser(
                 external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
-            boost::program_options::variables_map external_options;
-            boost::program_options::store(parsed, external_options);
+            po::variables_map external_options;
+            po::store(parsed, external_options);
 
             try
             {
@@ -1672,6 +1715,8 @@ public:
             max_client_network_bandwidth = options["max_client_network_bandwidth"].as<int>();
         if (options.count("compression"))
             config().setBool("compression", options["compression"].as<bool>());
+        if (options.count("server_logs_file"))
+            server_logs_file = options["server_logs_file"].as<std::string>();
     }
 };
 
@@ -1691,6 +1736,11 @@ int mainEntryClickHouseClient(int argc, char ** argv)
         std::cerr << "Bad arguments: " << e.what() << std::endl;
         return 1;
     }
+    catch (...)
+    {
+        std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
+        return 1;
+    }
 
     return client.run();
 }

dbms/programs/client/TestHint.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <memory>
+#include <sstream>
+#include <iostream>
+#include <Core/Types.h>
+#include <Common/Exception.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+
+/// Checks expected server and client error codes in testmode.
+/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }".
+class TestHint
+{
+public:
+    TestHint(bool enabled_, const String & query)
+    :   enabled(enabled_)
+    {
+        if (!enabled_)
+            return;
+
+        /// TODO: This is absolutely wrong. Fragment may be contained inside string literal.
+        size_t pos = query.find("--");
+
+        if (pos != String::npos && query.find("--", pos + 2) != String::npos)
+            return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query.
+
+        if (pos != String::npos)
+        {
+            /// TODO: This is also wrong. Comment may already have ended by line break.
+            pos = query.find('{', pos + 2);
+
+            if (pos != String::npos)
+            {
+                String hint = query.substr(pos + 1);
+
+                /// TODO: And this is wrong for the same reason.
+                pos = hint.find('}');
+                hint.resize(pos);
+                parse(hint);
+            }
+        }
+    }
+
+    /// @returns true if it's possible to continue without reconnect
+    bool checkActual(int & actual_server_error, int & actual_client_error,
+                     bool & got_exception, std::unique_ptr<Exception> & last_exception) const
+    {
+        if (!enabled)
+            return true;
+
+        if (allErrorsExpected(actual_server_error, actual_client_error))
+        {
+            got_exception = false;
+            last_exception.reset();
+            actual_server_error = 0;
+            actual_client_error = 0;
+            return false;
+        }
+
+        if (lostExpectedError(actual_server_error, actual_client_error))
+        {
+            std::cerr << "Success when error expected. It expects server error "
+                << server_error << ", client error " << client_error << "." << std::endl;
+            got_exception = true;
+            last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS
+            return false;
+        }
+
+        return true;
+    }
+
+    int serverError() const { return server_error; }
+    int clientError() const { return client_error; }
+
+private:
+    bool enabled = false;
+    int server_error = 0;
+    int client_error = 0;
+
+    void parse(const String & hint)
+    {
+        std::stringstream ss;
+        ss << hint;
+        while (!ss.eof())
+        {
+            String item;
+            ss >> item;
+            if (item.empty())
+                break;
+
+            if (item == "serverError")
+                ss >> server_error;
+            else if (item == "clientError")
+                ss >> client_error;
+        }
+    }
+
+    bool allErrorsExpected(int actual_server_error, int actual_client_error) const
+    {
+        return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error);
+    }
+
+    bool lostExpectedError(int actual_server_error, int actual_client_error) const
+    {
+        return (server_error && !actual_server_error) || (client_error && !actual_client_error);
+    }
+};
+
+}

dbms/programs/copier/ClusterCopier.cpp

@@ -30,6 +30,7 @@
 #include <Common/ClickHouseRevision.h>
 #include <Common/formatReadable.h>
 #include <Common/DNSResolver.h>
+#include <Common/CurrentThread.h>
 #include <Common/escapeForFileName.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Client/Connection.h>
@@ -2143,6 +2144,9 @@ void ClusterCopierApp::mainImpl()
     context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
     context->setCurrentDatabase(default_database);
 
+    /// Initialize query scope just in case.
+    CurrentThread::QueryScope query_scope(*context);
+
     auto copier = std::make_unique<ClusterCopier>(task_path, host_id, default_database, *context);
     copier->setSafeMode(is_safe_mode);
     copier->setCopyFaultProbability(copy_fault_probability);

dbms/programs/local/LocalServer.cpp

@@ -25,6 +25,7 @@
 #include <Parsers/IAST.h>
 #include <common/ErrorHandlers.h>
 #include <Common/StatusFile.h>
+#include <Common/ThreadStatus.h>
 #include <Functions/registerFunctions.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <TableFunctions/registerTableFunctions.h>
@@ -270,6 +271,9 @@ void LocalServer::processQueries()
     context->setCurrentQueryId("");
     applyCmdSettings(*context);
 
+    /// Use the same query_id (and thread group) for all queries
+    CurrentThread::QueryScope query_scope_holder(*context);
+
     bool echo_query = config().hasOption("echo") || config().hasOption("verbose");
     std::exception_ptr exception;
 

dbms/programs/main.cpp

@@ -56,6 +56,10 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
 int mainEntryClickHouseObfuscator(int argc, char ** argv);
 #endif
+#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+int mainEntryClickHouseODBCBridge(int argc, char ** argv);
+#endif
+
 
 #if USE_EMBEDDED_COMPILER
     int mainEntryClickHouseClang(int argc, char ** argv);
@@ -101,6 +105,10 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_OBFUSCATOR
     {"obfuscator", mainEntryClickHouseObfuscator},
 #endif
+#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+    {"odbc-bridge", mainEntryClickHouseODBCBridge},
+#endif
+
 #if USE_EMBEDDED_COMPILER
     {"clang", mainEntryClickHouseClang},
     {"clang++", mainEntryClickHouseClang},

dbms/programs/odbc-bridge/CMakeLists.txt

@@ -0,0 +1,31 @@
+add_library (clickhouse-odbc-bridge-lib
+    PingHandler.cpp
+    MainHandler.cpp
+    ColumnInfoHandler.cpp
+    HandlerFactory.cpp
+    ODBCBridge.cpp
+    validateODBCConnectionString.cpp
+)
+
+target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
+target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
+
+if (USE_POCO_SQLODBC)
+    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
+    target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
+endif ()
+
+if (USE_POCO_DATAODBC)
+    target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
+    target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
+endif()
+
+
+if (ENABLE_TESTS)
+    add_subdirectory (tests)
+endif ()
+
+if (CLICKHOUSE_SPLIT_BINARY)
+    add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
+    target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
+endif ()

dbms/programs/odbc-bridge/ColumnInfoHandler.cpp

@@ -0,0 +1,123 @@
+#include "ColumnInfoHandler.h"
+#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
+#include <Poco/Data/ODBC/ODBCException.h>
+#include <Poco/Data/ODBC/SessionImpl.h>
+#include <Poco/Data/ODBC/Utility.h>
+#include <Poco/Net/HTTPServerRequest.h>
+#include <Poco/Net/HTTPServerResponse.h>
+
+#include <DataTypes/DataTypeFactory.h>
+#include <IO/WriteBufferFromHTTPServerResponse.h>
+#include <IO/WriteHelpers.h>
+#include <Common/HTMLForm.h>
+#include <common/logger_useful.h>
+#include <ext/scope_guard.h>
+#include "validateODBCConnectionString.h"
+
+namespace DB
+{
+namespace
+{
+    DataTypePtr getDataType(SQLSMALLINT type)
+    {
+        const auto & factory = DataTypeFactory::instance();
+
+        switch (type)
+        {
+            case SQL_INTEGER:
+                return factory.get("Int32");
+            case SQL_SMALLINT:
+                return factory.get("Int16");
+            case SQL_FLOAT:
+                return factory.get("Float32");
+            case SQL_REAL:
+                return factory.get("Float32");
+            case SQL_DOUBLE:
+                return factory.get("Float64");
+            case SQL_DATETIME:
+                return factory.get("DateTime");
+            case SQL_TYPE_TIMESTAMP:
+                return factory.get("DateTime");
+            case SQL_TYPE_DATE:
+                return factory.get("Date");
+            default:
+                return factory.get("String");
+        }
+    }
+}
+
+void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response)
+{
+    Poco::Net::HTMLForm params(request, request.stream());
+    LOG_TRACE(log, "Request URI: " + request.getURI());
+
+    auto process_error = [&response, this](const std::string & message) {
+        response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
+        if (!response.sent())
+            response.send() << message << std::endl;
+        LOG_WARNING(log, message);
+    };
+
+    if (!params.has("table"))
+    {
+        process_error("No 'table' param in request URL");
+        return;
+    }
+    if (!params.has("connection_string"))
+    {
+        process_error("No 'connection_string' in request URL");
+        return;
+    }
+    std::string table_name = params.get("table");
+    std::string connection_string = params.get("connection_string");
+    LOG_TRACE(log, "Will fetch info for table '" << table_name << "'");
+    LOG_TRACE(log, "Got connection str '" << connection_string << "'");
+
+    try
+    {
+        Poco::Data::ODBC::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC);
+        SQLHDBC hdbc = session.dbc().handle();
+
+        SQLHSTMT hstmt = nullptr;
+
+        if (Poco::Data::ODBC::Utility::isError(SQLAllocStmt(hdbc, &hstmt)))
+            throw Poco::Data::ODBC::ODBCException("Could not allocate connection handle.");
+
+        SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP));
+
+        /// TODO Why not do SQLColumns instead?
+        std::string query = "SELECT * FROM " + table_name + " WHERE 1 = 0";
+        if (Poco::Data::ODBC::Utility::isError(Poco::Data::ODBC::SQLPrepare(hstmt, reinterpret_cast<SQLCHAR *>(&query[0]), query.size())))
+            throw Poco::Data::ODBC::DescriptorException(session.dbc());
+
+        if (Poco::Data::ODBC::Utility::isError(SQLExecute(hstmt)))
+            throw Poco::Data::ODBC::StatementException(hstmt);
+
+        SQLSMALLINT cols = 0;
+        if (Poco::Data::ODBC::Utility::isError(SQLNumResultCols(hstmt, &cols)))
+            throw Poco::Data::ODBC::StatementException(hstmt);
+
+        /// TODO cols not checked
+
+        NamesAndTypesList columns;
+        for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol)
+        {
+            SQLSMALLINT type = 0;
+            /// TODO Why 301?
+            SQLCHAR column_name[301];
+            /// TODO Result is not checked.
+            Poco::Data::ODBC::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), NULL, &type, NULL, NULL, NULL);
+            columns.emplace_back(reinterpret_cast<char *>(column_name), getDataType(type));
+        }
+
+        WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout);
+        writeStringBinary(columns.toString(), out);
+    }
+    catch (...)
+    {
+        process_error("Error getting columns from ODBC '" + getCurrentExceptionMessage(false) + "'");
+        tryLogCurrentException(log);
+    }
+}
+}
+#endif

dbms/programs/odbc-bridge/ColumnInfoHandler.h

@@ -0,0 +1,30 @@
+#pragma once
+#include <Common/config.h>
+#include <Interpreters/Context.h>
+#include <Poco/Logger.h>
+#include <Poco/Net/HTTPRequestHandler.h>
+
+#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
+/** The structure of the table is taken from the query "SELECT * FROM table WHERE 1=0".
+  * TODO: It would be much better to utilize ODBC methods dedicated for columns description.
+  * If there is no such table, an exception is thrown.
+  */
+namespace DB
+{
+class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler
+{
+public:
+    ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr<Context> context_)
+        : log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_)
+    {
+    }
+
+    void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
+
+private:
+    Poco::Logger * log;
+    size_t keep_alive_timeout;
+    std::shared_ptr<Context> context;
+};
+}
+#endif

dbms/programs/odbc-bridge/HandlerFactory.cpp

@@ -0,0 +1,34 @@
+#include "HandlerFactory.h"
+#include "PingHandler.h"
+#include "ColumnInfoHandler.h"
+#include <Common/HTMLForm.h>
+
+#include <Poco/Ext/SessionPoolHelpers.h>
+#include <Poco/Net/HTTPServerRequest.h>
+#include <common/logger_useful.h>
+
+namespace DB
+{
+Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request)
+{
+    Poco::URI uri{request.getURI()};
+    LOG_TRACE(log, "Request URI: " + uri.toString());
+
+    if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
+        return new PingHandler(keep_alive_timeout);
+
+    if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
+    {
+
+        if (uri.getPath() == "/columns_info")
+#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
+            return new ODBCColumnsInfoHandler(keep_alive_timeout, context);
+#else
+            return nullptr;
+#endif
+        else
+            return new ODBCHandler(pool_map, keep_alive_timeout, context);
+    }
+    return nullptr;
+}
+}

dbms/programs/odbc-bridge/HandlerFactory.h

@@ -0,0 +1,38 @@
+#pragma once
+#include <Interpreters/Context.h>
+#include <Poco/Logger.h>
+#include <Poco/Net/HTTPRequestHandler.h>
+#include <Poco/Net/HTTPRequestHandlerFactory.h>
+#include "MainHandler.h"
+#include "ColumnInfoHandler.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+    #include <Poco/Data/SessionPool.h>
+#pragma GCC diagnostic pop
+
+
+namespace DB
+{
+/** Factory for '/ping', '/' and '/columns_info' handlers.
+  * Also stores Session pools for ODBC connections
+  */
+class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
+{
+public:
+    HandlerFactory(const std::string & name_, size_t keep_alive_timeout_, std::shared_ptr<Context> context_)
+        : log(&Poco::Logger::get(name_)), name(name_), keep_alive_timeout(keep_alive_timeout_), context(context_)
+    {
+        pool_map = std::make_shared<ODBCHandler::PoolMap>();
+    }
+
+    Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override;
+
+private:
+    Poco::Logger * log;
+    std::string name;
+    size_t keep_alive_timeout;
+    std::shared_ptr<Context> context;
+    std::shared_ptr<ODBCHandler::PoolMap> pool_map;
+};
+}

dbms/programs/odbc-bridge/MainHandler.cpp

@@ -0,0 +1,126 @@
+#include "MainHandler.h"
+
+#include "validateODBCConnectionString.h"
+
+#include <memory>
+#include <DataStreams/copyData.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <Dictionaries/ODBCBlockInputStream.h>
+#include <Formats/BinaryRowInputStream.h>
+#include <Formats/FormatFactory.h>
+#include <IO/WriteBufferFromHTTPServerResponse.h>
+#include <IO/WriteHelpers.h>
+#include <Interpreters/Context.h>
+#include <Poco/Ext/SessionPoolHelpers.h>
+#include <Poco/Net/HTTPServerRequest.h>
+#include <Poco/Net/HTTPServerResponse.h>
+#include <Common/HTMLForm.h>
+#include <common/logger_useful.h>
+
+namespace DB
+{
+namespace
+{
+    std::unique_ptr<Block> parseColumns(std::string && column_string)
+    {
+        std::unique_ptr<Block> sample_block = std::make_unique<Block>();
+        auto names_and_types = NamesAndTypesList::parse(column_string);
+        for (const NameAndTypePair & column_data : names_and_types)
+            sample_block->insert({column_data.type, column_data.name});
+        return sample_block;
+    }
+}
+
+
+ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str)
+{
+    std::lock_guard lock(mutex);
+    if (!pool_map->count(connection_str))
+    {
+        pool_map->emplace(connection_str, createAndCheckResizePocoSessionPool([connection_str] {
+            return std::make_shared<Poco::Data::SessionPool>("ODBC", validateODBCConnectionString(connection_str));
+        }));
+    }
+    return pool_map->at(connection_str);
+}
+
+void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response)
+{
+    Poco::Net::HTMLForm params(request, request.stream());
+    LOG_TRACE(log, "Request URI: " + request.getURI());
+
+    auto process_error = [&response, this](const std::string & message) {
+        response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
+        if (!response.sent())
+            response.send() << message << std::endl;
+        LOG_WARNING(log, message);
+    };
+
+    if (!params.has("query"))
+    {
+        process_error("No 'query' in request body");
+        return;
+    }
+
+    if (!params.has("columns"))
+    {
+        process_error("No 'columns' in request URL");
+        return;
+    }
+
+    if (!params.has("connection_string"))
+    {
+        process_error("No 'connection_string' in request URL");
+        return;
+    }
+
+    size_t max_block_size = DEFAULT_BLOCK_SIZE;
+    if (params.has("max_block_size"))
+    {
+        std::string max_block_size_str = params.get("max_block_size", "");
+        if (max_block_size_str.empty())
+        {
+            process_error("Empty max_block_size specified");
+            return;
+        }
+        max_block_size = parse<size_t>(max_block_size_str);
+    }
+
+    std::string columns = params.get("columns");
+    std::unique_ptr<Block> sample_block;
+    try
+    {
+        sample_block = parseColumns(std::move(columns));
+    }
+    catch (const Exception & ex)
+    {
+        process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'");
+        LOG_WARNING(log, ex.getStackTrace().toString());
+        return;
+    }
+
+    std::string format = params.get("format", "RowBinary");
+    std::string query = params.get("query");
+    LOG_TRACE(log, "Query: " << query);
+
+    std::string connection_string = params.get("connection_string");
+    LOG_TRACE(log, "Connection string: '" << connection_string << "'");
+
+    WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout);
+    try
+    {
+        BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context);
+        auto pool = getPool(connection_string);
+        ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size);
+        copyData(inp, *writer);
+    }
+    catch (...)
+    {
+        auto message = getCurrentExceptionMessage(true);
+        response.setStatusAndReason(
+            Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, bacause of too soon response sending
+        writeStringBinary(message, out);
+        tryLogCurrentException(log);
+    }
+}
+}

dbms/programs/odbc-bridge/MainHandler.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <Interpreters/Context.h>
+#include <Poco/Logger.h>
+#include <Poco/Net/HTTPRequestHandler.h>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+    #include <Poco/Data/SessionPool.h>
+#pragma GCC diagnostic pop
+
+namespace DB
+{
+/** Main handler for requests to ODBC driver
+  * requires connection_string and columns in request params
+  * and also query in request body
+  * response in RowBinary format
+  */
+class ODBCHandler : public Poco::Net::HTTPRequestHandler
+{
+public:
+    using PoolPtr = std::shared_ptr<Poco::Data::SessionPool>;
+    using PoolMap = std::unordered_map<std::string, PoolPtr>;
+
+    ODBCHandler(std::shared_ptr<PoolMap> pool_map_,
+        size_t keep_alive_timeout_,
+        std::shared_ptr<Context> context_)
+        : log(&Poco::Logger::get("ODBCHandler"))
+        , pool_map(pool_map_)
+        , keep_alive_timeout(keep_alive_timeout_)
+        , context(context_)
+    {
+    }
+
+    void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
+
+private:
+    Poco::Logger * log;
+
+    std::shared_ptr<PoolMap> pool_map;
+    size_t keep_alive_timeout;
+    std::shared_ptr<Context> context;
+
+    static inline std::mutex mutex;
+
+    PoolPtr getPool(const std::string & connection_str);
+};
+
+}

dbms/programs/odbc-bridge/ODBCBridge.cpp

@@ -0,0 +1,205 @@
+#include "ODBCBridge.h"
+#include "HandlerFactory.h"
+
+#include <string>
+#include <errno.h>
+#include <IO/ReadHelpers.h>
+#include <boost/program_options.hpp>
+#include <Poco/Net/HTTPServer.h>
+#include <Poco/Net/NetException.h>
+#include <Poco/String.h>
+#include <Poco/Util/HelpFormatter.h>
+#include <Common/Exception.h>
+#include <Common/StringUtils/StringUtils.h>
+#include <Common/config.h>
+#include <common/logger_useful.h>
+#include <ext/scope_guard.h>
+#include <ext/range.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ARGUMENT_OUT_OF_BOUND;
+}
+
+namespace
+{
+    Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log)
+    {
+        Poco::Net::SocketAddress socket_address;
+        try
+        {
+            socket_address = Poco::Net::SocketAddress(host, port);
+        }
+        catch (const Poco::Net::DNSException & e)
+        {
+            const auto code = e.code();
+            if (code == EAI_FAMILY
+#if defined(EAI_ADDRFAMILY)
+                || code == EAI_ADDRFAMILY
+#endif
+            )
+            {
+                LOG_ERROR(log,
+                    "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message()
+                                                   << ". "
+                                                      "If it is an IPv6 address and your host has disabled IPv6, then consider to "
+                                                      "specify IPv4 address to listen in <listen_host> element of configuration "
+                                                      "file. Example: <listen_host>0.0.0.0</listen_host>");
+            }
+
+            throw;
+        }
+        return socket_address;
+    }
+
+    Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, Poco::Logger * log)
+    {
+        auto address = makeSocketAddress(host, port, log);
+#if POCO_VERSION < 0x01080000
+        socket.bind(address, /* reuseAddress = */ true);
+#else
+        socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ false);
+#endif
+
+        socket.listen(/* backlog = */ 64);
+
+        return address;
+    };
+}
+
+void ODBCBridge::handleHelp(const std::string &, const std::string &)
+{
+    Poco::Util::HelpFormatter helpFormatter(options());
+    helpFormatter.setCommand(commandName());
+    helpFormatter.setHeader("HTTP-proxy for odbc requests");
+    helpFormatter.setUsage("--http-port <port>");
+    helpFormatter.format(std::cerr);
+
+    stopOptionsProcessing();
+}
+
+
+void ODBCBridge::defineOptions(Poco::Util::OptionSet & options)
+{
+    options.addOption(Poco::Util::Option("http-port", "", "port to listen").argument("http-port", true).binding("http-port"));
+    options.addOption(
+        Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host"));
+    options.addOption(
+        Poco::Util::Option("http-timeout", "", "http timout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
+
+    options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024")
+                          .argument("max-server-connections")
+                          .binding("max-server-connections"));
+    options.addOption(Poco::Util::Option("keep-alive-timeout", "", "keepalive timeout, default 10")
+                          .argument("keep-alive-timeout")
+                          .binding("keep-alive-timeout"));
+
+    options.addOption(Poco::Util::Option("log-level", "", "sets log level, default info").argument("log-level").binding("logger.level"));
+
+    options.addOption(
+        Poco::Util::Option("log-path", "", "log path for all logs, default console").argument("log-path").binding("logger.log"));
+
+    options.addOption(Poco::Util::Option("err-log-path", "", "err log path for all logs, default no")
+                          .argument("err-log-path")
+                          .binding("logger.errorlog"));
+
+    using Me = std::decay_t<decltype(*this)>;
+    options.addOption(Poco::Util::Option("help", "", "produce this help message")
+                          .binding("help")
+                          .callback(Poco::Util::OptionCallback<Me>(this, &Me::handleHelp)));
+
+    ServerApplication::defineOptions(options); /// Don't need complex BaseDaemon's .xml config
+}
+
+void ODBCBridge::initialize(Application & self)
+{
+    BaseDaemon::closeFDs();
+    is_help = config().has("help");
+
+    if (is_help)
+        return;
+
+    if (!config().has("logger.log"))
+        config().setBool("logger.console", true);
+
+    config().setString("logger", "ODBCBridge");
+
+    buildLoggers(config());
+    log = &logger();
+    hostname = config().getString("listen-host", "localhost");
+    port = config().getUInt("http-port");
+    if (port > 0xFFFF)
+        throw Exception("Out of range 'http-port': " + std::to_string(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+    http_timeout = config().getUInt("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT);
+    max_server_connections = config().getUInt("max-server-connections", 1024);
+    keep_alive_timeout = config().getUInt("keep-alive-timeout", 10);
+
+    initializeTerminationAndSignalProcessing();
+
+    ServerApplication::initialize(self);
+}
+
+void ODBCBridge::uninitialize()
+{
+    BaseDaemon::uninitialize();
+}
+
+int ODBCBridge::main(const std::vector<std::string> & /*args*/)
+{
+    if (is_help)
+        return Application::EXIT_OK;
+
+    LOG_INFO(log, "Starting up");
+    Poco::Net::ServerSocket socket;
+    auto address = socketBindListen(socket, hostname, port, log);
+    socket.setReceiveTimeout(http_timeout);
+    socket.setSendTimeout(http_timeout);
+    Poco::ThreadPool server_pool(3, max_server_connections);
+    Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
+    http_params->setTimeout(http_timeout);
+    http_params->setKeepAliveTimeout(keep_alive_timeout);
+
+    context = std::make_shared<Context>(Context::createGlobal());
+    context->setGlobalContext(*context);
+
+    auto server = Poco::Net::HTTPServer(
+        new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params);
+    server.start();
+
+    LOG_INFO(log, "Listening http://" + address.toString());
+
+    SCOPE_EXIT({
+        LOG_DEBUG(log, "Received termination signal.");
+        LOG_DEBUG(log, "Waiting for current connections to close.");
+        server.stop();
+        for (size_t count : ext::range(1, 6))
+        {
+            if (server.currentConnections() == 0)
+                break;
+            LOG_DEBUG(log, "Waiting for " << server.currentConnections() << " connections, try " << count);
+            std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+        }
+    });
+
+    waitForTerminationRequest();
+    return Application::EXIT_OK;
+}
+}
+
+int mainEntryClickHouseODBCBridge(int argc, char ** argv)
+{
+    DB::ODBCBridge app;
+    try
+    {
+        return app.run(argc, argv);
+    }
+    catch (...)
+    {
+        std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
+        auto code = DB::getCurrentExceptionCode();
+        return code ? code : 1;
+    }
+}

dbms/programs/odbc-bridge/ODBCBridge.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <Interpreters/Context.h>
+#include <Poco/Logger.h>
+#include <daemon/BaseDaemon.h>
+
+namespace DB
+{
+/** Class represents clickhouse-odbc-bridge server, which listen
+  * incoming HTTP POST and GET requests on specified port and host.
+  * Has two handlers '/' for all incoming POST requests to ODBC driver
+  * and /ping for GET request about service status
+  */
+class ODBCBridge : public BaseDaemon
+{
+public:
+    void defineOptions(Poco::Util::OptionSet & options) override;
+
+protected:
+    void initialize(Application & self) override;
+
+    void uninitialize() override;
+
+    int main(const std::vector<std::string> & args) override;
+
+private:
+    void handleHelp(const std::string &, const std::string &);
+
+    bool is_help;
+    std::string hostname;
+    size_t port;
+    size_t http_timeout;
+    std::string log_level;
+    size_t max_server_connections;
+    size_t keep_alive_timeout;
+
+    Poco::Logger * log;
+
+    std::shared_ptr<Context> context; /// need for settings only
+};
+}

dbms/programs/odbc-bridge/PingHandler.cpp

@@ -0,0 +1,22 @@
+#include "PingHandler.h"
+#include <Poco/Net/HTTPServerRequest.h>
+#include <Poco/Net/HTTPServerResponse.h>
+#include <Common/Exception.h>
+#include <IO/HTTPCommon.h>
+
+namespace DB
+{
+void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response)
+{
+    try
+    {
+        setResponseDefaultHeaders(response, keep_alive_timeout);
+        const char * data = "Ok.\n";
+        response.sendBuffer(data, strlen(data));
+    }
+    catch (...)
+    {
+        tryLogCurrentException("PingHandler");
+    }
+}
+}

dbms/programs/odbc-bridge/PingHandler.h

@@ -0,0 +1,17 @@
+#pragma once
+#include <Poco/Net/HTTPRequestHandler.h>
+
+namespace DB
+{
+/** Simple ping handler, answers "Ok." to GET request
+ */
+class PingHandler : public Poco::Net::HTTPRequestHandler
+{
+public:
+    PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {}
+    void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
+
+private:
+    size_t keep_alive_timeout;
+};
+}

dbms/programs/odbc-bridge/README.md

@@ -0,0 +1,38 @@
+# clickhouse-odbc-bridge
+
+Simple HTTP-server which works like a proxy for ODBC driver. The main motivation
+was possible segfaults or another faults in ODBC implementations, which can
+crash whole clickhouse-server process.
+
+This tool works via HTTP, not via pipes, shared memory, or TCP because:
+- It's simplier to implement
+- It's simplier to debug
+- jdbc-bridge can be implemented in the same way
+
+## Usage
+
+`clickhouse-server` use this tool inside odbc table function and StorageODBC.
+However it can be used as standalone tool from command line with the following
+parameters in POST-request URL:
+- `connection_string` -- ODBC connection string.
+- `columns` -- columns in ClickHouse NamesAndTypesList format, name in backticks,
+  type as string. Name and type are space separated, rows separated with
+  newline.
+- `max_block_size` -- optional parameter, sets maximum size of single block.
+Query is send in post body. Response is returned in RowBinary format.
+
+## Example:
+
+```bash
+$ clickhouse-odbc-bridge --http-port 9018 --daemon
+
+$ curl -d "query=SELECT PageID, ImpID, AdType FROM Keys ORDER BY PageID, ImpID" --data-urlencode "connection_string=DSN=ClickHouse;DATABASE=stat" --data-urlencode "columns=columns format version: 1
+3 columns:
+\`PageID\` String
+\`ImpID\` String
+\`AdType\` String
+"  "http://localhost:9018/" > result.txt
+
+$ cat result.txt
+12246623837185725195925621517
+```

dbms/programs/odbc-bridge/odbc-bridge.cpp

@@ -0,0 +1,2 @@
+int mainEntryClickHouseODBCBridge(int argc, char ** argv);
+int main(int argc_, char ** argv_) { return mainEntryClickHouseODBCBridge(argc_, argv_); }

dbms/programs/odbc-bridge/tests/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
+target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)

dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp

@@ -1,6 +1,6 @@
 #include <iostream>
 #include <Common/Exception.h>
-#include <Dictionaries/validateODBCConnectionString.h>
+#include "../validateODBCConnectionString.h"
 
 
 using namespace DB;

dbms/programs/odbc-bridge/validateODBCConnectionString.cpp

@@ -5,7 +5,7 @@
 #include <common/find_first_symbols.h>
 #include <Common/Exception.h>
 #include <Common/StringUtils/StringUtils.h>
-#include <Dictionaries/validateODBCConnectionString.h>
+#include "validateODBCConnectionString.h"
 
 
 namespace DB

dbms/programs/server/HTTPHandler.cpp

@@ -9,10 +9,11 @@
 
 #include <ext/scope_guard.h>
 
-#include <Common/ExternalTable.h>
+#include <Core/ExternalTable.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/escapeForFileName.h>
 #include <Common/getFQDNOrHostName.h>
+#include <Common/CurrentThread.h>
 #include <IO/ReadBufferFromIStream.h>
 #include <IO/ZlibInflatingReadBuffer.h>
 #include <IO/ReadBufferFromString.h>
@@ -208,6 +209,13 @@ void HTTPHandler::processQuery(
     Poco::Net::HTTPServerResponse & response,
     Output & used_output)
 {
+    Context context = server.context();
+    context.setGlobalContext(server.context());
+
+    /// It will forcibly detach query even if unexpected error ocurred and detachQuery() was not called
+    /// Normal detaching is happen in BlockIO callbacks
+    CurrentThread::QueryScope query_scope_holder(context);
+
     LOG_TRACE(log, "Request URI: " << request.getURI());
 
     std::istream & istr = request.stream();
@@ -257,14 +265,9 @@ void HTTPHandler::processQuery(
     }
 
     std::string query_id = params.get("query_id", "");
-
-    const auto & config = server.config();
-
-    Context context = server.context();
-    context.setGlobalContext(server.context());
-
     context.setUser(user, password, request.clientAddress(), quota_key);
     context.setCurrentQueryId(query_id);
+    CurrentThread::attachQueryContext(context);
 
     /// The user could specify session identifier and session timeout.
     /// It allows to modify settings, create temporary tables and reuse them in subsequent requests.
@@ -273,6 +276,7 @@ void HTTPHandler::processQuery(
     String session_id;
     std::chrono::steady_clock::duration session_timeout;
     bool session_is_set = params.has("session_id");
+    const auto & config = server.config();
 
     if (session_is_set)
     {
@@ -421,34 +425,45 @@ void HTTPHandler::processQuery(
 
     std::unique_ptr<ReadBuffer> in;
 
-    // Used in case of POST request with form-data, but it not to be expectd to be deleted after that scope
+    static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
+        "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"};
+
+    Names reserved_param_suffixes;
+
+    auto param_could_be_skipped = [&] (const String & name)
+    {
+        if (reserved_param_names.count(name))
+            return true;
+
+        for (const String & suffix : reserved_param_suffixes)
+        {
+            if (endsWith(name, suffix))
+                return true;
+        }
+
+        return false;
+    };
+
+    /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
     std::string full_query;
 
     /// Support for "external data for query processing".
     if (startsWith(request.getContentType().data(), "multipart/form-data"))
     {
         ExternalTablesHandler handler(context, params);
-
-        /// Params are of both form params POST and uri (GET params)
         params.load(request, istr, handler);
 
-        for (const auto & it : params)
-        {
-            if (it.first == "query")
-            {
-                full_query += it.second;
-            }
-        }
-        in = std::make_unique<ReadBufferFromString>(full_query);
+        /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
+        reserved_param_suffixes.emplace_back("_format");
+        reserved_param_suffixes.emplace_back("_types");
+        reserved_param_suffixes.emplace_back("_structure");
 
-        /// Erase unneeded parameters to avoid confusing them later with context settings or query
-        /// parameters.
-        for (const auto & it : handler.names)
-        {
-            params.erase(it + "_format");
-            params.erase(it + "_types");
-            params.erase(it + "_structure");
-        }
+        /// Params are of both form params POST and uri (GET params)
+        for (const auto & it : params)
+            if (it.first == "query")
+                full_query += it.second;
+
+        in = std::make_unique<ReadBufferFromString>(full_query);
     }
     else
         in = std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
@@ -475,11 +490,6 @@ void HTTPHandler::processQuery(
 
     auto readonly_before_query = settings.readonly;
 
-    NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
-        "buffer_size", "wait_end_of_query",
-        "session_id", "session_timeout", "session_check"
-    };
-
     for (auto it = params.begin(); it != params.end(); ++it)
     {
         if (it->first == "database")
@@ -490,7 +500,7 @@ void HTTPHandler::processQuery(
         {
             context.setDefaultFormat(it->second);
         }
-        else if (reserved_param_names.find(it->first) != reserved_param_names.end())
+        else if (param_could_be_skipped(it->first))
         {
         }
         else

dbms/programs/server/MetricsTransmitter.cpp

@@ -81,7 +81,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
     {
         for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
         {
-            const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed);
+            const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
             const auto counter_increment = counter - prev_counters[i];
             prev_counters[i] = counter;
 

dbms/programs/server/Server.cpp

@@ -22,6 +22,7 @@
 #include <Common/getFQDNOrHostName.h>
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/getNumberOfPhysicalCPUCores.h>
+#include <Common/TaskStatsInfoGetter.h>
 #include <IO/HTTPCommon.h>
 #include <Interpreters/AsynchronousMetrics.h>
 #include <Interpreters/DDLWorker.h>
@@ -365,6 +366,13 @@ int Server::main(const std::vector<std::string> & /*args*/)
         dns_cache_updater = std::make_unique<DNSCacheUpdater>(*global_context);
     }
 
+    if (!TaskStatsInfoGetter::checkProcessHasRequiredPermissions())
+    {
+        LOG_INFO(log, "It looks like the process has not CAP_NET_ADMIN capability, some performance statistics will be disabled."
+                      " It could happen due to incorrect clickhouse package installation."
+                      " You could resolve the problem manually calling 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'");
+    }
+
     {
         Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
 

dbms/programs/server/TCPHandler.cpp

@@ -1,7 +1,15 @@
-#include "TCPHandler.h"
-
 #include <iomanip>
+#include <ext/scope_guard.h>
 #include <Poco/Net/NetException.h>
+#include <daemon/OwnSplitChannel.h>
+
+#include <Common/ClickHouseRevision.h>
+#include <Common/CurrentThread.h>
+#include <Common/Stopwatch.h>
+#include <Common/ClickHouseRevision.h>
+#include <Common/Stopwatch.h>
+#include <Common/NetException.h>
+#include <Common/config_version.h>
 #include <IO/Progress.h>
 #include <IO/CompressedReadBuffer.h>
 #include <IO/CompressedWriteBuffer.h>
@@ -15,14 +23,13 @@
 #include <Interpreters/executeQuery.h>
 #include <Interpreters/Quota.h>
 #include <Interpreters/TablesStatus.h>
+#include <Interpreters/InternalTextLogsQueue.h>
 #include <Storages/StorageMemory.h>
 #include <Storages/StorageReplicatedMergeTree.h>
-#include <Common/ClickHouseRevision.h>
-#include <Common/Stopwatch.h>
-#include <Common/ExternalTable.h>
-#include <Common/NetException.h>
-#include <Common/config_version.h>
-#include <ext/scope_guard.h>
+#include <Core/ExternalTable.h>
+
+#include "TCPHandler.h"
+
 
 namespace DB
 {
@@ -140,13 +147,29 @@ void TCPHandler::runImpl()
             if (!receivePacket())
                 continue;
 
-            /// Get blocks of temporary tables
-            readData(global_settings);
+            CurrentThread::initializeQuery();
 
-            /// Reset the input stream, as we received an empty block while receiving external table data.
-            /// So, the stream has been marked as cancelled and we can't read from it anymore.
-            state.block_in.reset();
-            state.maybe_compressed_in.reset();  /// For more accurate accounting by MemoryTracker.
+            /// Should we send internal logs to client?
+            if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS
+                && query_context.getSettingsRef().send_logs_level.value != "none")
+            {
+                state.logs_queue = std::make_shared<InternalTextLogsQueue>();
+                state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context.getSettingsRef().send_logs_level.value);
+                CurrentThread::attachInternalTextLogsQueue(state.logs_queue);
+            }
+
+            query_context.setExternalTablesInitializer([&global_settings, this] (Context & context) {
+                if (&context != &query_context)
+                    throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR);
+
+                /// Get blocks of temporary tables
+                readData(global_settings);
+
+                /// Reset the input stream, as we received an empty block while receiving external table data.
+                /// So, the stream has been marked as cancelled and we can't read from it anymore.
+                state.block_in.reset();
+                state.maybe_compressed_in.reset();  /// For more accurate accounting by MemoryTracker.
+            });
 
             /// Processing Query
             state.io = executeQuery(state.query, query_context, false, state.stage);
@@ -163,8 +186,9 @@ void TCPHandler::runImpl()
             else
                 processOrdinaryQuery();
 
-            sendEndOfStream();
+            sendLogs();
 
+            sendEndOfStream();
             state.reset();
         }
         catch (const Exception & e)
@@ -209,7 +233,20 @@ void TCPHandler::runImpl()
         try
         {
             if (exception)
+            {
+                try
+                {
+                    /// Try to send logs to client, but it could be risky too
+                    /// Assume that we can't break output here
+                    sendLogs();
+                }
+                catch (...)
+                {
+                    tryLogCurrentException(log, "Can't send logs to client");
+                }
+
                 sendException(*exception);
+            }
         }
         catch (...)
         {
@@ -220,6 +257,9 @@ void TCPHandler::runImpl()
 
         try
         {
+            /// It will forcibly detach query even if unexpected error ocсurred and detachQuery() was not called
+            CurrentThread::detachQueryIfNotDetached();
+
             state.reset();
         }
         catch (...)
@@ -252,12 +292,14 @@ void TCPHandler::readData(const Settings & global_settings)
     constexpr size_t min_poll_interval = 5000; // 5 ms
     size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval));
 
-    while (1)
+    sendLogs();
+
+    while (true)
     {
         Stopwatch watch(CLOCK_MONOTONIC_COARSE);
 
         /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down.
-        while (1)
+        while (true)
         {
             if (static_cast<ReadBufferFromPocoSocket &>(*in).poll(poll_interval))
                 break;
@@ -289,6 +331,8 @@ void TCPHandler::readData(const Settings & global_settings)
         /// We accept and process data. And if they are over, then we leave.
         if (!receivePacket())
             break;
+
+        sendLogs();
     }
 }
 
@@ -346,6 +390,8 @@ void TCPHandler::processOrdinaryQuery()
                         sendProgress();
                     }
 
+                    sendLogs();
+
                     if (async_in.poll(query_context.getSettingsRef().interactive_delay / 1000))
                     {
                         /// There is the following result block.
@@ -368,6 +414,7 @@ void TCPHandler::processOrdinaryQuery()
                 sendExtremes();
                 sendProfileInfo();
                 sendProgress();
+                sendLogs();
             }
 
             sendData(block);
@@ -692,11 +739,14 @@ void TCPHandler::initBlockOutput(const Block & block)
 {
     if (!state.block_out)
     {
-        if (state.compression == Protocol::Compression::Enable)
-            state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
-                *out, CompressionSettings(query_context.getSettingsRef()));
-        else
-            state.maybe_compressed_out = out;
+        if (!state.maybe_compressed_out)
+        {
+            if (state.compression == Protocol::Compression::Enable)
+                state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
+                    *out, CompressionSettings(query_context.getSettingsRef()));
+            else
+                state.maybe_compressed_out = out;
+        }
 
         state.block_out = std::make_shared<NativeBlockOutputStream>(
             *state.maybe_compressed_out,
@@ -705,6 +755,18 @@ void TCPHandler::initBlockOutput(const Block & block)
     }
 }
 
+void TCPHandler::initLogsBlockOutput(const Block & block)
+{
+    if (!state.logs_block_out)
+    {
+        /// Use uncompressed stream since log blocks usually contain only one row
+        state.logs_block_out = std::make_shared<NativeBlockOutputStream>(
+            *out,
+            client_revision,
+            block.cloneEmpty());
+    }
+}
+
 
 bool TCPHandler::isQueryCancelled()
 {
@@ -745,6 +807,7 @@ void TCPHandler::sendData(const Block & block)
     initBlockOutput(block);
 
     writeVarUInt(Protocol::Server::Data, *out);
+    /// Send external table name (empty name is the main table)
     writeStringBinary("", *out);
 
     state.block_out->write(block);
@@ -753,6 +816,19 @@ void TCPHandler::sendData(const Block & block)
 }
 
 
+void TCPHandler::sendLogData(const Block & block)
+{
+    initLogsBlockOutput(block);
+
+    writeVarUInt(Protocol::Server::Log, *out);
+    /// Send log tag (empty tag is the default tag)
+    writeStringBinary("", *out);
+
+    state.logs_block_out->write(block);
+    out->next();
+}
+
+
 void TCPHandler::sendException(const Exception & e)
 {
     writeVarUInt(Protocol::Server::Exception, *out);
@@ -784,6 +860,37 @@ void TCPHandler::sendProgress()
 }
 
 
+void TCPHandler::sendLogs()
+{
+    if (!state.logs_queue)
+        return;
+
+    MutableColumns logs_columns;
+    MutableColumns curr_logs_columns;
+    size_t rows = 0;
+
+    for (; state.logs_queue->tryPop(curr_logs_columns); ++rows)
+    {
+        if (rows == 0)
+        {
+            logs_columns = std::move(curr_logs_columns);
+        }
+        else
+        {
+            for (size_t j = 0; j < logs_columns.size(); ++j)
+                logs_columns[j]->insertRangeFrom(*curr_logs_columns[j], 0, curr_logs_columns[j]->size());
+        }
+    }
+
+    if (rows > 0)
+    {
+        Block block = InternalTextLogsQueue::getSampleBlock();
+        block.setColumns(std::move(logs_columns));
+        sendLogData(block);
+    }
+}
+
+
 void TCPHandler::run()
 {
     try

dbms/programs/server/TCPHandler.h

@@ -5,16 +5,18 @@
 #include <Common/getFQDNOrHostName.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Stopwatch.h>
-#include <IO/Progress.h>
 #include <Core/Protocol.h>
 #include <Core/QueryProcessingStage.h>
-#include <DataStreams/BlockIO.h>
+#include <IO/Progress.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <DataStreams/BlockIO.h>
+#include <Interpreters/InternalTextLogsQueue.h>
 #include <Client/TimeoutSetter.h>
 
 #include "IServer.h"
 
+
 namespace CurrentMetrics
 {
     extern const Metric TCPConnection;
@@ -63,6 +65,9 @@ struct QueryState
     /// Timeouts setter for current query
     std::unique_ptr<TimeoutSetter> timeout_setter;
 
+    /// A queue with internal logs that will be passed to client
+    InternalTextLogsQueuePtr logs_queue;
+    BlockOutputStreamPtr logs_block_out;
 
     void reset()
     {
@@ -140,8 +145,10 @@ private:
 
     void sendHello();
     void sendData(const Block & block);    /// Write a block to the network.
+    void sendLogData(const Block & block);
     void sendException(const Exception & e);
     void sendProgress();
+    void sendLogs();
     void sendEndOfStream();
     void sendProfileInfo();
     void sendTotals();
@@ -150,6 +157,7 @@ private:
     /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled.
     void initBlockInput();
     void initBlockOutput(const Block & block);
+    void initLogsBlockOutput(const Block & block);
 
     bool isQueryCancelled();
 

dbms/programs/server/config.d/listen.xml

@@ -0,0 +1 @@
+<yandex><listen_host>0.0.0.0</listen_host></yandex>

dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp

@@ -0,0 +1,30 @@
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionRetention.h>
+#include <AggregateFunctions/Helpers.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+
+
+namespace DB
+{
+
+namespace
+{
+
+AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, const DataTypes & arguments, const Array & params)
+{
+    assertNoParameters(name, params);
+
+    if (arguments.size() > AggregateFunctionRetentionData::max_events )
+        throw Exception("Too many event arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+
+    return std::make_shared<AggregateFunctionRetention>(arguments);
+}
+
+}
+
+void registerAggregateFunctionRetention(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive);
+}
+
+}

dbms/src/AggregateFunctions/AggregateFunctionRetention.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#include <iostream>
+#include <sstream>
+#include <unordered_set>
+#include <Columns/ColumnsNumber.h>
+#include <Columns/ColumnArray.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeArray.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <Common/ArenaAllocator.h>
+#include <Common/typeid_cast.h>
+#include <ext/range.h>
+#include <bitset>
+
+#include <AggregateFunctions/IAggregateFunction.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
+}
+
+struct AggregateFunctionRetentionData
+{
+    static constexpr auto max_events = 32;
+
+    using Events = std::bitset<max_events>;
+
+    Events events;
+
+    void add(UInt8 event)
+    {
+        events.set(event);
+    }
+
+    void merge(const AggregateFunctionRetentionData & other)
+    {
+        events |= other.events;
+    }
+
+    void serialize(WriteBuffer & buf) const
+    {
+        UInt32 event_value = events.to_ulong();
+        writeBinary(event_value, buf);
+    }
+
+    void deserialize(ReadBuffer & buf)
+    {
+        UInt32 event_value;
+        readBinary(event_value, buf);
+        events = event_value;
+    }
+};
+
+/**
+  * The max size of events is 32, that's enough for retention analytics
+  *
+  * Usage:
+  * - retention(cond1, cond2, cond3, ....)
+  * - returns [cond1_flag, cond1_flag && cond2_flag, cond1_flag && cond3_flag, ...]
+  */
+class AggregateFunctionRetention final
+        : public IAggregateFunctionDataHelper<AggregateFunctionRetentionData, AggregateFunctionRetention>
+{
+private:
+    UInt8 events_size;
+
+public:
+    String getName() const override
+    {
+        return "retention";
+    }
+
+    AggregateFunctionRetention(const DataTypes & arguments)
+    {
+        for (const auto i : ext::range(0, arguments.size()))
+        {
+            auto cond_arg = arguments[i].get();
+            if (!typeid_cast<const DataTypeUInt8 *>(cond_arg))
+                throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i) + " of aggregate function "
+                        + getName() + ", must be UInt8",
+                        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
+        }
+
+        events_size = arguments.size();
+    }
+
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt8>());
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
+    {
+        for (const auto i : ext::range(0, events_size))
+        {
+            auto event = static_cast<const ColumnVector<UInt8> *>(columns[i])->getData()[row_num];
+            if (event)
+            {
+                this->data(place).add(i);
+                break;
+            }
+        }
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).merge(this->data(rhs));
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).serialize(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).deserialize(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        auto & data_to = static_cast<ColumnArray &>(to).getData();
+        auto & offsets_to = static_cast<ColumnArray &>(to).getOffsets();
+
+        const bool first_flag = this->data(place).events.test(0);
+        data_to.insert(first_flag ? Field(static_cast<UInt64>(1)) : Field(static_cast<UInt64>(0)));
+        for (const auto i : ext::range(1, events_size))
+        {
+            if (first_flag && this->data(place).events.test(i))
+                data_to.insert(Field(static_cast<UInt64>(1)));
+            else
+                data_to.insert(Field(static_cast<UInt64>(0)));
+        }
+        offsets_to.push_back(offsets_to.size() == 0 ? events_size : offsets_to.back() +  events_size);
+    }
+
+    const char * getHeaderFilePath() const override
+    {
+        return __FILE__;
+    }
+};
+
+}

dbms/src/AggregateFunctions/registerAggregateFunctions.cpp

@@ -34,6 +34,7 @@ void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory
 void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory &);
 
 void registerAggregateFunctionHistogram(AggregateFunctionFactory & factory);
+void registerAggregateFunctionRetention(AggregateFunctionFactory & factory);
 
 void registerAggregateFunctions()
 {
@@ -59,6 +60,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionsBitwise(factory);
         registerAggregateFunctionsMaxIntersections(factory);
         registerAggregateFunctionHistogram(factory);
+        registerAggregateFunctionRetention(factory);
     }
 
     {

dbms/src/Client/Connection.cpp

@@ -114,6 +114,7 @@ void Connection::disconnect()
     //LOG_TRACE(log_wrapper.get(), "Disconnecting");
 
     in = nullptr;
+    last_input_packet_type.reset();
     out = nullptr; // can write to socket
     if (socket)
         socket->close();
@@ -379,6 +380,7 @@ void Connection::sendQuery(
     maybe_compressed_in.reset();
     maybe_compressed_out.reset();
     block_in.reset();
+    block_logs_in.reset();
     block_out.reset();
 
     /// Send empty block which means end of data.
@@ -506,20 +508,50 @@ bool Connection::poll(size_t timeout_microseconds)
 }
 
 
-bool Connection::hasReadBufferPendingData() const
+bool Connection::hasReadPendingData() const
 {
-    return static_cast<const ReadBufferFromPocoSocket &>(*in).hasPendingData();
+    return last_input_packet_type.has_value() || static_cast<const ReadBufferFromPocoSocket &>(*in).hasPendingData();
+}
+
+
+std::optional<UInt64> Connection::checkPacket(size_t timeout_microseconds)
+{
+    if (last_input_packet_type.has_value())
+        return last_input_packet_type;
+
+    if (hasReadPendingData() || poll(timeout_microseconds))
+    {
+        // LOG_TRACE(log_wrapper.get(), "Receiving packet type");
+        UInt64 packet_type;
+        readVarUInt(packet_type, *in);
+
+        last_input_packet_type.emplace(packet_type);
+        return last_input_packet_type;
+    }
+
+    return {};
 }
 
 
 Connection::Packet Connection::receivePacket()
 {
-    //LOG_TRACE(log_wrapper.get(), "Receiving packet");
-
     try
     {
         Packet res;
-        readVarUInt(res.type, *in);
+
+        /// Have we already read packet type?
+        if (last_input_packet_type)
+        {
+            res.type = *last_input_packet_type;
+            last_input_packet_type.reset();
+        }
+        else
+        {
+            //LOG_TRACE(log_wrapper.get(), "Receiving packet type");
+            readVarUInt(res.type, *in);
+        }
+
+        //LOG_TRACE(log_wrapper.get(), "Receiving packet " << res.type << " " << Protocol::Server::toString(res.type));
 
         switch (res.type)
         {
@@ -549,6 +581,10 @@ Connection::Packet Connection::receivePacket()
                 res.block = receiveData();
                 return res;
 
+            case Protocol::Server::Log:
+                res.block = receiveLogData();
+                return res;
+
             case Protocol::Server::EndOfStream:
                 return res;
 
@@ -576,14 +612,26 @@ Block Connection::receiveData()
     //LOG_TRACE(log_wrapper.get(), "Receiving data");
 
     initBlockInput();
+    return receiveDataImpl(block_in);
+}
 
+
+Block Connection::receiveLogData()
+{
+    initBlockLogsInput();
+    return receiveDataImpl(block_logs_in);
+}
+
+
+Block Connection::receiveDataImpl(BlockInputStreamPtr & stream)
+{
     String external_table_name;
     readStringBinary(external_table_name, *in);
 
     size_t prev_bytes = in->count();
 
     /// Read one block from network.
-    Block res = block_in->read();
+    Block res = stream->read();
 
     if (throttler)
         throttler->add(in->count() - prev_bytes);
@@ -592,20 +640,39 @@ Block Connection::receiveData()
 }
 
 
+void Connection::initInputBuffers()
+{
+
+}
+
+
 void Connection::initBlockInput()
 {
     if (!block_in)
     {
-        if (compression == Protocol::Compression::Enable)
-            maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
-        else
-            maybe_compressed_in = in;
+        if (!maybe_compressed_in)
+        {
+            if (compression == Protocol::Compression::Enable)
+                maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
+            else
+                maybe_compressed_in = in;
+        }
 
         block_in = std::make_shared<NativeBlockInputStream>(*maybe_compressed_in, server_revision);
     }
 }
 
 
+void Connection::initBlockLogsInput()
+{
+    if (!block_logs_in)
+    {
+        /// Have to return superset of SystemLogsQueue::getSampleBlock() columns
+        block_logs_in = std::make_shared<NativeBlockInputStream>(*in, server_revision);
+    }
+}
+
+
 void Connection::setDescription()
 {
     auto resolved_address = getResolvedAddress();

dbms/src/Client/Connection.h

@@ -23,6 +23,7 @@
 #include <Interpreters/TablesStatus.h>
 
 #include <atomic>
+#include <optional>
 
 
 namespace DB
@@ -138,7 +139,10 @@ public:
     bool poll(size_t timeout_microseconds = 0);
 
     /// Check, if has data in read buffer.
-    bool hasReadBufferPendingData() const;
+    bool hasReadPendingData() const;
+
+    /// Checks if there is input data in connection and reads packet ID.
+    std::optional<UInt64> checkPacket(size_t timeout_microseconds = 0);
 
     /// Receive packet from server.
     Packet receivePacket();
@@ -195,6 +199,7 @@ private:
     std::unique_ptr<Poco::Net::StreamSocket> socket;
     std::shared_ptr<ReadBuffer> in;
     std::shared_ptr<WriteBuffer> out;
+    std::optional<UInt64> last_input_packet_type;
 
     String query_id;
     Protocol::Compression compression;        /// Enable data compression for communication.
@@ -214,6 +219,7 @@ private:
     /// From where to read query execution result.
     std::shared_ptr<ReadBuffer> maybe_compressed_in;
     BlockInputStreamPtr block_in;
+    BlockInputStreamPtr block_logs_in;
 
     /// Where to write data for INSERT.
     std::shared_ptr<WriteBuffer> maybe_compressed_out;
@@ -249,11 +255,16 @@ private:
     bool ping();
 
     Block receiveData();
+    Block receiveLogData();
+    Block receiveDataImpl(BlockInputStreamPtr & stream);
+
     std::unique_ptr<Exception> receiveException();
     Progress receiveProgress();
     BlockStreamProfileInfo receiveProfileInfo();
 
+    void initInputBuffers();
     void initBlockInput();
+    void initBlockLogsInput();
 
     void throwUnexpectedPacket(UInt64 packet_type, const char * expected) const;
 };

dbms/src/Client/MultiplexedConnections.cpp

@@ -247,6 +247,7 @@ Connection::Packet MultiplexedConnections::receivePacketUnlocked()
         case Protocol::Server::ProfileInfo:
         case Protocol::Server::Totals:
         case Protocol::Server::Extremes:
+        case Protocol::Server::Log:
             break;
 
         case Protocol::Server::EndOfStream:
@@ -276,7 +277,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
     for (const ReplicaState & state : replica_states)
     {
         Connection * connection = state.connection;
-        if ((connection != nullptr) && connection->hasReadBufferPendingData())
+        if ((connection != nullptr) && connection->hasReadPendingData())
             read_list.push_back(*connection->socket);
     }
 

dbms/src/Columns/ColumnConst.cpp

@@ -30,6 +30,11 @@ ColumnPtr ColumnConst::convertToFullColumn() const
     return data->replicate(Offsets(1, s));
 }
 
+ColumnPtr ColumnConst::removeLowCardinality() const
+{
+    return ColumnConst::create(data->convertToFullColumnIfWithDictionary(), s);
+}
+
 ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
 {
     if (s != filt.size())

dbms/src/Columns/ColumnConst.h

@@ -36,6 +36,8 @@ public:
         return convertToFullColumn();
     }
 
+    ColumnPtr removeLowCardinality() const;
+
     std::string getName() const override
     {
         return "Const(" + data->getName() + ")";

dbms/src/Columns/ColumnString.cpp

@@ -175,7 +175,6 @@ ColumnPtr ColumnString::indexImpl(const PaddedPODArray<Type> & indexes, size_t l
     Chars_t & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
 
-
     size_t new_chars_size = 0;
     for (size_t i = 0; i < limit; ++i)
         new_chars_size += sizeAt(indexes[i]);

dbms/src/Columns/ColumnVector.cpp

@@ -118,7 +118,7 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
         memcpy(&new_col.data[0], &data[0], count * sizeof(data[0]));
 
         if (size > count)
-            memset(&new_col.data[count], static_cast<int>(value_type()), (size - count) * sizeof(value_type));
+            memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(value_type()), (size - count) * sizeof(value_type));
     }
 
     return std::move(res);
@@ -331,7 +331,7 @@ template class ColumnVector<Int128>;
 template class ColumnVector<Float32>;
 template class ColumnVector<Float64>;
 
-template class ColumnVector<Dec32>;
-template class ColumnVector<Dec64>;
-template class ColumnVector<Dec128>;
+template class ColumnVector<Decimal32>;
+template class ColumnVector<Decimal64>;
+template class ColumnVector<Decimal128>;
 }

dbms/src/Columns/ColumnVector.h

@@ -124,39 +124,25 @@ template <> inline UInt64 unionCastToUInt64(Float32 x)
 
 
 /// PaddedPODArray extended by Decimal scale
-template <typename T, size_t INITIAL_SIZE = 4096>
-class DecPaddedPODArray : public PODArray<T, INITIAL_SIZE, Allocator<false>, sizeof(T)-1>
+template <typename T>
+class DecimalPaddedPODArray : public PaddedPODArray<T>
 {
 public:
-    using Base = PODArray<T, INITIAL_SIZE, Allocator<false>, sizeof(T)-1>;
+    using Base = PaddedPODArray<T>;
     using Base::operator[];
+    using Base::Base;
 
-    DecPaddedPODArray()
+    DecimalPaddedPODArray(std::initializer_list<T> il)
+        : DecimalPaddedPODArray(std::begin(il), std::end(il))
     {}
 
-    DecPaddedPODArray(size_t n)
-    :   Base(n)
-    {}
-
-    DecPaddedPODArray(size_t n, const T & x)
-    :   Base(n, x)
-    {}
-
-    DecPaddedPODArray(typename Base::const_iterator from_begin, typename Base::const_iterator from_end)
-    :   Base(from_begin, from_end)
-    {}
-
-    DecPaddedPODArray(std::initializer_list<T> il)
-    :   DecPaddedPODArray(std::begin(il), std::end(il))
-    {}
-
-    DecPaddedPODArray(DecPaddedPODArray && other)
+    DecimalPaddedPODArray(DecimalPaddedPODArray && other)
     {
         this->swap(other);
         std::swap(scale, other.scale);
     }
 
-    DecPaddedPODArray & operator= (DecPaddedPODArray && other)
+    DecimalPaddedPODArray & operator=(DecimalPaddedPODArray && other)
     {
         this->swap(other);
         std::swap(scale, other.scale);
@@ -167,7 +153,7 @@ public:
     UInt32 getScale() const { return scale; }
 
 private:
-    UInt32 scale = std::numeric_limits<UInt32>::max();
+    UInt32 scale = DecimalField::wrongScale();
 };
 
 
@@ -185,7 +171,7 @@ private:
 
 public:
     using value_type = T;
-    using Container = std::conditional_t<decTrait<T>(), DecPaddedPODArray<value_type>, PaddedPODArray<value_type>>;
+    using Container = std::conditional_t<IsDecimalNumber<T>, DecimalPaddedPODArray<value_type>, PaddedPODArray<value_type>>;
 
 private:
     ColumnVector() {}
@@ -269,12 +255,12 @@ public:
 
     Field operator[](size_t n) const override
     {
-        if constexpr (decTrait<T>())
+        if constexpr (IsDecimalNumber<T>)
         {
             UInt32 scale = data.getScale();
-            if (scale == std::numeric_limits<UInt32>::max())
+            if (scale == DecimalField::wrongScale())
                 throw Exception("Extracting Decimal field with unknown scale. Scale is lost.", ErrorCodes::LOGICAL_ERROR);
-            return DecField(data[n], scale);
+            return DecimalField(data[n], scale);
         }
         else
             return typename NearestFieldType<T>::Type(data[n]);

dbms/src/Common/ActionBlocker.h

@@ -1,14 +1,15 @@
 #pragma once
-
 #include <atomic>
 #include <memory>
 #include <Common/ActionLock.h>
 
+
 namespace DB
 {
 
-/// An atomic variable that is used to block and interrupt certain actions
-/// If it is not zero then actions related with it should be considered as interrupted
+/// An atomic variable that is used to block and interrupt certain actions.
+/// If it is not zero then actions related with it should be considered as interrupted.
+/// Uses shared_ptr and the lock uses weak_ptr to be able to "hold" a lock when an object with blocker has already died.
 class ActionBlocker
 {
 public:
@@ -33,4 +34,5 @@ private:
     CounterPtr counter;
 };
 
+
 }

dbms/src/Common/ActionLock.h

@@ -1,7 +1,7 @@
 #pragma once
 #include <memory>
 #include <atomic>
-#include <Core/Types.h>
+
 
 namespace DB
 {

dbms/src/Common/CurrentThread.cpp

@@ -0,0 +1,80 @@
+#include "CurrentThread.h"
+#include <common/logger_useful.h>
+#include <Common/ThreadStatus.h>
+#include <Interpreters/ProcessList.h>
+#include <Interpreters/Context.h>
+#include <Poco/Ext/ThreadNumber.h>
+#include <Poco/Logger.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+void CurrentThread::updatePerformanceCounters()
+{
+    get()->updatePerformanceCounters();
+}
+
+ThreadStatusPtr CurrentThread::get()
+{
+#ifndef NDEBUG
+    if (!current_thread || current_thread.use_count() <= 0)
+        throw Exception("Thread #" + std::to_string(Poco::ThreadNumber::get()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR);
+
+    if (Poco::ThreadNumber::get() != current_thread->thread_number)
+        throw Exception("Current thread has different thread number", ErrorCodes::LOGICAL_ERROR);
+#endif
+
+    return current_thread;
+}
+
+ProfileEvents::Counters & CurrentThread::getProfileEvents()
+{
+    return current_thread->performance_counters;
+}
+
+MemoryTracker & CurrentThread::getMemoryTracker()
+{
+    return current_thread->memory_tracker;
+}
+
+void CurrentThread::updateProgressIn(const Progress & value)
+{
+    current_thread->progress_in.incrementPiecewiseAtomically(value);
+}
+
+void CurrentThread::updateProgressOut(const Progress & value)
+{
+    current_thread->progress_out.incrementPiecewiseAtomically(value);
+}
+
+void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr<InternalTextLogsQueue> & logs_queue)
+{
+    get()->attachInternalTextLogsQueue(logs_queue);
+}
+
+std::shared_ptr<InternalTextLogsQueue> CurrentThread::getInternalTextLogsQueue()
+{
+    /// NOTE: this method could be called at early server startup stage
+    /// NOTE: this method could be called in ThreadStatus destructor, therefore we make use_count() check just in case
+
+    if (!current_thread || current_thread.use_count() <= 0)
+        return nullptr;
+
+    if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died)
+        return nullptr;
+
+    return current_thread->getInternalTextLogsQueue();
+}
+
+ThreadGroupStatusPtr CurrentThread::getGroup()
+{
+    return get()->getThreadGroup();
+}
+
+}

dbms/src/Common/CurrentThread.h

@@ -0,0 +1,83 @@
+#pragma once
+#include <memory>
+#include <string>
+
+
+namespace ProfileEvents
+{
+class Counters;
+}
+
+class MemoryTracker;
+
+
+namespace DB
+{
+
+class Context;
+class QueryStatus;
+class ThreadStatus;
+struct Progress;
+using ThreadStatusPtr = std::shared_ptr<ThreadStatus>;
+class InternalTextLogsQueue;
+class ThreadGroupStatus;
+using ThreadGroupStatusPtr = std::shared_ptr<ThreadGroupStatus>;
+
+
+class CurrentThread
+{
+public:
+
+    /// Handler to current thread
+    static ThreadStatusPtr get();
+    /// Group to which belongs current thread
+    static ThreadGroupStatusPtr getGroup();
+
+    /// A logs queue used by TCPHandler to pass logs to a client
+    static void attachInternalTextLogsQueue(const std::shared_ptr<InternalTextLogsQueue> & logs_queue);
+    static std::shared_ptr<InternalTextLogsQueue> getInternalTextLogsQueue();
+
+    /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats
+    static void updatePerformanceCounters();
+
+    static ProfileEvents::Counters & getProfileEvents();
+    static MemoryTracker & getMemoryTracker();
+
+    /// Update read and write rows (bytes) statistics (used in system.query_thread_log)
+    static void updateProgressIn(const Progress & value);
+    static void updateProgressOut(const Progress & value);
+
+    /// Query management:
+
+    /// Call from master thread as soon as possible (e.g. when thread accepted connection)
+    static void initializeQuery();
+
+    /// Sets query_context for current thread group
+    static void attachQueryContext(Context & query_context);
+
+    /// You must call one of these methods when create a query child thread:
+    /// Add current thread to a group associated with the thread group
+    static void attachTo(const ThreadGroupStatusPtr & thread_group);
+    /// Is useful for a ThreadPool tasks
+    static void attachToIfDetached(const ThreadGroupStatusPtr & thread_group);
+
+    /// Update ProfileEvents and dumps info to system.query_thread_log
+    static void finalizePerformanceCounters();
+
+    /// Returns a non-empty string if the thread is attached to a query
+    static std::string getCurrentQueryID();
+
+    /// Non-master threads call this method in destructor automatically
+    static void detachQuery();
+    static void detachQueryIfNotDetached();
+
+    /// Initializes query with current thread as master thread in constructor, and detaches it in desstructor
+    struct QueryScope
+    {
+        explicit QueryScope(Context & query_context);
+        ~QueryScope();
+    };
+};
+
+}
+

dbms/src/Common/DNSResolver.cpp

@@ -7,6 +7,7 @@
 #include <Poco/NumberParser.h>
 #include <arpa/inet.h>
 #include <atomic>
+#include <optional>
 
 
 namespace DB
@@ -79,6 +80,10 @@ struct DNSResolver::Impl
 {
     SimpleCache<decltype(resolveIPAddressImpl), &resolveIPAddressImpl> cache_host;
 
+    /// Cached server host name
+    std::mutex mutex;
+    std::optional<String> host_name;
+
     /// If disabled, will not make cache lookups, will resolve addresses manually on each call
     std::atomic<bool> disable_cache{false};
 };
@@ -108,6 +113,9 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U
 void DNSResolver::dropCache()
 {
     impl->cache_host.drop();
+
+    std::unique_lock lock(impl->mutex);
+    impl->host_name.reset();
 }
 
 void DNSResolver::setDisableCacheFlag(bool is_disabled)
@@ -115,6 +123,19 @@ void DNSResolver::setDisableCacheFlag(bool is_disabled)
     impl->disable_cache = is_disabled;
 }
 
+String DNSResolver::getHostName()
+{
+    if (impl->disable_cache)
+        return Poco::Net::DNS::hostName();
+
+    std::unique_lock lock(impl->mutex);
+
+    if (!impl->host_name.has_value())
+        impl->host_name.emplace(Poco::Net::DNS::hostName());
+
+    return *impl->host_name;
+}
+
 DNSResolver::~DNSResolver() = default;
 
 

dbms/src/Common/DNSResolver.h

@@ -25,6 +25,9 @@ public:
 
     Poco::Net::SocketAddress resolveAddress(const std::string & host, UInt16 port);
 
+    /// Get this server host name
+    String getHostName();
+
     /// Disables caching
     void setDisableCacheFlag(bool is_disabled = true);
 

dbms/src/Common/ErrorCodes.cpp

@@ -382,6 +382,13 @@ namespace ErrorCodes
     extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT = 405;
     extern const int TOP_AND_LIMIT_TOGETHER = 406;
     extern const int DECIMAL_OVERFLOW = 407;
+    extern const int BAD_REQUEST_PARAMETER = 408;
+    extern const int EXTERNAL_EXECUTABLE_NOT_FOUND = 409;
+    extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING = 410;
+    extern const int PTHREAD_ERROR = 411;
+    extern const int NETLINK_ERROR = 412;
+    extern const int CANNOT_SET_SIGNAL_HANDLER = 413;
+    extern const int CANNOT_READLINE = 414;
 
     extern const int KEEPER_EXCEPTION = 999;
     extern const int POCO_EXCEPTION = 1000;

dbms/src/Common/ExternalTable.h

@@ -1,225 +0,0 @@
-#pragma once
-
-#include <boost/program_options.hpp>
-#include <boost/algorithm/string.hpp>
-#include <DataStreams/AsynchronousBlockInputStream.h>
-#include <DataTypes/DataTypeFactory.h>
-#include <Interpreters/Context.h>
-#include <IO/copyData.h>
-#include <IO/ReadBufferFromIStream.h>
-#include <IO/ReadBufferFromFile.h>
-#include <Storages/StorageMemory.h>
-#include <Client/Connection.h>
-#include <Poco/Net/HTMLForm.h>
-#include <Poco/Net/PartHandler.h>
-#include <Poco/Net/MessageHeader.h>
-#include <Common/HTMLForm.h>
-
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
-
-/// The base class containing the basic information about external table and
-/// basic functions for extracting this information from text fields.
-class BaseExternalTable
-{
-public:
-    std::string file;       /// File with data or '-' if stdin
-    std::string name;       /// The name of the table
-    std::string format;     /// Name of the data storage format
-
-    /// Description of the table structure: (column name, data type name)
-    std::vector<std::pair<std::string, std::string>> structure;
-
-    std::unique_ptr<ReadBuffer> read_buffer;
-    Block sample_block;
-
-    virtual ~BaseExternalTable() {}
-
-    /// Initialize read_buffer, depending on the data source. By default, does nothing.
-    virtual void initReadBuffer() {}
-
-    /// Get the table data - a pair (a thread with the contents of the table, the name of the table)
-    ExternalTableData getData(const Context & context)
-    {
-        initReadBuffer();
-        initSampleBlock();
-        ExternalTableData res = std::make_pair(std::make_shared<AsynchronousBlockInputStream>(context.getInputFormat(
-            format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE)), name);
-        return res;
-    }
-
-protected:
-    /// Clear all accumulated information
-    void clean()
-    {
-        name = "";
-        file = "";
-        format = "";
-        structure.clear();
-        sample_block = Block();
-        read_buffer.reset();
-    }
-
-    /// Function for debugging information output
-    void write()
-    {
-        std::cerr << "file " << file << std::endl;
-        std::cerr << "name " << name << std::endl;
-        std::cerr << "format " << format << std::endl;
-        std::cerr << "structure: \n";
-        for (size_t i = 0; i < structure.size(); ++i)
-            std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl;
-    }
-
-    static std::vector<std::string> split(const std::string & s, const std::string & d)
-    {
-        std::vector<std::string> res;
-        boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on);
-        return res;
-    }
-
-    /// Construct the `structure` vector from the text field `structure`
-    virtual void parseStructureFromStructureField(const std::string & argument)
-    {
-        std::vector<std::string> vals = split(argument, " ,");
-
-        if (vals.size() & 1)
-            throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS);
-
-        for (size_t i = 0; i < vals.size(); i += 2)
-            structure.emplace_back(vals[i], vals[i + 1]);
-    }
-
-    /// Construct the `structure` vector from the text field `types`
-    virtual void parseStructureFromTypesField(const std::string & argument)
-    {
-        std::vector<std::string> vals = split(argument, " ,");
-
-        for (size_t i = 0; i < vals.size(); ++i)
-            structure.emplace_back("_" + toString(i + 1), vals[i]);
-    }
-
-private:
-    /// Initialize sample_block according to the structure of the table stored in the `structure`
-    void initSampleBlock()
-    {
-        const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
-
-        for (size_t i = 0; i < structure.size(); ++i)
-        {
-            ColumnWithTypeAndName column;
-            column.name = structure[i].first;
-            column.type = data_type_factory.get(structure[i].second);
-            column.column = column.type->createColumn();
-            sample_block.insert(std::move(column));
-        }
-    }
-};
-
-
-/// Parsing of external table used in the tcp client.
-class ExternalTable : public BaseExternalTable
-{
-public:
-    void initReadBuffer() override
-    {
-        if (file == "-")
-            read_buffer = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
-        else
-            read_buffer = std::make_unique<ReadBufferFromFile>(file);
-    }
-
-    /// Extract parameters from variables_map, which is built on the client command line
-    ExternalTable(const boost::program_options::variables_map & external_options)
-    {
-        if (external_options.count("file"))
-            file = external_options["file"].as<std::string>();
-        else
-            throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
-
-        if (external_options.count("name"))
-            name = external_options["name"].as<std::string>();
-        else
-            throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
-
-        if (external_options.count("format"))
-            format = external_options["format"].as<std::string>();
-        else
-            throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
-
-        if (external_options.count("structure"))
-            parseStructureFromStructureField(external_options["structure"].as<std::string>());
-        else if (external_options.count("types"))
-            parseStructureFromTypesField(external_options["types"].as<std::string>());
-        else
-            throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
-    }
-};
-
-/// Parsing of external table used when sending tables via http
-/// The `handlePart` function will be called for each table passed,
-/// so it's also necessary to call `clean` at the end of the `handlePart`.
-class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
-{
-public:
-    std::vector<std::string> names;
-
-    ExternalTablesHandler(Context & context_, Poco::Net::NameValueCollection params_) : context(context_), params(params_) { }
-
-    void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream)
-    {
-        /// The buffer is initialized here, not in the virtual function initReadBuffer
-        read_buffer = std::make_unique<ReadBufferFromIStream>(stream);
-
-        /// Retrieve a collection of parameters from MessageHeader
-        Poco::Net::NameValueCollection content;
-        std::string label;
-        Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content);
-
-        /// Get parameters
-        name = content.get("name", "_data");
-        format = params.get(name + "_format", "TabSeparated");
-
-        if (params.has(name + "_structure"))
-            parseStructureFromStructureField(params.get(name + "_structure"));
-        else if (params.has(name + "_types"))
-            parseStructureFromTypesField(params.get(name + "_types"));
-        else
-            throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS);
-
-        ExternalTableData data = getData(context);
-
-        /// Create table
-        NamesAndTypesList columns = sample_block.getNamesAndTypesList();
-        StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
-        storage->startup();
-        context.addExternalTable(data.second, storage);
-        BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
-
-        /// Write data
-        data.first->readPrefix();
-        output->writePrefix();
-        while(Block block = data.first->read())
-            output->write(block);
-        data.first->readSuffix();
-        output->writeSuffix();
-
-        names.push_back(name);
-        /// We are ready to receive the next file, for this we clear all the information received
-        clean();
-    }
-
-private:
-    Context & context;
-    Poco::Net::NameValueCollection params;
-};
-
-
-}

dbms/src/Common/MemoryTracker.cpp

@@ -1,11 +1,10 @@
+#include "MemoryTracker.h"
 #include <common/likely.h>
 #include <common/logger_useful.h>
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>
+#include <Common/CurrentThread.h>
 #include <IO/WriteHelpers.h>
-#include <iomanip>
-
-#include <Common/MemoryTracker.h>
 
 
 namespace DB
@@ -19,7 +18,7 @@ namespace DB
 
 MemoryTracker::~MemoryTracker()
 {
-    if (peak)
+    if (static_cast<int>(level) < static_cast<int>(VariableContext::Process) && peak)
     {
         try
         {
@@ -56,13 +55,16 @@ void MemoryTracker::logPeakMemoryUsage() const
 
 void MemoryTracker::alloc(Int64 size)
 {
+    if (blocker.isCancelled())
+        return;
+
     /** Using memory_order_relaxed means that if allocations are done simultaneously,
       *  we allow exception about memory limit exceeded to be thrown only on next allocation.
       * So, we allow over-allocations.
       */
     Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed);
 
-    if (!next.load(std::memory_order_relaxed))
+    if (!parent.load(std::memory_order_relaxed))
         CurrentMetrics::add(metric, size);
 
     Int64 current_limit = limit.load(std::memory_order_relaxed);
@@ -102,45 +104,62 @@ void MemoryTracker::alloc(Int64 size)
     if (will_be > peak.load(std::memory_order_relaxed))        /// Races doesn't matter. Could rewrite with CAS, but not worth.
         peak.store(will_be, std::memory_order_relaxed);
 
-    if (auto loaded_next = next.load(std::memory_order_relaxed))
+    if (auto loaded_next = parent.load(std::memory_order_relaxed))
         loaded_next->alloc(size);
 }
 
 
 void MemoryTracker::free(Int64 size)
 {
-    Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
+    if (blocker.isCancelled())
+        return;
 
-    /** Sometimes, query could free some data, that was allocated outside of query context.
-      * Example: cache eviction.
-      * To avoid negative memory usage, we "saturate" amount.
-      * Memory usage will be calculated with some error.
-      * NOTE The code is not atomic. Not worth to fix.
-      */
-    if (new_amount < 0)
+    if (level == VariableContext::Thread)
     {
-        amount.fetch_sub(new_amount);
-        size += new_amount;
+        /// Could become negative if memory allocated in this thread is freed in another one
+        amount.fetch_sub(size, std::memory_order_relaxed);
+    }
+    else
+    {
+        Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
+
+        /** Sometimes, query could free some data, that was allocated outside of query context.
+          * Example: cache eviction.
+          * To avoid negative memory usage, we "saturate" amount.
+          * Memory usage will be calculated with some error.
+          * NOTE: The code is not atomic. Not worth to fix.
+          */
+        if (unlikely(new_amount < 0))
+        {
+            amount.fetch_sub(new_amount);
+            size += new_amount;
+        }
     }
 
-    if (auto loaded_next = next.load(std::memory_order_relaxed))
+    if (auto loaded_next = parent.load(std::memory_order_relaxed))
         loaded_next->free(size);
     else
         CurrentMetrics::sub(metric, size);
 }
 
 
-void MemoryTracker::reset()
+void MemoryTracker::resetCounters()
 {
-    if (!next.load(std::memory_order_relaxed))
-        CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
-
     amount.store(0, std::memory_order_relaxed);
     peak.store(0, std::memory_order_relaxed);
     limit.store(0, std::memory_order_relaxed);
 }
 
 
+void MemoryTracker::reset()
+{
+    if (!parent.load(std::memory_order_relaxed))
+        CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
+
+    resetCounters();
+}
+
+
 void MemoryTracker::setOrRaiseLimit(Int64 value)
 {
     /// This is just atomic set to maximum.
@@ -149,29 +168,26 @@ void MemoryTracker::setOrRaiseLimit(Int64 value)
         ;
 }
 
-#if __APPLE__ && __clang__
-__thread MemoryTracker * current_memory_tracker = nullptr;
-#else
-thread_local MemoryTracker * current_memory_tracker = nullptr;
-#endif
 
 namespace CurrentMemoryTracker
 {
     void alloc(Int64 size)
     {
-        if (current_memory_tracker)
-            current_memory_tracker->alloc(size);
+        DB::CurrentThread::getMemoryTracker().alloc(size);
     }
 
     void realloc(Int64 old_size, Int64 new_size)
     {
-        if (current_memory_tracker)
-            current_memory_tracker->alloc(new_size - old_size);
+        DB::CurrentThread::getMemoryTracker().alloc(new_size - old_size);
     }
 
     void free(Int64 size)
     {
-        if (current_memory_tracker)
-            current_memory_tracker->free(size);
+        DB::CurrentThread::getMemoryTracker().free(size);
     }
 }
+
+DB::SimpleActionLock getCurrentMemoryTrackerActionLock()
+{
+    return DB::CurrentThread::getMemoryTracker().blocker.cancel();
+}

dbms/src/Common/MemoryTracker.h

@@ -3,6 +3,8 @@
 #include <atomic>
 #include <common/Types.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/SimpleActionBlocker.h>
+#include <Common/VariableContext.h>
 
 
 namespace CurrentMetrics
@@ -26,7 +28,7 @@ class MemoryTracker
 
     /// Singly-linked list. All information will be passed to subsequent memory trackers also (it allows to implement trackers hierarchy).
     /// In terms of tree nodes it is the list of parents. Lifetime of these trackers should "include" lifetime of current tracker.
-    std::atomic<MemoryTracker *> next {};
+    std::atomic<MemoryTracker *> parent {};
 
     /// You could specify custom metric to track memory usage.
     CurrentMetrics::Metric metric = CurrentMetrics::MemoryTracking;
@@ -35,11 +37,14 @@ class MemoryTracker
     const char * description = nullptr;
 
 public:
-    MemoryTracker() {}
-    MemoryTracker(Int64 limit_) : limit(limit_) {}
+    MemoryTracker(VariableContext level = VariableContext::Thread) : level(level) {}
+    MemoryTracker(Int64 limit_, VariableContext level = VariableContext::Thread) : limit(limit_), level(level) {}
+    MemoryTracker(MemoryTracker * parent_, VariableContext level = VariableContext::Thread) : parent(parent_), level(level) {}
 
     ~MemoryTracker();
 
+    VariableContext level;
+
     /** Call the following functions before calling of corresponding operations with memory allocators.
       */
     void alloc(Int64 size);
@@ -79,9 +84,15 @@ public:
     }
 
     /// next should be changed only once: from nullptr to some value.
-    void setNext(MemoryTracker * elem)
+    /// NOTE: It is not true in MergeListElement
+    void setParent(MemoryTracker * elem)
     {
-        next.store(elem, std::memory_order_relaxed);
+        parent.store(elem, std::memory_order_relaxed);
+    }
+
+    MemoryTracker * getParent()
+    {
+        return parent.load(std::memory_order_relaxed);
     }
 
     /// The memory consumption could be shown in realtime via CurrentMetrics counter
@@ -95,26 +106,21 @@ public:
         description = description_;
     }
 
-    /// Reset the accumulated data.
+    /// Reset the accumulated data
+    void resetCounters();
+
+    /// Reset the accumulated data and the parent.
     void reset();
 
     /// Prints info about peak memory consumption into log.
     void logPeakMemoryUsage() const;
+
+    /// To be able to temporarily stop memory tracker
+    DB::SimpleActionBlocker blocker;
 };
 
 
-/** The MemoryTracker object is quite difficult to pass to all places where significant amounts of memory are allocated.
-  * Therefore, a thread-local pointer to used MemoryTracker is set, or nullptr if MemoryTracker does not need to be used.
-  * This pointer is set when memory consumption is monitored in current thread.
-  * So, you just need to pass it to all the threads that handle one request.
-  */
-#if defined(__APPLE__) && defined(__clang__)
-extern __thread MemoryTracker * current_memory_tracker;
-#else
-extern thread_local MemoryTracker * current_memory_tracker;
-#endif
-
-/// Convenience methods, that use current_memory_tracker if it is available.
+/// Convenience methods, that use current thread's memory_tracker if it is available.
 namespace CurrentMemoryTracker
 {
     void alloc(Int64 size);
@@ -123,20 +129,4 @@ namespace CurrentMemoryTracker
 }
 
 
-#include <boost/noncopyable.hpp>
-
-struct TemporarilyDisableMemoryTracker : private boost::noncopyable
-{
-    MemoryTracker * memory_tracker;
-
-    TemporarilyDisableMemoryTracker()
-    {
-        memory_tracker = current_memory_tracker;
-        current_memory_tracker = nullptr;
-    }
-
-    ~TemporarilyDisableMemoryTracker()
-    {
-        current_memory_tracker = memory_tracker;
-    }
-};
+DB::SimpleActionLock getCurrentMemoryTrackerActionLock();

dbms/src/Common/NaNUtils.h

@@ -50,7 +50,7 @@ std::enable_if_t<std::is_class_v<T>, T> NaNOrZero()
 
 #if 1 /// __int128
 template <typename T>
-std::enable_if_t<std::is_same_v<T, __int128>, __int128> NaNOrZero()
+std::enable_if_t<std::is_same_v<T, __int128> && !std::numeric_limits<T>::is_integer, __int128> NaNOrZero()
 {
     return __int128(0);
 }

dbms/src/Common/ODBCBridgeHelper.cpp

@@ -0,0 +1,131 @@
+#include <Common/ODBCBridgeHelper.h>
+
+#include <sstream>
+#include <IO/ReadHelpers.h>
+#include <IO/ReadWriteBufferFromHTTP.h>
+#include <Poco/Net/HTTPRequest.h>
+#include <Poco/Path.h>
+#include <Poco/Util/AbstractConfiguration.h>
+#include <Common/ShellCommand.h>
+#include <common/logger_useful.h>
+#include <ext/range.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING;
+}
+ODBCBridgeHelper::ODBCBridgeHelper(
+    const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
+    : config(config_), http_timeout(http_timeout_), connection_string(connection_string_)
+{
+    size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
+    std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
+
+    ping_url.setHost(bridge_host);
+    ping_url.setPort(bridge_port);
+    ping_url.setScheme("http");
+    ping_url.setPath(PING_HANDLER);
+}
+void ODBCBridgeHelper::startODBCBridge() const
+{
+    Poco::Path path{config.getString("application.dir", "")};
+    path.setFileName("clickhouse-odbc-bridge");
+
+    if (!path.isFile())
+        throw Exception("clickhouse-odbc-bridge is not found", ErrorCodes::EXTERNAL_EXECUTABLE_NOT_FOUND);
+
+    std::stringstream command;
+    command << path.toString() << ' ';
+    command << "--http-port " << config.getUInt("odbc_bridge.port", DEFAULT_PORT) << ' ';
+    command << "--listen-host " << config.getString("odbc_bridge.listen_host", DEFAULT_HOST) << ' ';
+    command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
+    if (config.has("logger.odbc_bridge_log"))
+        command << "--log-path " << config.getString("logger.odbc_bridge_log") << ' ';
+    if (config.has("logger.odbc_bridge_errlog"))
+        command << "--err-log-path " << config.getString("logger.odbc_bridge_errlog") << ' ';
+    if (config.has("logger.odbc_bridge_level"))
+        command << "--log-level " << config.getString("logger.odbc_bridge_level") << ' ';
+    command << "&"; /// we don't want to wait this process
+
+    auto command_str = command.str();
+    LOG_TRACE(log, "Starting clickhouse-odbc-bridge with command: " << command_str);
+
+    auto cmd = ShellCommand::execute(command_str);
+    cmd->wait();
+}
+
+std::vector<std::pair<std::string, std::string>> ODBCBridgeHelper::getURLParams(const std::string & cols, size_t max_block_size) const
+{
+    std::vector<std::pair<std::string, std::string>> result;
+
+    result.emplace_back("connection_string", connection_string); /// already validated
+    result.emplace_back("columns", cols);
+    result.emplace_back("max_block_size", std::to_string(max_block_size));
+
+    return result;
+}
+
+bool ODBCBridgeHelper::checkODBCBridgeIsRunning() const
+{
+    try
+    {
+        ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr);
+        return checkString(ODBCBridgeHelper::PING_OK_ANSWER, buf);
+    }
+    catch (...)
+    {
+        return false;
+    }
+}
+
+void ODBCBridgeHelper::startODBCBridgeSync() const
+{
+    if (!checkODBCBridgeIsRunning())
+    {
+        LOG_TRACE(log, "clickhouse-odbc-bridge is not running, will try to start it");
+        startODBCBridge();
+        bool started = false;
+        for (size_t counter : ext::range(1, 20))
+        {
+            LOG_TRACE(log, "Checking clickhouse-odbc-bridge is running, try " << counter);
+            if (checkODBCBridgeIsRunning())
+            {
+                started = true;
+                break;
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
+        if (!started)
+            throw Exception("ODBCBridgeHelper: clickhouse-odbc-bridge is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
+    }
+}
+
+Poco::URI ODBCBridgeHelper::getMainURI() const
+{
+    size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
+    std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
+
+    Poco::URI main_uri;
+    main_uri.setHost(bridge_host);
+    main_uri.setPort(bridge_port);
+    main_uri.setScheme("http");
+    main_uri.setPath(MAIN_HANDLER);
+    return main_uri;
+}
+
+Poco::URI ODBCBridgeHelper::getColumnsInfoURI() const
+{
+    size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
+    std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
+
+    Poco::URI columns_info_uri;
+    columns_info_uri.setHost(bridge_host);
+    columns_info_uri.setPort(bridge_port);
+    columns_info_uri.setScheme("http");
+    columns_info_uri.setPath(COL_INFO_HANDLER);
+    return columns_info_uri;
+}
+}

dbms/src/Common/ODBCBridgeHelper.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <Interpreters/Context.h>
+#include <Poco/Logger.h>
+#include <Poco/URI.h>
+#include <Poco/Util/AbstractConfiguration.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int EXTERNAL_EXECUTABLE_NOT_FOUND;
+}
+/** Helper for odbc-bridge, provide utility methods, not main request
+  */
+class ODBCBridgeHelper
+{
+private:
+
+    using Configuration = Poco::Util::AbstractConfiguration;
+
+    const Configuration & config;
+    Poco::Timespan http_timeout;
+
+    std::string connection_string;
+
+    Poco::URI ping_url;
+
+    Poco::Logger * log = &Poco::Logger::get("ODBCBridgeHelper");
+
+public:
+    static constexpr inline size_t DEFAULT_PORT = 9018;
+
+    static constexpr inline auto DEFAULT_HOST = "localhost";
+    static constexpr inline auto DEFAULT_FORMAT = "RowBinary";
+    static constexpr inline auto PING_HANDLER = "/ping";
+    static constexpr inline auto MAIN_HANDLER = "/";
+    static constexpr inline auto COL_INFO_HANDLER = "/columns_info";
+    static constexpr inline auto PING_OK_ANSWER = "Ok.";
+
+    ODBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_);
+
+    std::vector<std::pair<std::string, std::string>> getURLParams(const std::string & cols, size_t max_block_size) const;
+    bool checkODBCBridgeIsRunning() const;
+
+    void startODBCBridge() const;
+    void startODBCBridgeSync() const;
+
+    Poco::URI getMainURI() const;
+    Poco::URI getColumnsInfoURI() const;
+};
+}

dbms/src/Common/ProfileEvents.cpp

@@ -1,5 +1,7 @@
 #include <Common/ProfileEvents.h>
-
+#include <Common/CurrentThread.h>
+#include <Common/typeid_cast.h>
+#include <Columns/ColumnArray.h>
 
 /// Available events. Add something here as you wish.
 #define APPLY_FOR_EVENTS(M) \
@@ -37,6 +39,11 @@
     M(CreatedReadBufferAIO) \
     M(CreatedWriteBufferOrdinary) \
     M(CreatedWriteBufferAIO) \
+    M(DiskReadElapsedMicroseconds) \
+    M(DiskWriteElapsedMicroseconds) \
+    M(NetworkReceiveElapsedMicroseconds) \
+    M(NetworkSendElapsedMicroseconds) \
+    M(ThrottlerSleepMicroseconds) \
     \
     M(ReplicatedPartFetches) \
     M(ReplicatedPartFailedFetches) \
@@ -67,7 +74,9 @@
     M(ZooKeeperCheck) \
     M(ZooKeeperClose) \
     M(ZooKeeperWatchResponse) \
-    M(ZooKeeperExceptions) \
+    M(ZooKeeperUserExceptions) \
+    M(ZooKeeperHardwareExceptions) \
+    M(ZooKeeperOtherExceptions) \
     M(ZooKeeperWaitMicroseconds) \
     M(ZooKeeperBytesSent) \
     M(ZooKeeperBytesReceived) \
@@ -143,31 +152,93 @@
     M(RWLockAcquiredWriteLocks) \
     M(RWLockReadersWaitMilliseconds) \
     M(RWLockWritersWaitMilliseconds) \
+    M(NetworkErrors) \
     \
-    M(NetworkErrors)
+    M(RealTimeMicroseconds) \
+    M(UserTimeMicroseconds) \
+    M(SystemTimeMicroseconds) \
+    M(SoftPageFaults) \
+    M(HardPageFaults) \
+    M(VoluntaryContextSwitches) \
+    M(InvoluntaryContextSwitches) \
+    \
+    M(OSIOWaitMicroseconds) \
+    M(OSCPUWaitMicroseconds) \
+    M(OSCPUVirtualTimeMicroseconds) \
+    M(OSReadBytes) \
+    M(OSWriteBytes) \
+    M(OSReadChars) \
+    M(OSWriteChars) \
+
 
 namespace ProfileEvents
 {
-    #define M(NAME) extern const Event NAME = __COUNTER__;
+
+#define M(NAME) extern const Event NAME = __COUNTER__;
+    APPLY_FOR_EVENTS(M)
+#undef M
+constexpr Event END = __COUNTER__;
+
+/// Global variable, initialized by zeros.
+Counter global_counters_array[END] {};
+/// Initialize global counters statically
+Counters global_counters(global_counters_array);
+
+const Event Counters::num_counters = END;
+
+
+Counters::Counters(VariableContext level, Counters * parent)
+    : counters_holder(new Counter[num_counters] {}),
+      parent(parent),
+      level(level)
+{
+    counters = counters_holder.get();
+}
+
+void Counters::resetCounters()
+{
+    if (counters)
+    {
+        for (Event i = 0; i < num_counters; ++i)
+            counters[i].store(0, std::memory_order_relaxed);
+    }
+}
+
+void Counters::reset()
+{
+    parent = nullptr;
+    resetCounters();
+}
+
+Counters Counters::getPartiallyAtomicSnapshot() const
+{
+    Counters res(VariableContext::Snapshot, nullptr);
+    for (Event i = 0; i < num_counters; ++i)
+        res.counters[i].store(counters[i].load(std::memory_order_relaxed), std::memory_order_relaxed);
+    return res;
+}
+
+const char * getDescription(Event event)
+{
+    static const char * descriptions[] =
+    {
+    #define M(NAME) #NAME,
         APPLY_FOR_EVENTS(M)
     #undef M
-    constexpr Event END = __COUNTER__;
+    };
 
-    std::atomic<Count> counters[END] {};    /// Global variable, initialized by zeros.
+    return descriptions[event];
+}
 
-    const char * getDescription(Event event)
-    {
-        static const char * descriptions[] =
-        {
-        #define M(NAME) #NAME,
-            APPLY_FOR_EVENTS(M)
-        #undef M
-        };
 
-        return descriptions[event];
-    }
+Event end() { return END; }
+
+
+void increment(Event event, Count amount)
+{
+    DB::CurrentThread::getProfileEvents().increment(event, amount);
+}
 
-    Event end() { return END; }
 }
 
 #undef APPLY_FOR_EVENTS

dbms/src/Common/ProfileEvents.h

@@ -1,8 +1,9 @@
 #pragma once
 
-#include <stddef.h>
+#include <Common/VariableContext.h>
 #include <atomic>
-
+#include <memory>
+#include <stddef.h>
 
 /** Implements global counters for various events happening in the application
   *  - for high level profiling.
@@ -14,19 +15,80 @@ namespace ProfileEvents
     /// Event identifier (index in array).
     using Event = size_t;
     using Count = size_t;
+    using Counter = std::atomic<Count>;
+    class Counters;
+
+    /// Counters - how many times each event happened
+    extern Counters global_counters;
+
+    class Counters
+    {
+        Counter * counters = nullptr;
+        std::unique_ptr<Counter[]> counters_holder;
+        /// Used to propagate increments
+        Counters * parent = nullptr;
+
+    public:
+
+        VariableContext level = VariableContext::Thread;
+
+        /// By default, any instance have to increment global counters
+        Counters(VariableContext level = VariableContext::Thread, Counters * parent = &global_counters);
+
+        /// Global level static initializer
+        Counters(Counter * allocated_counters)
+            :  counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
+
+        Counter & operator[] (Event event)
+        {
+            return counters[event];
+        }
+
+        const Counter & operator[] (Event event) const
+        {
+            return counters[event];
+        }
+
+        inline void increment(Event event, Count amount = 1)
+        {
+            Counters * current = this;
+            do
+            {
+                current->counters[event].fetch_add(amount, std::memory_order_relaxed);
+                current = current->parent;
+            } while (current != nullptr);
+        }
+
+        /// Every single value is fetched atomically, but not all values as a whole.
+        Counters getPartiallyAtomicSnapshot() const;
+
+        /// Reset all counters to zero and reset parent.
+        void reset();
+
+        /// Get parent (thread unsafe)
+        Counters * getParent()
+        {
+            return parent;
+        }
+
+        /// Set parent (thread unsafe)
+        void setParent(Counters * parent_)
+        {
+            parent = parent_;
+        }
+
+        /// Set all counters to zero
+        void resetCounters();
+
+        static const Event num_counters;
+    };
+
+    /// Increment a counter for event. Thread-safe.
+    void increment(Event event, Count amount = 1);
 
     /// Get text description of event by identifier. Returns statically allocated string.
     const char * getDescription(Event event);
 
-    /// Counters - how many times each event happened.
-    extern std::atomic<Count> counters[];
-
-    /// Increment a counter for event. Thread-safe.
-    inline void increment(Event event, Count amount = 1)
-    {
-        counters[event].fetch_add(amount, std::memory_order_relaxed);
-    }
-
     /// Get index just after last event identifier.
     Event end();
 }

dbms/src/Common/RWLockFIFO.cpp

@@ -86,7 +86,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
 
         handler_ptr->it_client->info += "; " + client.info;
 
-        return  handler_ptr;
+        return handler_ptr;
     }
 
     if (type == Type::Write || queue.empty() || queue.back().type == Type::Write)

dbms/src/Common/SimpleActionBlocker.h

@@ -0,0 +1,79 @@
+#pragma once
+#include <atomic>
+
+
+namespace DB
+{
+
+class SimpleActionLock;
+
+
+/// Similar to ActionBlocker, but without weak_ptr magic
+class SimpleActionBlocker
+{
+    using Counter = std::atomic<int>;
+    Counter counter = 0;
+
+public:
+
+    SimpleActionBlocker() = default;
+
+    bool isCancelled() const { return counter > 0; }
+
+    /// Temporarily blocks corresponding actions (while the returned object is alive)
+    friend class SimpleActionLock;
+    inline SimpleActionLock cancel();
+
+    /// Cancel the actions forever.
+    void cancelForever() { ++counter; }
+};
+
+
+/// Blocks related action while a SimpleActionLock instance exists
+class SimpleActionLock
+{
+    SimpleActionBlocker * block = nullptr;
+
+public:
+
+    SimpleActionLock() = default;
+
+    explicit SimpleActionLock(SimpleActionBlocker & block_) : block(&block_)
+    {
+        ++block->counter;
+    }
+
+    SimpleActionLock(const SimpleActionLock &) = delete;
+
+    SimpleActionLock(SimpleActionLock && rhs) noexcept
+    {
+        *this = std::move(rhs);
+    }
+
+    SimpleActionLock & operator=(const SimpleActionLock &) = delete;
+
+    SimpleActionLock & operator=(SimpleActionLock && rhs) noexcept
+    {
+        if (block)
+            --block->counter;
+
+        block = rhs.block;
+        rhs.block = nullptr;
+
+        return *this;
+    }
+
+    ~SimpleActionLock()
+    {
+        if (block)
+            --block->counter;
+    }
+};
+
+
+SimpleActionLock SimpleActionBlocker::cancel()
+{
+    return SimpleActionLock(*this);
+}
+
+}

dbms/src/Common/StatusFile.cpp

@@ -30,7 +30,7 @@ StatusFile::StatusFile(const std::string & path_)
         std::string contents;
         {
             ReadBufferFromFile in(path, 1024);
-            LimitReadBuffer limit_in(in, 1024);
+            LimitReadBuffer limit_in(in, 1024, false);
             readStringUntilEOF(contents, limit_in);
         }
 

dbms/src/Common/Stopwatch.cpp

@@ -0,0 +1,14 @@
+#include <sys/resource.h>
+#include "Stopwatch.h"
+
+StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current()
+{
+    StopwatchRUsage::Timestamp res;
+
+    ::rusage rusage;
+    ::getrusage(RUSAGE_THREAD, &rusage);
+
+    res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
+    res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
+    return res;
+}

dbms/src/Common/Stopwatch.h

@@ -32,9 +32,11 @@ public:
     void stop()                         { stop_ns = nanoseconds(); is_running = false; }
     void reset()                        { start_ns = 0; stop_ns = 0; is_running = false; }
     void restart()                      { start(); }
-    UInt64 elapsed() const              { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
-    UInt64 elapsedMilliseconds() const  { return elapsed() / 1000000UL; }
-    double elapsedSeconds() const       { return static_cast<double>(elapsed()) / 1000000000ULL; }
+    UInt64 elapsed() const              { return elapsedNanoseconds(); }
+    UInt64 elapsedNanoseconds() const   { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
+    UInt64 elapsedMicroseconds() const  { return elapsedNanoseconds() / 1000U; }
+    UInt64 elapsedMilliseconds() const  { return elapsedNanoseconds() / 1000000UL; }
+    double elapsedSeconds() const       { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
 
 private:
     UInt64 start_ns = 0;
@@ -131,3 +133,59 @@ private:
     /// Most significant bit is a lock. When it is set, compareAndRestartDeferred method will return false.
     UInt64 nanoseconds() const { return StopWatchDetail::nanoseconds(clock_type) & 0x7FFFFFFFFFFFFFFFULL; }
 };
+
+
+/// Like ordinary StopWatch, but uses getrusage() system call
+struct StopwatchRUsage
+{
+    StopwatchRUsage() = default;
+
+    void start()    { start_ts = Timestamp::current(); is_running = true; }
+    void stop()     { stop_ts = Timestamp::current(); is_running = false; }
+    void reset()    { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
+    void restart()  { start(); }
+
+    UInt64 elapsed(bool count_user = true, bool count_sys = true) const
+    {
+        return elapsedNanoseconds(count_user, count_sys);
+    }
+
+    UInt64 elapsedNanoseconds(bool count_user = true, bool count_sys = true) const
+    {
+        return (is_running ? Timestamp::current() : stop_ts).nanoseconds(count_user, count_sys) - start_ts.nanoseconds(count_user, count_sys);
+    }
+
+    UInt64 elapsedMicroseconds(bool count_user = true, bool count_sys = true) const
+    {
+        return elapsedNanoseconds(count_user, count_sys) / 1000UL;
+    }
+
+    UInt64 elapsedMilliseconds(bool count_user = true, bool count_sys = true) const
+    {
+        return elapsedNanoseconds(count_user, count_sys) / 1000000UL;
+    }
+
+    double elapsedSeconds(bool count_user = true, bool count_sys = true) const
+    {
+        return static_cast<double>(elapsedNanoseconds(count_user, count_sys)) / 1000000000.0;
+    }
+
+private:
+
+    struct Timestamp
+    {
+        UInt64 user_ns = 0;
+        UInt64 sys_ns = 0;
+
+        static Timestamp current();
+
+        UInt64 nanoseconds(bool count_user = true, bool count_sys = true) const
+        {
+            return (count_user ? user_ns : 0) + (count_sys ? sys_ns : 0);
+        }
+    };
+
+    Timestamp start_ts;
+    Timestamp stop_ts;
+    bool is_running = false;
+};

dbms/src/Common/TaskStatsInfoGetter.cpp

@@ -0,0 +1,254 @@
+#include <Common/TaskStatsInfoGetter.h>
+#include <Common/Exception.h>
+#include <Core/Types.h>
+
+#include <errno.h>
+#include <linux/genetlink.h>
+#include <linux/netlink.h>
+#include <linux/taskstats.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <syscall.h>
+
+/// Basic idea is motivated by "iotop" tool.
+/// More info: https://www.kernel.org/doc/Documentation/accounting/taskstats.txt
+
+#define GENLMSG_DATA(glh)       ((void *)((char*)NLMSG_DATA(glh) + GENL_HDRLEN))
+#define GENLMSG_PAYLOAD(glh)    (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN)
+#define NLA_DATA(na)            ((void *)((char*)(na) + NLA_HDRLEN))
+#define NLA_PAYLOAD(len)        (len - NLA_HDRLEN)
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int NETLINK_ERROR;
+}
+
+
+namespace
+{
+
+static size_t constexpr MAX_MSG_SIZE = 1024;
+
+
+struct NetlinkMessage
+{
+    ::nlmsghdr n;
+    ::genlmsghdr g;
+    char buf[MAX_MSG_SIZE];
+};
+
+
+int sendCommand(
+    int sock_fd,
+    UInt16 nlmsg_type,
+    UInt32 nlmsg_pid,
+    UInt8 genl_cmd,
+    UInt16 nla_type,
+    void * nla_data,
+    int nla_len) noexcept
+{
+    NetlinkMessage msg{};
+
+    msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+    msg.n.nlmsg_type = nlmsg_type;
+    msg.n.nlmsg_flags = NLM_F_REQUEST;
+    msg.n.nlmsg_seq = 0;
+    msg.n.nlmsg_pid = nlmsg_pid;
+    msg.g.cmd = genl_cmd;
+    msg.g.version = 1;
+
+    ::nlattr * attr = static_cast<::nlattr *>(GENLMSG_DATA(&msg));
+    attr->nla_type = nla_type;
+    attr->nla_len = nla_len + 1 + NLA_HDRLEN;
+
+    memcpy(NLA_DATA(attr), nla_data, nla_len);
+    msg.n.nlmsg_len += NLMSG_ALIGN(attr->nla_len);
+
+    char * buf = reinterpret_cast<char *>(&msg);
+    ssize_t buflen = msg.n.nlmsg_len;
+
+    ::sockaddr_nl nladdr{};
+    nladdr.nl_family = AF_NETLINK;
+
+    while (true)
+    {
+        ssize_t r = ::sendto(sock_fd, buf, buflen, 0, reinterpret_cast<const ::sockaddr *>(&nladdr), sizeof(nladdr));
+
+        if (r >= buflen)
+            break;
+
+        if (r > 0)
+        {
+            buf += r;
+            buflen -= r;
+        }
+        else if (errno != EAGAIN)
+            return -1;
+    }
+
+    return 0;
+}
+
+
+UInt16 getFamilyId(int nl_sock_fd) noexcept
+{
+    struct
+    {
+        ::nlmsghdr header;
+        ::genlmsghdr ge_header;
+        char buf[256];
+    } answer;
+
+    static char name[] = TASKSTATS_GENL_NAME;
+
+    if (sendCommand(
+        nl_sock_fd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY,
+        CTRL_ATTR_FAMILY_NAME, (void *) name,
+        strlen(TASKSTATS_GENL_NAME) + 1))
+        return 0;
+
+    UInt16 id = 0;
+    ssize_t rep_len = ::recv(nl_sock_fd, &answer, sizeof(answer), 0);
+    if (answer.header.nlmsg_type == NLMSG_ERROR || (rep_len < 0) || !NLMSG_OK((&answer.header), rep_len))
+        return 0;
+
+    const ::nlattr * attr;
+    attr = static_cast<const ::nlattr *>(GENLMSG_DATA(&answer));
+    attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(attr) + NLA_ALIGN(attr->nla_len));
+    if (attr->nla_type == CTRL_ATTR_FAMILY_ID)
+        id = *static_cast<const UInt16 *>(NLA_DATA(attr));
+
+    return id;
+}
+
+}
+
+
+TaskStatsInfoGetter::TaskStatsInfoGetter() = default;
+
+void TaskStatsInfoGetter::init()
+{
+    if (netlink_socket_fd >= 0)
+        return;
+
+    netlink_socket_fd = ::socket(PF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+    if (netlink_socket_fd < 0)
+        throwFromErrno("Can't create PF_NETLINK socket");
+
+    ::sockaddr_nl addr{};
+    addr.nl_family = AF_NETLINK;
+
+    if (::bind(netlink_socket_fd, reinterpret_cast<const ::sockaddr *>(&addr), sizeof(addr)) < 0)
+        throwFromErrno("Can't bind PF_NETLINK socket");
+
+    netlink_family_id = getFamilyId(netlink_socket_fd);
+}
+
+bool TaskStatsInfoGetter::getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error)
+{
+    init();
+
+    if (sendCommand(netlink_socket_fd, netlink_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(pid_t)))
+        throwFromErrno("Can't send a Netlink command");
+
+    NetlinkMessage msg;
+    ssize_t rv = ::recv(netlink_socket_fd, &msg, sizeof(msg), 0);
+
+    if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), rv))
+    {
+        const ::nlmsgerr * err = static_cast<const ::nlmsgerr *>(NLMSG_DATA(&msg));
+        if (throw_on_error)
+            throw Exception("Can't get Netlink response, error: " + std::to_string(err->error), ErrorCodes::NETLINK_ERROR);
+        else
+            return false;
+    }
+
+    rv = GENLMSG_PAYLOAD(&msg.n);
+
+    const ::nlattr * attr = static_cast<const ::nlattr *>(GENLMSG_DATA(&msg));
+    ssize_t len = 0;
+
+    while (len < rv)
+    {
+        len += NLA_ALIGN(attr->nla_len);
+
+        if (attr->nla_type == TASKSTATS_TYPE_AGGR_TGID || attr->nla_type == TASKSTATS_TYPE_AGGR_PID)
+        {
+            int aggr_len = NLA_PAYLOAD(attr->nla_len);
+            int len2 = 0;
+
+            attr = static_cast<const ::nlattr *>(NLA_DATA(attr));
+            while (len2 < aggr_len)
+            {
+                if (attr->nla_type == TASKSTATS_TYPE_STATS)
+                {
+                    const ::taskstats * ts = static_cast<const ::taskstats *>(NLA_DATA(attr));
+                    out_stats = *ts;
+                }
+
+                len2 += NLA_ALIGN(attr->nla_len);
+                attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(attr) + len2);
+            }
+        }
+
+        attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(GENLMSG_DATA(&msg)) + len);
+    }
+
+    return true;
+}
+
+void TaskStatsInfoGetter::getStat(::taskstats & stat, int tid)
+{
+    tid = tid < 0 ? getDefaultTID() : tid;
+    getStatImpl(tid, stat, true);
+}
+
+bool TaskStatsInfoGetter::tryGetStat(::taskstats & stat, int tid)
+{
+    tid = tid < 0 ? getDefaultTID() : tid;
+    return getStatImpl(tid, stat, false);
+}
+
+TaskStatsInfoGetter::~TaskStatsInfoGetter()
+{
+    if (netlink_socket_fd >= 0)
+        close(netlink_socket_fd);
+}
+
+int TaskStatsInfoGetter::getCurrentTID()
+{
+    /// This call is always successful. - man gettid
+    return static_cast<int>(syscall(SYS_gettid));
+}
+
+int TaskStatsInfoGetter::getDefaultTID()
+{
+    if (default_tid < 0)
+        default_tid = getCurrentTID();
+
+    return default_tid;
+}
+
+static bool tryGetTaskStats()
+{
+    TaskStatsInfoGetter getter;
+    ::taskstats stat;
+    return getter.tryGetStat(stat);
+}
+
+bool TaskStatsInfoGetter::checkProcessHasRequiredPermissions()
+{
+    /// It is thread- and exception- safe since C++11
+    static bool res = tryGetTaskStats();
+    return res;
+}
+
+}

dbms/src/Common/TaskStatsInfoGetter.h

@@ -0,0 +1,43 @@
+#pragma once
+#include <Core/Types.h>
+
+struct taskstats;
+
+
+namespace DB
+{
+
+class Exception;
+
+
+/// Get taskstat info from OS kernel via Netlink protocol.
+class TaskStatsInfoGetter
+{
+public:
+    TaskStatsInfoGetter();
+    TaskStatsInfoGetter(const TaskStatsInfoGetter &) = delete;
+
+    void getStat(::taskstats & stat, int tid = -1);
+    bool tryGetStat(::taskstats & stat, int tid = -1);
+
+    ~TaskStatsInfoGetter();
+
+    /// Make a syscall and returns Linux thread id
+    static int getCurrentTID();
+
+    /// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info
+    static bool checkProcessHasRequiredPermissions();
+
+private:
+    /// Caches current thread tid to avoid extra sys calls
+    int getDefaultTID();
+    int default_tid = -1;
+
+    bool getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error = false);
+    void init();
+
+    int netlink_socket_fd = -1;
+    UInt16 netlink_family_id = 0;
+};
+
+}

dbms/src/Common/ThreadProfileEvents.h

@@ -0,0 +1,144 @@
+#pragma once
+#include <Common/TaskStatsInfoGetter.h>
+#include <Common/ProfileEvents.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <pthread.h>
+#include <linux/taskstats.h>
+
+
+namespace ProfileEvents
+{
+    extern const Event RealTimeMicroseconds;
+    extern const Event UserTimeMicroseconds;
+    extern const Event SystemTimeMicroseconds;
+    extern const Event SoftPageFaults;
+    extern const Event HardPageFaults;
+    extern const Event VoluntaryContextSwitches;
+    extern const Event InvoluntaryContextSwitches;
+
+    extern const Event OSIOWaitMicroseconds;
+    extern const Event OSCPUWaitMicroseconds;
+    extern const Event OSCPUVirtualTimeMicroseconds;
+    extern const Event OSReadChars;
+    extern const Event OSWriteChars;
+    extern const Event OSReadBytes;
+    extern const Event OSWriteBytes;
+}
+
+
+namespace DB
+{
+
+/// Handles overflow
+template <typename TUInt>
+inline TUInt safeDiff(TUInt prev, TUInt curr)
+{
+    return curr >= prev ? curr - prev : 0;
+}
+
+
+inline UInt64 getCurrentTimeNanoseconds(clockid_t clock_type = CLOCK_MONOTONIC)
+{
+    struct timespec ts;
+    clock_gettime(clock_type, &ts);
+    return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
+}
+
+
+struct RUsageCounters
+{
+    /// In nanoseconds
+    UInt64 real_time = 0;
+    UInt64 user_time = 0;
+    UInt64 sys_time = 0;
+
+    UInt64 soft_page_faults = 0;
+    UInt64 hard_page_faults = 0;
+
+    RUsageCounters() = default;
+    RUsageCounters(const ::rusage & rusage_, UInt64 real_time_)
+    {
+        set(rusage_, real_time_);
+    }
+
+    void set(const ::rusage & rusage, UInt64 real_time_)
+    {
+        real_time = real_time_;
+        user_time = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
+        sys_time = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
+
+        soft_page_faults = static_cast<UInt64>(rusage.ru_minflt);
+        hard_page_faults = static_cast<UInt64>(rusage.ru_majflt);
+    }
+
+    static RUsageCounters zeros(UInt64 real_time_ = getCurrentTimeNanoseconds())
+    {
+        RUsageCounters res;
+        res.real_time = real_time_;
+        return res;
+    }
+
+    static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds())
+    {
+        ::rusage rusage;
+        ::getrusage(RUSAGE_THREAD, &rusage);
+        return RUsageCounters(rusage, real_time_);
+    }
+
+    static void incrementProfileEvents(const RUsageCounters & prev, const RUsageCounters & curr, ProfileEvents::Counters & profile_events)
+    {
+        profile_events.increment(ProfileEvents::RealTimeMicroseconds,   (curr.real_time - prev.real_time) / 1000U);
+        profile_events.increment(ProfileEvents::UserTimeMicroseconds,   (curr.user_time - prev.user_time) / 1000U);
+        profile_events.increment(ProfileEvents::SystemTimeMicroseconds, (curr.sys_time - prev.sys_time) / 1000U);
+
+        profile_events.increment(ProfileEvents::SoftPageFaults, curr.soft_page_faults - prev.soft_page_faults);
+        profile_events.increment(ProfileEvents::HardPageFaults, curr.hard_page_faults - prev.hard_page_faults);
+    }
+
+    static void updateProfileEvents(RUsageCounters & last_counters, ProfileEvents::Counters & profile_events)
+    {
+        auto current_counters = current();
+        incrementProfileEvents(last_counters, current_counters, profile_events);
+        last_counters = current_counters;
+    }
+};
+
+
+struct TasksStatsCounters
+{
+    ::taskstats stat;
+
+    TasksStatsCounters() = default;
+
+    static TasksStatsCounters current();
+
+    static void incrementProfileEvents(const TasksStatsCounters & prev, const TasksStatsCounters & curr, ProfileEvents::Counters & profile_events)
+    {
+        profile_events.increment(ProfileEvents::OSCPUWaitMicroseconds,
+                                 safeDiff(prev.stat.cpu_delay_total, curr.stat.cpu_delay_total) / 1000U);
+        profile_events.increment(ProfileEvents::OSIOWaitMicroseconds,
+                                 safeDiff(prev.stat.blkio_delay_total, curr.stat.blkio_delay_total) / 1000U);
+        profile_events.increment(ProfileEvents::OSCPUVirtualTimeMicroseconds,
+                                 safeDiff(prev.stat.cpu_run_virtual_total, curr.stat.cpu_run_virtual_total) / 1000U);
+
+        /// Too old struct version, do not read new fields
+        if (curr.stat.version < TASKSTATS_VERSION)
+            return;
+
+        profile_events.increment(ProfileEvents::OSReadChars,  safeDiff(prev.stat.read_char, curr.stat.read_char));
+        profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char));
+        profile_events.increment(ProfileEvents::OSReadBytes,  safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
+        profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes));
+    }
+
+    static void updateProfileEvents(TasksStatsCounters & last_counters, ProfileEvents::Counters & profile_events)
+    {
+        auto current_counters = current();
+        incrementProfileEvents(last_counters, current_counters, profile_events);
+        last_counters = current_counters;
+    }
+};
+
+}

dbms/src/Common/ThreadStatus.cpp

@@ -0,0 +1,118 @@
+#include "ThreadStatus.h"
+#include <common/logger_useful.h>
+#include <Common/CurrentThread.h>
+#include <Common/Exception.h>
+#include <Common/ThreadProfileEvents.h>
+
+#include <Poco/Thread.h>
+#include <Poco/Ext/ThreadNumber.h>
+
+
+namespace DB
+{
+
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int PTHREAD_ERROR;
+}
+
+
+/// Order of current_thread and current_thread_scope matters
+thread_local ThreadStatusPtr current_thread = ThreadStatus::create();
+thread_local ThreadStatus::CurrentThreadScope current_thread_scope;
+
+
+TasksStatsCounters TasksStatsCounters::current()
+{
+    TasksStatsCounters res;
+    current_thread->taskstats_getter->getStat(res.stat, current_thread->os_thread_id);
+    return res;
+}
+
+
+ThreadStatus::ThreadStatus()
+{
+    thread_number = Poco::ThreadNumber::get();
+    os_thread_id = TaskStatsInfoGetter::getCurrentTID();
+
+    last_rusage = std::make_unique<RUsageCounters>();
+    last_taskstats = std::make_unique<TasksStatsCounters>();
+    taskstats_getter = std::make_unique<TaskStatsInfoGetter>();
+
+    memory_tracker.setDescription("(for thread)");
+    log = &Poco::Logger::get("ThreadStatus");
+
+    /// NOTE: It is important not to do any non-trivial actions (like updating ProfileEvents or logging) before ThreadStatus is created
+    /// Otherwise it could lead to SIGSEGV due to current_thread dereferencing
+}
+
+ThreadStatusPtr ThreadStatus::create()
+{
+    return ThreadStatusPtr(new ThreadStatus);
+}
+
+ThreadStatus::~ThreadStatus() = default;
+
+void ThreadStatus::initPerformanceCounters()
+{
+    performance_counters_finalized = false;
+
+    /// Clear stats from previous query if a new query is started
+    /// TODO: make separate query_thread_performance_counters and thread_performance_counters
+    performance_counters.resetCounters();
+    memory_tracker.resetCounters();
+    memory_tracker.setDescription("(for thread)");
+
+    query_start_time_nanoseconds = getCurrentTimeNanoseconds();
+    query_start_time = time(nullptr);
+    ++queries_started;
+
+    *last_rusage = RUsageCounters::current(query_start_time_nanoseconds);
+    has_permissions_for_taskstats = TaskStatsInfoGetter::checkProcessHasRequiredPermissions();
+    if (has_permissions_for_taskstats)
+        *last_taskstats = TasksStatsCounters::current();
+}
+
+void ThreadStatus::updatePerformanceCounters()
+{
+    try
+    {
+        RUsageCounters::updateProfileEvents(*last_rusage, performance_counters);
+        if (has_permissions_for_taskstats)
+            TasksStatsCounters::updateProfileEvents(*last_taskstats, performance_counters);
+    }
+    catch (...)
+    {
+        tryLogCurrentException(log);
+    }
+}
+
+void ThreadStatus::assertState(const std::initializer_list<int> & permitted_states, const char * description)
+{
+    for (auto permitted_state : permitted_states)
+    {
+        if (getCurrentState() == permitted_state)
+            return;
+    }
+
+    std::stringstream ss;
+    ss << "Unexpected thread state " << getCurrentState();
+    if (description)
+        ss << ": " << description;
+    throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR);
+}
+
+void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue)
+{
+    logs_queue_ptr = logs_queue;
+
+    if (!thread_group)
+        return;
+
+    std::unique_lock lock(thread_group->mutex);
+    thread_group->logs_queue_ptr = logs_queue;
+}
+
+}

dbms/src/Common/ThreadStatus.h

@@ -0,0 +1,197 @@
+#pragma once
+#include <Common/ProfileEvents.h>
+#include <Common/MemoryTracker.h>
+#include <IO/Progress.h>
+#include <memory>
+#include <map>
+#include <mutex>
+#include <shared_mutex>
+
+
+namespace Poco
+{
+    class Logger;
+}
+
+
+namespace DB
+{
+
+class Context;
+class QueryStatus;
+class ThreadStatus;
+using ThreadStatusPtr = std::shared_ptr<ThreadStatus>;
+class QueryThreadLog;
+struct TasksStatsCounters;
+struct RUsageCounters;
+class TaskStatsInfoGetter;
+class InternalTextLogsQueue;
+using InternalTextLogsQueuePtr = std::shared_ptr<InternalTextLogsQueue>;
+using InternalTextLogsQueueWeakPtr = std::weak_ptr<InternalTextLogsQueue>;
+
+
+class ThreadGroupStatus
+{
+public:
+
+    mutable std::shared_mutex mutex;
+
+    ProfileEvents::Counters performance_counters{VariableContext::Process};
+    MemoryTracker memory_tracker{VariableContext::Process};
+
+    Context * query_context = nullptr;
+    Context * global_context = nullptr;
+
+    InternalTextLogsQueueWeakPtr logs_queue_ptr;
+
+    /// Key is Poco's thread_id
+    using QueryThreadStatuses = std::map<UInt32, ThreadStatusPtr>;
+    QueryThreadStatuses thread_statuses;
+
+    /// The first thread created this thread group
+    ThreadStatusPtr master_thread;
+
+    String query;
+};
+
+using ThreadGroupStatusPtr = std::shared_ptr<ThreadGroupStatus>;
+
+
+class ThreadStatus : public std::enable_shared_from_this<ThreadStatus>
+{
+public:
+
+    /// Poco's thread number (the same number is used in logs)
+    UInt32 thread_number = 0;
+    /// Linux's PID (or TGID) (the same id is shown by ps util)
+    Int32 os_thread_id = -1;
+
+    /// TODO: merge them into common entity
+    ProfileEvents::Counters performance_counters{VariableContext::Thread};
+    MemoryTracker memory_tracker{VariableContext::Thread};
+
+    /// Statistics of read and write rows/bytes
+    Progress progress_in;
+    Progress progress_out;
+
+public:
+
+    static ThreadStatusPtr create();
+
+    ThreadGroupStatusPtr getThreadGroup() const
+    {
+        return thread_group;
+    }
+
+    enum ThreadState
+    {
+        DetachedFromQuery = 0,  /// We just created thread or it is a background thread
+        AttachedToQuery,        /// Thread executes enqueued query
+        Died,                   /// Thread does not exist
+    };
+
+    int getCurrentState() const
+    {
+        return thread_state.load(std::memory_order_relaxed);
+    }
+
+    String getQueryID();
+
+    /// Starts new query and create new thread group for it, current thread becomes master thread of the query
+    void initializeQuery();
+
+    /// Attaches slave thread to existing thread group
+    void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true);
+
+    InternalTextLogsQueuePtr getInternalTextLogsQueue() const
+    {
+        return thread_state == Died ? nullptr : logs_queue_ptr.lock();
+    }
+
+    void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue);
+
+    /// Sets query context for current thread and its thread group
+    /// NOTE: query_context have to be alive until detachQuery() is called
+    void attachQueryContext(Context & query_context);
+
+    /// Update several ProfileEvents counters
+    void updatePerformanceCounters();
+
+    /// Update ProfileEvents and dumps info to system.query_thread_log
+    void finalizePerformanceCounters();
+
+    /// Detaches thread from the thread group and the query, dumps performance counters if they have not been dumped
+    void detachQuery(bool exit_if_already_detached = false, bool thread_exits = false);
+
+    ~ThreadStatus();
+
+protected:
+
+    ThreadStatus();
+
+    void initPerformanceCounters();
+
+    void logToQueryThreadLog(QueryThreadLog & thread_log);
+
+    void assertState(const std::initializer_list<int> & permitted_states, const char * description = nullptr);
+
+    ThreadGroupStatusPtr thread_group;
+
+    std::atomic<int> thread_state{ThreadState::DetachedFromQuery};
+
+    /// Is set once
+    Context * global_context = nullptr;
+    /// Use it only from current thread
+    Context * query_context = nullptr;
+
+    /// A logs queue used by TCPHandler to pass logs to a client
+    InternalTextLogsQueueWeakPtr logs_queue_ptr;
+
+    bool performance_counters_finalized = false;
+    UInt64 query_start_time_nanoseconds = 0;
+    time_t query_start_time = 0;
+    size_t queries_started = 0;
+
+    Poco::Logger * log = nullptr;
+
+    friend class CurrentThread;
+    friend struct TasksStatsCounters;
+
+    /// Use ptr not to add extra dependencies in the header
+    std::unique_ptr<RUsageCounters> last_rusage;
+    std::unique_ptr<TasksStatsCounters> last_taskstats;
+    std::unique_ptr<TaskStatsInfoGetter> taskstats_getter;
+    bool has_permissions_for_taskstats = false;
+
+public:
+
+    /// Implicitly finalizes current thread in the destructor
+    class CurrentThreadScope
+    {
+    public:
+        void (*deleter)() = nullptr;
+
+        CurrentThreadScope() = default;
+        ~CurrentThreadScope()
+        {
+            try
+            {
+                if (deleter)
+                    deleter();
+            }
+            catch (...)
+            {
+                std::terminate();
+            }
+        }
+    };
+
+private:
+    static void defaultThreadDeleter();
+};
+
+
+extern thread_local ThreadStatusPtr current_thread;
+extern thread_local ThreadStatus::CurrentThreadScope current_thread_scope;
+
+}

dbms/src/Common/Throttler.h

@@ -5,10 +5,17 @@
 #include <memory>
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>
+#include <Common/ProfileEvents.h>
 #include <IO/WriteHelpers.h>
 #include <port/clock.h>
 
 
+namespace ProfileEvents
+{
+    extern const Event ThrottlerSleepMicroseconds;
+}
+
+
 namespace DB
 {
 
@@ -69,10 +76,14 @@ public:
             if (desired_ns > elapsed_ns)
             {
                 UInt64 sleep_ns = desired_ns - elapsed_ns;
-                timespec sleep_ts;
+                ::timespec sleep_ts;
                 sleep_ts.tv_sec = sleep_ns / 1000000000;
                 sleep_ts.tv_nsec = sleep_ns % 1000000000;
-                nanosleep(&sleep_ts, nullptr);    /// NOTE Returns early in case of a signal. This is considered normal.
+
+                /// NOTE: Returns early in case of a signal. This is considered normal.
+                ::nanosleep(&sleep_ts, nullptr);
+
+                ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_ns / 1000UL);
             }
         }
 

dbms/src/Common/UInt128.h

@@ -28,9 +28,17 @@ struct UInt128
     UInt64 high;
 
     UInt128() = default;
-    explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
     explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) {}
 
+#if 1
+    explicit UInt128(const unsigned __int128 rhs)
+    :   low(rhs & 0xffffffffffffffffll),
+        high(rhs >> 64)
+    {}
+#else
+    explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
+#endif
+
     auto tuple() const { return std::tie(high, low); }
 
     bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); }
@@ -65,7 +73,7 @@ template <typename T> bool inline operator<  (T a, const UInt128 b) { return UIn
 
 template <> constexpr bool IsNumber<UInt128> = true;
 template <> struct TypeName<UInt128> { static const char * get() { return "UInt128"; } };
-template <> struct TypeNumber<UInt128> { static constexpr const size_t value = 5; };
+template <> struct TypeId<UInt128> { static constexpr const size_t value = 5; };
 
 struct UInt128Hash
 {

dbms/src/Common/VariableContext.h

@@ -0,0 +1,12 @@
+#pragma once
+
+/// Used in ProfileEvents and MemoryTracker to determine their hierarchy level
+/// The less value the higher level (zero level is the root)
+enum class VariableContext
+{
+    Global = 0,
+    User,           /// Group of processes
+    Process,        /// For example, a query or a merge
+    Thread,         /// A thread of a process
+    Snapshot        /// Does not belong to anybody
+};

dbms/src/Common/ZooKeeper/KeeperException.h

@@ -7,27 +7,6 @@ namespace zkutil
 {
 
 
-/// You should reinitialize ZooKeeper session in case of these errors
-inline bool isHardwareError(int32_t zk_return_code)
-{
-    return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE
-        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED
-        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED
-        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS
-        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT;
-}
-
-/// Valid errors sent from server
-inline bool isUserError(int32_t zk_return_code)
-{
-    return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE
-           || zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION
-           || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS
-           || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS
-           || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY;
-}
-
-
 using KeeperException = ZooKeeperImpl::Exception;
 
 

dbms/src/Common/ZooKeeper/LeaderElection.h

@@ -6,7 +6,7 @@
 #include <memory>
 #include <common/logger_useful.h>
 #include <Common/CurrentMetrics.h>
-#include <Common/BackgroundSchedulePool.h>
+#include <Core/BackgroundSchedulePool.h>
 
 
 namespace ProfileEvents

dbms/src/Common/ZooKeeper/ZooKeeper.cpp

@@ -493,7 +493,7 @@ Responses ZooKeeper::multi(const Requests & requests)
 int32_t ZooKeeper::tryMulti(const Requests & requests, Responses & responses)
 {
     int32_t code = multiImpl(requests, responses);
-    if (code && !isUserError(code))
+    if (code && !ZooKeeperImpl::ZooKeeper::isUserError(code))
         throw KeeperException(code);
     return code;
 }
@@ -824,7 +824,7 @@ size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Responses & re
         if (responses[index]->error)
             return index;
 
-    if (!isUserError(code))
+    if (!ZooKeeperImpl::ZooKeeper::isUserError(code))
         throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(code) + "' is not valid response code for that",
             DB::ErrorCodes::LOGICAL_ERROR);
 
@@ -850,7 +850,7 @@ void KeeperMultiException::check(int32_t code, const Requests & requests, const
     if (!code)
         return;
 
-    if (isUserError(code))
+    if (ZooKeeperImpl::ZooKeeper::isUserError(code))
         throw KeeperMultiException(code, requests, responses);
     else
         throw KeeperException(code);

dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp

@@ -25,7 +25,9 @@ namespace DB
 
 namespace ProfileEvents
 {
-    extern const Event ZooKeeperExceptions;
+    extern const Event ZooKeeperUserExceptions;
+    extern const Event ZooKeeperHardwareExceptions;
+    extern const Event ZooKeeperOtherExceptions;
     extern const Event ZooKeeperInit;
     extern const Event ZooKeeperTransactions;
     extern const Event ZooKeeperCreate;
@@ -267,7 +269,12 @@ namespace ZooKeeperImpl
 Exception::Exception(const std::string & msg, const int32_t code, int)
     : DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code)
 {
-    ProfileEvents::increment(ProfileEvents::ZooKeeperExceptions);
+    if (ZooKeeper::isUserError(code))
+        ProfileEvents::increment(ProfileEvents::ZooKeeperUserExceptions);
+    else if (ZooKeeper::isHardwareError(code))
+        ProfileEvents::increment(ProfileEvents::ZooKeeperHardwareExceptions);
+    else
+        ProfileEvents::increment(ProfileEvents::ZooKeeperOtherExceptions);
 }
 
 Exception::Exception(const std::string & msg, const int32_t code)
@@ -515,6 +522,25 @@ const char * ZooKeeper::errorMessage(int32_t code)
     return "unknown error";
 }
 
+bool ZooKeeper::isHardwareError(int32_t zk_return_code)
+{
+    return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZMARSHALLINGERROR
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT;
+}
+
+bool ZooKeeper::isUserError(int32_t zk_return_code)
+{
+    return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS
+        || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY;
+}
+
 
 ZooKeeper::~ZooKeeper()
 {

dbms/src/Common/ZooKeeper/ZooKeeperImpl.h

@@ -549,6 +549,12 @@ public:
         ZSESSIONMOVED = -118                /// Session moved to another server, so operation is ignored
     };
 
+    /// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors
+    static bool isHardwareError(int32_t code);
+
+    /// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here.
+    static bool isUserError(int32_t code);
+
     static const char * errorMessage(int32_t code);
 
     /// For watches.

dbms/src/Common/setThreadName.cpp

@@ -6,10 +6,20 @@
 #else
 #include <sys/prctl.h>
 #endif
+#include <pthread.h>
+#include <cstring>
 
 #include <Common/Exception.h>
 #include <Common/setThreadName.h>
 
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int PTHREAD_ERROR;
+}
+}
+
 
 void setThreadName(const char * name)
 {
@@ -22,5 +32,21 @@ void setThreadName(const char * name)
 #else
     if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))
 #endif
-        DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME...)");
+        DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)");
+}
+
+std::string getThreadName()
+{
+    std::string name(16, '\0');
+
+#if defined(__FreeBSD__) || defined(__APPLE__)
+    if (pthread_get_name_np(pthread_self(), name.data(), name.size());
+        throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
+#else
+    if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
+#endif
+        DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
+
+    name.resize(std::strlen(name.data()));
+    return name;
 }

dbms/src/Common/setThreadName.h

@@ -1,7 +1,10 @@
 #pragma once
+#include <string>
 
 /** Sets the thread name (maximum length is 15 bytes),
   *  which will be visible in ps, gdb, /proc,
   *  for convenience of observation and debugging.
   */
 void setThreadName(const char * name);
+
+std::string getThreadName();

dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp

@@ -32,7 +32,7 @@ TEST(Common, RWLockFIFO_1)
 
     auto func = [&] (size_t threads, int round)
     {
-        for (int  i = 0; i < cycles; ++i)
+        for (int i = 0; i < cycles; ++i)
         {
             auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockFIFO::Read : RWLockFIFO::Write;
             auto sleep_for = std::chrono::duration<int, std::micro>(std::uniform_int_distribution<>(1, 100)(gen));

dbms/src/Core/AccurateComparison.h

@@ -332,10 +332,32 @@ inline bool equalsOp<DB::Float32, DB::UInt128>(DB::Float32 f, DB::UInt128 u)
     return equalsOp(static_cast<DB::Float64>(f), u);
 }
 
-inline bool greaterOp(DB::Int128 i, DB::Float64 f) { return static_cast<DB::Int128>(f) < i; }
-inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return static_cast<DB::Int128>(f) < i; }
-inline bool greaterOp(DB::Float64 f, DB::Int128 i) { return static_cast<DB::Int128>(f) > i; }
-inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return static_cast<DB::Int128>(f) > i; }
+inline bool greaterOp(DB::Int128 i, DB::Float64 f)
+{
+    static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
+    static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
+
+    if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
+        return static_cast<DB::Float64>(i) > f;
+
+    return (f < static_cast<DB::Float64>(min_int128))
+        || (f < static_cast<DB::Float64>(max_int128) && i > static_cast<DB::Int128>(f));
+}
+
+inline bool greaterOp(DB::Float64 f, DB::Int128 i)
+{
+    static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
+    static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
+
+    if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
+        return f > static_cast<DB::Float64>(i);
+
+    return (f >= static_cast<DB::Float64>(max_int128))
+        || (f > static_cast<DB::Float64>(min_int128) && static_cast<DB::Int128>(f) > i);
+}
+
+inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return greaterOp(i, static_cast<DB::Float64>(f)); }
+inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return greaterOp(static_cast<DB::Float64>(f), i); }
 
 inline bool equalsOp(DB::Int128 i, DB::Float64 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float64>(i) == f; }
 inline bool equalsOp(DB::Int128 i, DB::Float32 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float32>(i) == f; }

dbms/src/Core/BackgroundSchedulePool.cpp

@@ -1,11 +1,14 @@
-#include <Common/BackgroundSchedulePool.h>
+#include "BackgroundSchedulePool.h"
 #include <Common/MemoryTracker.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/Exception.h>
 #include <Common/setThreadName.h>
 #include <Common/Stopwatch.h>
+#include <Common/CurrentThread.h>
 #include <common/logger_useful.h>
 #include <chrono>
+#include <ext/scope_guard.h>
+
 
 namespace CurrentMetrics
 {
@@ -140,6 +143,12 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size)
 {
     LOG_INFO(&Logger::get("BackgroundSchedulePool"), "Create BackgroundSchedulePool with " << size << " threads");
 
+    /// Put all threads of both thread pools to one thread group
+    /// The master thread exits immediately
+    CurrentThread::initializeQuery();
+    thread_group = CurrentThread::getGroup();
+    CurrentThread::detachQuery();
+
     threads.resize(size);
     for (auto & thread : threads)
         thread = std::thread([this] { threadFunction(); });
@@ -212,9 +221,11 @@ void BackgroundSchedulePool::threadFunction()
 {
     setThreadName("BackgrSchedPool");
 
-    MemoryTracker memory_tracker;
-    memory_tracker.setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool);
-    current_memory_tracker = &memory_tracker;
+    /// Put all threads to one thread pool
+    CurrentThread::attachTo(thread_group);
+    SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); });
+
+    CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool);
 
     while (!shutdown)
     {
@@ -224,8 +235,6 @@ void BackgroundSchedulePool::threadFunction()
             task_notification.execute();
         }
     }
-
-    current_memory_tracker = nullptr;
 }
 
 
@@ -233,6 +242,10 @@ void BackgroundSchedulePool::delayExecutionThreadFunction()
 {
     setThreadName("BckSchPoolDelay");
 
+    /// Put all threads to one thread pool
+    CurrentThread::attachTo(thread_group);
+    SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); });
+
     while (!shutdown)
     {
         TaskInfoPtr task;

dbms/src/Core/BackgroundSchedulePool.h

@@ -12,6 +12,7 @@
 #include <functional>
 #include <boost/noncopyable.hpp>
 #include <Common/ZooKeeper/Types.h>
+#include <Common/CurrentThread.h>
 
 namespace DB
 {
@@ -138,6 +139,9 @@ private:
     std::thread delayed_thread;
     /// Tasks ordered by scheduled time.
     DelayedTasks delayed_tasks;
+
+    /// Thread group used for profiling purposes
+    ThreadGroupStatusPtr thread_group;
 };
 
 using BackgroundSchedulePoolPtr = std::shared_ptr<BackgroundSchedulePool>;

dbms/src/Core/Defines.h

@@ -46,6 +46,7 @@
 #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
 #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
 #define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401
+#define DBMS_MIN_REVISION_WITH_SERVER_LOGS 54406
 
 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
 #define DBMS_TCP_PROTOCOL_VERSION 54226

dbms/src/Core/ExternalTable.cpp

@@ -0,0 +1,182 @@
+#include <boost/program_options.hpp>
+#include <boost/algorithm/string.hpp>
+#include <DataStreams/AsynchronousBlockInputStream.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <Interpreters/Context.h>
+#include <IO/copyData.h>
+#include <IO/ReadBufferFromIStream.h>
+#include <IO/ReadBufferFromFile.h>
+#include <IO/LimitReadBuffer.h>
+#include <Storages/StorageMemory.h>
+#include <Poco/Net/HTMLForm.h>
+#include <Poco/Net/MessageHeader.h>
+#include <Common/HTMLForm.h>
+
+#include <Core/ExternalTable.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+
+ExternalTableData BaseExternalTable::getData(const Context & context)
+{
+    initReadBuffer();
+    initSampleBlock();
+    auto input = context.getInputFormat(format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE);
+    return std::make_pair(std::make_shared<AsynchronousBlockInputStream>(input), name);
+}
+
+void BaseExternalTable::clean()
+{
+    name = "";
+    file = "";
+    format = "";
+    structure.clear();
+    sample_block = Block();
+    read_buffer.reset();
+}
+
+/// Function for debugging information output
+void BaseExternalTable::write()
+{
+    std::cerr << "file " << file << std::endl;
+    std::cerr << "name " << name << std::endl;
+    std::cerr << "format " << format << std::endl;
+    std::cerr << "structure: \n";
+    for (size_t i = 0; i < structure.size(); ++i)
+        std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl;
+}
+
+std::vector<std::string> BaseExternalTable::split(const std::string & s, const std::string & d)
+{
+    std::vector<std::string> res;
+    boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on);
+    return res;
+}
+
+void BaseExternalTable::parseStructureFromStructureField(const std::string & argument)
+{
+    std::vector<std::string> vals = split(argument, " ,");
+
+    if (vals.size() & 1)
+        throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS);
+
+    for (size_t i = 0; i < vals.size(); i += 2)
+        structure.emplace_back(vals[i], vals[i + 1]);
+}
+
+void BaseExternalTable::parseStructureFromTypesField(const std::string & argument)
+{
+    std::vector<std::string> vals = split(argument, " ,");
+
+    for (size_t i = 0; i < vals.size(); ++i)
+        structure.emplace_back("_" + toString(i + 1), vals[i]);
+}
+
+void BaseExternalTable::initSampleBlock()
+{
+    const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
+
+    for (size_t i = 0; i < structure.size(); ++i)
+    {
+        ColumnWithTypeAndName column;
+        column.name = structure[i].first;
+        column.type = data_type_factory.get(structure[i].second);
+        column.column = column.type->createColumn();
+        sample_block.insert(std::move(column));
+    }
+}
+
+
+void ExternalTable::initReadBuffer()
+{
+    if (file == "-")
+        read_buffer = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
+    else
+        read_buffer = std::make_unique<ReadBufferFromFile>(file);
+}
+
+ExternalTable::ExternalTable(const boost::program_options::variables_map & external_options)
+{
+    if (external_options.count("file"))
+        file = external_options["file"].as<std::string>();
+    else
+        throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
+
+    if (external_options.count("name"))
+        name = external_options["name"].as<std::string>();
+    else
+        throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
+
+    if (external_options.count("format"))
+        format = external_options["format"].as<std::string>();
+    else
+        throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
+
+    if (external_options.count("structure"))
+        parseStructureFromStructureField(external_options["structure"].as<std::string>());
+    else if (external_options.count("types"))
+        parseStructureFromTypesField(external_options["types"].as<std::string>());
+    else
+        throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
+}
+
+
+void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, std::istream & stream)
+{
+    const Settings & settings = context.getSettingsRef();
+
+    /// The buffer is initialized here, not in the virtual function initReadBuffer
+    read_buffer_impl = std::make_unique<ReadBufferFromIStream>(stream);
+
+    if (settings.http_max_multipart_form_data_size)
+        read_buffer = std::make_unique<LimitReadBuffer>(
+            *read_buffer_impl, settings.http_max_multipart_form_data_size,
+            true, "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting");
+    else
+        read_buffer = std::move(read_buffer_impl);
+
+    /// Retrieve a collection of parameters from MessageHeader
+    Poco::Net::NameValueCollection content;
+    std::string label;
+    Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content);
+
+    /// Get parameters
+    name = content.get("name", "_data");
+    format = params.get(name + "_format", "TabSeparated");
+
+    if (params.has(name + "_structure"))
+        parseStructureFromStructureField(params.get(name + "_structure"));
+    else if (params.has(name + "_types"))
+        parseStructureFromTypesField(params.get(name + "_types"));
+    else
+        throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS);
+
+    ExternalTableData data = getData(context);
+
+    /// Create table
+    NamesAndTypesList columns = sample_block.getNamesAndTypesList();
+    StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
+    storage->startup();
+    context.addExternalTable(data.second, storage);
+    BlockOutputStreamPtr output = storage->write(ASTPtr(), settings);
+
+    /// Write data
+    data.first->readPrefix();
+    output->writePrefix();
+    while(Block block = data.first->read())
+        output->write(block);
+    data.first->readSuffix();
+    output->writeSuffix();
+
+    /// We are ready to receive the next file, for this we clear all the information received
+    clean();
+}
+
+}

dbms/src/Core/ExternalTable.h

@@ -0,0 +1,111 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <iosfwd>
+
+#include <Poco/Net/PartHandler.h>
+
+#include <Core/Block.h>
+#include <Client/Connection.h>
+#include <IO/ReadBuffer.h>
+
+
+namespace Poco
+{
+    namespace Net
+    {
+        class NameValueCollection;
+        class MessageHeader;
+    }
+}
+
+namespace boost
+{
+    namespace program_options
+    {
+        class variables_map;
+    }
+}
+
+
+namespace DB
+{
+
+class Context;
+
+
+/// The base class containing the basic information about external table and
+/// basic functions for extracting this information from text fields.
+class BaseExternalTable
+{
+public:
+    std::string file;       /// File with data or '-' if stdin
+    std::string name;       /// The name of the table
+    std::string format;     /// Name of the data storage format
+
+    /// Description of the table structure: (column name, data type name)
+    std::vector<std::pair<std::string, std::string>> structure;
+
+    std::unique_ptr<ReadBuffer> read_buffer;
+    Block sample_block;
+
+    virtual ~BaseExternalTable() {}
+
+    /// Initialize read_buffer, depending on the data source. By default, does nothing.
+    virtual void initReadBuffer() {}
+
+    /// Get the table data - a pair (a stream with the contents of the table, the name of the table)
+    ExternalTableData getData(const Context & context);
+
+protected:
+    /// Clear all accumulated information
+    void clean();
+
+    /// Function for debugging information output
+    void write();
+
+    static std::vector<std::string> split(const std::string & s, const std::string & d);
+
+    /// Construct the `structure` vector from the text field `structure`
+    virtual void parseStructureFromStructureField(const std::string & argument);
+
+    /// Construct the `structure` vector from the text field `types`
+    virtual void parseStructureFromTypesField(const std::string & argument);
+
+private:
+    /// Initialize sample_block according to the structure of the table stored in the `structure`
+    void initSampleBlock();
+};
+
+
+/// Parsing of external table used in the tcp client.
+class ExternalTable : public BaseExternalTable
+{
+public:
+    void initReadBuffer() override;
+
+    /// Extract parameters from variables_map, which is built on the client command line
+    ExternalTable(const boost::program_options::variables_map & external_options);
+};
+
+
+/// Parsing of external table used when sending tables via http
+/// The `handlePart` function will be called for each table passed,
+/// so it's also necessary to call `clean` at the end of the `handlePart`.
+class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
+{
+public:
+    ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {}
+
+    void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream);
+
+private:
+    Context & context;
+    const Poco::Net::NameValueCollection & params;
+    std::unique_ptr<ReadBuffer> read_buffer_impl;
+};
+
+
+}

dbms/src/Core/Field.cpp

@@ -273,21 +273,21 @@ namespace DB
     }
 
 
-    bool DecField::operator < (const DecField & r) const
+    bool DecimalField::operator < (const DecimalField & r) const
     {
-        using Comparator = DecimalComparison<Dec128, Dec128, LessOp>;
-        return Comparator::compare(Dec128(dec), Dec128(r.dec), scale, r.scale);
+        using Comparator = DecimalComparison<Decimal128, Decimal128, LessOp>;
+        return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale);
     }
 
-    bool DecField::operator <= (const DecField & r) const
+    bool DecimalField::operator <= (const DecimalField & r) const
     {
-        using Comparator = DecimalComparison<Dec128, Dec128, LessOrEqualsOp>;
-        return Comparator::compare(Dec128(dec), Dec128(r.dec), scale, r.scale);
+        using Comparator = DecimalComparison<Decimal128, Decimal128, LessOrEqualsOp>;
+        return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale);
     }
 
-    bool DecField::operator == (const DecField & r) const
+    bool DecimalField::operator == (const DecimalField & r) const
     {
-        using Comparator = DecimalComparison<Dec128, Dec128, EqualsOp>;
-        return Comparator::compare(Dec128(dec), Dec128(r.dec), scale, r.scale);
+        using Comparator = DecimalComparison<Decimal128, Decimal128, EqualsOp>;
+        return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale);
     }
 }

dbms/src/Core/Field.h

@@ -28,29 +28,29 @@ using TupleBackend = std::vector<Field>;
 STRONG_TYPEDEF(TupleBackend, Tuple) /// Array and Tuple are different types with equal representation inside Field.
 
 
-class DecField
+class DecimalField
 {
 public:
     static constexpr UInt32 wrongScale() { return std::numeric_limits<UInt32>::max(); }
 
-    DecField(Int128 value, UInt32 scale_ = wrongScale())
+    DecimalField(Int128 value, UInt32 scale_ = wrongScale())
     :   dec(value),
         scale(scale_)
     {}
 
-    operator Dec32() const { return dec; }
-    operator Dec64() const { return dec; }
-    operator Dec128() const { return dec; }
+    operator Decimal32() const { return dec; }
+    operator Decimal64() const { return dec; }
+    operator Decimal128() const { return dec; }
 
     UInt32 getScale() const { return scale; }
 
-    bool operator < (const DecField & r) const;
-    bool operator <= (const DecField & r) const;
-    bool operator == (const DecField & r) const;
+    bool operator < (const DecimalField & r) const;
+    bool operator <= (const DecimalField & r) const;
+    bool operator == (const DecimalField & r) const;
 
-    bool operator > (const DecField & r) const { return r < *this; }
-    bool operator >= (const DecField & r) const { return r <= * this; }
-    bool operator != (const DecField & r) const { return !(*this == r); }
+    bool operator > (const DecimalField & r) const { return r < *this; }
+    bool operator >= (const DecimalField & r) const { return r <= * this; }
+    bool operator != (const DecimalField & r) const { return !(*this == r); }
 
 private:
     Int128 dec;
@@ -294,7 +294,7 @@ public:
             case Types::String:  return get<String>()  < rhs.get<String>();
             case Types::Array:   return get<Array>()   < rhs.get<Array>();
             case Types::Tuple:   return get<Tuple>()   < rhs.get<Tuple>();
-            case Types::Decimal: return get<DecField>()  < rhs.get<DecField>();
+            case Types::Decimal: return get<DecimalField>()  < rhs.get<DecimalField>();
         }
 
         throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@@ -323,7 +323,7 @@ public:
             case Types::String:  return get<String>()  <= rhs.get<String>();
             case Types::Array:   return get<Array>()   <= rhs.get<Array>();
             case Types::Tuple:   return get<Tuple>()   <= rhs.get<Tuple>();
-            case Types::Decimal: return get<DecField>()  <= rhs.get<DecField>();
+            case Types::Decimal: return get<DecimalField>()  <= rhs.get<DecimalField>();
         }
 
         throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@@ -350,7 +350,7 @@ public:
             case Types::Tuple:   return get<Tuple>()   == rhs.get<Tuple>();
             case Types::UInt128: return get<UInt128>() == rhs.get<UInt128>();
             case Types::Int128:  return get<Int128>()  == rhs.get<Int128>();
-            case Types::Decimal: return get<DecField>()  == rhs.get<DecField>();
+            case Types::Decimal: return get<DecimalField>()  == rhs.get<DecimalField>();
         }
 
         throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@@ -363,7 +363,7 @@ public:
 
 private:
     std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
-        Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple, DecField
+        Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple, DecimalField
         > storage;
 
     Types::Which which;
@@ -412,7 +412,7 @@ private:
             case Types::String:  f(field.template get<String>());  return;
             case Types::Array:   f(field.template get<Array>());   return;
             case Types::Tuple:   f(field.template get<Tuple>());   return;
-            case Types::Decimal: f(field.template get<DecField>()); return;
+            case Types::Decimal: f(field.template get<DecimalField>()); return;
 
             default:
                 throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@@ -496,7 +496,7 @@ template <> struct Field::TypeToEnum<Float64> { static const Types::Which value
 template <> struct Field::TypeToEnum<String>  { static const Types::Which value = Types::String; };
 template <> struct Field::TypeToEnum<Array>   { static const Types::Which value = Types::Array; };
 template <> struct Field::TypeToEnum<Tuple>   { static const Types::Which value = Types::Tuple; };
-template <> struct Field::TypeToEnum<DecField>{ static const Types::Which value = Types::Decimal; };
+template <> struct Field::TypeToEnum<DecimalField>{ static const Types::Which value = Types::Decimal; };
 
 template <> struct Field::EnumToType<Field::Types::Null>    { using Type = Null; };
 template <> struct Field::EnumToType<Field::Types::UInt64>  { using Type = UInt64; };
@@ -507,7 +507,7 @@ template <> struct Field::EnumToType<Field::Types::Float64> { using Type = Float
 template <> struct Field::EnumToType<Field::Types::String>  { using Type = String; };
 template <> struct Field::EnumToType<Field::Types::Array>   { using Type = Array; };
 template <> struct Field::EnumToType<Field::Types::Tuple>   { using Type = Tuple; };
-template <> struct Field::EnumToType<Field::Types::Decimal> { using Type = DecField; };
+template <> struct Field::EnumToType<Field::Types::Decimal> { using Type = DecimalField; };
 
 
 template <typename T>
@@ -551,9 +551,9 @@ template <> struct NearestFieldType<Int16>   { using Type = Int64; };
 template <> struct NearestFieldType<Int32>   { using Type = Int64; };
 template <> struct NearestFieldType<Int64>   { using Type = Int64; };
 template <> struct NearestFieldType<Int128>  { using Type = Int128; };
-template <> struct NearestFieldType<Dec32>   { using Type = DecField; };
-template <> struct NearestFieldType<Dec64>   { using Type = DecField; };
-template <> struct NearestFieldType<Dec128>  { using Type = DecField; };
+template <> struct NearestFieldType<Decimal32>   { using Type = DecimalField; };
+template <> struct NearestFieldType<Decimal64>   { using Type = DecimalField; };
+template <> struct NearestFieldType<Decimal128>  { using Type = DecimalField; };
 template <> struct NearestFieldType<Float32> { using Type = Float64; };
 template <> struct NearestFieldType<Float64> { using Type = Float64; };
 template <> struct NearestFieldType<String>  { using Type = String; };

dbms/src/Core/Protocol.h

@@ -69,6 +69,7 @@ namespace Protocol
             Totals = 7,               /// A block with totals (compressed or not).
             Extremes = 8,             /// A block with minimums and maximums (compressed or not).
             TablesStatusResponse = 9, /// A response to TablesStatus request.
+            Log = 10                  /// System logs of the query execution
         };
 
         /// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10
@@ -77,8 +78,8 @@ namespace Protocol
         /// See https://www.securecoding.cert.org/confluence/display/cplusplus/INT36-CPP.+Do+not+use+out-of-range+enumeration+values
         inline const char * toString(UInt64 packet)
         {
-            static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse" };
-            return packet < 10
+            static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse", "Log" };
+            return packet < 11
                 ? data[packet]
                 : "Unknown packet";
         }
@@ -97,6 +98,7 @@ namespace Protocol
             Cancel = 3,              /// Cancel the query execution.
             Ping = 4,                /// Check that connection to the server is alive.
             TablesStatusRequest = 5, /// Check status of tables on the server.
+            KeepAlive = 6            /// Keep the connection alive
         };
 
         inline const char * toString(UInt64 packet)

dbms/src/Core/Types.h

@@ -57,21 +57,21 @@ template <> struct TypeName<Float32> { static const char * get() { return "Float
 template <> struct TypeName<Float64> { static const char * get() { return "Float64"; } };
 template <> struct TypeName<String>  { static const char * get() { return "String";  } };
 
-template <typename T> struct TypeNumber;
+template <typename T> struct TypeId;
 
 /// 0 reserved for types without number
-template <> struct TypeNumber<UInt8>    { static constexpr const size_t value = 1;  };
-template <> struct TypeNumber<UInt16>   { static constexpr const size_t value = 2;  };
-template <> struct TypeNumber<UInt32>   { static constexpr const size_t value = 3;  };
-template <> struct TypeNumber<UInt64>   { static constexpr const size_t value = 4;  };
-/// 5 reserved for TypeNumber<UInt128>
-template <> struct TypeNumber<Float32>  { static constexpr const size_t value = 7;  };
-template <> struct TypeNumber<Float64>  { static constexpr const size_t value = 8;  };
-template <> struct TypeNumber<Int8>     { static constexpr const size_t value = 9;  };
-template <> struct TypeNumber<Int16>    { static constexpr const size_t value = 10; };
-template <> struct TypeNumber<Int32>    { static constexpr const size_t value = 11; };
-template <> struct TypeNumber<Int64>    { static constexpr const size_t value = 12; };
-/// 13 reserved for TypeNumber<Int128>
+template <> struct TypeId<UInt8>    { static constexpr const size_t value = 1;  };
+template <> struct TypeId<UInt16>   { static constexpr const size_t value = 2;  };
+template <> struct TypeId<UInt32>   { static constexpr const size_t value = 3;  };
+template <> struct TypeId<UInt64>   { static constexpr const size_t value = 4;  };
+/// 5 reserved for TypeId<UInt128>
+template <> struct TypeId<Float32>  { static constexpr const size_t value = 7;  };
+template <> struct TypeId<Float64>  { static constexpr const size_t value = 8;  };
+template <> struct TypeId<Int8>     { static constexpr const size_t value = 9;  };
+template <> struct TypeId<Int16>    { static constexpr const size_t value = 10; };
+template <> struct TypeId<Int32>    { static constexpr const size_t value = 11; };
+template <> struct TypeId<Int64>    { static constexpr const size_t value = 12; };
+/// 13 reserved for TypeId<Int128>
 
 /// Not a data type in database, defined just for convenience.
 using Strings = std::vector<String>;
@@ -85,7 +85,7 @@ namespace DB
 using Int128 = __int128;
 template <> constexpr bool IsNumber<Int128> = true;
 template <> struct TypeName<Int128>  { static const char * get() { return "Int128";  } };
-template <> struct TypeNumber<Int128>   { static constexpr const size_t value = 13; };
+template <> struct TypeId<Int128>   { static constexpr const size_t value = 13; };
 
 }
 
@@ -118,62 +118,55 @@ template <> struct is_arithmetic<__int128>
 
 namespace DB
 {
-    /// Own FieldType for Decimal
+    /// Own FieldType for Decimal.
+    /// It is only a "storage" for decimal. To perform operations, you also have to provide a scale (number of digits after point).
     template <typename T>
-    class Dec
+    struct Decimal
     {
-    public:
         using NativeType = T;
 
-        Dec() = default;
-        Dec(Dec<T> &&) = default;
-        Dec(const Dec<T> &) = default;
+        Decimal() = default;
+        Decimal(Decimal<T> &&) = default;
+        Decimal(const Decimal<T> &) = default;
 
-        Dec(const T & value_)
+        Decimal(const T & value_)
         :   value(value_)
         {}
 
         template <typename U>
-        Dec(const Dec<U> & x)
+        Decimal(const Decimal<U> & x)
         :   value(x)
         {}
 
-        constexpr Dec<T> & operator = (Dec<T> &&) = default;
-        constexpr Dec<T> & operator = (const Dec<T> &) = default;
+        constexpr Decimal<T> & operator = (Decimal<T> &&) = default;
+        constexpr Decimal<T> & operator = (const Decimal<T> &) = default;
 
         operator T () const { return value; }
 
-        const Dec<T> & operator += (const T & x) { value += x; return *this; }
-        const Dec<T> & operator -= (const T & x) { value -= x; return *this; }
-        const Dec<T> & operator *= (const T & x) { value *= x; return *this; }
-        const Dec<T> & operator /= (const T & x) { value /= x; return *this; }
-        const Dec<T> & operator %= (const T & x) { value %= x; return *this; }
+        const Decimal<T> & operator += (const T & x) { value += x; return *this; }
+        const Decimal<T> & operator -= (const T & x) { value -= x; return *this; }
+        const Decimal<T> & operator *= (const T & x) { value *= x; return *this; }
+        const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
+        const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
 
-    private:
         T value;
     };
 
-    using Dec32 = Dec<Int32>;
-    using Dec64 = Dec<Int64>;
-    using Dec128 = Dec<Int128>;
+    using Decimal32 = Decimal<Int32>;
+    using Decimal64 = Decimal<Int64>;
+    using Decimal128 = Decimal<Int128>;
 
-    template <> struct TypeName<Dec32>   { static const char * get() { return "Dec32";   } };
-    template <> struct TypeName<Dec64>   { static const char * get() { return "Dec64";   } };
-    template <> struct TypeName<Dec128>  { static const char * get() { return "Dec128";  } };
+    template <> struct TypeName<Decimal32>   { static const char * get() { return "Decimal32";   } };
+    template <> struct TypeName<Decimal64>   { static const char * get() { return "Decimal64";   } };
+    template <> struct TypeName<Decimal128>  { static const char * get() { return "Decimal128";  } };
 
-    template <> struct TypeNumber<Dec32>    { static constexpr const size_t value = 16; };
-    template <> struct TypeNumber<Dec64>    { static constexpr const size_t value = 17; };
-    template <> struct TypeNumber<Dec128>   { static constexpr const size_t value = 18; };
+    template <> struct TypeId<Decimal32>    { static constexpr const size_t value = 16; };
+    template <> struct TypeId<Decimal64>    { static constexpr const size_t value = 17; };
+    template <> struct TypeId<Decimal128>   { static constexpr const size_t value = 18; };
 
     template <typename T>
-    inline constexpr bool decTrait() { return false; }
-    template <> constexpr bool decTrait<Dec32>() { return true; }
-    template <> constexpr bool decTrait<Dec64>() { return true; }
-    template <> constexpr bool decTrait<Dec128>() { return true; }
-
-    template <typename T>
-    inline constexpr bool decBaseTrait() { return false; }
-    template <> constexpr bool decBaseTrait<Int32>() { return true; }
-    template <> constexpr bool decBaseTrait<Int64>() { return true; }
-    template <> constexpr bool decBaseTrait<Int128>() { return true; }
+    constexpr bool IsDecimalNumber = false;
+    template <> constexpr bool IsDecimalNumber<Decimal32> = true;
+    template <> constexpr bool IsDecimalNumber<Decimal64> = true;
+    template <> constexpr bool IsDecimalNumber<Decimal128> = true;
 }

dbms/src/DataStreams/AsynchronousBlockInputStream.cpp

@@ -0,0 +1,84 @@
+#include "AsynchronousBlockInputStream.h"
+#include <Common/CurrentThread.h>
+
+
+namespace DB
+{
+
+Block AsynchronousBlockInputStream::readImpl()
+{
+    /// If there were no calculations yet, calculate the first block synchronously
+    if (!started)
+    {
+        calculate();
+        started = true;
+    }
+    else    /// If the calculations are already in progress - wait for the result
+        pool.wait();
+
+    if (exception)
+        std::rethrow_exception(exception);
+
+    Block res = block;
+    if (!res)
+        return res;
+
+    /// Start the next block calculation
+    block.clear();
+    next();
+
+    return res;
+}
+
+
+void AsynchronousBlockInputStream::next()
+{
+    ready.reset();
+
+    pool.schedule([this, thread_group=CurrentThread::getGroup()] ()
+    {
+        CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
+
+        try
+        {
+            if (first)
+                setThreadName("AsyncBlockInput");
+
+            /// AsynchronousBlockInputStream is used in Client which does not create queries and thread groups
+            if (thread_group)
+                CurrentThread::attachToIfDetached(thread_group);
+        }
+        catch (...)
+        {
+            exception = std::current_exception();
+            ready.set();
+            return;
+        }
+
+        calculate();
+    });
+}
+
+
+void AsynchronousBlockInputStream::calculate()
+{
+    try
+    {
+        if (first)
+        {
+            first = false;
+            children.back()->readPrefix();
+        }
+
+        block = children.back()->read();
+    }
+    catch (...)
+    {
+        exception = std::current_exception();
+    }
+
+    ready.set();
+}
+
+}
+

dbms/src/DataStreams/AsynchronousBlockInputStream.h

@@ -7,6 +7,7 @@
 #include <Common/CurrentMetrics.h>
 #include <common/ThreadPool.h>
 #include <Common/MemoryTracker.h>
+#include <Poco/Ext/ThreadNumber.h>
 
 
 namespace CurrentMetrics
@@ -91,64 +92,12 @@ protected:
     Block block;
     std::exception_ptr exception;
 
+    Block readImpl() override;
 
-    Block readImpl() override
-    {
-        /// If there were no calculations yet, calculate the first block synchronously
-        if (!started)
-        {
-            calculate(current_memory_tracker);
-            started = true;
-        }
-        else    /// If the calculations are already in progress - wait for the result
-            pool.wait();
-
-        if (exception)
-            std::rethrow_exception(exception);
-
-        Block res = block;
-        if (!res)
-            return res;
-
-        /// Start the next block calculation
-        block.clear();
-        next();
-
-        return res;
-    }
-
-
-    void next()
-    {
-        ready.reset();
-        pool.schedule(std::bind(&AsynchronousBlockInputStream::calculate, this, current_memory_tracker));
-    }
-
+    void next();
 
     /// Calculations that can be performed in a separate thread
-    void calculate(MemoryTracker * memory_tracker)
-    {
-        CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
-
-        try
-        {
-            if (first)
-            {
-                first = false;
-                setThreadName("AsyncBlockInput");
-                current_memory_tracker = memory_tracker;
-                children.back()->readPrefix();
-            }
-
-            block = children.back()->read();
-        }
-        catch (...)
-        {
-            exception = std::current_exception();
-        }
-
-        ready.set();
-    }
+    void calculate();
 };
 
 }

dbms/src/DataStreams/BlockIO.h

@@ -12,8 +12,7 @@ class ProcessListEntry;
 struct BlockIO
 {
     /** process_list_entry should be destroyed after in and after out,
-      *  since in and out contain pointer to an object inside process_list_entry
-      *  (MemoryTracker * current_memory_tracker),
+      *  since in and out contain pointer to objects inside process_list_entry (query-level MemoryTracker for example),
       *  which could be used before destroying of in and out.
       */
     std::shared_ptr<ProcessListEntry> process_list_entry;
@@ -38,12 +37,17 @@ struct BlockIO
             exception_callback();
     }
 
+    /// We provide the correct order of destruction.
+    void reset()
+    {
+        out.reset();
+        in.reset();
+        process_list_entry.reset();
+    }
+
     BlockIO & operator= (const BlockIO & rhs)
     {
-        /// We provide the correct order of destruction.
-        out                     = nullptr;
-        in                      = nullptr;
-        process_list_entry      = nullptr;
+        reset();
 
         process_list_entry      = rhs.process_list_entry;
         in                      = rhs.in;

dbms/src/DataStreams/ConvertColumnWithDictionaryToFullBlockInputStream.h

@@ -3,6 +3,7 @@
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <Columns/ColumnWithDictionary.h>
 #include <DataTypes/DataTypeWithDictionary.h>
+#include <Columns/ColumnConst.h>
 
 namespace DB
 {
@@ -32,22 +33,13 @@ private:
     {
         for (auto & column : block)
         {
-            auto * type_with_dict = typeid_cast<const DataTypeWithDictionary *>(column.type.get());
-            auto * col_with_dict = typeid_cast<const ColumnWithDictionary *>(column.column.get());
+            if (auto * column_const = typeid_cast<const ColumnConst *>(column.column.get()))
+                column.column = column_const->removeLowCardinality();
+            else
+                column.column = column.column->convertToFullColumnIfWithDictionary();
 
-            if (type_with_dict && !col_with_dict)
-                throw Exception("Invalid column for " + type_with_dict->getName() + ": " + column.column->getName(),
-                                ErrorCodes::LOGICAL_ERROR);
-
-            if (!type_with_dict && col_with_dict)
-                throw Exception("Invalid type for " + col_with_dict->getName() + ": " + column.type->getName(),
-                                ErrorCodes::LOGICAL_ERROR);
-
-            if (type_with_dict && col_with_dict)
-            {
-                column.column = col_with_dict->convertToFullColumn();
-                column.type = type_with_dict->getDictionaryType();
-            }
+            if (auto * low_cardinality_type = typeid_cast<const DataTypeWithDictionary *>(column.type.get()))
+                column.type = low_cardinality_type->getDictionaryType();
         }
 
         return std::move(block);