diff --git a/.gitmodules b/.gitmodules index 742e4616276..c43b754dba8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,7 +15,7 @@ url = https://github.com/google/cctz.git [submodule "contrib/zlib-ng"] path = contrib/zlib-ng - url = https://github.com/Dead2/zlib-ng.git + url = https://github.com/ClickHouse-Extras/zlib-ng.git [submodule "contrib/googletest"] path = contrib/googletest url = https://github.com/google/googletest.git diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 93c681b0336..5c7dc6cef09 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -1 +1,31 @@ ## RU + +## ClickHouse release 18.10.3, 2018-08-13 + +### Новые возможности: +* поддержка межсерверной репликации по HTTPS +* MurmurHash +* ODBCDriver2 с поддержкой NULL-ов +* поддержка UUID в ключевых колонках (экспериментально) + +### Улучшения: +* добавлена поддержка SETTINGS для движка Kafka +* поддежка пустых кусков после мержей в движках Summing, Collapsing and VersionedCollapsing +* удаление старых записей о полностью выполнившихся мутациях +* исправлена логика REPLACE PARTITION для движка RplicatedMergeTree +* добавлена системная таблица system.merge_tree_settings +* в системную таблицу system.tables добавлены столбцы зависимостей: dependencies_database и dependencies_table +* заменен аллокатор, теперь используется jemalloc вместо tcmalloc +* улучшена валидация connection string ODBC +* удалена поддержка CHECK TABLE для распределенных таблиц +* добавлены stateful тесты (пока без данных) +* добавлена опция конфига max_partition_size_to_drop +* добавлена настройка output_format_json_escape_slashes +* добавлена настройка max_fetch_partition_retries_count +* добавлена настройка prefer_localhost_replica +* добавлены libressl, unixodbc и mariadb-connector-c как сабмодули + +### Исправление ошибок: +* #2786 +* #2777 +* #2795 diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fff4641e24..2ae51712f6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ project (ClickHouse) -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 3.3) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/") @@ -218,7 +218,7 @@ else () set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc") endif () -option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF) +option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) set(NOT_UNBUNDLED 0) else () diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 65361386035..abc30d99e32 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -7,9 +7,9 @@ endif () if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386") set (ARCH_I386 1) endif () -if ( ( ARCH_ARM AND NOT ARCH_AARCH64 ) OR ARCH_I386) +if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386) set (ARCH_32 1) - message (WARNING "Support for 32bit platforms is highly experimental") + message (FATAL_ERROR "32bit platforms are not supported") endif () if (CMAKE_SYSTEM MATCHES "Linux") diff --git a/cmake/find_rdkafka.cmake b/cmake/find_rdkafka.cmake index c62c32bb64f..dc8e9913bc7 100644 --- a/cmake/find_rdkafka.cmake +++ b/cmake/find_rdkafka.cmake @@ -2,7 +2,9 @@ option (ENABLE_RDKAFKA "Enable kafka" ON) if (ENABLE_RDKAFKA) -option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) +if (OS_LINUX) + option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) +endif () if (USE_INTERNAL_RDKAFKA_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt") message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive") diff --git a/cmake/find_zlib.cmake b/cmake/find_zlib.cmake index 0e198c9bb0f..f1f7c128d7b 100644 --- a/cmake/find_zlib.cmake +++ b/cmake/find_zlib.cmake @@ -1,4 +1,6 @@ -option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED}) +if (NOT OS_FREEBSD AND NOT APPLE) + option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED}) +endif () if (NOT USE_INTERNAL_ZLIB_LIBRARY) find_package (ZLIB) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 15213d980ab..b905dbc1ba7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,10 +1,10 @@ # Third-party libraries may have substandard code. if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -std=c++1z") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z") endif () @@ -151,7 +151,7 @@ endif () if (USE_INTERNAL_LLVM_LIBRARY) # ld: unknown option: --color-diagnostics - if (APPLE AND COMPILER_GCC) + if (APPLE) set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") endif () add_subdirectory (llvm/llvm) diff --git a/contrib/ssl b/contrib/ssl index 994687ca6c7..de02224a42c 160000 --- a/contrib/ssl +++ b/contrib/ssl @@ -1 +1 @@ -Subproject commit 994687ca6c7b5a2b7e4346bf835a54068b3530a4 +Subproject commit de02224a42c69e3d8c9112c82018816f821878d0 diff --git a/contrib/zlib-ng b/contrib/zlib-ng index e07a52dbaa3..9173b89d467 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit e07a52dbaa35d003f5659b221b29d220c091667b +Subproject commit 9173b89d46799582d20a30578e0aa9788bc7d6e1 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 8dba0991944..573b0e4ea0f 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -157,6 +157,7 @@ target_link_libraries (dbms ${RE2_LIBRARY} ${RE2_ST_LIBRARY} ${BTRIE_LIBRARIES} + ${Boost_PROGRAM_OPTIONS_LIBRARY} ) if (NOT USE_INTERNAL_RE2_LIBRARY) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 659f73b2f26..38c7dbc1022 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54405 CACHE STRING "") +set(VERSION_REVISION 54406 CACHE STRING "") set(VERSION_MAJOR 18 CACHE STRING "") -set(VERSION_MINOR 10 CACHE STRING "") -set(VERSION_PATCH 1 CACHE STRING "") -set(VERSION_GITHASH 419bc587c0079b51a906a65af9a10da3300ddaf2 CACHE STRING "") -set(VERSION_DESCRIBE v18.10.1-testing CACHE STRING "") -set(VERSION_STRING 18.10.1 CACHE STRING "") +set(VERSION_MINOR 11 CACHE STRING "") +set(VERSION_PATCH 0 CACHE STRING "") +set(VERSION_GITHASH 76af46ed5d223b3a7af92e31eae291174da16355 CACHE STRING "") +set(VERSION_DESCRIBE v18.11.0-testing CACHE STRING "") +set(VERSION_STRING 18.11.0 CACHE STRING "") # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt index a5692d81c09..136616ca44b 100644 --- a/dbms/programs/CMakeLists.txt +++ b/dbms/programs/CMakeLists.txt @@ -13,6 +13,7 @@ option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLI option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) @@ -27,10 +28,11 @@ add_subdirectory (copier) add_subdirectory (format) add_subdirectory (clang) add_subdirectory (obfuscator) +add_subdirectory (odbc-bridge) if (CLICKHOUSE_SPLIT_BINARY) set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test - clickhouse-extract-from-config clickhouse-format clickhouse-copier) + clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier clickhouse-odbc-bridge) if (USE_EMBEDDED_COMPILER) list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld) @@ -83,6 +85,9 @@ else () if (USE_EMBEDDED_COMPILER) target_link_libraries (clickhouse clickhouse-compiler-lib) endif () + if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) + target_link_libraries (clickhouse clickhouse-odbc-bridge-lib) + endif() set (CLICKHOUSE_BUNDLE) if (ENABLE_CLICKHOUSE_SERVER) @@ -135,6 +140,12 @@ else () install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator) endif () + if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) + add_custom_target (clickhouse-odbc-bridge ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-odbc-bridge DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-odbc-bridge DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge) + endif () + # install always because depian package want this files: add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse) diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt index 88862dcf269..4844cb37c93 100644 --- a/dbms/programs/clang/CMakeLists.txt +++ b/dbms/programs/clang/CMakeLists.txt @@ -13,10 +13,10 @@ if (CLICKHOUSE_SPLIT_BINARY) endif () endif () -set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/headers") +set(TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}") # Make and install empty dir for debian package if compiler disabled add_custom_target(make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR}) -install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse) +install(DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse) # TODO: fix on macos copy_headers.sh: sed --posix if (USE_EMBEDDED_COMPILER AND NOT APPLE) add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index a52dadcc5dc..664f5d79df5 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1,3 +1,5 @@ +#include "TestHint.h" + #include #include #include @@ -20,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -87,105 +90,11 @@ namespace ErrorCodes extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED; extern const int LOGICAL_ERROR; + extern const int CANNOT_SET_SIGNAL_HANDLER; + extern const int CANNOT_READLINE; } -/// Checks expected server and client error codes in testmode. -/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }". -class TestHint -{ -public: - TestHint(bool enabled_, const String & query) - : enabled(enabled_), - server_error(0), - client_error(0) - { - if (!enabled_) - return; - - size_t pos = query.find("--"); - if (pos != String::npos && query.find("--", pos + 2) != String::npos) - return; /// It's not last comment. Hint belongs to commented query. - - if (pos != String::npos) - { - pos = query.find('{', pos + 2); - if (pos != String::npos) - { - String hint = query.substr(pos + 1); - pos = hint.find('}'); - hint.resize(pos); - parse(hint); - } - } - } - - /// @returns true if it's possible to continue without reconnect - bool checkActual(int & actual_server_error, int & actual_client_error, - bool & got_exception, std::unique_ptr & last_exception) const - { - if (!enabled) - return true; - - if (allErrorsExpected(actual_server_error, actual_client_error)) - { - got_exception = false; - last_exception.reset(); - actual_server_error = 0; - actual_client_error = 0; - return false; - } - - if (lostExpectedError(actual_server_error, actual_client_error)) - { - std::cerr << "Success when error expected. It expects server error " - << server_error << ", client error " << client_error << "." << std::endl; - got_exception = true; - last_exception = std::make_unique("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS - return false; - } - - return true; - } - - int serverError() const { return server_error; } - int clientError() const { return client_error; } - -private: - bool enabled; - int server_error; - int client_error; - - void parse(const String & hint) - { - std::stringstream ss; - ss << hint; - while (!ss.eof()) - { - String item; - ss >> item; - if (item.empty()) - break; - - if (item == "serverError") - ss >> server_error; - else if (item == "clientError") - ss >> client_error; - } - } - - bool allErrorsExpected(int actual_server_error, int actual_client_error) const - { - return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); - } - - bool lostExpectedError(int actual_server_error, int actual_client_error) const - { - return (server_error && !actual_server_error) || (client_error && !actual_client_error); - } -}; - - class Client : public Poco::Util::Application { public: @@ -235,6 +144,11 @@ private: std::optional out_file_buf; BlockOutputStreamPtr block_out_stream; + /// The user could specify special file for server logs (stderr by default) + std::unique_ptr out_logs_buf; + String server_logs_file; + BlockOutputStreamPtr logs_out_stream; + String home_path; String current_profile; @@ -408,20 +322,10 @@ private: /// If exception code isn't zero, we should return non-zero return code anyway. return e.code() ? e.code() : -1; } - catch (const Poco::Exception & e) - { - std::cerr << "Poco::Exception: " << e.displayText() << std::endl; - return ErrorCodes::POCO_EXCEPTION; - } - catch (const std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return ErrorCodes::STD_EXCEPTION; - } catch (...) { - std::cerr << "Unknown exception" << std::endl; - return ErrorCodes::UNKNOWN_EXCEPTION; + std::cerr << getCurrentExceptionMessage(false) << std::endl; + return getCurrentExceptionCode(); } } @@ -469,7 +373,12 @@ private: format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size); insert_format = "Values"; - insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); + + /// Setting value from cmd arg overrides one from config + if (context.getSettingsRef().max_insert_block_size.changed) + insert_format_max_block_size = context.getSettingsRef().max_insert_block_size; + else + insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); if (!is_interactive) { @@ -782,6 +691,7 @@ private: { const char * pos = begin; ASTPtr ast = parseQuery(pos, end, true); + if (!ast) { if (ignore_error) @@ -797,7 +707,7 @@ private: return true; } - ASTInsertQuery * insert = typeid_cast(&*ast); + ASTInsertQuery * insert = typeid_cast(ast.get()); if (insert && insert->data) { @@ -990,7 +900,7 @@ private: /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. sendData(sample); - receivePacket(); + receiveEndOfQuery(); } } @@ -1072,6 +982,11 @@ private: connection->sendData(block); processed_rows += block.rows(); + /// Check if server send Log packet + auto packet_type = connection->checkPacket(); + if (packet_type && *packet_type == Protocol::Server::Log) + receiveAndProcessPacket(); + if (!block) break; } @@ -1083,18 +998,28 @@ private: /// Flush all buffers. void resetOutput() { - block_out_stream = nullptr; + block_out_stream.reset(); + logs_out_stream.reset(); + if (pager_cmd) { pager_cmd->in.close(); pager_cmd->wait(); } pager_cmd = nullptr; + if (out_file_buf) { out_file_buf->next(); out_file_buf.reset(); } + + if (out_logs_buf) + { + out_logs_buf->next(); + out_logs_buf.reset(); + } + std_out.next(); } @@ -1127,7 +1052,7 @@ private: continue; /// If there is no new data, continue checking whether the query was cancelled after a timeout. } - if (!receivePacket()) + if (!receiveAndProcessPacket()) break; } @@ -1138,7 +1063,7 @@ private: /// Receive a part of the result, or progress info or an exception and process it. /// Returns true if one should continue receiving packets. - bool receivePacket() + bool receiveAndProcessPacket() { Connection::Packet packet = connection->receivePacket(); @@ -1169,6 +1094,10 @@ private: last_exception = std::move(packet.exception); return false; + case Protocol::Server::Log: + onLogData(packet.block); + return true; + case Protocol::Server::EndOfStream: onEndOfStream(); return false; @@ -1182,22 +1111,59 @@ private: /// Receive the block that serves as an example of the structure of table where data will be inserted. bool receiveSampleBlock(Block & out) { - Connection::Packet packet = connection->receivePacket(); - - switch (packet.type) + while (true) { - case Protocol::Server::Data: - out = packet.block; - return true; + Connection::Packet packet = connection->receivePacket(); - case Protocol::Server::Exception: - onException(*packet.exception); - last_exception = std::move(packet.exception); - return false; + switch (packet.type) + { + case Protocol::Server::Data: + out = packet.block; + return true; - default: - throw NetException("Unexpected packet from server (expected Data, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Data, Exception or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } + } + } + + + /// Process Log packets, exit when recieve Exception or EndOfStream + bool receiveEndOfQuery() + { + while (true) + { + Connection::Packet packet = connection->receivePacket(); + + switch (packet.type) + { + case Protocol::Server::EndOfStream: + onEndOfStream(); + return true; + + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Exception, EndOfStream or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } } } @@ -1253,6 +1219,38 @@ private: } + void initLogsOutputStream() + { + if (!logs_out_stream) + { + WriteBuffer * wb = out_logs_buf.get(); + + if (!out_logs_buf) + { + if (server_logs_file.empty()) + { + /// Use stderr by default + out_logs_buf = std::make_unique(STDERR_FILENO); + wb = out_logs_buf.get(); + } + else if (server_logs_file == "-") + { + /// Use stdout if --server_logs_file=- specified + wb = &std_out; + } + else + { + out_logs_buf = std::make_unique(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); + wb = out_logs_buf.get(); + } + } + + logs_out_stream = std::make_shared(*wb); + logs_out_stream->writePrefix(); + } + } + + void onData(Block & block) { if (written_progress_chars) @@ -1276,6 +1274,14 @@ private: } + void onLogData(Block & block) + { + initLogsOutputStream(); + logs_out_stream->write(block); + logs_out_stream->flush(); + } + + void onTotals(Block & block) { initBlockOutputStream(block); @@ -1436,6 +1442,9 @@ private: if (block_out_stream) block_out_stream->writeSuffix(); + if (logs_out_stream) + logs_out_stream->writeSuffix(); + resetOutput(); if (is_interactive && !written_first_block) @@ -1509,9 +1518,38 @@ public: } } +#if USE_READLINE + if (rl_initialize()) + throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); + + auto clear_prompt_or_exit = [](int) + { + /// This is signal safe. + ssize_t res = write(STDOUT_FILENO, "\n", 1); + + if (res == 1 && rl_line_buffer[0]) + { + rl_replace_line("", 0); + if (rl_forced_update_display()) + _exit(0); + } + else + { + /// A little dirty, but we struggle to find better way to correctly + /// force readline to exit after returning from the signal handler. + _exit(0); + } + }; + + if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR) + throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); +#endif + ioctl(0, TIOCGWINSZ, &terminal_size); - unsigned line_length = boost::program_options::options_description::m_default_line_length; + namespace po = boost::program_options; + + unsigned line_length = po::options_description::m_default_line_length; unsigned min_description_length = line_length / 2; if (!stdin_is_not_tty) { @@ -1519,55 +1557,58 @@ public: min_description_length = std::min(min_description_length, line_length - 2); } -#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value (), DESCRIPTION) +#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value (), DESCRIPTION) /// Main commandline options related to client functionality and all parameters from Settings. - boost::program_options::options_description main_description("Main options", line_length, min_description_length); + po::options_description main_description("Main options", line_length, min_description_length); main_description.add_options() ("help", "produce help message") - ("config-file,c", boost::program_options::value(), "config-file path") - ("host,h", boost::program_options::value()->default_value("localhost"), "server host") - ("port", boost::program_options::value()->default_value(9000), "server port") + ("config-file,c", po::value(), "config-file path") + ("host,h", po::value()->default_value("localhost"), "server host") + ("port", po::value()->default_value(9000), "server port") ("secure,s", "secure") - ("user,u", boost::program_options::value()->default_value("default"), "user") - ("password", boost::program_options::value(), "password") + ("user,u", po::value()->default_value("default"), "user") + ("password", po::value(), "password") ("ask-password", "ask-password") - ("query_id", boost::program_options::value(), "query_id") - ("query,q", boost::program_options::value(), "query") - ("database,d", boost::program_options::value(), "database") - ("pager", boost::program_options::value(), "pager") + ("query_id", po::value(), "query_id") + ("query,q", po::value(), "query") + ("database,d", po::value(), "database") + ("pager", po::value(), "pager") ("multiline,m", "multiline") ("multiquery,n", "multiquery") + ("format,f", po::value(), "default output format") ("testmode,T", "enable test hints in comments") ("ignore-error", "do not stop processing in multiquery mode") - ("format,f", boost::program_options::value(), "default output format") ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command") ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)") ("stacktrace", "print stack traces of exceptions") ("progress", "print progress even in non-interactive mode") ("version,V", "print version information and exit") + ("version-clean", "print version in machine-readable format and exit") ("echo", "in batch mode, print query before execution") - ("max_client_network_bandwidth", boost::program_options::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") - ("compression", boost::program_options::value(), "enable or disable compression") + ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") + ("compression", po::value(), "enable or disable compression") + ("log-level", po::value(), "client log level") + ("server_logs_file", po::value(), "put server logs into specified file") APPLY_FOR_SETTINGS(DECLARE_SETTING) ; #undef DECLARE_SETTING /// Commandline options related to external tables. - boost::program_options::options_description external_description("External tables options"); + po::options_description external_description("External tables options"); external_description.add_options() - ("file", boost::program_options::value(), "data file or - for stdin") - ("name", boost::program_options::value()->default_value("_data"), "name of the table") - ("format", boost::program_options::value()->default_value("TabSeparated"), "data format") - ("structure", boost::program_options::value(), "structure") - ("types", boost::program_options::value(), "types") + ("file", po::value(), "data file or - for stdin") + ("name", po::value()->default_value("_data"), "name of the table") + ("format", po::value()->default_value("TabSeparated"), "data format") + ("structure", po::value(), "structure") + ("types", po::value(), "types") ; /// Parse main commandline options. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( common_arguments.size(), common_arguments.data()).options(main_description).run(); - boost::program_options::variables_map options; - boost::program_options::store(parsed, options); + po::variables_map options; + po::store(parsed, options); if (options.count("version") || options.count("V")) { @@ -1575,6 +1616,12 @@ public: exit(0); } + if (options.count("version-clean")) + { + std::cout << VERSION_STRING; + exit(0); + } + /// Output of help message. if (options.count("help") || (options.count("host") && options["host"].as() == "elp")) /// If user writes -help instead of --help. @@ -1584,14 +1631,17 @@ public: exit(0); } + if (options.count("log-level")) + Poco::Logger::root().setLevel(options["log-level"].as()); + size_t number_of_external_tables_with_stdin_source = 0; for (size_t i = 0; i < external_tables_arguments.size(); ++i) { /// Parse commandline options related to external tables. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run(); - boost::program_options::variables_map external_options; - boost::program_options::store(parsed, external_options); + po::variables_map external_options; + po::store(parsed, external_options); try { @@ -1665,6 +1715,8 @@ public: max_client_network_bandwidth = options["max_client_network_bandwidth"].as(); if (options.count("compression")) config().setBool("compression", options["compression"].as()); + if (options.count("server_logs_file")) + server_logs_file = options["server_logs_file"].as(); } }; @@ -1684,6 +1736,11 @@ int mainEntryClickHouseClient(int argc, char ** argv) std::cerr << "Bad arguments: " << e.what() << std::endl; return 1; } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; + return 1; + } return client.run(); } diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h new file mode 100644 index 00000000000..790e58ee7fe --- /dev/null +++ b/dbms/programs/client/TestHint.h @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +/// Checks expected server and client error codes in testmode. +/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }". +class TestHint +{ +public: + TestHint(bool enabled_, const String & query) + : enabled(enabled_) + { + if (!enabled_) + return; + + /// TODO: This is absolutely wrong. Fragment may be contained inside string literal. + size_t pos = query.find("--"); + + if (pos != String::npos && query.find("--", pos + 2) != String::npos) + return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query. + + if (pos != String::npos) + { + /// TODO: This is also wrong. Comment may already have ended by line break. + pos = query.find('{', pos + 2); + + if (pos != String::npos) + { + String hint = query.substr(pos + 1); + + /// TODO: And this is wrong for the same reason. + pos = hint.find('}'); + hint.resize(pos); + parse(hint); + } + } + } + + /// @returns true if it's possible to continue without reconnect + bool checkActual(int & actual_server_error, int & actual_client_error, + bool & got_exception, std::unique_ptr & last_exception) const + { + if (!enabled) + return true; + + if (allErrorsExpected(actual_server_error, actual_client_error)) + { + got_exception = false; + last_exception.reset(); + actual_server_error = 0; + actual_client_error = 0; + return false; + } + + if (lostExpectedError(actual_server_error, actual_client_error)) + { + std::cerr << "Success when error expected. It expects server error " + << server_error << ", client error " << client_error << "." << std::endl; + got_exception = true; + last_exception = std::make_unique("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS + return false; + } + + return true; + } + + int serverError() const { return server_error; } + int clientError() const { return client_error; } + +private: + bool enabled = false; + int server_error = 0; + int client_error = 0; + + void parse(const String & hint) + { + std::stringstream ss; + ss << hint; + while (!ss.eof()) + { + String item; + ss >> item; + if (item.empty()) + break; + + if (item == "serverError") + ss >> server_error; + else if (item == "clientError") + ss >> client_error; + } + } + + bool allErrorsExpected(int actual_server_error, int actual_client_error) const + { + return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); + } + + bool lostExpectedError(int actual_server_error, int actual_client_error) const + { + return (server_error && !actual_server_error) || (client_error && !actual_client_error); + } +}; + +} diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index a000401f02c..fab1d5af4c5 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -2143,6 +2144,9 @@ void ClusterCopierApp::mainImpl() context->addDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); + /// Initialize query scope just in case. + CurrentThread::QueryScope query_scope(*context); + auto copier = std::make_unique(task_path, host_id, default_database, *context); copier->setSafeMode(is_safe_mode); copier->setCopyFaultProbability(copy_fault_probability); diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index ae9e9b7b720..4528ad40128 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -270,6 +271,9 @@ void LocalServer::processQueries() context->setCurrentQueryId(""); applyCmdSettings(*context); + /// Use the same query_id (and thread group) for all queries + CurrentThread::QueryScope query_scope_holder(*context); + bool echo_query = config().hasOption("echo") || config().hasOption("verbose"); std::exception_ptr exception; diff --git a/dbms/programs/main.cpp b/dbms/programs/main.cpp index aba03a87a83..3bf1fa1d6ed 100644 --- a/dbms/programs/main.cpp +++ b/dbms/programs/main.cpp @@ -56,6 +56,10 @@ int mainEntryClickHouseClusterCopier(int argc, char ** argv); #if ENABLE_CLICKHOUSE_OBFUSCATOR int mainEntryClickHouseObfuscator(int argc, char ** argv); #endif +#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE) +int mainEntryClickHouseODBCBridge(int argc, char ** argv); +#endif + #if USE_EMBEDDED_COMPILER int mainEntryClickHouseClang(int argc, char ** argv); @@ -101,6 +105,10 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_OBFUSCATOR {"obfuscator", mainEntryClickHouseObfuscator}, #endif +#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE) + {"odbc-bridge", mainEntryClickHouseODBCBridge}, +#endif + #if USE_EMBEDDED_COMPILER {"clang", mainEntryClickHouseClang}, {"clang++", mainEntryClickHouseClang}, diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt new file mode 100644 index 00000000000..df68a8c546c --- /dev/null +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library (clickhouse-odbc-bridge-lib + PingHandler.cpp + MainHandler.cpp + ColumnInfoHandler.cpp + HandlerFactory.cpp + ODBCBridge.cpp + validateODBCConnectionString.cpp +) + +target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms) +target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) + +if (USE_POCO_SQLODBC) + target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS}) +endif () + +if (USE_POCO_DATAODBC) + target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) +endif() + + +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-odbc-bridge odbc-bridge.cpp) + target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib) +endif () diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp new file mode 100644 index 00000000000..3b4c7f42cea --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -0,0 +1,123 @@ +#include "ColumnInfoHandler.h" +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "validateODBCConnectionString.h" + +namespace DB +{ +namespace +{ + DataTypePtr getDataType(SQLSMALLINT type) + { + const auto & factory = DataTypeFactory::instance(); + + switch (type) + { + case SQL_INTEGER: + return factory.get("Int32"); + case SQL_SMALLINT: + return factory.get("Int16"); + case SQL_FLOAT: + return factory.get("Float32"); + case SQL_REAL: + return factory.get("Float32"); + case SQL_DOUBLE: + return factory.get("Float64"); + case SQL_DATETIME: + return factory.get("DateTime"); + case SQL_TYPE_TIMESTAMP: + return factory.get("DateTime"); + case SQL_TYPE_DATE: + return factory.get("Date"); + default: + return factory.get("String"); + } + } +} + +void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("table")) + { + process_error("No 'table' param in request URL"); + return; + } + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + std::string table_name = params.get("table"); + std::string connection_string = params.get("connection_string"); + LOG_TRACE(log, "Will fetch info for table '" << table_name << "'"); + LOG_TRACE(log, "Got connection str '" << connection_string << "'"); + + try + { + Poco::Data::ODBC::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + SQLHDBC hdbc = session.dbc().handle(); + + SQLHSTMT hstmt = nullptr; + + if (Poco::Data::ODBC::Utility::isError(SQLAllocStmt(hdbc, &hstmt))) + throw Poco::Data::ODBC::ODBCException("Could not allocate connection handle."); + + SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP)); + + /// TODO Why not do SQLColumns instead? + std::string query = "SELECT * FROM " + table_name + " WHERE 1 = 0"; + if (Poco::Data::ODBC::Utility::isError(Poco::Data::ODBC::SQLPrepare(hstmt, reinterpret_cast(&query[0]), query.size()))) + throw Poco::Data::ODBC::DescriptorException(session.dbc()); + + if (Poco::Data::ODBC::Utility::isError(SQLExecute(hstmt))) + throw Poco::Data::ODBC::StatementException(hstmt); + + SQLSMALLINT cols = 0; + if (Poco::Data::ODBC::Utility::isError(SQLNumResultCols(hstmt, &cols))) + throw Poco::Data::ODBC::StatementException(hstmt); + + /// TODO cols not checked + + NamesAndTypesList columns; + for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol) + { + SQLSMALLINT type = 0; + /// TODO Why 301? + SQLCHAR column_name[301]; + /// TODO Result is not checked. + Poco::Data::ODBC::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), NULL, &type, NULL, NULL, NULL); + columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); + } + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + writeStringBinary(columns.toString(), out); + } + catch (...) + { + process_error("Error getting columns from ODBC '" + getCurrentExceptionMessage(false) + "'"); + tryLogCurrentException(log); + } +} +} +#endif diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.h b/dbms/programs/odbc-bridge/ColumnInfoHandler.h new file mode 100644 index 00000000000..426cea15b34 --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include +#include + +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +/** The structure of the table is taken from the query "SELECT * FROM table WHERE 1=0". + * TODO: It would be much better to utilize ODBC methods dedicated for columns description. + * If there is no such table, an exception is thrown. + */ +namespace DB +{ +class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler +{ +public: + ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + size_t keep_alive_timeout; + std::shared_ptr context; +}; +} +#endif diff --git a/dbms/programs/odbc-bridge/HandlerFactory.cpp b/dbms/programs/odbc-bridge/HandlerFactory.cpp new file mode 100644 index 00000000000..f552203174e --- /dev/null +++ b/dbms/programs/odbc-bridge/HandlerFactory.cpp @@ -0,0 +1,34 @@ +#include "HandlerFactory.h" +#include "PingHandler.h" +#include "ColumnInfoHandler.h" +#include + +#include +#include +#include + +namespace DB +{ +Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) +{ + Poco::URI uri{request.getURI()}; + LOG_TRACE(log, "Request URI: " + uri.toString()); + + if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) + return new PingHandler(keep_alive_timeout); + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + { + + if (uri.getPath() == "/columns_info") +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + return new ODBCColumnsInfoHandler(keep_alive_timeout, context); +#else + return nullptr; +#endif + else + return new ODBCHandler(pool_map, keep_alive_timeout, context); + } + return nullptr; +} +} diff --git a/dbms/programs/odbc-bridge/HandlerFactory.h b/dbms/programs/odbc-bridge/HandlerFactory.h new file mode 100644 index 00000000000..4fe00ffca98 --- /dev/null +++ b/dbms/programs/odbc-bridge/HandlerFactory.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include +#include +#include "MainHandler.h" +#include "ColumnInfoHandler.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" + #include +#pragma GCC diagnostic pop + + +namespace DB +{ +/** Factory for '/ping', '/' and '/columns_info' handlers. + * Also stores Session pools for ODBC connections + */ +class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + HandlerFactory(const std::string & name_, size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get(name_)), name(name_), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + pool_map = std::make_shared(); + } + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; + +private: + Poco::Logger * log; + std::string name; + size_t keep_alive_timeout; + std::shared_ptr context; + std::shared_ptr pool_map; +}; +} diff --git a/dbms/programs/odbc-bridge/MainHandler.cpp b/dbms/programs/odbc-bridge/MainHandler.cpp new file mode 100644 index 00000000000..e7764907dbe --- /dev/null +++ b/dbms/programs/odbc-bridge/MainHandler.cpp @@ -0,0 +1,126 @@ +#include "MainHandler.h" + +#include "validateODBCConnectionString.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace +{ + std::unique_ptr parseColumns(std::string && column_string) + { + std::unique_ptr sample_block = std::make_unique(); + auto names_and_types = NamesAndTypesList::parse(column_string); + for (const NameAndTypePair & column_data : names_and_types) + sample_block->insert({column_data.type, column_data.name}); + return sample_block; + } +} + + +ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str) +{ + std::lock_guard lock(mutex); + if (!pool_map->count(connection_str)) + { + pool_map->emplace(connection_str, createAndCheckResizePocoSessionPool([connection_str] { + return std::make_shared("ODBC", validateODBCConnectionString(connection_str)); + })); + } + return pool_map->at(connection_str); +} + +void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("query")) + { + process_error("No 'query' in request body"); + return; + } + + if (!params.has("columns")) + { + process_error("No 'columns' in request URL"); + return; + } + + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + + size_t max_block_size = DEFAULT_BLOCK_SIZE; + if (params.has("max_block_size")) + { + std::string max_block_size_str = params.get("max_block_size", ""); + if (max_block_size_str.empty()) + { + process_error("Empty max_block_size specified"); + return; + } + max_block_size = parse(max_block_size_str); + } + + std::string columns = params.get("columns"); + std::unique_ptr sample_block; + try + { + sample_block = parseColumns(std::move(columns)); + } + catch (const Exception & ex) + { + process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'"); + LOG_WARNING(log, ex.getStackTrace().toString()); + return; + } + + std::string format = params.get("format", "RowBinary"); + std::string query = params.get("query"); + LOG_TRACE(log, "Query: " << query); + + std::string connection_string = params.get("connection_string"); + LOG_TRACE(log, "Connection string: '" << connection_string << "'"); + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + try + { + BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context); + auto pool = getPool(connection_string); + ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); + copyData(inp, *writer); + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + response.setStatusAndReason( + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, bacause of too soon response sending + writeStringBinary(message, out); + tryLogCurrentException(log); + } +} +} diff --git a/dbms/programs/odbc-bridge/MainHandler.h b/dbms/programs/odbc-bridge/MainHandler.h new file mode 100644 index 00000000000..ae139f393f8 --- /dev/null +++ b/dbms/programs/odbc-bridge/MainHandler.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" + #include +#pragma GCC diagnostic pop + +namespace DB +{ +/** Main handler for requests to ODBC driver + * requires connection_string and columns in request params + * and also query in request body + * response in RowBinary format + */ +class ODBCHandler : public Poco::Net::HTTPRequestHandler +{ +public: + using PoolPtr = std::shared_ptr; + using PoolMap = std::unordered_map; + + ODBCHandler(std::shared_ptr pool_map_, + size_t keep_alive_timeout_, + std::shared_ptr context_) + : log(&Poco::Logger::get("ODBCHandler")) + , pool_map(pool_map_) + , keep_alive_timeout(keep_alive_timeout_) + , context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + + std::shared_ptr pool_map; + size_t keep_alive_timeout; + std::shared_ptr context; + + static inline std::mutex mutex; + + PoolPtr getPool(const std::string & connection_str); +}; + +} diff --git a/dbms/programs/odbc-bridge/ODBCBridge.cpp b/dbms/programs/odbc-bridge/ODBCBridge.cpp new file mode 100644 index 00000000000..bab58250fa4 --- /dev/null +++ b/dbms/programs/odbc-bridge/ODBCBridge.cpp @@ -0,0 +1,205 @@ +#include "ODBCBridge.h" +#include "HandlerFactory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log) + { + Poco::Net::SocketAddress socket_address; + try + { + socket_address = Poco::Net::SocketAddress(host, port); + } + catch (const Poco::Net::DNSException & e) + { + const auto code = e.code(); + if (code == EAI_FAMILY +#if defined(EAI_ADDRFAMILY) + || code == EAI_ADDRFAMILY +#endif + ) + { + LOG_ERROR(log, + "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() + << ". " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0"); + } + + throw; + } + return socket_address; + } + + Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, Poco::Logger * log) + { + auto address = makeSocketAddress(host, port, log); +#if POCO_VERSION < 0x01080000 + socket.bind(address, /* reuseAddress = */ true); +#else + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ false); +#endif + + socket.listen(/* backlog = */ 64); + + return address; + }; +} + +void ODBCBridge::handleHelp(const std::string &, const std::string &) +{ + Poco::Util::HelpFormatter helpFormatter(options()); + helpFormatter.setCommand(commandName()); + helpFormatter.setHeader("HTTP-proxy for odbc requests"); + helpFormatter.setUsage("--http-port "); + helpFormatter.format(std::cerr); + + stopOptionsProcessing(); +} + + +void ODBCBridge::defineOptions(Poco::Util::OptionSet & options) +{ + options.addOption(Poco::Util::Option("http-port", "", "port to listen").argument("http-port", true).binding("http-port")); + options.addOption( + Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host")); + options.addOption( + Poco::Util::Option("http-timeout", "", "http timout for socket, default 1800").argument("http-timeout").binding("http-timeout")); + + options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024") + .argument("max-server-connections") + .binding("max-server-connections")); + options.addOption(Poco::Util::Option("keep-alive-timeout", "", "keepalive timeout, default 10") + .argument("keep-alive-timeout") + .binding("keep-alive-timeout")); + + options.addOption(Poco::Util::Option("log-level", "", "sets log level, default info").argument("log-level").binding("logger.level")); + + options.addOption( + Poco::Util::Option("log-path", "", "log path for all logs, default console").argument("log-path").binding("logger.log")); + + options.addOption(Poco::Util::Option("err-log-path", "", "err log path for all logs, default no") + .argument("err-log-path") + .binding("logger.errorlog")); + + using Me = std::decay_t; + options.addOption(Poco::Util::Option("help", "", "produce this help message") + .binding("help") + .callback(Poco::Util::OptionCallback(this, &Me::handleHelp))); + + ServerApplication::defineOptions(options); /// Don't need complex BaseDaemon's .xml config +} + +void ODBCBridge::initialize(Application & self) +{ + BaseDaemon::closeFDs(); + is_help = config().has("help"); + + if (is_help) + return; + + if (!config().has("logger.log")) + config().setBool("logger.console", true); + + config().setString("logger", "ODBCBridge"); + + buildLoggers(config()); + log = &logger(); + hostname = config().getString("listen-host", "localhost"); + port = config().getUInt("http-port"); + if (port > 0xFFFF) + throw Exception("Out of range 'http-port': " + std::to_string(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + http_timeout = config().getUInt("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT); + max_server_connections = config().getUInt("max-server-connections", 1024); + keep_alive_timeout = config().getUInt("keep-alive-timeout", 10); + + initializeTerminationAndSignalProcessing(); + + ServerApplication::initialize(self); +} + +void ODBCBridge::uninitialize() +{ + BaseDaemon::uninitialize(); +} + +int ODBCBridge::main(const std::vector & /*args*/) +{ + if (is_help) + return Application::EXIT_OK; + + LOG_INFO(log, "Starting up"); + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, hostname, port, log); + socket.setReceiveTimeout(http_timeout); + socket.setSendTimeout(http_timeout); + Poco::ThreadPool server_pool(3, max_server_connections); + Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; + http_params->setTimeout(http_timeout); + http_params->setKeepAliveTimeout(keep_alive_timeout); + + context = std::make_shared(Context::createGlobal()); + context->setGlobalContext(*context); + + auto server = Poco::Net::HTTPServer( + new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params); + server.start(); + + LOG_INFO(log, "Listening http://" + address.toString()); + + SCOPE_EXIT({ + LOG_DEBUG(log, "Received termination signal."); + LOG_DEBUG(log, "Waiting for current connections to close."); + server.stop(); + for (size_t count : ext::range(1, 6)) + { + if (server.currentConnections() == 0) + break; + LOG_DEBUG(log, "Waiting for " << server.currentConnections() << " connections, try " << count); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + }); + + waitForTerminationRequest(); + return Application::EXIT_OK; +} +} + +int mainEntryClickHouseODBCBridge(int argc, char ** argv) +{ + DB::ODBCBridge app; + try + { + return app.run(argc, argv); + } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + return code ? code : 1; + } +} diff --git a/dbms/programs/odbc-bridge/ODBCBridge.h b/dbms/programs/odbc-bridge/ODBCBridge.h new file mode 100644 index 00000000000..4ae11ad7301 --- /dev/null +++ b/dbms/programs/odbc-bridge/ODBCBridge.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +/** Class represents clickhouse-odbc-bridge server, which listen + * incoming HTTP POST and GET requests on specified port and host. + * Has two handlers '/' for all incoming POST requests to ODBC driver + * and /ping for GET request about service status + */ +class ODBCBridge : public BaseDaemon +{ +public: + void defineOptions(Poco::Util::OptionSet & options) override; + +protected: + void initialize(Application & self) override; + + void uninitialize() override; + + int main(const std::vector & args) override; + +private: + void handleHelp(const std::string &, const std::string &); + + bool is_help; + std::string hostname; + size_t port; + size_t http_timeout; + std::string log_level; + size_t max_server_connections; + size_t keep_alive_timeout; + + Poco::Logger * log; + + std::shared_ptr context; /// need for settings only +}; +} diff --git a/dbms/programs/odbc-bridge/PingHandler.cpp b/dbms/programs/odbc-bridge/PingHandler.cpp new file mode 100644 index 00000000000..b0313e46bf3 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.cpp @@ -0,0 +1,22 @@ +#include "PingHandler.h" +#include +#include +#include +#include + +namespace DB +{ +void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response) +{ + try + { + setResponseDefaultHeaders(response, keep_alive_timeout); + const char * data = "Ok.\n"; + response.sendBuffer(data, strlen(data)); + } + catch (...) + { + tryLogCurrentException("PingHandler"); + } +} +} diff --git a/dbms/programs/odbc-bridge/PingHandler.h b/dbms/programs/odbc-bridge/PingHandler.h new file mode 100644 index 00000000000..d8109a50bb6 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace DB +{ +/** Simple ping handler, answers "Ok." to GET request + */ +class PingHandler : public Poco::Net::HTTPRequestHandler +{ +public: + PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {} + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + size_t keep_alive_timeout; +}; +} diff --git a/dbms/programs/odbc-bridge/README.md b/dbms/programs/odbc-bridge/README.md new file mode 100644 index 00000000000..91a6e476733 --- /dev/null +++ b/dbms/programs/odbc-bridge/README.md @@ -0,0 +1,38 @@ +# clickhouse-odbc-bridge + +Simple HTTP-server which works like a proxy for ODBC driver. The main motivation +was possible segfaults or another faults in ODBC implementations, which can +crash whole clickhouse-server process. + +This tool works via HTTP, not via pipes, shared memory, or TCP because: +- It's simplier to implement +- It's simplier to debug +- jdbc-bridge can be implemented in the same way + +## Usage + +`clickhouse-server` use this tool inside odbc table function and StorageODBC. +However it can be used as standalone tool from command line with the following +parameters in POST-request URL: +- `connection_string` -- ODBC connection string. +- `columns` -- columns in ClickHouse NamesAndTypesList format, name in backticks, + type as string. Name and type are space separated, rows separated with + newline. +- `max_block_size` -- optional parameter, sets maximum size of single block. +Query is send in post body. Response is returned in RowBinary format. + +## Example: + +```bash +$ clickhouse-odbc-bridge --http-port 9018 --daemon + +$ curl -d "query=SELECT PageID, ImpID, AdType FROM Keys ORDER BY PageID, ImpID" --data-urlencode "connection_string=DSN=ClickHouse;DATABASE=stat" --data-urlencode "columns=columns format version: 1 +3 columns: +\`PageID\` String +\`ImpID\` String +\`AdType\` String +" "http://localhost:9018/" > result.txt + +$ cat result.txt +12246623837185725195925621517 +``` diff --git a/dbms/programs/odbc-bridge/odbc-bridge.cpp b/dbms/programs/odbc-bridge/odbc-bridge.cpp new file mode 100644 index 00000000000..af42eef8647 --- /dev/null +++ b/dbms/programs/odbc-bridge/odbc-bridge.cpp @@ -0,0 +1,2 @@ +int mainEntryClickHouseODBCBridge(int argc, char ** argv); +int main(int argc_, char ** argv_) { return mainEntryClickHouseODBCBridge(argc_, argv_); } diff --git a/dbms/programs/odbc-bridge/tests/CMakeLists.txt b/dbms/programs/odbc-bridge/tests/CMakeLists.txt new file mode 100644 index 00000000000..5240a917429 --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) +target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib) diff --git a/dbms/src/Dictionaries/tests/validate-odbc-connection-string.cpp b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp similarity index 88% rename from dbms/src/Dictionaries/tests/validate-odbc-connection-string.cpp rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp index 766a709d8fd..c4558811f77 100644 --- a/dbms/src/Dictionaries/tests/validate-odbc-connection-string.cpp +++ b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include "../validateODBCConnectionString.h" using namespace DB; diff --git a/dbms/src/Dictionaries/tests/validate-odbc-connection-string.reference b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference similarity index 100% rename from dbms/src/Dictionaries/tests/validate-odbc-connection-string.reference rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference diff --git a/dbms/src/Dictionaries/tests/validate-odbc-connection-string.sh b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh similarity index 100% rename from dbms/src/Dictionaries/tests/validate-odbc-connection-string.sh rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh diff --git a/dbms/src/Dictionaries/validateODBCConnectionString.cpp b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp similarity index 99% rename from dbms/src/Dictionaries/validateODBCConnectionString.cpp rename to dbms/programs/odbc-bridge/validateODBCConnectionString.cpp index b37a15c3a70..a817a01c288 100644 --- a/dbms/src/Dictionaries/validateODBCConnectionString.cpp +++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include "validateODBCConnectionString.h" namespace DB diff --git a/dbms/src/Dictionaries/validateODBCConnectionString.h b/dbms/programs/odbc-bridge/validateODBCConnectionString.h similarity index 100% rename from dbms/src/Dictionaries/validateODBCConnectionString.h rename to dbms/programs/odbc-bridge/validateODBCConnectionString.h diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index f2a114c4702..0db338e0fbf 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -9,10 +9,11 @@ #include -#include +#include #include #include #include +#include #include #include #include @@ -208,6 +209,13 @@ void HTTPHandler::processQuery( Poco::Net::HTTPServerResponse & response, Output & used_output) { + Context context = server.context(); + context.setGlobalContext(server.context()); + + /// It will forcibly detach query even if unexpected error ocurred and detachQuery() was not called + /// Normal detaching is happen in BlockIO callbacks + CurrentThread::QueryScope query_scope_holder(context); + LOG_TRACE(log, "Request URI: " << request.getURI()); std::istream & istr = request.stream(); @@ -257,14 +265,9 @@ void HTTPHandler::processQuery( } std::string query_id = params.get("query_id", ""); - - const auto & config = server.config(); - - Context context = server.context(); - context.setGlobalContext(server.context()); - context.setUser(user, password, request.clientAddress(), quota_key); context.setCurrentQueryId(query_id); + CurrentThread::attachQueryContext(context); /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. @@ -273,6 +276,7 @@ void HTTPHandler::processQuery( String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); + const auto & config = server.config(); if (session_is_set) { @@ -421,34 +425,45 @@ void HTTPHandler::processQuery( std::unique_ptr in; - // Used in case of POST request with form-data, but it not to be expectd to be deleted after that scope + static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", + "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"}; + + Names reserved_param_suffixes; + + auto param_could_be_skipped = [&] (const String & name) + { + if (reserved_param_names.count(name)) + return true; + + for (const String & suffix : reserved_param_suffixes) + { + if (endsWith(name, suffix)) + return true; + } + + return false; + }; + + /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope. std::string full_query; /// Support for "external data for query processing". if (startsWith(request.getContentType().data(), "multipart/form-data")) { ExternalTablesHandler handler(context, params); - - /// Params are of both form params POST and uri (GET params) params.load(request, istr, handler); - for (const auto & it : params) - { - if (it.first == "query") - { - full_query += it.second; - } - } - in = std::make_unique(full_query); + /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters. + reserved_param_suffixes.emplace_back("_format"); + reserved_param_suffixes.emplace_back("_types"); + reserved_param_suffixes.emplace_back("_structure"); - /// Erase unneeded parameters to avoid confusing them later with context settings or query - /// parameters. - for (const auto & it : handler.names) - { - params.erase(it + "_format"); - params.erase(it + "_types"); - params.erase(it + "_structure"); - } + /// Params are of both form params POST and uri (GET params) + for (const auto & it : params) + if (it.first == "query") + full_query += it.second; + + in = std::make_unique(full_query); } else in = std::make_unique(*in_param, *in_post_maybe_compressed); @@ -475,11 +490,6 @@ void HTTPHandler::processQuery( auto readonly_before_query = settings.readonly; - NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", - "session_id", "session_timeout", "session_check" - }; - for (auto it = params.begin(); it != params.end(); ++it) { if (it->first == "database") @@ -490,7 +500,7 @@ void HTTPHandler::processQuery( { context.setDefaultFormat(it->second); } - else if (reserved_param_names.find(it->first) != reserved_param_names.end()) + else if (param_could_be_skipped(it->first)) { } else diff --git a/dbms/programs/server/MetricsTransmitter.cpp b/dbms/programs/server/MetricsTransmitter.cpp index 278347a6774..ed87eb5cf9f 100644 --- a/dbms/programs/server/MetricsTransmitter.cpp +++ b/dbms/programs/server/MetricsTransmitter.cpp @@ -81,7 +81,7 @@ void MetricsTransmitter::transmit(std::vector & prev_count { for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) { - const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed); + const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed); const auto counter_increment = counter - prev_counters[i]; prev_counters[i] = counter; diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 153f48c9aef..3a55add70af 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -365,6 +366,13 @@ int Server::main(const std::vector & /*args*/) dns_cache_updater = std::make_unique(*global_context); } + if (!TaskStatsInfoGetter::checkProcessHasRequiredPermissions()) + { + LOG_INFO(log, "It looks like the process has not CAP_NET_ADMIN capability, some performance statistics will be disabled." + " It could happen due to incorrect clickhouse package installation." + " You could resolve the problem manually calling 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'"); + } + { Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 4f75f7dd6a7..c2f8d34b5d2 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -1,7 +1,15 @@ -#include "TCPHandler.h" - #include +#include #include +#include + +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -15,14 +23,13 @@ #include #include #include +#include #include #include -#include -#include -#include -#include -#include -#include +#include + +#include "TCPHandler.h" + namespace DB { @@ -140,13 +147,29 @@ void TCPHandler::runImpl() if (!receivePacket()) continue; - /// Get blocks of temporary tables - readData(global_settings); + CurrentThread::initializeQuery(); - /// Reset the input stream, as we received an empty block while receiving external table data. - /// So, the stream has been marked as cancelled and we can't read from it anymore. - state.block_in.reset(); - state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + /// Should we send internal logs to client? + if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS + && query_context.getSettingsRef().send_logs_level.value != "none") + { + state.logs_queue = std::make_shared(); + state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context.getSettingsRef().send_logs_level.value); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue); + } + + query_context.setExternalTablesInitializer([&global_settings, this] (Context & context) { + if (&context != &query_context) + throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); + + /// Get blocks of temporary tables + readData(global_settings); + + /// Reset the input stream, as we received an empty block while receiving external table data. + /// So, the stream has been marked as cancelled and we can't read from it anymore. + state.block_in.reset(); + state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + }); /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage); @@ -163,8 +186,9 @@ void TCPHandler::runImpl() else processOrdinaryQuery(); - sendEndOfStream(); + sendLogs(); + sendEndOfStream(); state.reset(); } catch (const Exception & e) @@ -209,7 +233,20 @@ void TCPHandler::runImpl() try { if (exception) + { + try + { + /// Try to send logs to client, but it could be risky too + /// Assume that we can't break output here + sendLogs(); + } + catch (...) + { + tryLogCurrentException(log, "Can't send logs to client"); + } + sendException(*exception); + } } catch (...) { @@ -220,6 +257,9 @@ void TCPHandler::runImpl() try { + /// It will forcibly detach query even if unexpected error ocсurred and detachQuery() was not called + CurrentThread::detachQueryIfNotDetached(); + state.reset(); } catch (...) @@ -252,12 +292,14 @@ void TCPHandler::readData(const Settings & global_settings) constexpr size_t min_poll_interval = 5000; // 5 ms size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); - while (1) + sendLogs(); + + while (true) { Stopwatch watch(CLOCK_MONOTONIC_COARSE); /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. - while (1) + while (true) { if (static_cast(*in).poll(poll_interval)) break; @@ -289,6 +331,8 @@ void TCPHandler::readData(const Settings & global_settings) /// We accept and process data. And if they are over, then we leave. if (!receivePacket()) break; + + sendLogs(); } } @@ -346,6 +390,8 @@ void TCPHandler::processOrdinaryQuery() sendProgress(); } + sendLogs(); + if (async_in.poll(query_context.getSettingsRef().interactive_delay / 1000)) { /// There is the following result block. @@ -368,6 +414,7 @@ void TCPHandler::processOrdinaryQuery() sendExtremes(); sendProfileInfo(); sendProgress(); + sendLogs(); } sendData(block); @@ -692,11 +739,14 @@ void TCPHandler::initBlockOutput(const Block & block) { if (!state.block_out) { - if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_out = std::make_shared( - *out, CompressionSettings(query_context.getSettingsRef())); - else - state.maybe_compressed_out = out; + if (!state.maybe_compressed_out) + { + if (state.compression == Protocol::Compression::Enable) + state.maybe_compressed_out = std::make_shared( + *out, CompressionSettings(query_context.getSettingsRef())); + else + state.maybe_compressed_out = out; + } state.block_out = std::make_shared( *state.maybe_compressed_out, @@ -705,6 +755,18 @@ void TCPHandler::initBlockOutput(const Block & block) } } +void TCPHandler::initLogsBlockOutput(const Block & block) +{ + if (!state.logs_block_out) + { + /// Use uncompressed stream since log blocks usually contain only one row + state.logs_block_out = std::make_shared( + *out, + client_revision, + block.cloneEmpty()); + } +} + bool TCPHandler::isQueryCancelled() { @@ -745,6 +807,7 @@ void TCPHandler::sendData(const Block & block) initBlockOutput(block); writeVarUInt(Protocol::Server::Data, *out); + /// Send external table name (empty name is the main table) writeStringBinary("", *out); state.block_out->write(block); @@ -753,6 +816,19 @@ void TCPHandler::sendData(const Block & block) } +void TCPHandler::sendLogData(const Block & block) +{ + initLogsBlockOutput(block); + + writeVarUInt(Protocol::Server::Log, *out); + /// Send log tag (empty tag is the default tag) + writeStringBinary("", *out); + + state.logs_block_out->write(block); + out->next(); +} + + void TCPHandler::sendException(const Exception & e) { writeVarUInt(Protocol::Server::Exception, *out); @@ -784,6 +860,37 @@ void TCPHandler::sendProgress() } +void TCPHandler::sendLogs() +{ + if (!state.logs_queue) + return; + + MutableColumns logs_columns; + MutableColumns curr_logs_columns; + size_t rows = 0; + + for (; state.logs_queue->tryPop(curr_logs_columns); ++rows) + { + if (rows == 0) + { + logs_columns = std::move(curr_logs_columns); + } + else + { + for (size_t j = 0; j < logs_columns.size(); ++j) + logs_columns[j]->insertRangeFrom(*curr_logs_columns[j], 0, curr_logs_columns[j]->size()); + } + } + + if (rows > 0) + { + Block block = InternalTextLogsQueue::getSampleBlock(); + block.setColumns(std::move(logs_columns)); + sendLogData(block); + } +} + + void TCPHandler::run() { try diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index af122513cf7..1969d02b48b 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -5,16 +5,18 @@ #include #include #include -#include #include #include -#include +#include #include #include +#include +#include #include #include "IServer.h" + namespace CurrentMetrics { extern const Metric TCPConnection; @@ -63,6 +65,9 @@ struct QueryState /// Timeouts setter for current query std::unique_ptr timeout_setter; + /// A queue with internal logs that will be passed to client + InternalTextLogsQueuePtr logs_queue; + BlockOutputStreamPtr logs_block_out; void reset() { @@ -140,8 +145,10 @@ private: void sendHello(); void sendData(const Block & block); /// Write a block to the network. + void sendLogData(const Block & block); void sendException(const Exception & e); void sendProgress(); + void sendLogs(); void sendEndOfStream(); void sendProfileInfo(); void sendTotals(); @@ -150,6 +157,7 @@ private: /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled. void initBlockInput(); void initBlockOutput(const Block & block); + void initLogsBlockOutput(const Block & block); bool isQueryCancelled(); diff --git a/dbms/programs/server/config.d/listen.xml b/dbms/programs/server/config.d/listen.xml new file mode 100644 index 00000000000..24c64bbb60a --- /dev/null +++ b/dbms/programs/server/config.d/listen.xml @@ -0,0 +1 @@ +0.0.0.0 \ No newline at end of file diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp b/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp new file mode 100644 index 00000000000..e5cf72590f0 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, const DataTypes & arguments, const Array & params) +{ + assertNoParameters(name, params); + + if (arguments.size() > AggregateFunctionRetentionData::max_events ) + throw Exception("Too many event arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(arguments); +} + +} + +void registerAggregateFunctionRetention(AggregateFunctionFactory & factory) +{ + factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h new file mode 100644 index 00000000000..be5e0810d13 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h @@ -0,0 +1,150 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; +} + +struct AggregateFunctionRetentionData +{ + static constexpr auto max_events = 32; + + using Events = std::bitset; + + Events events; + + void add(UInt8 event) + { + events.set(event); + } + + void merge(const AggregateFunctionRetentionData & other) + { + events |= other.events; + } + + void serialize(WriteBuffer & buf) const + { + UInt32 event_value = events.to_ulong(); + writeBinary(event_value, buf); + } + + void deserialize(ReadBuffer & buf) + { + UInt32 event_value; + readBinary(event_value, buf); + events = event_value; + } +}; + +/** + * The max size of events is 32, that's enough for retention analytics + * + * Usage: + * - retention(cond1, cond2, cond3, ....) + * - returns [cond1_flag, cond1_flag && cond2_flag, cond1_flag && cond3_flag, ...] + */ +class AggregateFunctionRetention final + : public IAggregateFunctionDataHelper +{ +private: + UInt8 events_size; + +public: + String getName() const override + { + return "retention"; + } + + AggregateFunctionRetention(const DataTypes & arguments) + { + for (const auto i : ext::range(0, arguments.size())) + { + auto cond_arg = arguments[i].get(); + if (!typeid_cast(cond_arg)) + throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i) + " of aggregate function " + + getName() + ", must be UInt8", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + events_size = arguments.size(); + } + + + DataTypePtr getReturnType() const override + { + return std::make_shared(std::make_shared()); + } + + void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override + { + for (const auto i : ext::range(0, events_size)) + { + auto event = static_cast *>(columns[i])->getData()[row_num]; + if (event) + { + this->data(place).add(i); + break; + } + } + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + this->data(place).serialize(buf); + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + { + auto & data_to = static_cast(to).getData(); + auto & offsets_to = static_cast(to).getOffsets(); + + const bool first_flag = this->data(place).events.test(0); + data_to.insert(first_flag ? Field(static_cast(1)) : Field(static_cast(0))); + for (const auto i : ext::range(1, events_size)) + { + if (first_flag && this->data(place).events.test(i)) + data_to.insert(Field(static_cast(1))); + else + data_to.insert(Field(static_cast(0))); + } + offsets_to.push_back(offsets_to.size() == 0 ? events_size : offsets_to.back() + events_size); + } + + const char * getHeaderFilePath() const override + { + return __FILE__; + } +}; + +} diff --git a/dbms/src/AggregateFunctions/QuantileExact.h b/dbms/src/AggregateFunctions/QuantileExact.h index 568ad8d0950..86be2f84921 100644 --- a/dbms/src/AggregateFunctions/QuantileExact.h +++ b/dbms/src/AggregateFunctions/QuantileExact.h @@ -77,7 +77,7 @@ struct QuantileExact return array[n]; } - return Value(); + return std::numeric_limits::quiet_NaN(); } /// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address. diff --git a/dbms/src/AggregateFunctions/QuantileExactWeighted.h b/dbms/src/AggregateFunctions/QuantileExactWeighted.h index 76e65f07dac..eca89f30aa9 100644 --- a/dbms/src/AggregateFunctions/QuantileExactWeighted.h +++ b/dbms/src/AggregateFunctions/QuantileExactWeighted.h @@ -72,7 +72,7 @@ struct QuantileExactWeighted size_t size = map.size(); if (0 == size) - return Value(); + return std::numeric_limits::quiet_NaN(); /// Copy the data to a temporary array to get the element you need in order. using Pair = typename Map::value_type; diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp index 5efb1e5bf1b..3517ad57a73 100644 --- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -34,6 +34,7 @@ void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory &); void registerAggregateFunctionHistogram(AggregateFunctionFactory & factory); +void registerAggregateFunctionRetention(AggregateFunctionFactory & factory); void registerAggregateFunctions() { @@ -59,6 +60,7 @@ void registerAggregateFunctions() registerAggregateFunctionsBitwise(factory); registerAggregateFunctionsMaxIntersections(factory); registerAggregateFunctionHistogram(factory); + registerAggregateFunctionRetention(factory); } { diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index affd89b1c28..1ed186085f7 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -114,6 +114,7 @@ void Connection::disconnect() //LOG_TRACE(log_wrapper.get(), "Disconnecting"); in = nullptr; + last_input_packet_type.reset(); out = nullptr; // can write to socket if (socket) socket->close(); @@ -379,6 +380,7 @@ void Connection::sendQuery( maybe_compressed_in.reset(); maybe_compressed_out.reset(); block_in.reset(); + block_logs_in.reset(); block_out.reset(); /// Send empty block which means end of data. @@ -506,20 +508,50 @@ bool Connection::poll(size_t timeout_microseconds) } -bool Connection::hasReadBufferPendingData() const +bool Connection::hasReadPendingData() const { - return static_cast(*in).hasPendingData(); + return last_input_packet_type.has_value() || static_cast(*in).hasPendingData(); +} + + +std::optional Connection::checkPacket(size_t timeout_microseconds) +{ + if (last_input_packet_type.has_value()) + return last_input_packet_type; + + if (hasReadPendingData() || poll(timeout_microseconds)) + { + // LOG_TRACE(log_wrapper.get(), "Receiving packet type"); + UInt64 packet_type; + readVarUInt(packet_type, *in); + + last_input_packet_type.emplace(packet_type); + return last_input_packet_type; + } + + return {}; } Connection::Packet Connection::receivePacket() { - //LOG_TRACE(log_wrapper.get(), "Receiving packet"); - try { Packet res; - readVarUInt(res.type, *in); + + /// Have we already read packet type? + if (last_input_packet_type) + { + res.type = *last_input_packet_type; + last_input_packet_type.reset(); + } + else + { + //LOG_TRACE(log_wrapper.get(), "Receiving packet type"); + readVarUInt(res.type, *in); + } + + //LOG_TRACE(log_wrapper.get(), "Receiving packet " << res.type << " " << Protocol::Server::toString(res.type)); switch (res.type) { @@ -549,6 +581,10 @@ Connection::Packet Connection::receivePacket() res.block = receiveData(); return res; + case Protocol::Server::Log: + res.block = receiveLogData(); + return res; + case Protocol::Server::EndOfStream: return res; @@ -576,14 +612,26 @@ Block Connection::receiveData() //LOG_TRACE(log_wrapper.get(), "Receiving data"); initBlockInput(); + return receiveDataImpl(block_in); +} + +Block Connection::receiveLogData() +{ + initBlockLogsInput(); + return receiveDataImpl(block_logs_in); +} + + +Block Connection::receiveDataImpl(BlockInputStreamPtr & stream) +{ String external_table_name; readStringBinary(external_table_name, *in); size_t prev_bytes = in->count(); /// Read one block from network. - Block res = block_in->read(); + Block res = stream->read(); if (throttler) throttler->add(in->count() - prev_bytes); @@ -592,20 +640,39 @@ Block Connection::receiveData() } +void Connection::initInputBuffers() +{ + +} + + void Connection::initBlockInput() { if (!block_in) { - if (compression == Protocol::Compression::Enable) - maybe_compressed_in = std::make_shared(*in); - else - maybe_compressed_in = in; + if (!maybe_compressed_in) + { + if (compression == Protocol::Compression::Enable) + maybe_compressed_in = std::make_shared(*in); + else + maybe_compressed_in = in; + } block_in = std::make_shared(*maybe_compressed_in, server_revision); } } +void Connection::initBlockLogsInput() +{ + if (!block_logs_in) + { + /// Have to return superset of SystemLogsQueue::getSampleBlock() columns + block_logs_in = std::make_shared(*in, server_revision); + } +} + + void Connection::setDescription() { auto resolved_address = getResolvedAddress(); diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index dabb50b53a9..ad98df3cc8f 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -23,6 +23,7 @@ #include #include +#include namespace DB @@ -138,7 +139,10 @@ public: bool poll(size_t timeout_microseconds = 0); /// Check, if has data in read buffer. - bool hasReadBufferPendingData() const; + bool hasReadPendingData() const; + + /// Checks if there is input data in connection and reads packet ID. + std::optional checkPacket(size_t timeout_microseconds = 0); /// Receive packet from server. Packet receivePacket(); @@ -195,6 +199,7 @@ private: std::unique_ptr socket; std::shared_ptr in; std::shared_ptr out; + std::optional last_input_packet_type; String query_id; Protocol::Compression compression; /// Enable data compression for communication. @@ -214,6 +219,7 @@ private: /// From where to read query execution result. std::shared_ptr maybe_compressed_in; BlockInputStreamPtr block_in; + BlockInputStreamPtr block_logs_in; /// Where to write data for INSERT. std::shared_ptr maybe_compressed_out; @@ -249,11 +255,16 @@ private: bool ping(); Block receiveData(); + Block receiveLogData(); + Block receiveDataImpl(BlockInputStreamPtr & stream); + std::unique_ptr receiveException(); Progress receiveProgress(); BlockStreamProfileInfo receiveProfileInfo(); + void initInputBuffers(); void initBlockInput(); + void initBlockLogsInput(); void throwUnexpectedPacket(UInt64 packet_type, const char * expected) const; }; diff --git a/dbms/src/Client/MultiplexedConnections.cpp b/dbms/src/Client/MultiplexedConnections.cpp index 8fe27ecf7fa..3e88a20caa3 100644 --- a/dbms/src/Client/MultiplexedConnections.cpp +++ b/dbms/src/Client/MultiplexedConnections.cpp @@ -247,6 +247,7 @@ Connection::Packet MultiplexedConnections::receivePacketUnlocked() case Protocol::Server::ProfileInfo: case Protocol::Server::Totals: case Protocol::Server::Extremes: + case Protocol::Server::Log: break; case Protocol::Server::EndOfStream: @@ -276,7 +277,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead for (const ReplicaState & state : replica_states) { Connection * connection = state.connection; - if ((connection != nullptr) && connection->hasReadBufferPendingData()) + if ((connection != nullptr) && connection->hasReadPendingData()) read_list.push_back(*connection->socket); } diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp index 607b6651499..95c21786484 100644 --- a/dbms/src/Columns/ColumnConst.cpp +++ b/dbms/src/Columns/ColumnConst.cpp @@ -30,6 +30,11 @@ ColumnPtr ColumnConst::convertToFullColumn() const return data->replicate(Offsets(1, s)); } +ColumnPtr ColumnConst::removeLowCardinality() const +{ + return ColumnConst::create(data->convertToFullColumnIfWithDictionary(), s); +} + ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/) const { if (s != filt.size()) diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index d85c00eb604..bd40246b1e5 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -36,6 +36,8 @@ public: return convertToFullColumn(); } + ColumnPtr removeLowCardinality() const; + std::string getName() const override { return "Const(" + data->getName() + ")"; diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 2bea3d68953..488c8dd52bc 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -175,7 +175,6 @@ ColumnPtr ColumnString::indexImpl(const PaddedPODArray & indexes, size_t l Chars_t & res_chars = res->chars; Offsets & res_offsets = res->offsets; - size_t new_chars_size = 0; for (size_t i = 0; i < limit; ++i) new_chars_size += sizeAt(indexes[i]); diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 4fc22d28e70..5f86623c49a 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -118,7 +118,7 @@ MutableColumnPtr ColumnVector::cloneResized(size_t size) const memcpy(&new_col.data[0], &data[0], count * sizeof(data[0])); if (size > count) - memset(&new_col.data[count], static_cast(value_type()), (size - count) * sizeof(value_type)); + memset(static_cast(&new_col.data[count]), static_cast(value_type()), (size - count) * sizeof(value_type)); } return std::move(res); @@ -327,6 +327,11 @@ template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; +template class ColumnVector; template class ColumnVector; template class ColumnVector; + +template class ColumnVector; +template class ColumnVector; +template class ColumnVector; } diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index cfda75c9162..7240a9627ac 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -8,6 +8,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + /** Stuff for comparing numbers. * Integer values are compared as usual. * Floating-point numbers are compared this way that NaNs always end up at the end @@ -117,22 +123,55 @@ template <> inline UInt64 unionCastToUInt64(Float32 x) } +/// PaddedPODArray extended by Decimal scale +template +class DecimalPaddedPODArray : public PaddedPODArray +{ +public: + using Base = PaddedPODArray; + using Base::operator[]; + using Base::Base; + + DecimalPaddedPODArray(std::initializer_list il) + : DecimalPaddedPODArray(std::begin(il), std::end(il)) + {} + + DecimalPaddedPODArray(DecimalPaddedPODArray && other) + { + this->swap(other); + std::swap(scale, other.scale); + } + + DecimalPaddedPODArray & operator=(DecimalPaddedPODArray && other) + { + this->swap(other); + std::swap(scale, other.scale); + return *this; + } + + void setScale(UInt32 s) { scale = s; } + UInt32 getScale() const { return scale; } + +private: + UInt32 scale = DecimalField::wrongScale(); +}; + + /** A template for columns that use a simple array to store. - */ + */ template class ColumnVector final : public COWPtrHelper> { private: - friend class COWPtrHelper>; - using Self = ColumnVector; + friend class COWPtrHelper; struct less; struct greater; public: using value_type = T; - using Container = PaddedPODArray; + using Container = std::conditional_t, DecimalPaddedPODArray, PaddedPODArray>; private: ColumnVector() {} @@ -216,12 +255,20 @@ public: Field operator[](size_t n) const override { - return typename NearestFieldType::Type(data[n]); + if constexpr (IsDecimalNumber) + { + UInt32 scale = data.getScale(); + if (scale == DecimalField::wrongScale()) + throw Exception("Extracting Decimal field with unknown scale. Scale is lost.", ErrorCodes::LOGICAL_ERROR); + return DecimalField(data[n], scale); + } + else + return typename NearestFieldType::Type(data[n]); } void get(size_t n, Field & res) const override { - res = typename NearestFieldType::Type(data[n]); + res = (*this)[n]; } UInt64 get64(size_t n) const override; diff --git a/dbms/src/Common/ActionBlocker.h b/dbms/src/Common/ActionBlocker.h index 2ddea2d4359..055d266c72e 100644 --- a/dbms/src/Common/ActionBlocker.h +++ b/dbms/src/Common/ActionBlocker.h @@ -1,14 +1,15 @@ #pragma once - #include #include #include + namespace DB { -/// An atomic variable that is used to block and interrupt certain actions -/// If it is not zero then actions related with it should be considered as interrupted +/// An atomic variable that is used to block and interrupt certain actions. +/// If it is not zero then actions related with it should be considered as interrupted. +/// Uses shared_ptr and the lock uses weak_ptr to be able to "hold" a lock when an object with blocker has already died. class ActionBlocker { public: @@ -33,4 +34,5 @@ private: CounterPtr counter; }; + } diff --git a/dbms/src/Common/ActionLock.h b/dbms/src/Common/ActionLock.h index 3d6bfc8ada7..1167a23b8dd 100644 --- a/dbms/src/Common/ActionLock.h +++ b/dbms/src/Common/ActionLock.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include + namespace DB { diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp new file mode 100644 index 00000000000..38a42cb65c8 --- /dev/null +++ b/dbms/src/Common/CurrentThread.cpp @@ -0,0 +1,80 @@ +#include "CurrentThread.h" +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +void CurrentThread::updatePerformanceCounters() +{ + get()->updatePerformanceCounters(); +} + +ThreadStatusPtr CurrentThread::get() +{ +#ifndef NDEBUG + if (!current_thread || current_thread.use_count() <= 0) + throw Exception("Thread #" + std::to_string(Poco::ThreadNumber::get()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR); + + if (Poco::ThreadNumber::get() != current_thread->thread_number) + throw Exception("Current thread has different thread number", ErrorCodes::LOGICAL_ERROR); +#endif + + return current_thread; +} + +ProfileEvents::Counters & CurrentThread::getProfileEvents() +{ + return current_thread->performance_counters; +} + +MemoryTracker & CurrentThread::getMemoryTracker() +{ + return current_thread->memory_tracker; +} + +void CurrentThread::updateProgressIn(const Progress & value) +{ + current_thread->progress_in.incrementPiecewiseAtomically(value); +} + +void CurrentThread::updateProgressOut(const Progress & value) +{ + current_thread->progress_out.incrementPiecewiseAtomically(value); +} + +void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue) +{ + get()->attachInternalTextLogsQueue(logs_queue); +} + +std::shared_ptr CurrentThread::getInternalTextLogsQueue() +{ + /// NOTE: this method could be called at early server startup stage + /// NOTE: this method could be called in ThreadStatus destructor, therefore we make use_count() check just in case + + if (!current_thread || current_thread.use_count() <= 0) + return nullptr; + + if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died) + return nullptr; + + return current_thread->getInternalTextLogsQueue(); +} + +ThreadGroupStatusPtr CurrentThread::getGroup() +{ + return get()->getThreadGroup(); +} + +} diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h new file mode 100644 index 00000000000..afde7ad8bf2 --- /dev/null +++ b/dbms/src/Common/CurrentThread.h @@ -0,0 +1,83 @@ +#pragma once +#include +#include + + +namespace ProfileEvents +{ +class Counters; +} + +class MemoryTracker; + + +namespace DB +{ + +class Context; +class QueryStatus; +class ThreadStatus; +struct Progress; +using ThreadStatusPtr = std::shared_ptr; +class InternalTextLogsQueue; +class ThreadGroupStatus; +using ThreadGroupStatusPtr = std::shared_ptr; + + +class CurrentThread +{ +public: + + /// Handler to current thread + static ThreadStatusPtr get(); + /// Group to which belongs current thread + static ThreadGroupStatusPtr getGroup(); + + /// A logs queue used by TCPHandler to pass logs to a client + static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue); + static std::shared_ptr getInternalTextLogsQueue(); + + /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats + static void updatePerformanceCounters(); + + static ProfileEvents::Counters & getProfileEvents(); + static MemoryTracker & getMemoryTracker(); + + /// Update read and write rows (bytes) statistics (used in system.query_thread_log) + static void updateProgressIn(const Progress & value); + static void updateProgressOut(const Progress & value); + + /// Query management: + + /// Call from master thread as soon as possible (e.g. when thread accepted connection) + static void initializeQuery(); + + /// Sets query_context for current thread group + static void attachQueryContext(Context & query_context); + + /// You must call one of these methods when create a query child thread: + /// Add current thread to a group associated with the thread group + static void attachTo(const ThreadGroupStatusPtr & thread_group); + /// Is useful for a ThreadPool tasks + static void attachToIfDetached(const ThreadGroupStatusPtr & thread_group); + + /// Update ProfileEvents and dumps info to system.query_thread_log + static void finalizePerformanceCounters(); + + /// Returns a non-empty string if the thread is attached to a query + static std::string getCurrentQueryID(); + + /// Non-master threads call this method in destructor automatically + static void detachQuery(); + static void detachQueryIfNotDetached(); + + /// Initializes query with current thread as master thread in constructor, and detaches it in desstructor + struct QueryScope + { + explicit QueryScope(Context & query_context); + ~QueryScope(); + }; +}; + +} + diff --git a/dbms/src/Common/DNSResolver.cpp b/dbms/src/Common/DNSResolver.cpp index e3d9442deea..c82f302d7b2 100644 --- a/dbms/src/Common/DNSResolver.cpp +++ b/dbms/src/Common/DNSResolver.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -79,6 +80,10 @@ struct DNSResolver::Impl { SimpleCache cache_host; + /// Cached server host name + std::mutex mutex; + std::optional host_name; + /// If disabled, will not make cache lookups, will resolve addresses manually on each call std::atomic disable_cache{false}; }; @@ -108,6 +113,9 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U void DNSResolver::dropCache() { impl->cache_host.drop(); + + std::unique_lock lock(impl->mutex); + impl->host_name.reset(); } void DNSResolver::setDisableCacheFlag(bool is_disabled) @@ -115,6 +123,19 @@ void DNSResolver::setDisableCacheFlag(bool is_disabled) impl->disable_cache = is_disabled; } +String DNSResolver::getHostName() +{ + if (impl->disable_cache) + return Poco::Net::DNS::hostName(); + + std::unique_lock lock(impl->mutex); + + if (!impl->host_name.has_value()) + impl->host_name.emplace(Poco::Net::DNS::hostName()); + + return *impl->host_name; +} + DNSResolver::~DNSResolver() = default; diff --git a/dbms/src/Common/DNSResolver.h b/dbms/src/Common/DNSResolver.h index fb3892e101f..097e646fa65 100644 --- a/dbms/src/Common/DNSResolver.h +++ b/dbms/src/Common/DNSResolver.h @@ -25,6 +25,9 @@ public: Poco::Net::SocketAddress resolveAddress(const std::string & host, UInt16 port); + /// Get this server host name + String getHostName(); + /// Disables caching void setDisableCacheFlag(bool is_disabled = true); diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index e719f8be4b3..928d011fb75 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -380,6 +380,15 @@ namespace ErrorCodes extern const int INVALID_JOIN_ON_EXPRESSION = 403; extern const int BAD_ODBC_CONNECTION_STRING = 404; extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT = 405; + extern const int TOP_AND_LIMIT_TOGETHER = 406; + extern const int DECIMAL_OVERFLOW = 407; + extern const int BAD_REQUEST_PARAMETER = 408; + extern const int EXTERNAL_EXECUTABLE_NOT_FOUND = 409; + extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING = 410; + extern const int PTHREAD_ERROR = 411; + extern const int NETLINK_ERROR = 412; + extern const int CANNOT_SET_SIGNAL_HANDLER = 413; + extern const int CANNOT_READLINE = 414; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h deleted file mode 100644 index 6331ecba5ca..00000000000 --- a/dbms/src/Common/ExternalTable.h +++ /dev/null @@ -1,225 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - - -/// The base class containing the basic information about external table and -/// basic functions for extracting this information from text fields. -class BaseExternalTable -{ -public: - std::string file; /// File with data or '-' if stdin - std::string name; /// The name of the table - std::string format; /// Name of the data storage format - - /// Description of the table structure: (column name, data type name) - std::vector> structure; - - std::unique_ptr read_buffer; - Block sample_block; - - virtual ~BaseExternalTable() {} - - /// Initialize read_buffer, depending on the data source. By default, does nothing. - virtual void initReadBuffer() {} - - /// Get the table data - a pair (a thread with the contents of the table, the name of the table) - ExternalTableData getData(const Context & context) - { - initReadBuffer(); - initSampleBlock(); - ExternalTableData res = std::make_pair(std::make_shared(context.getInputFormat( - format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE)), name); - return res; - } - -protected: - /// Clear all accumulated information - void clean() - { - name = ""; - file = ""; - format = ""; - structure.clear(); - sample_block = Block(); - read_buffer.reset(); - } - - /// Function for debugging information output - void write() - { - std::cerr << "file " << file << std::endl; - std::cerr << "name " << name << std::endl; - std::cerr << "format " << format << std::endl; - std::cerr << "structure: \n"; - for (size_t i = 0; i < structure.size(); ++i) - std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl; - } - - static std::vector split(const std::string & s, const std::string & d) - { - std::vector res; - boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on); - return res; - } - - /// Construct the `structure` vector from the text field `structure` - virtual void parseStructureFromStructureField(const std::string & argument) - { - std::vector vals = split(argument, " ,"); - - if (vals.size() & 1) - throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS); - - for (size_t i = 0; i < vals.size(); i += 2) - structure.emplace_back(vals[i], vals[i + 1]); - } - - /// Construct the `structure` vector from the text field `types` - virtual void parseStructureFromTypesField(const std::string & argument) - { - std::vector vals = split(argument, " ,"); - - for (size_t i = 0; i < vals.size(); ++i) - structure.emplace_back("_" + toString(i + 1), vals[i]); - } - -private: - /// Initialize sample_block according to the structure of the table stored in the `structure` - void initSampleBlock() - { - const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - - for (size_t i = 0; i < structure.size(); ++i) - { - ColumnWithTypeAndName column; - column.name = structure[i].first; - column.type = data_type_factory.get(structure[i].second); - column.column = column.type->createColumn(); - sample_block.insert(std::move(column)); - } - } -}; - - -/// Parsing of external table used in the tcp client. -class ExternalTable : public BaseExternalTable -{ -public: - void initReadBuffer() override - { - if (file == "-") - read_buffer = std::make_unique(STDIN_FILENO); - else - read_buffer = std::make_unique(file); - } - - /// Extract parameters from variables_map, which is built on the client command line - ExternalTable(const boost::program_options::variables_map & external_options) - { - if (external_options.count("file")) - file = external_options["file"].as(); - else - throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("name")) - name = external_options["name"].as(); - else - throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("format")) - format = external_options["format"].as(); - else - throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("structure")) - parseStructureFromStructureField(external_options["structure"].as()); - else if (external_options.count("types")) - parseStructureFromTypesField(external_options["types"].as()); - else - throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - } -}; - -/// Parsing of external table used when sending tables via http -/// The `handlePart` function will be called for each table passed, -/// so it's also necessary to call `clean` at the end of the `handlePart`. -class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable -{ -public: - std::vector names; - - ExternalTablesHandler(Context & context_, Poco::Net::NameValueCollection params_) : context(context_), params(params_) { } - - void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) - { - /// The buffer is initialized here, not in the virtual function initReadBuffer - read_buffer = std::make_unique(stream); - - /// Retrieve a collection of parameters from MessageHeader - Poco::Net::NameValueCollection content; - std::string label; - Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content); - - /// Get parameters - name = content.get("name", "_data"); - format = params.get(name + "_format", "TabSeparated"); - - if (params.has(name + "_structure")) - parseStructureFromStructureField(params.get(name + "_structure")); - else if (params.has(name + "_types")) - parseStructureFromTypesField(params.get(name + "_types")); - else - throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS); - - ExternalTableData data = getData(context); - - /// Create table - NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); - storage->startup(); - context.addExternalTable(data.second, storage); - BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef()); - - /// Write data - data.first->readPrefix(); - output->writePrefix(); - while(Block block = data.first->read()) - output->write(block); - data.first->readSuffix(); - output->writeSuffix(); - - names.push_back(name); - /// We are ready to receive the next file, for this we clear all the information received - clean(); - } - -private: - Context & context; - Poco::Net::NameValueCollection params; -}; - - -} diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 2c4d781056d..7e957ae1ae4 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -1,11 +1,10 @@ +#include "MemoryTracker.h" #include #include #include #include +#include #include -#include - -#include namespace DB @@ -19,7 +18,7 @@ namespace DB MemoryTracker::~MemoryTracker() { - if (peak) + if (static_cast(level) < static_cast(VariableContext::Process) && peak) { try { @@ -56,13 +55,16 @@ void MemoryTracker::logPeakMemoryUsage() const void MemoryTracker::alloc(Int64 size) { + if (blocker.isCancelled()) + return; + /** Using memory_order_relaxed means that if allocations are done simultaneously, * we allow exception about memory limit exceeded to be thrown only on next allocation. * So, we allow over-allocations. */ Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed); - if (!next.load(std::memory_order_relaxed)) + if (!parent.load(std::memory_order_relaxed)) CurrentMetrics::add(metric, size); Int64 current_limit = limit.load(std::memory_order_relaxed); @@ -102,45 +104,62 @@ void MemoryTracker::alloc(Int64 size) if (will_be > peak.load(std::memory_order_relaxed)) /// Races doesn't matter. Could rewrite with CAS, but not worth. peak.store(will_be, std::memory_order_relaxed); - if (auto loaded_next = next.load(std::memory_order_relaxed)) + if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->alloc(size); } void MemoryTracker::free(Int64 size) { - Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size; + if (blocker.isCancelled()) + return; - /** Sometimes, query could free some data, that was allocated outside of query context. - * Example: cache eviction. - * To avoid negative memory usage, we "saturate" amount. - * Memory usage will be calculated with some error. - * NOTE The code is not atomic. Not worth to fix. - */ - if (new_amount < 0) + if (level == VariableContext::Thread) { - amount.fetch_sub(new_amount); - size += new_amount; + /// Could become negative if memory allocated in this thread is freed in another one + amount.fetch_sub(size, std::memory_order_relaxed); + } + else + { + Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size; + + /** Sometimes, query could free some data, that was allocated outside of query context. + * Example: cache eviction. + * To avoid negative memory usage, we "saturate" amount. + * Memory usage will be calculated with some error. + * NOTE: The code is not atomic. Not worth to fix. + */ + if (unlikely(new_amount < 0)) + { + amount.fetch_sub(new_amount); + size += new_amount; + } } - if (auto loaded_next = next.load(std::memory_order_relaxed)) + if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); else CurrentMetrics::sub(metric, size); } -void MemoryTracker::reset() +void MemoryTracker::resetCounters() { - if (!next.load(std::memory_order_relaxed)) - CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed)); - amount.store(0, std::memory_order_relaxed); peak.store(0, std::memory_order_relaxed); limit.store(0, std::memory_order_relaxed); } +void MemoryTracker::reset() +{ + if (!parent.load(std::memory_order_relaxed)) + CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed)); + + resetCounters(); +} + + void MemoryTracker::setOrRaiseLimit(Int64 value) { /// This is just atomic set to maximum. @@ -149,29 +168,26 @@ void MemoryTracker::setOrRaiseLimit(Int64 value) ; } -#if __APPLE__ && __clang__ -__thread MemoryTracker * current_memory_tracker = nullptr; -#else -thread_local MemoryTracker * current_memory_tracker = nullptr; -#endif namespace CurrentMemoryTracker { void alloc(Int64 size) { - if (current_memory_tracker) - current_memory_tracker->alloc(size); + DB::CurrentThread::getMemoryTracker().alloc(size); } void realloc(Int64 old_size, Int64 new_size) { - if (current_memory_tracker) - current_memory_tracker->alloc(new_size - old_size); + DB::CurrentThread::getMemoryTracker().alloc(new_size - old_size); } void free(Int64 size) { - if (current_memory_tracker) - current_memory_tracker->free(size); + DB::CurrentThread::getMemoryTracker().free(size); } } + +DB::SimpleActionLock getCurrentMemoryTrackerActionLock() +{ + return DB::CurrentThread::getMemoryTracker().blocker.cancel(); +} diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 32a93978fcd..68c145393fe 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace CurrentMetrics @@ -26,7 +28,7 @@ class MemoryTracker /// Singly-linked list. All information will be passed to subsequent memory trackers also (it allows to implement trackers hierarchy). /// In terms of tree nodes it is the list of parents. Lifetime of these trackers should "include" lifetime of current tracker. - std::atomic next {}; + std::atomic parent {}; /// You could specify custom metric to track memory usage. CurrentMetrics::Metric metric = CurrentMetrics::MemoryTracking; @@ -35,11 +37,14 @@ class MemoryTracker const char * description = nullptr; public: - MemoryTracker() {} - MemoryTracker(Int64 limit_) : limit(limit_) {} + MemoryTracker(VariableContext level = VariableContext::Thread) : level(level) {} + MemoryTracker(Int64 limit_, VariableContext level = VariableContext::Thread) : limit(limit_), level(level) {} + MemoryTracker(MemoryTracker * parent_, VariableContext level = VariableContext::Thread) : parent(parent_), level(level) {} ~MemoryTracker(); + VariableContext level; + /** Call the following functions before calling of corresponding operations with memory allocators. */ void alloc(Int64 size); @@ -79,9 +84,15 @@ public: } /// next should be changed only once: from nullptr to some value. - void setNext(MemoryTracker * elem) + /// NOTE: It is not true in MergeListElement + void setParent(MemoryTracker * elem) { - next.store(elem, std::memory_order_relaxed); + parent.store(elem, std::memory_order_relaxed); + } + + MemoryTracker * getParent() + { + return parent.load(std::memory_order_relaxed); } /// The memory consumption could be shown in realtime via CurrentMetrics counter @@ -95,26 +106,21 @@ public: description = description_; } - /// Reset the accumulated data. + /// Reset the accumulated data + void resetCounters(); + + /// Reset the accumulated data and the parent. void reset(); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage() const; + + /// To be able to temporarily stop memory tracker + DB::SimpleActionBlocker blocker; }; -/** The MemoryTracker object is quite difficult to pass to all places where significant amounts of memory are allocated. - * Therefore, a thread-local pointer to used MemoryTracker is set, or nullptr if MemoryTracker does not need to be used. - * This pointer is set when memory consumption is monitored in current thread. - * So, you just need to pass it to all the threads that handle one request. - */ -#if defined(__APPLE__) && defined(__clang__) -extern __thread MemoryTracker * current_memory_tracker; -#else -extern thread_local MemoryTracker * current_memory_tracker; -#endif - -/// Convenience methods, that use current_memory_tracker if it is available. +/// Convenience methods, that use current thread's memory_tracker if it is available. namespace CurrentMemoryTracker { void alloc(Int64 size); @@ -123,20 +129,4 @@ namespace CurrentMemoryTracker } -#include - -struct TemporarilyDisableMemoryTracker : private boost::noncopyable -{ - MemoryTracker * memory_tracker; - - TemporarilyDisableMemoryTracker() - { - memory_tracker = current_memory_tracker; - current_memory_tracker = nullptr; - } - - ~TemporarilyDisableMemoryTracker() - { - current_memory_tracker = memory_tracker; - } -}; +DB::SimpleActionLock getCurrentMemoryTrackerActionLock(); diff --git a/dbms/src/Common/NaNUtils.h b/dbms/src/Common/NaNUtils.h index b30b2336c97..af0b26266a7 100644 --- a/dbms/src/Common/NaNUtils.h +++ b/dbms/src/Common/NaNUtils.h @@ -47,3 +47,11 @@ std::enable_if_t, T> NaNOrZero() { return T{}; } + +#if 1 /// __int128 +template +std::enable_if_t && !std::numeric_limits::is_integer, __int128> NaNOrZero() +{ + return __int128(0); +} +#endif diff --git a/dbms/src/Common/ODBCBridgeHelper.cpp b/dbms/src/Common/ODBCBridgeHelper.cpp new file mode 100644 index 00000000000..785c457062d --- /dev/null +++ b/dbms/src/Common/ODBCBridgeHelper.cpp @@ -0,0 +1,131 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING; +} +ODBCBridgeHelper::ODBCBridgeHelper( + const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_) + : config(config_), http_timeout(http_timeout_), connection_string(connection_string_) +{ + size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); + std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); + + ping_url.setHost(bridge_host); + ping_url.setPort(bridge_port); + ping_url.setScheme("http"); + ping_url.setPath(PING_HANDLER); +} +void ODBCBridgeHelper::startODBCBridge() const +{ + Poco::Path path{config.getString("application.dir", "")}; + path.setFileName("clickhouse-odbc-bridge"); + + if (!path.isFile()) + throw Exception("clickhouse-odbc-bridge is not found", ErrorCodes::EXTERNAL_EXECUTABLE_NOT_FOUND); + + std::stringstream command; + command << path.toString() << ' '; + command << "--http-port " << config.getUInt("odbc_bridge.port", DEFAULT_PORT) << ' '; + command << "--listen-host " << config.getString("odbc_bridge.listen_host", DEFAULT_HOST) << ' '; + command << "--http-timeout " << http_timeout.totalMicroseconds() << ' '; + if (config.has("logger.odbc_bridge_log")) + command << "--log-path " << config.getString("logger.odbc_bridge_log") << ' '; + if (config.has("logger.odbc_bridge_errlog")) + command << "--err-log-path " << config.getString("logger.odbc_bridge_errlog") << ' '; + if (config.has("logger.odbc_bridge_level")) + command << "--log-level " << config.getString("logger.odbc_bridge_level") << ' '; + command << "&"; /// we don't want to wait this process + + auto command_str = command.str(); + LOG_TRACE(log, "Starting clickhouse-odbc-bridge with command: " << command_str); + + auto cmd = ShellCommand::execute(command_str); + cmd->wait(); +} + +std::vector> ODBCBridgeHelper::getURLParams(const std::string & cols, size_t max_block_size) const +{ + std::vector> result; + + result.emplace_back("connection_string", connection_string); /// already validated + result.emplace_back("columns", cols); + result.emplace_back("max_block_size", std::to_string(max_block_size)); + + return result; +} + +bool ODBCBridgeHelper::checkODBCBridgeIsRunning() const +{ + try + { + ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr); + return checkString(ODBCBridgeHelper::PING_OK_ANSWER, buf); + } + catch (...) + { + return false; + } +} + +void ODBCBridgeHelper::startODBCBridgeSync() const +{ + if (!checkODBCBridgeIsRunning()) + { + LOG_TRACE(log, "clickhouse-odbc-bridge is not running, will try to start it"); + startODBCBridge(); + bool started = false; + for (size_t counter : ext::range(1, 20)) + { + LOG_TRACE(log, "Checking clickhouse-odbc-bridge is running, try " << counter); + if (checkODBCBridgeIsRunning()) + { + started = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + if (!started) + throw Exception("ODBCBridgeHelper: clickhouse-odbc-bridge is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING); + } +} + +Poco::URI ODBCBridgeHelper::getMainURI() const +{ + size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); + std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); + + Poco::URI main_uri; + main_uri.setHost(bridge_host); + main_uri.setPort(bridge_port); + main_uri.setScheme("http"); + main_uri.setPath(MAIN_HANDLER); + return main_uri; +} + +Poco::URI ODBCBridgeHelper::getColumnsInfoURI() const +{ + size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); + std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); + + Poco::URI columns_info_uri; + columns_info_uri.setHost(bridge_host); + columns_info_uri.setPort(bridge_port); + columns_info_uri.setScheme("http"); + columns_info_uri.setPath(COL_INFO_HANDLER); + return columns_info_uri; +} +} diff --git a/dbms/src/Common/ODBCBridgeHelper.h b/dbms/src/Common/ODBCBridgeHelper.h new file mode 100644 index 00000000000..807782d73eb --- /dev/null +++ b/dbms/src/Common/ODBCBridgeHelper.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int EXTERNAL_EXECUTABLE_NOT_FOUND; +} +/** Helper for odbc-bridge, provide utility methods, not main request + */ +class ODBCBridgeHelper +{ +private: + + using Configuration = Poco::Util::AbstractConfiguration; + + const Configuration & config; + Poco::Timespan http_timeout; + + std::string connection_string; + + Poco::URI ping_url; + + Poco::Logger * log = &Poco::Logger::get("ODBCBridgeHelper"); + +public: + static constexpr inline size_t DEFAULT_PORT = 9018; + + static constexpr inline auto DEFAULT_HOST = "localhost"; + static constexpr inline auto DEFAULT_FORMAT = "RowBinary"; + static constexpr inline auto PING_HANDLER = "/ping"; + static constexpr inline auto MAIN_HANDLER = "/"; + static constexpr inline auto COL_INFO_HANDLER = "/columns_info"; + static constexpr inline auto PING_OK_ANSWER = "Ok."; + + ODBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_); + + std::vector> getURLParams(const std::string & cols, size_t max_block_size) const; + bool checkODBCBridgeIsRunning() const; + + void startODBCBridge() const; + void startODBCBridgeSync() const; + + Poco::URI getMainURI() const; + Poco::URI getColumnsInfoURI() const; +}; +} diff --git a/dbms/src/Common/ProfileEvents.cpp b/dbms/src/Common/ProfileEvents.cpp index adef2331609..b4cfd6e6852 100644 --- a/dbms/src/Common/ProfileEvents.cpp +++ b/dbms/src/Common/ProfileEvents.cpp @@ -1,5 +1,7 @@ #include - +#include +#include +#include /// Available events. Add something here as you wish. #define APPLY_FOR_EVENTS(M) \ @@ -37,6 +39,11 @@ M(CreatedReadBufferAIO) \ M(CreatedWriteBufferOrdinary) \ M(CreatedWriteBufferAIO) \ + M(DiskReadElapsedMicroseconds) \ + M(DiskWriteElapsedMicroseconds) \ + M(NetworkReceiveElapsedMicroseconds) \ + M(NetworkSendElapsedMicroseconds) \ + M(ThrottlerSleepMicroseconds) \ \ M(ReplicatedPartFetches) \ M(ReplicatedPartFailedFetches) \ @@ -67,7 +74,9 @@ M(ZooKeeperCheck) \ M(ZooKeeperClose) \ M(ZooKeeperWatchResponse) \ - M(ZooKeeperExceptions) \ + M(ZooKeeperUserExceptions) \ + M(ZooKeeperHardwareExceptions) \ + M(ZooKeeperOtherExceptions) \ M(ZooKeeperWaitMicroseconds) \ M(ZooKeeperBytesSent) \ M(ZooKeeperBytesReceived) \ @@ -143,31 +152,93 @@ M(RWLockAcquiredWriteLocks) \ M(RWLockReadersWaitMilliseconds) \ M(RWLockWritersWaitMilliseconds) \ + M(NetworkErrors) \ \ - M(NetworkErrors) + M(RealTimeMicroseconds) \ + M(UserTimeMicroseconds) \ + M(SystemTimeMicroseconds) \ + M(SoftPageFaults) \ + M(HardPageFaults) \ + M(VoluntaryContextSwitches) \ + M(InvoluntaryContextSwitches) \ + \ + M(OSIOWaitMicroseconds) \ + M(OSCPUWaitMicroseconds) \ + M(OSCPUVirtualTimeMicroseconds) \ + M(OSReadBytes) \ + M(OSWriteBytes) \ + M(OSReadChars) \ + M(OSWriteChars) \ + namespace ProfileEvents { - #define M(NAME) extern const Event NAME = __COUNTER__; + +#define M(NAME) extern const Event NAME = __COUNTER__; + APPLY_FOR_EVENTS(M) +#undef M +constexpr Event END = __COUNTER__; + +/// Global variable, initialized by zeros. +Counter global_counters_array[END] {}; +/// Initialize global counters statically +Counters global_counters(global_counters_array); + +const Event Counters::num_counters = END; + + +Counters::Counters(VariableContext level, Counters * parent) + : counters_holder(new Counter[num_counters] {}), + parent(parent), + level(level) +{ + counters = counters_holder.get(); +} + +void Counters::resetCounters() +{ + if (counters) + { + for (Event i = 0; i < num_counters; ++i) + counters[i].store(0, std::memory_order_relaxed); + } +} + +void Counters::reset() +{ + parent = nullptr; + resetCounters(); +} + +Counters Counters::getPartiallyAtomicSnapshot() const +{ + Counters res(VariableContext::Snapshot, nullptr); + for (Event i = 0; i < num_counters; ++i) + res.counters[i].store(counters[i].load(std::memory_order_relaxed), std::memory_order_relaxed); + return res; +} + +const char * getDescription(Event event) +{ + static const char * descriptions[] = + { + #define M(NAME) #NAME, APPLY_FOR_EVENTS(M) #undef M - constexpr Event END = __COUNTER__; + }; - std::atomic counters[END] {}; /// Global variable, initialized by zeros. + return descriptions[event]; +} - const char * getDescription(Event event) - { - static const char * descriptions[] = - { - #define M(NAME) #NAME, - APPLY_FOR_EVENTS(M) - #undef M - }; - return descriptions[event]; - } +Event end() { return END; } + + +void increment(Event event, Count amount) +{ + DB::CurrentThread::getProfileEvents().increment(event, amount); +} - Event end() { return END; } } #undef APPLY_FOR_EVENTS diff --git a/dbms/src/Common/ProfileEvents.h b/dbms/src/Common/ProfileEvents.h index 0cb88f0ceed..38d8a9df7b9 100644 --- a/dbms/src/Common/ProfileEvents.h +++ b/dbms/src/Common/ProfileEvents.h @@ -1,8 +1,9 @@ #pragma once -#include +#include #include - +#include +#include /** Implements global counters for various events happening in the application * - for high level profiling. @@ -14,19 +15,80 @@ namespace ProfileEvents /// Event identifier (index in array). using Event = size_t; using Count = size_t; + using Counter = std::atomic; + class Counters; + + /// Counters - how many times each event happened + extern Counters global_counters; + + class Counters + { + Counter * counters = nullptr; + std::unique_ptr counters_holder; + /// Used to propagate increments + Counters * parent = nullptr; + + public: + + VariableContext level = VariableContext::Thread; + + /// By default, any instance have to increment global counters + Counters(VariableContext level = VariableContext::Thread, Counters * parent = &global_counters); + + /// Global level static initializer + Counters(Counter * allocated_counters) + : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {} + + Counter & operator[] (Event event) + { + return counters[event]; + } + + const Counter & operator[] (Event event) const + { + return counters[event]; + } + + inline void increment(Event event, Count amount = 1) + { + Counters * current = this; + do + { + current->counters[event].fetch_add(amount, std::memory_order_relaxed); + current = current->parent; + } while (current != nullptr); + } + + /// Every single value is fetched atomically, but not all values as a whole. + Counters getPartiallyAtomicSnapshot() const; + + /// Reset all counters to zero and reset parent. + void reset(); + + /// Get parent (thread unsafe) + Counters * getParent() + { + return parent; + } + + /// Set parent (thread unsafe) + void setParent(Counters * parent_) + { + parent = parent_; + } + + /// Set all counters to zero + void resetCounters(); + + static const Event num_counters; + }; + + /// Increment a counter for event. Thread-safe. + void increment(Event event, Count amount = 1); /// Get text description of event by identifier. Returns statically allocated string. const char * getDescription(Event event); - /// Counters - how many times each event happened. - extern std::atomic counters[]; - - /// Increment a counter for event. Thread-safe. - inline void increment(Event event, Count amount = 1) - { - counters[event].fetch_add(amount, std::memory_order_relaxed); - } - /// Get index just after last event identifier. Event end(); } diff --git a/dbms/src/Common/RWLockFIFO.cpp b/dbms/src/Common/RWLockFIFO.cpp index 51a2f756475..a1211a0bb9d 100644 --- a/dbms/src/Common/RWLockFIFO.cpp +++ b/dbms/src/Common/RWLockFIFO.cpp @@ -86,7 +86,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C handler_ptr->it_client->info += "; " + client.info; - return handler_ptr; + return handler_ptr; } if (type == Type::Write || queue.empty() || queue.back().type == Type::Write) diff --git a/dbms/src/Common/SimpleActionBlocker.h b/dbms/src/Common/SimpleActionBlocker.h new file mode 100644 index 00000000000..4a96db0e09d --- /dev/null +++ b/dbms/src/Common/SimpleActionBlocker.h @@ -0,0 +1,79 @@ +#pragma once +#include + + +namespace DB +{ + +class SimpleActionLock; + + +/// Similar to ActionBlocker, but without weak_ptr magic +class SimpleActionBlocker +{ + using Counter = std::atomic; + Counter counter = 0; + +public: + + SimpleActionBlocker() = default; + + bool isCancelled() const { return counter > 0; } + + /// Temporarily blocks corresponding actions (while the returned object is alive) + friend class SimpleActionLock; + inline SimpleActionLock cancel(); + + /// Cancel the actions forever. + void cancelForever() { ++counter; } +}; + + +/// Blocks related action while a SimpleActionLock instance exists +class SimpleActionLock +{ + SimpleActionBlocker * block = nullptr; + +public: + + SimpleActionLock() = default; + + explicit SimpleActionLock(SimpleActionBlocker & block_) : block(&block_) + { + ++block->counter; + } + + SimpleActionLock(const SimpleActionLock &) = delete; + + SimpleActionLock(SimpleActionLock && rhs) noexcept + { + *this = std::move(rhs); + } + + SimpleActionLock & operator=(const SimpleActionLock &) = delete; + + SimpleActionLock & operator=(SimpleActionLock && rhs) noexcept + { + if (block) + --block->counter; + + block = rhs.block; + rhs.block = nullptr; + + return *this; + } + + ~SimpleActionLock() + { + if (block) + --block->counter; + } +}; + + +SimpleActionLock SimpleActionBlocker::cancel() +{ + return SimpleActionLock(*this); +} + +} diff --git a/dbms/src/Common/StatusFile.cpp b/dbms/src/Common/StatusFile.cpp index 463d04b62e7..4da9ea48aa7 100644 --- a/dbms/src/Common/StatusFile.cpp +++ b/dbms/src/Common/StatusFile.cpp @@ -30,7 +30,7 @@ StatusFile::StatusFile(const std::string & path_) std::string contents; { ReadBufferFromFile in(path, 1024); - LimitReadBuffer limit_in(in, 1024); + LimitReadBuffer limit_in(in, 1024, false); readStringUntilEOF(contents, limit_in); } diff --git a/dbms/src/Common/Stopwatch.cpp b/dbms/src/Common/Stopwatch.cpp new file mode 100644 index 00000000000..b6d7092c054 --- /dev/null +++ b/dbms/src/Common/Stopwatch.cpp @@ -0,0 +1,14 @@ +#include +#include "Stopwatch.h" + +StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current() +{ + StopwatchRUsage::Timestamp res; + + ::rusage rusage; + ::getrusage(RUSAGE_THREAD, &rusage); + + res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL; + res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL; + return res; +} diff --git a/dbms/src/Common/Stopwatch.h b/dbms/src/Common/Stopwatch.h index 334e1574dde..c1ec623e100 100644 --- a/dbms/src/Common/Stopwatch.h +++ b/dbms/src/Common/Stopwatch.h @@ -32,9 +32,11 @@ public: void stop() { stop_ns = nanoseconds(); is_running = false; } void reset() { start_ns = 0; stop_ns = 0; is_running = false; } void restart() { start(); } - UInt64 elapsed() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } - UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; } - double elapsedSeconds() const { return static_cast(elapsed()) / 1000000000ULL; } + UInt64 elapsed() const { return elapsedNanoseconds(); } + UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } + UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; } + UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; } + double elapsedSeconds() const { return static_cast(elapsedNanoseconds()) / 1000000000ULL; } private: UInt64 start_ns = 0; @@ -131,3 +133,59 @@ private: /// Most significant bit is a lock. When it is set, compareAndRestartDeferred method will return false. UInt64 nanoseconds() const { return StopWatchDetail::nanoseconds(clock_type) & 0x7FFFFFFFFFFFFFFFULL; } }; + + +/// Like ordinary StopWatch, but uses getrusage() system call +struct StopwatchRUsage +{ + StopwatchRUsage() = default; + + void start() { start_ts = Timestamp::current(); is_running = true; } + void stop() { stop_ts = Timestamp::current(); is_running = false; } + void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; } + void restart() { start(); } + + UInt64 elapsed(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys); + } + + UInt64 elapsedNanoseconds(bool count_user = true, bool count_sys = true) const + { + return (is_running ? Timestamp::current() : stop_ts).nanoseconds(count_user, count_sys) - start_ts.nanoseconds(count_user, count_sys); + } + + UInt64 elapsedMicroseconds(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys) / 1000UL; + } + + UInt64 elapsedMilliseconds(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys) / 1000000UL; + } + + double elapsedSeconds(bool count_user = true, bool count_sys = true) const + { + return static_cast(elapsedNanoseconds(count_user, count_sys)) / 1000000000.0; + } + +private: + + struct Timestamp + { + UInt64 user_ns = 0; + UInt64 sys_ns = 0; + + static Timestamp current(); + + UInt64 nanoseconds(bool count_user = true, bool count_sys = true) const + { + return (count_user ? user_ns : 0) + (count_sys ? sys_ns : 0); + } + }; + + Timestamp start_ts; + Timestamp stop_ts; + bool is_running = false; +}; diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp new file mode 100644 index 00000000000..8f4d1c6c4ee --- /dev/null +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -0,0 +1,254 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// Basic idea is motivated by "iotop" tool. +/// More info: https://www.kernel.org/doc/Documentation/accounting/taskstats.txt + +#define GENLMSG_DATA(glh) ((void *)((char*)NLMSG_DATA(glh) + GENL_HDRLEN)) +#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN) +#define NLA_DATA(na) ((void *)((char*)(na) + NLA_HDRLEN)) +#define NLA_PAYLOAD(len) (len - NLA_HDRLEN) + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NETLINK_ERROR; +} + + +namespace +{ + +static size_t constexpr MAX_MSG_SIZE = 1024; + + +struct NetlinkMessage +{ + ::nlmsghdr n; + ::genlmsghdr g; + char buf[MAX_MSG_SIZE]; +}; + + +int sendCommand( + int sock_fd, + UInt16 nlmsg_type, + UInt32 nlmsg_pid, + UInt8 genl_cmd, + UInt16 nla_type, + void * nla_data, + int nla_len) noexcept +{ + NetlinkMessage msg{}; + + msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + msg.n.nlmsg_type = nlmsg_type; + msg.n.nlmsg_flags = NLM_F_REQUEST; + msg.n.nlmsg_seq = 0; + msg.n.nlmsg_pid = nlmsg_pid; + msg.g.cmd = genl_cmd; + msg.g.version = 1; + + ::nlattr * attr = static_cast<::nlattr *>(GENLMSG_DATA(&msg)); + attr->nla_type = nla_type; + attr->nla_len = nla_len + 1 + NLA_HDRLEN; + + memcpy(NLA_DATA(attr), nla_data, nla_len); + msg.n.nlmsg_len += NLMSG_ALIGN(attr->nla_len); + + char * buf = reinterpret_cast(&msg); + ssize_t buflen = msg.n.nlmsg_len; + + ::sockaddr_nl nladdr{}; + nladdr.nl_family = AF_NETLINK; + + while (true) + { + ssize_t r = ::sendto(sock_fd, buf, buflen, 0, reinterpret_cast(&nladdr), sizeof(nladdr)); + + if (r >= buflen) + break; + + if (r > 0) + { + buf += r; + buflen -= r; + } + else if (errno != EAGAIN) + return -1; + } + + return 0; +} + + +UInt16 getFamilyId(int nl_sock_fd) noexcept +{ + struct + { + ::nlmsghdr header; + ::genlmsghdr ge_header; + char buf[256]; + } answer; + + static char name[] = TASKSTATS_GENL_NAME; + + if (sendCommand( + nl_sock_fd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY, + CTRL_ATTR_FAMILY_NAME, (void *) name, + strlen(TASKSTATS_GENL_NAME) + 1)) + return 0; + + UInt16 id = 0; + ssize_t rep_len = ::recv(nl_sock_fd, &answer, sizeof(answer), 0); + if (answer.header.nlmsg_type == NLMSG_ERROR || (rep_len < 0) || !NLMSG_OK((&answer.header), rep_len)) + return 0; + + const ::nlattr * attr; + attr = static_cast(GENLMSG_DATA(&answer)); + attr = reinterpret_cast(reinterpret_cast(attr) + NLA_ALIGN(attr->nla_len)); + if (attr->nla_type == CTRL_ATTR_FAMILY_ID) + id = *static_cast(NLA_DATA(attr)); + + return id; +} + +} + + +TaskStatsInfoGetter::TaskStatsInfoGetter() = default; + +void TaskStatsInfoGetter::init() +{ + if (netlink_socket_fd >= 0) + return; + + netlink_socket_fd = ::socket(PF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + if (netlink_socket_fd < 0) + throwFromErrno("Can't create PF_NETLINK socket"); + + ::sockaddr_nl addr{}; + addr.nl_family = AF_NETLINK; + + if (::bind(netlink_socket_fd, reinterpret_cast(&addr), sizeof(addr)) < 0) + throwFromErrno("Can't bind PF_NETLINK socket"); + + netlink_family_id = getFamilyId(netlink_socket_fd); +} + +bool TaskStatsInfoGetter::getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error) +{ + init(); + + if (sendCommand(netlink_socket_fd, netlink_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(pid_t))) + throwFromErrno("Can't send a Netlink command"); + + NetlinkMessage msg; + ssize_t rv = ::recv(netlink_socket_fd, &msg, sizeof(msg), 0); + + if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), rv)) + { + const ::nlmsgerr * err = static_cast(NLMSG_DATA(&msg)); + if (throw_on_error) + throw Exception("Can't get Netlink response, error: " + std::to_string(err->error), ErrorCodes::NETLINK_ERROR); + else + return false; + } + + rv = GENLMSG_PAYLOAD(&msg.n); + + const ::nlattr * attr = static_cast(GENLMSG_DATA(&msg)); + ssize_t len = 0; + + while (len < rv) + { + len += NLA_ALIGN(attr->nla_len); + + if (attr->nla_type == TASKSTATS_TYPE_AGGR_TGID || attr->nla_type == TASKSTATS_TYPE_AGGR_PID) + { + int aggr_len = NLA_PAYLOAD(attr->nla_len); + int len2 = 0; + + attr = static_cast(NLA_DATA(attr)); + while (len2 < aggr_len) + { + if (attr->nla_type == TASKSTATS_TYPE_STATS) + { + const ::taskstats * ts = static_cast(NLA_DATA(attr)); + out_stats = *ts; + } + + len2 += NLA_ALIGN(attr->nla_len); + attr = reinterpret_cast(reinterpret_cast(attr) + len2); + } + } + + attr = reinterpret_cast(reinterpret_cast(GENLMSG_DATA(&msg)) + len); + } + + return true; +} + +void TaskStatsInfoGetter::getStat(::taskstats & stat, int tid) +{ + tid = tid < 0 ? getDefaultTID() : tid; + getStatImpl(tid, stat, true); +} + +bool TaskStatsInfoGetter::tryGetStat(::taskstats & stat, int tid) +{ + tid = tid < 0 ? getDefaultTID() : tid; + return getStatImpl(tid, stat, false); +} + +TaskStatsInfoGetter::~TaskStatsInfoGetter() +{ + if (netlink_socket_fd >= 0) + close(netlink_socket_fd); +} + +int TaskStatsInfoGetter::getCurrentTID() +{ + /// This call is always successful. - man gettid + return static_cast(syscall(SYS_gettid)); +} + +int TaskStatsInfoGetter::getDefaultTID() +{ + if (default_tid < 0) + default_tid = getCurrentTID(); + + return default_tid; +} + +static bool tryGetTaskStats() +{ + TaskStatsInfoGetter getter; + ::taskstats stat; + return getter.tryGetStat(stat); +} + +bool TaskStatsInfoGetter::checkProcessHasRequiredPermissions() +{ + /// It is thread- and exception- safe since C++11 + static bool res = tryGetTaskStats(); + return res; +} + +} diff --git a/dbms/src/Common/TaskStatsInfoGetter.h b/dbms/src/Common/TaskStatsInfoGetter.h new file mode 100644 index 00000000000..c89194cf88a --- /dev/null +++ b/dbms/src/Common/TaskStatsInfoGetter.h @@ -0,0 +1,43 @@ +#pragma once +#include + +struct taskstats; + + +namespace DB +{ + +class Exception; + + +/// Get taskstat info from OS kernel via Netlink protocol. +class TaskStatsInfoGetter +{ +public: + TaskStatsInfoGetter(); + TaskStatsInfoGetter(const TaskStatsInfoGetter &) = delete; + + void getStat(::taskstats & stat, int tid = -1); + bool tryGetStat(::taskstats & stat, int tid = -1); + + ~TaskStatsInfoGetter(); + + /// Make a syscall and returns Linux thread id + static int getCurrentTID(); + + /// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info + static bool checkProcessHasRequiredPermissions(); + +private: + /// Caches current thread tid to avoid extra sys calls + int getDefaultTID(); + int default_tid = -1; + + bool getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error = false); + void init(); + + int netlink_socket_fd = -1; + UInt16 netlink_family_id = 0; +}; + +} diff --git a/dbms/src/Common/ThreadProfileEvents.h b/dbms/src/Common/ThreadProfileEvents.h new file mode 100644 index 00000000000..3a780f509a7 --- /dev/null +++ b/dbms/src/Common/ThreadProfileEvents.h @@ -0,0 +1,144 @@ +#pragma once +#include +#include + +#include +#include +#include +#include + + +namespace ProfileEvents +{ + extern const Event RealTimeMicroseconds; + extern const Event UserTimeMicroseconds; + extern const Event SystemTimeMicroseconds; + extern const Event SoftPageFaults; + extern const Event HardPageFaults; + extern const Event VoluntaryContextSwitches; + extern const Event InvoluntaryContextSwitches; + + extern const Event OSIOWaitMicroseconds; + extern const Event OSCPUWaitMicroseconds; + extern const Event OSCPUVirtualTimeMicroseconds; + extern const Event OSReadChars; + extern const Event OSWriteChars; + extern const Event OSReadBytes; + extern const Event OSWriteBytes; +} + + +namespace DB +{ + +/// Handles overflow +template +inline TUInt safeDiff(TUInt prev, TUInt curr) +{ + return curr >= prev ? curr - prev : 0; +} + + +inline UInt64 getCurrentTimeNanoseconds(clockid_t clock_type = CLOCK_MONOTONIC) +{ + struct timespec ts; + clock_gettime(clock_type, &ts); + return ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + + +struct RUsageCounters +{ + /// In nanoseconds + UInt64 real_time = 0; + UInt64 user_time = 0; + UInt64 sys_time = 0; + + UInt64 soft_page_faults = 0; + UInt64 hard_page_faults = 0; + + RUsageCounters() = default; + RUsageCounters(const ::rusage & rusage_, UInt64 real_time_) + { + set(rusage_, real_time_); + } + + void set(const ::rusage & rusage, UInt64 real_time_) + { + real_time = real_time_; + user_time = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL; + sys_time = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL; + + soft_page_faults = static_cast(rusage.ru_minflt); + hard_page_faults = static_cast(rusage.ru_majflt); + } + + static RUsageCounters zeros(UInt64 real_time_ = getCurrentTimeNanoseconds()) + { + RUsageCounters res; + res.real_time = real_time_; + return res; + } + + static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds()) + { + ::rusage rusage; + ::getrusage(RUSAGE_THREAD, &rusage); + return RUsageCounters(rusage, real_time_); + } + + static void incrementProfileEvents(const RUsageCounters & prev, const RUsageCounters & curr, ProfileEvents::Counters & profile_events) + { + profile_events.increment(ProfileEvents::RealTimeMicroseconds, (curr.real_time - prev.real_time) / 1000U); + profile_events.increment(ProfileEvents::UserTimeMicroseconds, (curr.user_time - prev.user_time) / 1000U); + profile_events.increment(ProfileEvents::SystemTimeMicroseconds, (curr.sys_time - prev.sys_time) / 1000U); + + profile_events.increment(ProfileEvents::SoftPageFaults, curr.soft_page_faults - prev.soft_page_faults); + profile_events.increment(ProfileEvents::HardPageFaults, curr.hard_page_faults - prev.hard_page_faults); + } + + static void updateProfileEvents(RUsageCounters & last_counters, ProfileEvents::Counters & profile_events) + { + auto current_counters = current(); + incrementProfileEvents(last_counters, current_counters, profile_events); + last_counters = current_counters; + } +}; + + +struct TasksStatsCounters +{ + ::taskstats stat; + + TasksStatsCounters() = default; + + static TasksStatsCounters current(); + + static void incrementProfileEvents(const TasksStatsCounters & prev, const TasksStatsCounters & curr, ProfileEvents::Counters & profile_events) + { + profile_events.increment(ProfileEvents::OSCPUWaitMicroseconds, + safeDiff(prev.stat.cpu_delay_total, curr.stat.cpu_delay_total) / 1000U); + profile_events.increment(ProfileEvents::OSIOWaitMicroseconds, + safeDiff(prev.stat.blkio_delay_total, curr.stat.blkio_delay_total) / 1000U); + profile_events.increment(ProfileEvents::OSCPUVirtualTimeMicroseconds, + safeDiff(prev.stat.cpu_run_virtual_total, curr.stat.cpu_run_virtual_total) / 1000U); + + /// Too old struct version, do not read new fields + if (curr.stat.version < TASKSTATS_VERSION) + return; + + profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char)); + profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char)); + profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes)); + profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes)); + } + + static void updateProfileEvents(TasksStatsCounters & last_counters, ProfileEvents::Counters & profile_events) + { + auto current_counters = current(); + incrementProfileEvents(last_counters, current_counters, profile_events); + last_counters = current_counters; + } +}; + +} diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp new file mode 100644 index 00000000000..d4cca1b326c --- /dev/null +++ b/dbms/src/Common/ThreadStatus.cpp @@ -0,0 +1,118 @@ +#include "ThreadStatus.h" +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int PTHREAD_ERROR; +} + + +/// Order of current_thread and current_thread_scope matters +thread_local ThreadStatusPtr current_thread = ThreadStatus::create(); +thread_local ThreadStatus::CurrentThreadScope current_thread_scope; + + +TasksStatsCounters TasksStatsCounters::current() +{ + TasksStatsCounters res; + current_thread->taskstats_getter->getStat(res.stat, current_thread->os_thread_id); + return res; +} + + +ThreadStatus::ThreadStatus() +{ + thread_number = Poco::ThreadNumber::get(); + os_thread_id = TaskStatsInfoGetter::getCurrentTID(); + + last_rusage = std::make_unique(); + last_taskstats = std::make_unique(); + taskstats_getter = std::make_unique(); + + memory_tracker.setDescription("(for thread)"); + log = &Poco::Logger::get("ThreadStatus"); + + /// NOTE: It is important not to do any non-trivial actions (like updating ProfileEvents or logging) before ThreadStatus is created + /// Otherwise it could lead to SIGSEGV due to current_thread dereferencing +} + +ThreadStatusPtr ThreadStatus::create() +{ + return ThreadStatusPtr(new ThreadStatus); +} + +ThreadStatus::~ThreadStatus() = default; + +void ThreadStatus::initPerformanceCounters() +{ + performance_counters_finalized = false; + + /// Clear stats from previous query if a new query is started + /// TODO: make separate query_thread_performance_counters and thread_performance_counters + performance_counters.resetCounters(); + memory_tracker.resetCounters(); + memory_tracker.setDescription("(for thread)"); + + query_start_time_nanoseconds = getCurrentTimeNanoseconds(); + query_start_time = time(nullptr); + ++queries_started; + + *last_rusage = RUsageCounters::current(query_start_time_nanoseconds); + has_permissions_for_taskstats = TaskStatsInfoGetter::checkProcessHasRequiredPermissions(); + if (has_permissions_for_taskstats) + *last_taskstats = TasksStatsCounters::current(); +} + +void ThreadStatus::updatePerformanceCounters() +{ + try + { + RUsageCounters::updateProfileEvents(*last_rusage, performance_counters); + if (has_permissions_for_taskstats) + TasksStatsCounters::updateProfileEvents(*last_taskstats, performance_counters); + } + catch (...) + { + tryLogCurrentException(log); + } +} + +void ThreadStatus::assertState(const std::initializer_list & permitted_states, const char * description) +{ + for (auto permitted_state : permitted_states) + { + if (getCurrentState() == permitted_state) + return; + } + + std::stringstream ss; + ss << "Unexpected thread state " << getCurrentState(); + if (description) + ss << ": " << description; + throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR); +} + +void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue) +{ + logs_queue_ptr = logs_queue; + + if (!thread_group) + return; + + std::unique_lock lock(thread_group->mutex); + thread_group->logs_queue_ptr = logs_queue; +} + +} diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h new file mode 100644 index 00000000000..b708b3dce03 --- /dev/null +++ b/dbms/src/Common/ThreadStatus.h @@ -0,0 +1,197 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco +{ + class Logger; +} + + +namespace DB +{ + +class Context; +class QueryStatus; +class ThreadStatus; +using ThreadStatusPtr = std::shared_ptr; +class QueryThreadLog; +struct TasksStatsCounters; +struct RUsageCounters; +class TaskStatsInfoGetter; +class InternalTextLogsQueue; +using InternalTextLogsQueuePtr = std::shared_ptr; +using InternalTextLogsQueueWeakPtr = std::weak_ptr; + + +class ThreadGroupStatus +{ +public: + + mutable std::shared_mutex mutex; + + ProfileEvents::Counters performance_counters{VariableContext::Process}; + MemoryTracker memory_tracker{VariableContext::Process}; + + Context * query_context = nullptr; + Context * global_context = nullptr; + + InternalTextLogsQueueWeakPtr logs_queue_ptr; + + /// Key is Poco's thread_id + using QueryThreadStatuses = std::map; + QueryThreadStatuses thread_statuses; + + /// The first thread created this thread group + ThreadStatusPtr master_thread; + + String query; +}; + +using ThreadGroupStatusPtr = std::shared_ptr; + + +class ThreadStatus : public std::enable_shared_from_this +{ +public: + + /// Poco's thread number (the same number is used in logs) + UInt32 thread_number = 0; + /// Linux's PID (or TGID) (the same id is shown by ps util) + Int32 os_thread_id = -1; + + /// TODO: merge them into common entity + ProfileEvents::Counters performance_counters{VariableContext::Thread}; + MemoryTracker memory_tracker{VariableContext::Thread}; + + /// Statistics of read and write rows/bytes + Progress progress_in; + Progress progress_out; + +public: + + static ThreadStatusPtr create(); + + ThreadGroupStatusPtr getThreadGroup() const + { + return thread_group; + } + + enum ThreadState + { + DetachedFromQuery = 0, /// We just created thread or it is a background thread + AttachedToQuery, /// Thread executes enqueued query + Died, /// Thread does not exist + }; + + int getCurrentState() const + { + return thread_state.load(std::memory_order_relaxed); + } + + String getQueryID(); + + /// Starts new query and create new thread group for it, current thread becomes master thread of the query + void initializeQuery(); + + /// Attaches slave thread to existing thread group + void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true); + + InternalTextLogsQueuePtr getInternalTextLogsQueue() const + { + return thread_state == Died ? nullptr : logs_queue_ptr.lock(); + } + + void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue); + + /// Sets query context for current thread and its thread group + /// NOTE: query_context have to be alive until detachQuery() is called + void attachQueryContext(Context & query_context); + + /// Update several ProfileEvents counters + void updatePerformanceCounters(); + + /// Update ProfileEvents and dumps info to system.query_thread_log + void finalizePerformanceCounters(); + + /// Detaches thread from the thread group and the query, dumps performance counters if they have not been dumped + void detachQuery(bool exit_if_already_detached = false, bool thread_exits = false); + + ~ThreadStatus(); + +protected: + + ThreadStatus(); + + void initPerformanceCounters(); + + void logToQueryThreadLog(QueryThreadLog & thread_log); + + void assertState(const std::initializer_list & permitted_states, const char * description = nullptr); + + ThreadGroupStatusPtr thread_group; + + std::atomic thread_state{ThreadState::DetachedFromQuery}; + + /// Is set once + Context * global_context = nullptr; + /// Use it only from current thread + Context * query_context = nullptr; + + /// A logs queue used by TCPHandler to pass logs to a client + InternalTextLogsQueueWeakPtr logs_queue_ptr; + + bool performance_counters_finalized = false; + UInt64 query_start_time_nanoseconds = 0; + time_t query_start_time = 0; + size_t queries_started = 0; + + Poco::Logger * log = nullptr; + + friend class CurrentThread; + friend struct TasksStatsCounters; + + /// Use ptr not to add extra dependencies in the header + std::unique_ptr last_rusage; + std::unique_ptr last_taskstats; + std::unique_ptr taskstats_getter; + bool has_permissions_for_taskstats = false; + +public: + + /// Implicitly finalizes current thread in the destructor + class CurrentThreadScope + { + public: + void (*deleter)() = nullptr; + + CurrentThreadScope() = default; + ~CurrentThreadScope() + { + try + { + if (deleter) + deleter(); + } + catch (...) + { + std::terminate(); + } + } + }; + +private: + static void defaultThreadDeleter(); +}; + + +extern thread_local ThreadStatusPtr current_thread; +extern thread_local ThreadStatus::CurrentThreadScope current_thread_scope; + +} diff --git a/dbms/src/Common/Throttler.h b/dbms/src/Common/Throttler.h index abc87ffae50..9c7b446dd96 100644 --- a/dbms/src/Common/Throttler.h +++ b/dbms/src/Common/Throttler.h @@ -5,10 +5,17 @@ #include #include #include +#include #include #include +namespace ProfileEvents +{ + extern const Event ThrottlerSleepMicroseconds; +} + + namespace DB { @@ -69,10 +76,14 @@ public: if (desired_ns > elapsed_ns) { UInt64 sleep_ns = desired_ns - elapsed_ns; - timespec sleep_ts; + ::timespec sleep_ts; sleep_ts.tv_sec = sleep_ns / 1000000000; sleep_ts.tv_nsec = sleep_ns % 1000000000; - nanosleep(&sleep_ts, nullptr); /// NOTE Returns early in case of a signal. This is considered normal. + + /// NOTE: Returns early in case of a signal. This is considered normal. + ::nanosleep(&sleep_ts, nullptr); + + ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_ns / 1000UL); } } diff --git a/dbms/src/Common/UInt128.h b/dbms/src/Common/UInt128.h index 8bda6e2b204..9a4f4b570f6 100644 --- a/dbms/src/Common/UInt128.h +++ b/dbms/src/Common/UInt128.h @@ -28,9 +28,17 @@ struct UInt128 UInt64 high; UInt128() = default; - explicit UInt128(const UInt64 rhs) : low(rhs), high() {} explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) {} +#if 1 + explicit UInt128(const unsigned __int128 rhs) + : low(rhs & 0xffffffffffffffffll), + high(rhs >> 64) + {} +#else + explicit UInt128(const UInt64 rhs) : low(rhs), high() {} +#endif + auto tuple() const { return std::tie(high, low); } bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); } @@ -65,6 +73,7 @@ template bool inline operator< (T a, const UInt128 b) { return UIn template <> constexpr bool IsNumber = true; template <> struct TypeName { static const char * get() { return "UInt128"; } }; +template <> struct TypeId { static constexpr const size_t value = 5; }; struct UInt128Hash { diff --git a/dbms/src/Common/VariableContext.h b/dbms/src/Common/VariableContext.h new file mode 100644 index 00000000000..2fe4ffb565a --- /dev/null +++ b/dbms/src/Common/VariableContext.h @@ -0,0 +1,12 @@ +#pragma once + +/// Used in ProfileEvents and MemoryTracker to determine their hierarchy level +/// The less value the higher level (zero level is the root) +enum class VariableContext +{ + Global = 0, + User, /// Group of processes + Process, /// For example, a query or a merge + Thread, /// A thread of a process + Snapshot /// Does not belong to anybody +}; diff --git a/dbms/src/Common/ZooKeeper/KeeperException.h b/dbms/src/Common/ZooKeeper/KeeperException.h index f2c88ea82fd..5e1c4616757 100644 --- a/dbms/src/Common/ZooKeeper/KeeperException.h +++ b/dbms/src/Common/ZooKeeper/KeeperException.h @@ -7,27 +7,6 @@ namespace zkutil { -/// You should reinitialize ZooKeeper session in case of these errors -inline bool isHardwareError(int32_t zk_return_code) -{ - return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT; -} - -/// Valid errors sent from server -inline bool isUserError(int32_t zk_return_code) -{ - return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS - || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY; -} - - using KeeperException = ZooKeeperImpl::Exception; diff --git a/dbms/src/Common/ZooKeeper/LeaderElection.h b/dbms/src/Common/ZooKeeper/LeaderElection.h index 12adba37bff..4447c2ccfd2 100644 --- a/dbms/src/Common/ZooKeeper/LeaderElection.h +++ b/dbms/src/Common/ZooKeeper/LeaderElection.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace ProfileEvents diff --git a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp index 622e09c485c..d0702e0604d 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeper.cpp @@ -493,7 +493,7 @@ Responses ZooKeeper::multi(const Requests & requests) int32_t ZooKeeper::tryMulti(const Requests & requests, Responses & responses) { int32_t code = multiImpl(requests, responses); - if (code && !isUserError(code)) + if (code && !ZooKeeperImpl::ZooKeeper::isUserError(code)) throw KeeperException(code); return code; } @@ -824,7 +824,7 @@ size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Responses & re if (responses[index]->error) return index; - if (!isUserError(code)) + if (!ZooKeeperImpl::ZooKeeper::isUserError(code)) throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(code) + "' is not valid response code for that", DB::ErrorCodes::LOGICAL_ERROR); @@ -850,7 +850,7 @@ void KeeperMultiException::check(int32_t code, const Requests & requests, const if (!code) return; - if (isUserError(code)) + if (ZooKeeperImpl::ZooKeeper::isUserError(code)) throw KeeperMultiException(code, requests, responses); else throw KeeperException(code); diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 8b80e74c96d..3a0978f415e 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -25,7 +25,9 @@ namespace DB namespace ProfileEvents { - extern const Event ZooKeeperExceptions; + extern const Event ZooKeeperUserExceptions; + extern const Event ZooKeeperHardwareExceptions; + extern const Event ZooKeeperOtherExceptions; extern const Event ZooKeeperInit; extern const Event ZooKeeperTransactions; extern const Event ZooKeeperCreate; @@ -267,7 +269,12 @@ namespace ZooKeeperImpl Exception::Exception(const std::string & msg, const int32_t code, int) : DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code) { - ProfileEvents::increment(ProfileEvents::ZooKeeperExceptions); + if (ZooKeeper::isUserError(code)) + ProfileEvents::increment(ProfileEvents::ZooKeeperUserExceptions); + else if (ZooKeeper::isHardwareError(code)) + ProfileEvents::increment(ProfileEvents::ZooKeeperHardwareExceptions); + else + ProfileEvents::increment(ProfileEvents::ZooKeeperOtherExceptions); } Exception::Exception(const std::string & msg, const int32_t code) @@ -515,6 +522,25 @@ const char * ZooKeeper::errorMessage(int32_t code) return "unknown error"; } +bool ZooKeeper::isHardwareError(int32_t zk_return_code) +{ + return zk_return_code == ZooKeeperImpl::ZooKeeper::ZINVALIDSTATE + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZSESSIONMOVED + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZCONNECTIONLOSS + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZMARSHALLINGERROR + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZOPERATIONTIMEOUT; +} + +bool ZooKeeper::isUserError(int32_t zk_return_code) +{ + return zk_return_code == ZooKeeperImpl::ZooKeeper::ZNONODE + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZBADVERSION + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOCHILDRENFOREPHEMERALS + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNODEEXISTS + || zk_return_code == ZooKeeperImpl::ZooKeeper::ZNOTEMPTY; +} + ZooKeeper::~ZooKeeper() { diff --git a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h index 5c027753371..8b6f57a8741 100644 --- a/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/dbms/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -549,6 +549,12 @@ public: ZSESSIONMOVED = -118 /// Session moved to another server, so operation is ignored }; + /// Network errors and similar. You should reinitialize ZooKeeper session in case of these errors + static bool isHardwareError(int32_t code); + + /// Valid errors sent from the server about database state (like "no node"). Logical and authentication errors (like "bad arguments") are not here. + static bool isUserError(int32_t code); + static const char * errorMessage(int32_t code); /// For watches. diff --git a/dbms/src/Common/setThreadName.cpp b/dbms/src/Common/setThreadName.cpp index 1a33c330027..d92b2e84715 100644 --- a/dbms/src/Common/setThreadName.cpp +++ b/dbms/src/Common/setThreadName.cpp @@ -6,10 +6,20 @@ #else #include #endif +#include +#include #include #include +namespace DB +{ +namespace ErrorCodes +{ + extern const int PTHREAD_ERROR; +} +} + void setThreadName(const char * name) { @@ -22,5 +32,21 @@ void setThreadName(const char * name) #else if (0 != prctl(PR_SET_NAME, name, 0, 0, 0)) #endif - DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME...)"); + DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)"); +} + +std::string getThreadName() +{ + std::string name(16, '\0'); + +#if defined(__FreeBSD__) || defined(__APPLE__) + if (pthread_get_name_np(pthread_self(), name.data(), name.size()); + throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR); +#else + if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0)) +#endif + DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)"); + + name.resize(std::strlen(name.data())); + return name; } diff --git a/dbms/src/Common/setThreadName.h b/dbms/src/Common/setThreadName.h index dc6af7336e0..cdcb6b46914 100644 --- a/dbms/src/Common/setThreadName.h +++ b/dbms/src/Common/setThreadName.h @@ -1,7 +1,10 @@ #pragma once +#include /** Sets the thread name (maximum length is 15 bytes), * which will be visible in ps, gdb, /proc, * for convenience of observation and debugging. */ void setThreadName(const char * name); + +std::string getThreadName(); diff --git a/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp b/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp index 9d1eda9178c..15b01367cac 100644 --- a/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp +++ b/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp @@ -32,7 +32,7 @@ TEST(Common, RWLockFIFO_1) auto func = [&] (size_t threads, int round) { - for (int i = 0; i < cycles; ++i) + for (int i = 0; i < cycles; ++i) { auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockFIFO::Read : RWLockFIFO::Write; auto sleep_for = std::chrono::duration(std::uniform_int_distribution<>(1, 100)(gen)); diff --git a/dbms/src/Core/AccurateComparison.h b/dbms/src/Core/AccurateComparison.h index c84405e2dd4..23876d8e306 100644 --- a/dbms/src/Core/AccurateComparison.h +++ b/dbms/src/Core/AccurateComparison.h @@ -139,7 +139,6 @@ inline bool_if_double_can_be_used equalsOpTmpl(TAFloat a, TAInt return static_cast(a) == static_cast(b); } - /* Final realiztions */ @@ -333,6 +332,37 @@ inline bool equalsOp(DB::Float32 f, DB::UInt128 u) return equalsOp(static_cast(f), u); } +inline bool greaterOp(DB::Int128 i, DB::Float64 f) +{ + static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; + static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return static_cast(i) > f; + + return (f < static_cast(min_int128)) + || (f < static_cast(max_int128) && i > static_cast(f)); +} + +inline bool greaterOp(DB::Float64 f, DB::Int128 i) +{ + static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; + static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return f > static_cast(i); + + return (f >= static_cast(max_int128)) + || (f > static_cast(min_int128) && static_cast(f) > i); +} + +inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return greaterOp(i, static_cast(f)); } +inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return greaterOp(static_cast(f), i); } + +inline bool equalsOp(DB::Int128 i, DB::Float64 f) { return i == static_cast(f) && static_cast(i) == f; } +inline bool equalsOp(DB::Int128 i, DB::Float32 f) { return i == static_cast(f) && static_cast(i) == f; } +inline bool equalsOp(DB::Float64 f, DB::Int128 i) { return equalsOp(i, f); } +inline bool equalsOp(DB::Float32 f, DB::Int128 i) { return equalsOp(i, f); } template inline bool_if_not_safe_conversion notEqualsOp(A a, B b) diff --git a/dbms/src/Common/BackgroundSchedulePool.cpp b/dbms/src/Core/BackgroundSchedulePool.cpp similarity index 90% rename from dbms/src/Common/BackgroundSchedulePool.cpp rename to dbms/src/Core/BackgroundSchedulePool.cpp index 9556c9a037b..9cdec4087a4 100644 --- a/dbms/src/Common/BackgroundSchedulePool.cpp +++ b/dbms/src/Core/BackgroundSchedulePool.cpp @@ -1,11 +1,14 @@ -#include +#include "BackgroundSchedulePool.h" #include #include #include #include #include +#include #include #include +#include + namespace CurrentMetrics { @@ -140,6 +143,12 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size) { LOG_INFO(&Logger::get("BackgroundSchedulePool"), "Create BackgroundSchedulePool with " << size << " threads"); + /// Put all threads of both thread pools to one thread group + /// The master thread exits immediately + CurrentThread::initializeQuery(); + thread_group = CurrentThread::getGroup(); + CurrentThread::detachQuery(); + threads.resize(size); for (auto & thread : threads) thread = std::thread([this] { threadFunction(); }); @@ -212,9 +221,11 @@ void BackgroundSchedulePool::threadFunction() { setThreadName("BackgrSchedPool"); - MemoryTracker memory_tracker; - memory_tracker.setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); - current_memory_tracker = &memory_tracker; + /// Put all threads to one thread pool + CurrentThread::attachTo(thread_group); + SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); + + CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); while (!shutdown) { @@ -224,8 +235,6 @@ void BackgroundSchedulePool::threadFunction() task_notification.execute(); } } - - current_memory_tracker = nullptr; } @@ -233,6 +242,10 @@ void BackgroundSchedulePool::delayExecutionThreadFunction() { setThreadName("BckSchPoolDelay"); + /// Put all threads to one thread pool + CurrentThread::attachTo(thread_group); + SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); + while (!shutdown) { TaskInfoPtr task; diff --git a/dbms/src/Common/BackgroundSchedulePool.h b/dbms/src/Core/BackgroundSchedulePool.h similarity index 97% rename from dbms/src/Common/BackgroundSchedulePool.h rename to dbms/src/Core/BackgroundSchedulePool.h index 0ebcc207b2c..f55cc95dbbc 100644 --- a/dbms/src/Common/BackgroundSchedulePool.h +++ b/dbms/src/Core/BackgroundSchedulePool.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB { @@ -138,6 +139,9 @@ private: std::thread delayed_thread; /// Tasks ordered by scheduled time. DelayedTasks delayed_tasks; + + /// Thread group used for profiling purposes + ThreadGroupStatusPtr thread_group; }; using BackgroundSchedulePoolPtr = std::shared_ptr; diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 331e226103d..d7a2ca419e3 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -46,6 +46,7 @@ #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 #define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401 +#define DBMS_MIN_REVISION_WITH_SERVER_LOGS 54406 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp new file mode 100644 index 00000000000..3d8d8077e97 --- /dev/null +++ b/dbms/src/Core/ExternalTable.cpp @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + + +ExternalTableData BaseExternalTable::getData(const Context & context) +{ + initReadBuffer(); + initSampleBlock(); + auto input = context.getInputFormat(format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE); + return std::make_pair(std::make_shared(input), name); +} + +void BaseExternalTable::clean() +{ + name = ""; + file = ""; + format = ""; + structure.clear(); + sample_block = Block(); + read_buffer.reset(); +} + +/// Function for debugging information output +void BaseExternalTable::write() +{ + std::cerr << "file " << file << std::endl; + std::cerr << "name " << name << std::endl; + std::cerr << "format " << format << std::endl; + std::cerr << "structure: \n"; + for (size_t i = 0; i < structure.size(); ++i) + std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl; +} + +std::vector BaseExternalTable::split(const std::string & s, const std::string & d) +{ + std::vector res; + boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on); + return res; +} + +void BaseExternalTable::parseStructureFromStructureField(const std::string & argument) +{ + std::vector vals = split(argument, " ,"); + + if (vals.size() & 1) + throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS); + + for (size_t i = 0; i < vals.size(); i += 2) + structure.emplace_back(vals[i], vals[i + 1]); +} + +void BaseExternalTable::parseStructureFromTypesField(const std::string & argument) +{ + std::vector vals = split(argument, " ,"); + + for (size_t i = 0; i < vals.size(); ++i) + structure.emplace_back("_" + toString(i + 1), vals[i]); +} + +void BaseExternalTable::initSampleBlock() +{ + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + + for (size_t i = 0; i < structure.size(); ++i) + { + ColumnWithTypeAndName column; + column.name = structure[i].first; + column.type = data_type_factory.get(structure[i].second); + column.column = column.type->createColumn(); + sample_block.insert(std::move(column)); + } +} + + +void ExternalTable::initReadBuffer() +{ + if (file == "-") + read_buffer = std::make_unique(STDIN_FILENO); + else + read_buffer = std::make_unique(file); +} + +ExternalTable::ExternalTable(const boost::program_options::variables_map & external_options) +{ + if (external_options.count("file")) + file = external_options["file"].as(); + else + throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("name")) + name = external_options["name"].as(); + else + throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("format")) + format = external_options["format"].as(); + else + throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("structure")) + parseStructureFromStructureField(external_options["structure"].as()); + else if (external_options.count("types")) + parseStructureFromTypesField(external_options["types"].as()); + else + throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); +} + + +void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) +{ + const Settings & settings = context.getSettingsRef(); + + /// The buffer is initialized here, not in the virtual function initReadBuffer + read_buffer_impl = std::make_unique(stream); + + if (settings.http_max_multipart_form_data_size) + read_buffer = std::make_unique( + *read_buffer_impl, settings.http_max_multipart_form_data_size, + true, "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting"); + else + read_buffer = std::move(read_buffer_impl); + + /// Retrieve a collection of parameters from MessageHeader + Poco::Net::NameValueCollection content; + std::string label; + Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content); + + /// Get parameters + name = content.get("name", "_data"); + format = params.get(name + "_format", "TabSeparated"); + + if (params.has(name + "_structure")) + parseStructureFromStructureField(params.get(name + "_structure")); + else if (params.has(name + "_types")) + parseStructureFromTypesField(params.get(name + "_types")); + else + throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS); + + ExternalTableData data = getData(context); + + /// Create table + NamesAndTypesList columns = sample_block.getNamesAndTypesList(); + StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); + storage->startup(); + context.addExternalTable(data.second, storage); + BlockOutputStreamPtr output = storage->write(ASTPtr(), settings); + + /// Write data + data.first->readPrefix(); + output->writePrefix(); + while(Block block = data.first->read()) + output->write(block); + data.first->readSuffix(); + output->writeSuffix(); + + /// We are ready to receive the next file, for this we clear all the information received + clean(); +} + +} diff --git a/dbms/src/Core/ExternalTable.h b/dbms/src/Core/ExternalTable.h new file mode 100644 index 00000000000..6927f19b524 --- /dev/null +++ b/dbms/src/Core/ExternalTable.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + + +namespace Poco +{ + namespace Net + { + class NameValueCollection; + class MessageHeader; + } +} + +namespace boost +{ + namespace program_options + { + class variables_map; + } +} + + +namespace DB +{ + +class Context; + + +/// The base class containing the basic information about external table and +/// basic functions for extracting this information from text fields. +class BaseExternalTable +{ +public: + std::string file; /// File with data or '-' if stdin + std::string name; /// The name of the table + std::string format; /// Name of the data storage format + + /// Description of the table structure: (column name, data type name) + std::vector> structure; + + std::unique_ptr read_buffer; + Block sample_block; + + virtual ~BaseExternalTable() {} + + /// Initialize read_buffer, depending on the data source. By default, does nothing. + virtual void initReadBuffer() {} + + /// Get the table data - a pair (a stream with the contents of the table, the name of the table) + ExternalTableData getData(const Context & context); + +protected: + /// Clear all accumulated information + void clean(); + + /// Function for debugging information output + void write(); + + static std::vector split(const std::string & s, const std::string & d); + + /// Construct the `structure` vector from the text field `structure` + virtual void parseStructureFromStructureField(const std::string & argument); + + /// Construct the `structure` vector from the text field `types` + virtual void parseStructureFromTypesField(const std::string & argument); + +private: + /// Initialize sample_block according to the structure of the table stored in the `structure` + void initSampleBlock(); +}; + + +/// Parsing of external table used in the tcp client. +class ExternalTable : public BaseExternalTable +{ +public: + void initReadBuffer() override; + + /// Extract parameters from variables_map, which is built on the client command line + ExternalTable(const boost::program_options::variables_map & external_options); +}; + + +/// Parsing of external table used when sending tables via http +/// The `handlePart` function will be called for each table passed, +/// so it's also necessary to call `clean` at the end of the `handlePart`. +class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable +{ +public: + ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {} + + void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream); + +private: + Context & context; + const Poco::Net::NameValueCollection & params; + std::unique_ptr read_buffer_impl; +}; + + +} diff --git a/dbms/src/Core/Field.cpp b/dbms/src/Core/Field.cpp index 1e14689d856..279dd90c52b 100644 --- a/dbms/src/Core/Field.cpp +++ b/dbms/src/Core/Field.cpp @@ -5,6 +5,7 @@ #include #include +#include namespace DB @@ -270,4 +271,23 @@ namespace DB DB::String res = applyVisitor(DB::FieldVisitorToString(), DB::Field(x)); buf.write(res.data(), res.size()); } + + + bool DecimalField::operator < (const DecimalField & r) const + { + using Comparator = DecimalComparison; + return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale); + } + + bool DecimalField::operator <= (const DecimalField & r) const + { + using Comparator = DecimalComparison; + return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale); + } + + bool DecimalField::operator == (const DecimalField & r) const + { + using Comparator = DecimalComparison; + return Comparator::compare(Decimal128(dec), Decimal128(r.dec), scale, r.scale); + } } diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index c8811fd0f29..a77b22a3da4 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -28,6 +28,35 @@ using TupleBackend = std::vector; STRONG_TYPEDEF(TupleBackend, Tuple) /// Array and Tuple are different types with equal representation inside Field. +class DecimalField +{ +public: + static constexpr UInt32 wrongScale() { return std::numeric_limits::max(); } + + DecimalField(Int128 value, UInt32 scale_ = wrongScale()) + : dec(value), + scale(scale_) + {} + + operator Decimal32() const { return dec; } + operator Decimal64() const { return dec; } + operator Decimal128() const { return dec; } + + UInt32 getScale() const { return scale; } + + bool operator < (const DecimalField & r) const; + bool operator <= (const DecimalField & r) const; + bool operator == (const DecimalField & r) const; + + bool operator > (const DecimalField & r) const { return r < *this; } + bool operator >= (const DecimalField & r) const { return r <= * this; } + bool operator != (const DecimalField & r) const { return !(*this == r); } + +private: + Int128 dec; + UInt32 scale; +}; + /** 32 is enough. Round number is used for alignment and for better arithmetic inside std::vector. * NOTE: Actually, sizeof(std::string) is 32 when using libc++, so Field is 40 bytes. */ @@ -55,12 +84,14 @@ public: Int64 = 2, Float64 = 3, UInt128 = 4, + Int128 = 5, /// Non-POD types. String = 16, Array = 17, Tuple = 18, + Decimal = 19, }; static const int MIN_NON_POD = 16; @@ -73,14 +104,15 @@ public: case UInt64: return "UInt64"; case UInt128: return "UInt128"; case Int64: return "Int64"; + case Int128: return "Int128"; case Float64: return "Float64"; case String: return "String"; case Array: return "Array"; case Tuple: return "Tuple"; - - default: - throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); + case Decimal: return "Decimal"; } + + throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); } }; @@ -257,14 +289,15 @@ public: case Types::UInt64: return get() < rhs.get(); case Types::UInt128: return get() < rhs.get(); case Types::Int64: return get() < rhs.get(); + case Types::Int128: return get() < rhs.get(); case Types::Float64: return get() < rhs.get(); case Types::String: return get() < rhs.get(); case Types::Array: return get() < rhs.get(); case Types::Tuple: return get() < rhs.get(); - - default: - throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); + case Types::Decimal: return get() < rhs.get(); } + + throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); } bool operator> (const Field & rhs) const @@ -285,15 +318,15 @@ public: case Types::UInt64: return get() <= rhs.get(); case Types::UInt128: return get() <= rhs.get(); case Types::Int64: return get() <= rhs.get(); + case Types::Int128: return get() <= rhs.get(); case Types::Float64: return get() <= rhs.get(); case Types::String: return get() <= rhs.get(); case Types::Array: return get() <= rhs.get(); case Types::Tuple: return get() <= rhs.get(); - - - default: - throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); + case Types::Decimal: return get() <= rhs.get(); } + + throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); } bool operator>= (const Field & rhs) const @@ -316,10 +349,11 @@ public: case Types::Array: return get() == rhs.get(); case Types::Tuple: return get() == rhs.get(); case Types::UInt128: return get() == rhs.get(); - - default: - throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); + case Types::Int128: return get() == rhs.get(); + case Types::Decimal: return get() == rhs.get(); } + + throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); } bool operator!= (const Field & rhs) const @@ -329,7 +363,7 @@ public: private: std::aligned_union_t storage; Types::Which which; @@ -370,6 +404,7 @@ private: case Types::UInt64: f(field.template get()); return; case Types::UInt128: f(field.template get()); return; case Types::Int64: f(field.template get()); return; + case Types::Int128: f(field.template get()); return; case Types::Float64: f(field.template get()); return; #if !__clang__ #pragma GCC diagnostic pop @@ -377,6 +412,7 @@ private: case Types::String: f(field.template get()); return; case Types::Array: f(field.template get()); return; case Types::Tuple: f(field.template get()); return; + case Types::Decimal: f(field.template get()); return; default: throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD); @@ -455,19 +491,23 @@ template <> struct Field::TypeToEnum { static const Types::Which value template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt64; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::UInt128; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int64; }; +template <> struct Field::TypeToEnum { static const Types::Which value = Types::Int128; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Float64; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::String; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Array; }; template <> struct Field::TypeToEnum { static const Types::Which value = Types::Tuple; }; +template <> struct Field::TypeToEnum{ static const Types::Which value = Types::Decimal; }; template <> struct Field::EnumToType { using Type = Null; }; template <> struct Field::EnumToType { using Type = UInt64; }; template <> struct Field::EnumToType { using Type = UInt128; }; template <> struct Field::EnumToType { using Type = Int64; }; +template <> struct Field::EnumToType { using Type = Int128; }; template <> struct Field::EnumToType { using Type = Float64; }; template <> struct Field::EnumToType { using Type = String; }; template <> struct Field::EnumToType { using Type = Array; }; template <> struct Field::EnumToType { using Type = Tuple; }; +template <> struct Field::EnumToType { using Type = DecimalField; }; template @@ -510,6 +550,10 @@ template <> struct NearestFieldType { using Type = Int64; }; template <> struct NearestFieldType { using Type = Int64; }; template <> struct NearestFieldType { using Type = Int64; }; template <> struct NearestFieldType { using Type = Int64; }; +template <> struct NearestFieldType { using Type = Int128; }; +template <> struct NearestFieldType { using Type = DecimalField; }; +template <> struct NearestFieldType { using Type = DecimalField; }; +template <> struct NearestFieldType { using Type = DecimalField; }; template <> struct NearestFieldType { using Type = Float64; }; template <> struct NearestFieldType { using Type = Float64; }; template <> struct NearestFieldType { using Type = String; }; diff --git a/dbms/src/Core/Protocol.h b/dbms/src/Core/Protocol.h index cd5456cca34..5451e1550f6 100644 --- a/dbms/src/Core/Protocol.h +++ b/dbms/src/Core/Protocol.h @@ -69,6 +69,7 @@ namespace Protocol Totals = 7, /// A block with totals (compressed or not). Extremes = 8, /// A block with minimums and maximums (compressed or not). TablesStatusResponse = 9, /// A response to TablesStatus request. + Log = 10 /// System logs of the query execution }; /// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10 @@ -77,8 +78,8 @@ namespace Protocol /// See https://www.securecoding.cert.org/confluence/display/cplusplus/INT36-CPP.+Do+not+use+out-of-range+enumeration+values inline const char * toString(UInt64 packet) { - static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse" }; - return packet < 10 + static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse", "Log" }; + return packet < 11 ? data[packet] : "Unknown packet"; } @@ -97,6 +98,7 @@ namespace Protocol Cancel = 3, /// Cancel the query execution. Ping = 4, /// Check that connection to the server is alive. TablesStatusRequest = 5, /// Check status of tables on the server. + KeepAlive = 6 /// Keep the connection alive }; inline const char * toString(UInt64 packet) diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index 1f09dcba17a..d967d4f52a3 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB @@ -12,15 +12,15 @@ namespace DB struct Null {}; -using UInt8 = Poco::UInt8; -using UInt16 = Poco::UInt16; -using UInt32 = Poco::UInt32; -using UInt64 = Poco::UInt64; +using UInt8 = uint8_t; +using UInt16 = uint16_t; +using UInt32 = uint32_t; +using UInt64 = uint64_t; -using Int8 = Poco::Int8; -using Int16 = Poco::Int16; -using Int32 = Poco::Int32; -using Int64 = Poco::Int64; +using Int8 = int8_t; +using Int16 = int16_t; +using Int32 = int32_t; +using Int64 = int64_t; using Float32 = float; using Float64 = double; @@ -57,8 +57,116 @@ template <> struct TypeName { static const char * get() { return "Float template <> struct TypeName { static const char * get() { return "Float64"; } }; template <> struct TypeName { static const char * get() { return "String"; } }; +template struct TypeId; + +/// 0 reserved for types without number +template <> struct TypeId { static constexpr const size_t value = 1; }; +template <> struct TypeId { static constexpr const size_t value = 2; }; +template <> struct TypeId { static constexpr const size_t value = 3; }; +template <> struct TypeId { static constexpr const size_t value = 4; }; +/// 5 reserved for TypeId +template <> struct TypeId { static constexpr const size_t value = 7; }; +template <> struct TypeId { static constexpr const size_t value = 8; }; +template <> struct TypeId { static constexpr const size_t value = 9; }; +template <> struct TypeId { static constexpr const size_t value = 10; }; +template <> struct TypeId { static constexpr const size_t value = 11; }; +template <> struct TypeId { static constexpr const size_t value = 12; }; +/// 13 reserved for TypeId /// Not a data type in database, defined just for convenience. using Strings = std::vector; } + +#if 1 /// __int128 +namespace DB +{ + +using Int128 = __int128; +template <> constexpr bool IsNumber = true; +template <> struct TypeName { static const char * get() { return "Int128"; } }; +template <> struct TypeId { static constexpr const size_t value = 13; }; + +} + +namespace std +{ + +template <> struct is_signed<__int128> +{ + static constexpr bool value = true; +}; + +template <> struct is_unsigned<__int128> +{ + static constexpr bool value = false; +}; + +template <> struct is_integral<__int128> +{ + static constexpr bool value = true; +}; + +template <> struct is_arithmetic<__int128> +{ + static constexpr bool value = true; +}; + +} +#endif + + +namespace DB +{ + /// Own FieldType for Decimal. + /// It is only a "storage" for decimal. To perform operations, you also have to provide a scale (number of digits after point). + template + struct Decimal + { + using NativeType = T; + + Decimal() = default; + Decimal(Decimal &&) = default; + Decimal(const Decimal &) = default; + + Decimal(const T & value_) + : value(value_) + {} + + template + Decimal(const Decimal & x) + : value(x) + {} + + constexpr Decimal & operator = (Decimal &&) = default; + constexpr Decimal & operator = (const Decimal &) = default; + + operator T () const { return value; } + + const Decimal & operator += (const T & x) { value += x; return *this; } + const Decimal & operator -= (const T & x) { value -= x; return *this; } + const Decimal & operator *= (const T & x) { value *= x; return *this; } + const Decimal & operator /= (const T & x) { value /= x; return *this; } + const Decimal & operator %= (const T & x) { value %= x; return *this; } + + T value; + }; + + using Decimal32 = Decimal; + using Decimal64 = Decimal; + using Decimal128 = Decimal; + + template <> struct TypeName { static const char * get() { return "Decimal32"; } }; + template <> struct TypeName { static const char * get() { return "Decimal64"; } }; + template <> struct TypeName { static const char * get() { return "Decimal128"; } }; + + template <> struct TypeId { static constexpr const size_t value = 16; }; + template <> struct TypeId { static constexpr const size_t value = 17; }; + template <> struct TypeId { static constexpr const size_t value = 18; }; + + template + constexpr bool IsDecimalNumber = false; + template <> constexpr bool IsDecimalNumber = true; + template <> constexpr bool IsDecimalNumber = true; + template <> constexpr bool IsDecimalNumber = true; +} diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp b/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp new file mode 100644 index 00000000000..ba31b45bfd2 --- /dev/null +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp @@ -0,0 +1,84 @@ +#include "AsynchronousBlockInputStream.h" +#include + + +namespace DB +{ + +Block AsynchronousBlockInputStream::readImpl() +{ + /// If there were no calculations yet, calculate the first block synchronously + if (!started) + { + calculate(); + started = true; + } + else /// If the calculations are already in progress - wait for the result + pool.wait(); + + if (exception) + std::rethrow_exception(exception); + + Block res = block; + if (!res) + return res; + + /// Start the next block calculation + block.clear(); + next(); + + return res; +} + + +void AsynchronousBlockInputStream::next() +{ + ready.reset(); + + pool.schedule([this, thread_group=CurrentThread::getGroup()] () + { + CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; + + try + { + if (first) + setThreadName("AsyncBlockInput"); + + /// AsynchronousBlockInputStream is used in Client which does not create queries and thread groups + if (thread_group) + CurrentThread::attachToIfDetached(thread_group); + } + catch (...) + { + exception = std::current_exception(); + ready.set(); + return; + } + + calculate(); + }); +} + + +void AsynchronousBlockInputStream::calculate() +{ + try + { + if (first) + { + first = false; + children.back()->readPrefix(); + } + + block = children.back()->read(); + } + catch (...) + { + exception = std::current_exception(); + } + + ready.set(); +} + +} + diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.h b/dbms/src/DataStreams/AsynchronousBlockInputStream.h index 0a80628cf2a..c790deb49c2 100644 --- a/dbms/src/DataStreams/AsynchronousBlockInputStream.h +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -91,64 +92,12 @@ protected: Block block; std::exception_ptr exception; + Block readImpl() override; - Block readImpl() override - { - /// If there were no calculations yet, calculate the first block synchronously - if (!started) - { - calculate(current_memory_tracker); - started = true; - } - else /// If the calculations are already in progress - wait for the result - pool.wait(); - - if (exception) - std::rethrow_exception(exception); - - Block res = block; - if (!res) - return res; - - /// Start the next block calculation - block.clear(); - next(); - - return res; - } - - - void next() - { - ready.reset(); - pool.schedule(std::bind(&AsynchronousBlockInputStream::calculate, this, current_memory_tracker)); - } - + void next(); /// Calculations that can be performed in a separate thread - void calculate(MemoryTracker * memory_tracker) - { - CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; - - try - { - if (first) - { - first = false; - setThreadName("AsyncBlockInput"); - current_memory_tracker = memory_tracker; - children.back()->readPrefix(); - } - - block = children.back()->read(); - } - catch (...) - { - exception = std::current_exception(); - } - - ready.set(); - } + void calculate(); }; } diff --git a/dbms/src/DataStreams/BlockIO.h b/dbms/src/DataStreams/BlockIO.h index 6d97e30e510..9f69f834e5f 100644 --- a/dbms/src/DataStreams/BlockIO.h +++ b/dbms/src/DataStreams/BlockIO.h @@ -12,8 +12,7 @@ class ProcessListEntry; struct BlockIO { /** process_list_entry should be destroyed after in and after out, - * since in and out contain pointer to an object inside process_list_entry - * (MemoryTracker * current_memory_tracker), + * since in and out contain pointer to objects inside process_list_entry (query-level MemoryTracker for example), * which could be used before destroying of in and out. */ std::shared_ptr process_list_entry; @@ -38,12 +37,17 @@ struct BlockIO exception_callback(); } + /// We provide the correct order of destruction. + void reset() + { + out.reset(); + in.reset(); + process_list_entry.reset(); + } + BlockIO & operator= (const BlockIO & rhs) { - /// We provide the correct order of destruction. - out = nullptr; - in = nullptr; - process_list_entry = nullptr; + reset(); process_list_entry = rhs.process_list_entry; in = rhs.in; diff --git a/dbms/src/DataStreams/ConvertColumnWithDictionaryToFullBlockInputStream.h b/dbms/src/DataStreams/ConvertColumnWithDictionaryToFullBlockInputStream.h index 784850feb6a..1e3ce882dab 100644 --- a/dbms/src/DataStreams/ConvertColumnWithDictionaryToFullBlockInputStream.h +++ b/dbms/src/DataStreams/ConvertColumnWithDictionaryToFullBlockInputStream.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -32,22 +33,13 @@ private: { for (auto & column : block) { - auto * type_with_dict = typeid_cast(column.type.get()); - auto * col_with_dict = typeid_cast(column.column.get()); + if (auto * column_const = typeid_cast(column.column.get())) + column.column = column_const->removeLowCardinality(); + else + column.column = column.column->convertToFullColumnIfWithDictionary(); - if (type_with_dict && !col_with_dict) - throw Exception("Invalid column for " + type_with_dict->getName() + ": " + column.column->getName(), - ErrorCodes::LOGICAL_ERROR); - - if (!type_with_dict && col_with_dict) - throw Exception("Invalid type for " + col_with_dict->getName() + ": " + column.type->getName(), - ErrorCodes::LOGICAL_ERROR); - - if (type_with_dict && col_with_dict) - { - column.column = col_with_dict->convertToFullColumn(); - column.type = type_with_dict->getDictionaryType(); - } + if (auto * low_cardinality_type = typeid_cast(column.type.get())) + column.type = low_cardinality_type->getDictionaryType(); } return std::move(block); diff --git a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp index 8313f5820e5..4c78aeb7ce5 100644 --- a/dbms/src/DataStreams/ConvertingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ConvertingBlockInputStream.cpp @@ -15,6 +15,20 @@ namespace ErrorCodes } +static ColumnPtr castColumnWithDiagnostic(const ColumnWithTypeAndName & src_elem, const ColumnWithTypeAndName & res_elem, const Context & context) +{ + try + { + return castColumn(src_elem, res_elem.type, context); + } + catch (Exception & e) + { + e.addMessage("while converting source column " + backQuoteIfNeed(src_elem.name) + " to destination column " + backQuoteIfNeed(res_elem.name)); + throw; + } +} + + ConvertingBlockInputStream::ConvertingBlockInputStream( const Context & context_, const BlockInputStreamPtr & input, @@ -69,7 +83,7 @@ ConvertingBlockInputStream::ConvertingBlockInputStream( /// Check conversion by dry run CAST function. - castColumn(src_elem, res_elem.type, context); + castColumnWithDiagnostic(src_elem, res_elem, context); } } @@ -87,7 +101,7 @@ Block ConvertingBlockInputStream::readImpl() const auto & src_elem = src.getByPosition(conversion[res_pos]); auto & res_elem = res.getByPosition(res_pos); - ColumnPtr converted = castColumn(src_elem, res_elem.type, context); + ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context); if (src_elem.column->isColumnConst() && !res_elem.column->isColumnConst()) converted = converted->convertToFullColumnIfConst(); diff --git a/dbms/src/DataStreams/CountingBlockOutputStream.h b/dbms/src/DataStreams/CountingBlockOutputStream.h index 12732473ac3..ea1b5ec037d 100644 --- a/dbms/src/DataStreams/CountingBlockOutputStream.h +++ b/dbms/src/DataStreams/CountingBlockOutputStream.h @@ -20,7 +20,7 @@ public: progress_callback = callback; } - void setProcessListElement(ProcessListElement * elem) + void setProcessListElement(QueryStatus * elem) { process_elem = elem; } @@ -43,7 +43,7 @@ protected: BlockOutputStreamPtr stream; Progress progress; ProgressCallback progress_callback; - ProcessListElement * process_elem = nullptr; + QueryStatus * process_elem = nullptr; }; } diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index bd75dda1293..39d0d0c9615 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -17,8 +17,9 @@ namespace ErrorCodes } -FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name) - : expression(expression_) +FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, + const String & filter_column_name, bool remove_filter) + : remove_filter(remove_filter), expression(expression_) { children.push_back(input); @@ -40,6 +41,9 @@ FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input FilterDescription filter_description_check(*column_elem.column); column_elem.column = column_elem.type->createColumnConst(header.rows(), UInt64(1)); } + + if (remove_filter) + header.erase(filter_column_name); } @@ -69,7 +73,7 @@ Block FilterBlockInputStream::readImpl() Block res; if (constant_filter_description.always_false) - return res; + return removeFilterIfNeed(std::move(res)); /// Until non-empty block after filtering or end of stream. while (1) @@ -81,7 +85,7 @@ Block FilterBlockInputStream::readImpl() expression->execute(res); if (constant_filter_description.always_true) - return res; + return removeFilterIfNeed(std::move(res)); size_t columns = res.columns(); ColumnPtr column = res.safeGetByPosition(filter_column).column; @@ -100,7 +104,7 @@ Block FilterBlockInputStream::readImpl() } if (constant_filter_description.always_true) - return res; + return removeFilterIfNeed(std::move(res)); FilterDescription filter_and_holder(*column); @@ -142,7 +146,7 @@ Block FilterBlockInputStream::readImpl() /// Replace the column with the filter by a constant. res.safeGetByPosition(filter_column).column = res.safeGetByPosition(filter_column).type->createColumnConst(filtered_rows, UInt64(1)); /// No need to touch the rest of the columns. - return res; + return removeFilterIfNeed(std::move(res)); } /// Filter the rest of the columns. @@ -170,9 +174,18 @@ Block FilterBlockInputStream::readImpl() current_column.column = current_column.column->filter(*filter_and_holder.data, -1); } - return res; + return removeFilterIfNeed(std::move(res)); } } +Block FilterBlockInputStream::removeFilterIfNeed(Block && block) +{ + if (block && remove_filter) + block.erase(static_cast(filter_column)); + + return std::move(block); +} + + } diff --git a/dbms/src/DataStreams/FilterBlockInputStream.h b/dbms/src/DataStreams/FilterBlockInputStream.h index 8089cf87420..ca63b34f45c 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.h +++ b/dbms/src/DataStreams/FilterBlockInputStream.h @@ -20,7 +20,8 @@ private: using ExpressionActionsPtr = std::shared_ptr; public: - FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name_); + FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, + const String & filter_column_name_, bool remove_filter = false); String getName() const override; Block getTotals() override; @@ -29,12 +30,16 @@ public: protected: Block readImpl() override; + bool remove_filter; + private: ExpressionActionsPtr expression; Block header; ssize_t filter_column; ConstantFilterDescription constant_filter_description; + + Block removeFilterIfNeed(Block && block); }; } diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index 5c4cf2aa219..998ea2b42db 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -1,6 +1,13 @@ #include #include #include +#include + + +namespace ProfileEvents +{ + extern const Event ThrottlerSleepMicroseconds; +} namespace DB @@ -286,21 +293,34 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value) size_t total_rows = progress.total_rows; - if (limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0)) - { - double total_elapsed = info.total_stopwatch.elapsedSeconds(); + constexpr UInt64 profile_events_update_period_microseconds = 10 * 1000; // 10 milliseconds + UInt64 total_elapsed_microseconds = info.total_stopwatch.elapsedMicroseconds(); - if (total_elapsed > limits.timeout_before_checking_execution_speed.totalMicroseconds() / 1000000.0) + if (last_profile_events_update_time + profile_events_update_period_microseconds < total_elapsed_microseconds) + { + CurrentThread::updatePerformanceCounters(); + last_profile_events_update_time = total_elapsed_microseconds; + } + + if ((limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0)) + && (static_cast(total_elapsed_microseconds) > limits.timeout_before_checking_execution_speed.totalMicroseconds())) + { + /// Do not count sleeps in throttlers + UInt64 throttler_sleep_microseconds = CurrentThread::getProfileEvents()[ProfileEvents::ThrottlerSleepMicroseconds]; + double elapsed_seconds = (throttler_sleep_microseconds > total_elapsed_microseconds) + ? 0.0 : (total_elapsed_microseconds - throttler_sleep_microseconds) / 1000000.0; + + if (elapsed_seconds > 0) { - if (limits.min_execution_speed && progress.rows / total_elapsed < limits.min_execution_speed) - throw Exception("Query is executing too slow: " + toString(progress.rows / total_elapsed) + if (limits.min_execution_speed && progress.rows / elapsed_seconds < limits.min_execution_speed) + throw Exception("Query is executing too slow: " + toString(progress.rows / elapsed_seconds) + " rows/sec., minimum: " + toString(limits.min_execution_speed), ErrorCodes::TOO_SLOW); /// If the predicted execution time is longer than `max_execution_time`. if (limits.max_execution_time != 0 && total_rows) { - double estimated_execution_time_seconds = total_elapsed * (static_cast(total_rows) / progress.rows); + double estimated_execution_time_seconds = elapsed_seconds * (static_cast(total_rows) / progress.rows); if (estimated_execution_time_seconds > limits.max_execution_time.totalSeconds()) throw Exception("Estimated query execution time (" + toString(estimated_execution_time_seconds) + " seconds)" @@ -363,7 +383,7 @@ void IProfilingBlockInputStream::setProgressCallback(const ProgressCallback & ca } -void IProfilingBlockInputStream::setProcessListElement(ProcessListElement * elem) +void IProfilingBlockInputStream::setProcessListElement(QueryStatus * elem) { process_list_elem = elem; diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.h b/dbms/src/DataStreams/IProfilingBlockInputStream.h index 5febcb18c56..ab0db8dd99d 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.h +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.h @@ -20,6 +20,7 @@ namespace ErrorCodes } class QuotaForIntervals; +class QueryStatus; class ProcessListElement; class IProfilingBlockInputStream; @@ -103,7 +104,7 @@ public: * Based on this information, the quota and some restrictions will be checked. * This information will also be available in the SHOW PROCESSLIST request. */ - void setProcessListElement(ProcessListElement * elem); + void setProcessListElement(QueryStatus * elem); /** Set the approximate total number of rows to read. */ @@ -178,7 +179,9 @@ protected: std::atomic is_cancelled{false}; std::atomic is_killed{false}; ProgressCallback progress_callback; - ProcessListElement * process_list_elem = nullptr; + QueryStatus * process_list_elem = nullptr; + /// According to total_stopwatch in microseconds + UInt64 last_profile_events_update_time = 0; /// Additional information that can be generated during the work process. diff --git a/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp new file mode 100644 index 00000000000..02d2f8cf440 --- /dev/null +++ b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp @@ -0,0 +1,82 @@ +#include "InternalTextLogsRowOutputStream.h" +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +Block InternalTextLogsRowOutputStream::getHeader() const +{ + return InternalTextLogsQueue::getSampleBlock(); +} + +void InternalTextLogsRowOutputStream::write(const Block & block) +{ + auto & array_event_time = typeid_cast(*block.getByName("event_time").column).getData(); + auto & array_microseconds = typeid_cast(*block.getByName("event_time_microseconds").column).getData(); + + auto & column_host_name = typeid_cast(*block.getByName("host_name").column); + auto & column_query_id = typeid_cast(*block.getByName("query_id").column); + + auto & array_thread_number = typeid_cast(*block.getByName("thread_number").column).getData(); + auto & array_priority = typeid_cast(*block.getByName("priority").column).getData(); + auto & column_source = typeid_cast(*block.getByName("source").column); + auto & column_text = typeid_cast(*block.getByName("text").column); + + for (size_t row_num = 0; row_num < block.rows(); ++row_num) + { + auto host_name = column_host_name.getDataAt(row_num); + if (host_name.size) + { + writeCString("[", wb); + writeString(host_name, wb); + writeCString("] ", wb); + } + + auto event_time = array_event_time[row_num]; + writeDateTimeText<'.', ':'>(event_time, wb); + + auto microseconds = array_microseconds[row_num]; + writeChar('.', wb); + writeChar('0' + ((microseconds / 100000) % 10), wb); + writeChar('0' + ((microseconds / 10000) % 10), wb); + writeChar('0' + ((microseconds / 1000) % 10), wb); + writeChar('0' + ((microseconds / 100) % 10), wb); + writeChar('0' + ((microseconds / 10) % 10), wb); + writeChar('0' + ((microseconds / 1) % 10), wb); + + auto query_id = column_query_id.getDataAt(row_num); + if (query_id.size) + { + writeCString(" {", wb); + writeString(query_id, wb); + writeCString("}", wb); + } + + UInt32 thread_number = array_thread_number[row_num]; + writeCString(" [ ", wb); + writeIntText(thread_number, wb); + writeCString(" ] <", wb); + + Int8 priority = array_priority[row_num]; + writeString(InternalTextLogsQueue::getPriorityName(priority), wb); + writeCString("> ", wb); + + auto source = column_source.getDataAt(row_num); + writeString(source, wb); + writeCString(": ", wb); + + auto text = column_text.getDataAt(row_num); + writeString(text, wb); + + writeChar('\n', wb); + } +} + +} diff --git a/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h new file mode 100644 index 00000000000..3f54a00e633 --- /dev/null +++ b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + + +namespace DB +{ + +/// Prints internal server logs +/// Input blocks have to have the same structure as SystemLogsQueue::getSampleBlock() +/// NOTE: IRowOutputStream does not suite well for this case +class InternalTextLogsRowOutputStream : public IBlockOutputStream +{ +public: + + InternalTextLogsRowOutputStream(WriteBuffer & buf_out) : wb(buf_out) {} + + Block getHeader() const override; + + void write(const Block & block) override; + + void flush() override + { + wb.next(); + } + +private: + + WriteBuffer & wb; +}; + +} diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 334c65d0c40..022366cbc04 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -175,10 +176,10 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start() { auto & child = children[i]; - auto memory_tracker = current_memory_tracker; - reading_pool->schedule([&child, memory_tracker] + auto thread_group = CurrentThread::getGroup(); + reading_pool->schedule([&child, thread_group] { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); setThreadName("MergeAggReadThr"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; child->readPrefix(); @@ -196,8 +197,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start() */ for (size_t i = 0; i < merging_threads; ++i) - pool.schedule(std::bind(&MergingAggregatedMemoryEfficientBlockInputStream::mergeThread, - this, current_memory_tracker)); + pool.schedule([this, thread_group=CurrentThread::getGroup()] () { mergeThread(thread_group); } ); } } @@ -293,14 +293,16 @@ void MergingAggregatedMemoryEfficientBlockInputStream::finalize() } -void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker * memory_tracker) +void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupStatusPtr thread_group) { - setThreadName("MergeAggMergThr"); - current_memory_tracker = memory_tracker; CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; try { + if (thread_group) + CurrentThread::attachTo(thread_group); + setThreadName("MergeAggMergThr"); + while (!parallel_merge_data->finish) { /** Receiving next blocks is processing by one thread pool, and merge is in another. @@ -480,10 +482,10 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate { if (need_that_input(input)) { - auto memory_tracker = current_memory_tracker; - reading_pool->schedule([&input, &read_from_input, memory_tracker] + auto thread_group = CurrentThread::getGroup(); + reading_pool->schedule([&input, &read_from_input, thread_group] { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); setThreadName("MergeAggReadThr"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; read_from_input(input); diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h index 837c10869cf..bdabd8cc1f8 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -151,7 +152,7 @@ private: std::unique_ptr parallel_merge_data; - void mergeThread(MemoryTracker * memory_tracker); + void mergeThread(ThreadGroupStatusPtr main_thread); void finalize(); }; diff --git a/dbms/src/DataStreams/ParallelInputsProcessor.h b/dbms/src/DataStreams/ParallelInputsProcessor.h index 115dc6e1a3a..a1c4e2ac480 100644 --- a/dbms/src/DataStreams/ParallelInputsProcessor.h +++ b/dbms/src/DataStreams/ParallelInputsProcessor.h @@ -12,6 +12,7 @@ #include #include #include +#include /** Allows to process multiple block input streams (sources) in parallel, using specified number of threads. @@ -105,8 +106,9 @@ public: { active_threads = max_threads; threads.reserve(max_threads); + auto thread_group = CurrentThread::getGroup(); for (size_t i = 0; i < max_threads; ++i) - threads.emplace_back(std::bind(&ParallelInputsProcessor::thread, this, current_memory_tracker, i)); + threads.emplace_back([=] () { thread(thread_group, i); } ); } /// Ask all sources to stop earlier than they run out. @@ -174,16 +176,16 @@ private: } } - void thread(MemoryTracker * memory_tracker, size_t thread_num) + void thread(ThreadGroupStatusPtr thread_group, size_t thread_num) { - current_memory_tracker = memory_tracker; std::exception_ptr exception; - - setThreadName("ParalInputsProc"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; try { + setThreadName("ParalInputsProc"); + CurrentThread::attachTo(thread_group); + while (!finish) { InputData unprepared_input; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 349c74732d6..670a70ad6bb 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include #include @@ -137,7 +139,7 @@ void RemoteBlockInputStream::sendExternalTables() for (const auto & table : external_tables) { StoragePtr cur = table.second; - QueryProcessingStage::Enum read_from_table_stage = QueryProcessingStage::Complete; + QueryProcessingStage::Enum read_from_table_stage = cur->getQueryProcessingStage(context); BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context, read_from_table_stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) @@ -232,6 +234,12 @@ Block RemoteBlockInputStream::readImpl() extremes = packet.block; break; + case Protocol::Server::Log: + /// Pass logs from remote server to client + if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) + log_queue->pushBlock(std::move(packet.block)); + break; + default: got_unknown_packet_from_replica = true; throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index d9095ec91b9..f1e68a6a0c1 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -4,6 +4,8 @@ #include #include +#include +#include namespace DB @@ -24,23 +26,33 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const */ connection.sendQuery(query, "", QueryProcessingStage::Complete, settings, nullptr); - Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::Data == packet.type) + while (true) { - header = packet.block; + Connection::Packet packet = connection.receivePacket(); - if (!header) - throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); + if (Protocol::Server::Data == packet.type) + { + header = packet.block; + + if (!header) + throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); + break; + } + else if (Protocol::Server::Exception == packet.type) + { + packet.exception->rethrow(); + break; + } + else if (Protocol::Server::Log == packet.type) + { + /// Pass logs from remote server to client + if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) + log_queue->pushBlock(std::move(packet.block)); + } + else + throw NetException("Unexpected packet from server (expected Data or Exception, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } - else if (Protocol::Server::Exception == packet.type) - { - packet.exception->rethrow(); - return; - } - else - throw NetException("Unexpected packet from server (expected Data or Exception, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } @@ -55,15 +67,11 @@ void RemoteBlockOutputStream::write(const Block & block) catch (const NetException &) { /// Try to get more detailed exception from server - if (connection.poll(0)) + auto packet_type = connection.checkPacket(); + if (packet_type && *packet_type == Protocol::Server::Exception) { Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::Exception == packet.type) - { - packet.exception->rethrow(); - return; - } + packet.exception->rethrow(); } throw; @@ -83,18 +91,23 @@ void RemoteBlockOutputStream::writeSuffix() /// Empty block means end of data. connection.sendData(Block()); - /// Receive EndOfStream packet. - Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::EndOfStream == packet.type) + /// Wait for EndOfStream or Exception packet, skip Log packets. + while (true) { - /// Do nothing. + Connection::Packet packet = connection.receivePacket(); + + if (Protocol::Server::EndOfStream == packet.type) + break; + else if (Protocol::Server::Exception == packet.type) + packet.exception->rethrow(); + else if (Protocol::Server::Log == packet.type) + { + // Do nothing + } + else + throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } - else if (Protocol::Server::Exception == packet.type) - packet.exception->rethrow(); - else - throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); finished = true; } diff --git a/dbms/src/DataStreams/tests/expression_stream.cpp b/dbms/src/DataStreams/tests/expression_stream.cpp index 445c2a4b0df..3556c06b026 100644 --- a/dbms/src/DataStreams/tests/expression_stream.cpp +++ b/dbms/src/DataStreams/tests/expression_stream.cpp @@ -47,7 +47,7 @@ try Names column_names; column_names.push_back("number"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreamPtr in; in = table->read(column_names, {}, context, stage, 8192, 1)[0]; diff --git a/dbms/src/DataStreams/tests/filter_stream.cpp b/dbms/src/DataStreams/tests/filter_stream.cpp index 5e634fcf9a8..f1e9494e874 100644 --- a/dbms/src/DataStreams/tests/filter_stream.cpp +++ b/dbms/src/DataStreams/tests/filter_stream.cpp @@ -52,7 +52,7 @@ try Names column_names; column_names.push_back("number"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0]; in = std::make_shared(in, expression, "equals(modulo(number, 3), 1)"); diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/union_stream2.cpp b/dbms/src/DataStreams/tests/union_stream2.cpp index fe0b225c376..377dd76bf2e 100644 --- a/dbms/src/DataStreams/tests/union_stream2.cpp +++ b/dbms/src/DataStreams/tests/union_stream2.cpp @@ -34,7 +34,7 @@ try StoragePtr table = context.getTable("default", "hits6"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreams streams = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads); for (size_t i = 0, size = streams.size(); i < size; ++i) diff --git a/dbms/src/DataTypes/DataTypeFactory.cpp b/dbms/src/DataTypes/DataTypeFactory.cpp index b9d25b09544..ad123a7431b 100644 --- a/dbms/src/DataTypes/DataTypeFactory.cpp +++ b/dbms/src/DataTypes/DataTypeFactory.cpp @@ -129,6 +129,7 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator } void registerDataTypeNumbers(DataTypeFactory & factory); +void registerDataTypeDecimal(DataTypeFactory & factory); void registerDataTypeDate(DataTypeFactory & factory); void registerDataTypeDateTime(DataTypeFactory & factory); void registerDataTypeString(DataTypeFactory & factory); @@ -148,6 +149,7 @@ void registerDataTypeWithDictionary(DataTypeFactory & factory); DataTypeFactory::DataTypeFactory() { registerDataTypeNumbers(*this); + registerDataTypeDecimal(*this); registerDataTypeDate(*this); registerDataTypeDateTime(*this); registerDataTypeString(*this); diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index 439b3ebc8db..ca365bfaa78 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -16,6 +16,7 @@ public: using FieldType = T; const char * getFamilyName() const override { return TypeName::get(); } + size_t getTypeId() const override { return TypeId::value; } void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; diff --git a/dbms/src/DataTypes/DataTypeWithDictionary.cpp b/dbms/src/DataTypes/DataTypeWithDictionary.cpp index fc904f43a8f..8877eb820f1 100644 --- a/dbms/src/DataTypes/DataTypeWithDictionary.cpp +++ b/dbms/src/DataTypes/DataTypeWithDictionary.cpp @@ -669,6 +669,7 @@ void DataTypeWithDictionary::deserializeBinaryBulkWithMultipleStreams( } }; + bool first_dictionary = true; while (limit) { if (state_with_dictionary->num_pending_rows == 0) @@ -681,8 +682,11 @@ void DataTypeWithDictionary::deserializeBinaryBulkWithMultipleStreams( index_type.deserialize(*indexes_stream); - if (index_type.need_global_dictionary && (!global_dictionary || index_type.need_update_dictionary)) + if (index_type.need_global_dictionary && (!global_dictionary || index_type.need_update_dictionary || (first_dictionary && !settings.continuous_reading))) + { readDictionary(); + first_dictionary = false; + } if (state_with_dictionary->index_type.has_additional_keys) readAdditionalKeys(); diff --git a/dbms/src/DataTypes/DataTypesDecimal.cpp b/dbms/src/DataTypes/DataTypesDecimal.cpp new file mode 100644 index 00000000000..b094619c914 --- /dev/null +++ b/dbms/src/DataTypes/DataTypesDecimal.cpp @@ -0,0 +1,220 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; +} + + +// + +template +std::string DataTypeDecimal::getName() const +{ + std::stringstream ss; + ss << "Decimal(" << precision << ", " << scale << ")"; + return ss.str(); +} + +template +bool DataTypeDecimal::equals(const IDataType & rhs) const +{ + if (auto * ptype = typeid_cast *>(&rhs)) + return scale == ptype->getScale(); + return false; +} + +template +void DataTypeDecimal::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const +{ + T value = static_cast(column).getData()[row_num]; + if (value < T(0)) + { + value *= T(-1); + writeChar('-', ostr); /// avoid crop leading minus when whole part is zero + } + + writeIntText(static_cast(wholePart(value)), ostr); + if (scale) + { + writeChar('.', ostr); + String str_fractional(scale, '0'); + for (Int32 pos = scale - 1; pos >= 0; --pos, value /= T(10)) + str_fractional[pos] += value % T(10); + ostr.write(str_fractional.data(), scale); + } +} + + +template +void DataTypeDecimal::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +{ + T x; + UInt32 unread_scale = scale; + readDecimalText(istr, x, precision, unread_scale); + x *= getScaleMultiplier(unread_scale); + static_cast(column).getData().push_back(x); +} + + +template +T DataTypeDecimal::parseFromString(const String & str) const +{ + ReadBufferFromMemory buf(str.data(), str.size()); + T x; + UInt32 unread_scale = scale; + readDecimalText(buf, x, precision, unread_scale, true); + x *= getScaleMultiplier(unread_scale); + return x; +} + + +template +void DataTypeDecimal::serializeBinary(const Field & field, WriteBuffer & ostr) const +{ + FieldType x = get(field); + writeBinary(x, ostr); +} + +template +void DataTypeDecimal::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const +{ + const FieldType & x = static_cast(column).getData()[row_num]; + writeBinary(x, ostr); +} + +template +void DataTypeDecimal::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const +{ + const typename ColumnType::Container & x = typeid_cast(column).getData(); + + size_t size = x.size(); + + if (limit == 0 || offset + limit > size) + limit = size - offset; + + ostr.write(reinterpret_cast(&x[offset]), sizeof(FieldType) * limit); +} + + +template +void DataTypeDecimal::deserializeBinary(Field & field, ReadBuffer & istr) const +{ + typename FieldType::NativeType x; + readBinary(x, istr); + field = DecimalField(T(x), scale); +} + +template +void DataTypeDecimal::deserializeBinary(IColumn & column, ReadBuffer & istr) const +{ + typename FieldType::NativeType x; + readBinary(x, istr); + static_cast(column).getData().push_back(FieldType(x)); +} + +template +void DataTypeDecimal::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double ) const +{ + typename ColumnType::Container & x = typeid_cast(column).getData(); + size_t initial_size = x.size(); + x.resize(initial_size + limit); + size_t size = istr.readBig(reinterpret_cast(&x[initial_size]), sizeof(FieldType) * limit); + x.resize(initial_size + size / sizeof(FieldType)); +} + + +template +Field DataTypeDecimal::getDefault() const +{ + return DecimalField(T(0), scale); +} + + +template +MutableColumnPtr DataTypeDecimal::createColumn() const +{ + auto column = ColumnType::create(); + column->getData().setScale(scale); + return column; +} + + +// + +static DataTypePtr create(const ASTPtr & arguments) +{ + if (!arguments || arguments->children.size() != 2) + throw Exception("Decimal data type family must have exactly two arguments: precision and scale", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const ASTLiteral * precision = typeid_cast(arguments->children[0].get()); + const ASTLiteral * scale = typeid_cast(arguments->children[1].get()); + + if (!precision || precision->value.getType() != Field::Types::UInt64 || + !scale || !(scale->value.getType() == Field::Types::Int64 || scale->value.getType() == Field::Types::UInt64)) + throw Exception("Decimal data type family must have a two numbers as its arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + UInt64 precision_value = precision->value.get(); + Int64 scale_value = scale->value.get(); + + if (precision_value < minDecimalPrecision() || precision_value > maxDecimalPrecision()) + throw Exception("Wrong precision", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + if (scale_value < 0 || static_cast(scale_value) > precision_value) + throw Exception("Negative scales and scales larger than presicion are not supported", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + if (precision_value <= maxDecimalPrecision()) + return std::make_shared>(precision_value, scale_value); + else if (precision_value <= maxDecimalPrecision()) + return std::make_shared>(precision_value, scale_value); + return std::make_shared>(precision_value, scale_value); +} + + +void registerDataTypeDecimal(DataTypeFactory & factory) +{ + factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive); + factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); +} + + +template <> +Decimal32 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) +{ + return common::exp10_i32(scale_); +} + +template <> +Decimal64 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) +{ + return common::exp10_i64(scale_); +} + +template <> +Decimal128 DataTypeDecimal::getScaleMultiplier(UInt32 scale_) +{ + return common::exp10_i128(scale_); +} + + +/// Explicit template instantiations. +template class DataTypeDecimal; +template class DataTypeDecimal; +template class DataTypeDecimal; + +} diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h new file mode 100644 index 00000000000..dc4ef6adedc --- /dev/null +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -0,0 +1,272 @@ +#pragma once +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int ARGUMENT_OUT_OF_BOUND; +} + +/// +class DataTypeSimpleSerialization : public IDataType +{ + void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override + { + serializeText(column, row_num, ostr, settings); + } + + void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override + { + serializeText(column, row_num, ostr, settings); + } + + void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override + { + serializeText(column, row_num, ostr, settings); + } + + void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override + { + serializeText(column, row_num, ostr, settings); + } + + void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override + { + deserializeText(column, istr, settings); + } + + void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override + { + deserializeText(column, istr, settings); + } + + void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override + { + deserializeText(column, istr, settings); + } + + void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override + { + deserializeText(column, istr, settings); + } + + virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; +}; + + +static constexpr size_t minDecimalPrecision() { return 1; } +template static constexpr size_t maxDecimalPrecision() { return 0; } +template <> constexpr size_t maxDecimalPrecision() { return 9; } +template <> constexpr size_t maxDecimalPrecision() { return 18; } +template <> constexpr size_t maxDecimalPrecision() { return 38; } + + +/// Implements Decimal(P, S), where P is precision, S is scale. +/// Maximum precisions for underlying types are: +/// Int32 9 +/// Int64 18 +/// Int128 38 +/// Operation between two decimals leads to Decimal(P, S), where +/// P is one of (9, 18, 38); equals to the maximum precision for the biggest underlying type of operands. +/// S is maximum scale of operands. +/// +/// NOTE: It's possible to set scale as a template parameter then most of functions become static. +template +class DataTypeDecimal final : public DataTypeSimpleSerialization +{ +public: + using FieldType = T; + using ColumnType = ColumnVector; + + static constexpr bool is_parametric = true; + + DataTypeDecimal(UInt32 precision_, UInt32 scale_) + : precision(precision_), + scale(scale_) + { + if (unlikely(precision < 1 || precision > maxDecimalPrecision())) + throw Exception("Precision is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + if (unlikely(scale < 0 || static_cast(scale) > maxDecimalPrecision())) + throw Exception("Scale is out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + + const char * getFamilyName() const override { return "Decimal"; } + std::string getName() const override; + size_t getTypeId() const override { return TypeId::value; } + + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + + void serializeBinary(const Field & field, WriteBuffer & ostr) const override; + void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override; + void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; + + void deserializeBinary(Field & field, ReadBuffer & istr) const override; + void deserializeBinary(IColumn & column, ReadBuffer & istr) const override; + void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override; + + Field getDefault() const override; + MutableColumnPtr createColumn() const override; + bool equals(const IDataType & rhs) const override; + + bool isParametric() const override { return true; } + bool haveSubtypes() const override { return false; } + bool shouldAlignRightInPrettyFormats() const override { return true; } + bool textCanContainOnlyValidUTF8() const override { return true; } + bool isComparable() const override { return true; } + bool isValueRepresentedByNumber() const override { return true; } + bool isValueRepresentedByInteger() const override { return true; } + bool isValueRepresentedByUnsignedInteger() const override { return false; } + bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; } + bool haveMaximumSizeOfValue() const override { return true; } + size_t getSizeOfValueInMemory() const override { return sizeof(T); } + bool isCategorial() const override { return isValueRepresentedByInteger(); } + + bool canBeUsedAsVersion() const override { return false; } + bool isSummable() const override { return true; } + bool canBeUsedInBitOperations() const override { return false; } + bool isUnsignedInteger() const override { return false; } + bool canBeUsedInBooleanContext() const override { return true; } + bool isNumber() const override { return true; } + bool isInteger() const override { return false; } + bool canBeInsideNullable() const override { return true; } + + /// Decimal specific + + UInt32 getPrecision() const { return precision; } + UInt32 getScale() const { return scale; } + T getScaleMultiplier() const { return getScaleMultiplier(scale); } + + T wholePart(T x) const + { + if (scale == 0) + return x; + return x / getScaleMultiplier(); + } + + T fractionalPart(T x) const + { + if (scale == 0) + return 0; + if (x < T(0)) + x *= T(-1); + return x % getScaleMultiplier(); + } + + T maxWholeValue() const { return getScaleMultiplier(maxDecimalPrecision() - scale) - T(1); } + + bool canStoreWhole(T x) const + { + T max = maxWholeValue(); + if (x > max || x < -max) + return false; + return true; + } + + /// @returns multiplier for U to become T with correct scale + template + T scaleFactorFor(const DataTypeDecimal & x, bool ) const + { + if (getScale() < x.getScale()) + throw Exception("Decimal result's scale is less then argiment's one", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + UInt32 scale_delta = getScale() - x.getScale(); /// scale_delta >= 0 + return getScaleMultiplier(scale_delta); + } + + template + T scaleFactorFor(const DataTypeNumber & , bool is_multiply_or_divisor) const + { + if (is_multiply_or_divisor) + return 1; + return getScaleMultiplier(); + } + + T parseFromString(const String & str) const; + + static T getScaleMultiplier(UInt32 scale); + +private: + const UInt32 precision; + const UInt32 scale; /// TODO: should we support scales out of [0, precision]? +}; + + +template +typename std::enable_if_t<(sizeof(T) >= sizeof(U)), const DataTypeDecimal> +decimalResultType(const DataTypeDecimal & tx, const DataTypeDecimal & ty, bool is_multiply, bool is_divide) +{ + UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale()); + if (is_multiply) + scale = tx.getScale() + ty.getScale(); + else if (is_divide) + scale = tx.getScale(); + return DataTypeDecimal(maxDecimalPrecision(), scale); +} + +template +typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DataTypeDecimal> +decimalResultType(const DataTypeDecimal & tx, const DataTypeDecimal & ty, bool is_multiply, bool is_divide) +{ + UInt32 scale = (tx.getScale() > ty.getScale() ? tx.getScale() : ty.getScale()); + if (is_multiply) + scale = tx.getScale() * ty.getScale(); + else if (is_divide) + scale = tx.getScale(); + return DataTypeDecimal(maxDecimalPrecision(), scale); +} + +template +const DataTypeDecimal decimalResultType(const DataTypeDecimal & tx, const DataTypeNumber &, bool, bool) +{ + return DataTypeDecimal(maxDecimalPrecision(), tx.getScale()); +} + +template +const DataTypeDecimal decimalResultType(const DataTypeNumber &, const DataTypeDecimal & ty, bool, bool) +{ + return DataTypeDecimal(maxDecimalPrecision(), ty.getScale()); +} + + +template +inline const DataTypeDecimal * checkDecimal(const IDataType & data_type) +{ + return typeid_cast *>(&data_type); +} + +inline bool isDecimal(const IDataType & data_type) +{ + if (typeid_cast *>(&data_type)) + return true; + if (typeid_cast *>(&data_type)) + return true; + if (typeid_cast *>(&data_type)) + return true; + return false; +} + +/// +inline bool notDecimalButComparableToDecimal(const IDataType & data_type) +{ + if (data_type.isInteger()) + return true; + return false; +} + +/// +inline bool comparableToDecimal(const IDataType & data_type) +{ + if (data_type.isInteger()) + return true; + return isDecimal(data_type); +} + +} diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 708fd72cc61..da55762c02f 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -45,6 +45,9 @@ public: /// Name of data type family (example: FixedString, Array). virtual const char * getFamilyName() const = 0; + /// Unique type number or zero + virtual size_t getTypeId() const { return 0; } + /** Binary serialization for range of values in column - for writing to disk/network, etc. * * Some data types are represented in multiple streams while being serialized. @@ -132,6 +135,9 @@ public: InputStreamGetter getter; SubstreamPath path; + /// True if continue reading from previous positions in file. False if made fseek to the start of new granule. + bool continuous_reading = true; + bool position_independent_encoding = true; /// If not zero, may be used to avoid reallocations while reading column of String type. double avg_value_size_hint = 0; diff --git a/dbms/src/DataTypes/tests/data_type_string.cpp b/dbms/src/DataTypes/tests/data_type_string.cpp index a8cd5ad126b..65a15d059b4 100644 --- a/dbms/src/DataTypes/tests/data_type_string.cpp +++ b/dbms/src/DataTypes/tests/data_type_string.cpp @@ -22,6 +22,7 @@ try size_t size = strlen(s) + 1; DataTypeString data_type; + { auto column = ColumnString::create(); ColumnString::Chars_t & data = column->getChars(); @@ -37,8 +38,14 @@ try WriteBufferFromFile out_buf("test"); + IDataType::SerializeBinaryBulkSettings settings; + IDataType::SerializeBinaryBulkStatePtr state; + settings.getter = [&](const IDataType::SubstreamPath &){ return &out_buf; }; + stopwatch.restart(); - data_type.serializeBinaryBulkWithMultipleStreams(*column, [&](const IDataType::SubstreamPath &){ return &out_buf; }, 0, 0, true, {}); + data_type.serializeBinaryBulkStatePrefix(settings, state); + data_type.serializeBinaryBulkWithMultipleStreams(*column, 0, 0, settings, state); + data_type.serializeBinaryBulkStateSuffix(settings, state); stopwatch.stop(); std::cout << "Writing, elapsed: " << stopwatch.elapsedSeconds() << std::endl; @@ -49,8 +56,13 @@ try ReadBufferFromFile in_buf("test"); + IDataType::DeserializeBinaryBulkSettings settings; + IDataType::DeserializeBinaryBulkStatePtr state; + settings.getter = [&](const IDataType::SubstreamPath &){ return &in_buf; }; + stopwatch.restart(); - data_type.deserializeBinaryBulkWithMultipleStreams(*column, [&](const IDataType::SubstreamPath &){ return &in_buf; }, n, 0, true, {}); + data_type.deserializeBinaryBulkStatePrefix(settings, state); + data_type.deserializeBinaryBulkWithMultipleStreams(*column, n, settings, state); stopwatch.stop(); std::cout << "Reading, elapsed: " << stopwatch.elapsedSeconds() << std::endl; diff --git a/dbms/src/DataTypes/tests/data_types_number_fixed.cpp b/dbms/src/DataTypes/tests/data_types_number_fixed.cpp index a88229c2a3f..998d0f44481 100644 --- a/dbms/src/DataTypes/tests/data_types_number_fixed.cpp +++ b/dbms/src/DataTypes/tests/data_types_number_fixed.cpp @@ -27,7 +27,12 @@ int main(int, char **) WriteBufferFromOStream out_buf(ostr); stopwatch.restart(); - data_type.serializeBinaryBulkWithMultipleStreams(*column, [&](const IDataType::SubstreamPath &){ return &out_buf; }, 0, 0, true, {}); + IDataType::SerializeBinaryBulkSettings settings; + settings.getter = [&](const IDataType::SubstreamPath &){ return &out_buf; }; + IDataType::SerializeBinaryBulkStatePtr state; + data_type.serializeBinaryBulkStatePrefix(settings, state); + data_type.serializeBinaryBulkWithMultipleStreams(*column, 0, 0, settings, state); + data_type.serializeBinaryBulkStateSuffix(settings, state); stopwatch.stop(); std::cout << "Elapsed: " << stopwatch.elapsedSeconds() << std::endl; diff --git a/dbms/src/Dictionaries/CMakeLists.txt b/dbms/src/Dictionaries/CMakeLists.txt index 65172356645..e69de29bb2d 100644 --- a/dbms/src/Dictionaries/CMakeLists.txt +++ b/dbms/src/Dictionaries/CMakeLists.txt @@ -1,3 +0,0 @@ -if (ENABLE_TESTS) - add_subdirectory (tests) -endif () diff --git a/dbms/src/Dictionaries/ODBCDictionarySource.cpp b/dbms/src/Dictionaries/ODBCDictionarySource.cpp index 2224bb8e223..4813c1a80ba 100644 --- a/dbms/src/Dictionaries/ODBCDictionarySource.cpp +++ b/dbms/src/Dictionaries/ODBCDictionarySource.cpp @@ -1,20 +1,58 @@ +#include #include #include #include #include +#include #include #include -#include -#include -#include +#include #include #include #include +#include +#include namespace DB { +namespace +{ + class ODBCBridgeBlockInputStream : public IProfilingBlockInputStream + { + public: + ODBCBridgeBlockInputStream(const Poco::URI & uri, + std::function callback, + const Block & sample_block, + const Context & context, + size_t max_block_size, + const ConnectionTimeouts & timeouts) + { + read_buf = std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, callback, timeouts); + reader = FormatFactory::instance().getInput(ODBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size); + } + + Block getHeader() const override + { + return reader->getHeader(); + } + + String getName() const override + { + return "ODBCBridgeBlockInputStream"; + } + + private: + Block readImpl() override + { + return reader->read(); + } + + std::unique_ptr read_buf; + BlockInputStreamPtr reader; + }; +} static const size_t max_block_size = 8192; @@ -32,20 +70,16 @@ ODBCDictionarySource::ODBCDictionarySource(const DictionaryStructure & dict_stru sample_block{sample_block}, query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::None}, /// NOTE Better to obtain quoting style via ODBC interface. load_all_query{query_builder.composeLoadAllQuery()}, - invalidate_query{config.getString(config_prefix + ".invalidate_query", "")} + invalidate_query{config.getString(config_prefix + ".invalidate_query", "")}, + odbc_bridge_helper{context.getConfigRef(), context.getSettingsRef().http_receive_timeout.value, config.getString(config_prefix + ".connection_string")}, + timeouts{ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef())}, + global_context(context) { - std::size_t field_size = context.getSettingsRef().odbc_max_field_size; + bridge_url = odbc_bridge_helper.getMainURI(); - pool = createAndCheckResizePocoSessionPool([&] - { - auto session = std::make_shared( - config.getString(config_prefix + ".connector", "ODBC"), - validateODBCConnectionString(config.getString(config_prefix + ".connection_string"))); - - /// Default POCO value is 1024. Set property manually to make possible reading of longer strings. - session->setProperty("maxFieldSize", Poco::Any(field_size)); - return session; - }); + auto url_params = odbc_bridge_helper.getURLParams(sample_block.getNamesAndTypesList().toString(), max_block_size); + for (const auto & [name, value] : url_params) + bridge_url.addQueryParameter(name, value); } /// copy-constructor is provided in order to support cloneability @@ -58,11 +92,14 @@ ODBCDictionarySource::ODBCDictionarySource(const ODBCDictionarySource & other) where{other.where}, update_field{other.update_field}, sample_block{other.sample_block}, - pool{other.pool}, query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::None}, load_all_query{other.load_all_query}, - invalidate_query{other.invalidate_query}, invalidate_query_response{other.invalidate_query_response} + invalidate_query{other.invalidate_query}, + invalidate_query_response{other.invalidate_query_response}, + odbc_bridge_helper{other.odbc_bridge_helper}, + global_context{other.global_context} { + } std::string ODBCDictionarySource::getUpdateFieldAndDate() @@ -86,7 +123,7 @@ std::string ODBCDictionarySource::getUpdateFieldAndDate() BlockInputStreamPtr ODBCDictionarySource::loadAll() { LOG_TRACE(log, load_all_query); - return std::make_shared(pool->get(), load_all_query, sample_block, max_block_size); + return loadBase(load_all_query); } BlockInputStreamPtr ODBCDictionarySource::loadUpdatedAll() @@ -94,20 +131,20 @@ BlockInputStreamPtr ODBCDictionarySource::loadUpdatedAll() std::string load_query_update = getUpdateFieldAndDate(); LOG_TRACE(log, load_query_update); - return std::make_shared(pool->get(), load_query_update, sample_block, max_block_size); + return loadBase(load_query_update); } BlockInputStreamPtr ODBCDictionarySource::loadIds(const std::vector & ids) { const auto query = query_builder.composeLoadIdsQuery(ids); - return std::make_shared(pool->get(), query, sample_block, max_block_size); + return loadBase(query); } BlockInputStreamPtr ODBCDictionarySource::loadKeys( const Columns & key_columns, const std::vector & requested_rows) { const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN); - return std::make_shared(pool->get(), query, sample_block, max_block_size); + return loadBase(query); } bool ODBCDictionarySource::supportsSelectiveLoad() const @@ -148,8 +185,28 @@ std::string ODBCDictionarySource::doInvalidateQuery(const std::string & request) Block invalidate_sample_block; ColumnPtr column(ColumnString::create()); invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared(), "Sample Block")); - ODBCBlockInputStream block_input_stream(pool->get(), request, invalidate_sample_block, 1); - return readInvalidateQuery(block_input_stream); + odbc_bridge_helper.startODBCBridgeSync(); + + ODBCBridgeBlockInputStream stream( + bridge_url, + [request](std::ostream & os) { os << "query=" << request; }, + invalidate_sample_block, + global_context, + max_block_size, + timeouts); + + return readInvalidateQuery(stream); +} + +BlockInputStreamPtr ODBCDictionarySource::loadBase(const std::string & query) const +{ + odbc_bridge_helper.startODBCBridgeSync(); + return std::make_shared(bridge_url, + [query](std::ostream & os) { os << "query=" << query; }, + sample_block, + global_context, + max_block_size, + timeouts); } } diff --git a/dbms/src/Dictionaries/ODBCDictionarySource.h b/dbms/src/Dictionaries/ODBCDictionarySource.h index 419a02f0ace..7d7a0ca51e0 100644 --- a/dbms/src/Dictionaries/ODBCDictionarySource.h +++ b/dbms/src/Dictionaries/ODBCDictionarySource.h @@ -1,11 +1,16 @@ #pragma once #include +#include + +#include #include #include #include +#include + namespace Poco { @@ -58,6 +63,8 @@ private: // execute invalidate_query. expects single cell in result std::string doInvalidateQuery(const std::string & request) const; + BlockInputStreamPtr loadBase(const std::string & query) const; + Poco::Logger * log; std::chrono::time_point update_time; @@ -67,11 +74,16 @@ private: const std::string where; const std::string update_field; Block sample_block; - std::shared_ptr pool = nullptr; ExternalQueryBuilder query_builder; const std::string load_all_query; std::string invalidate_query; mutable std::string invalidate_query_response; + + ODBCBridgeHelper odbc_bridge_helper; + Poco::URI bridge_url; + ConnectionTimeouts timeouts; + const Context & global_context; + }; diff --git a/dbms/src/Dictionaries/tests/CMakeLists.txt b/dbms/src/Dictionaries/tests/CMakeLists.txt index f0a4cf4ab68..e69de29bb2d 100644 --- a/dbms/src/Dictionaries/tests/CMakeLists.txt +++ b/dbms/src/Dictionaries/tests/CMakeLists.txt @@ -1,2 +0,0 @@ -add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) -target_link_libraries (validate-odbc-connection-string dbms) diff --git a/dbms/src/Functions/FunctionHelpers.h b/dbms/src/Functions/FunctionHelpers.h index aaaca0bf26a..6b1d89560df 100644 --- a/dbms/src/Functions/FunctionHelpers.h +++ b/dbms/src/Functions/FunctionHelpers.h @@ -7,6 +7,101 @@ #include #include +namespace common +{ + template + inline bool addOverflow(T x, T y, T & res) + { + return __builtin_add_overflow(x, y, &res); + } + + template <> + inline bool addOverflow(Int32 x, Int32 y, Int32 & res) + { + return __builtin_sadd_overflow(x, y, &res); + } + + template <> + inline bool addOverflow(long x, long y, long & res) + { + return __builtin_saddl_overflow(x, y, &res); + } + + template <> + inline bool addOverflow(long long x, long long y, long long & res) + { + return __builtin_saddll_overflow(x, y, &res); + } + + template <> + inline bool addOverflow(__int128 x, __int128 y, __int128 & res) + { + res = x + y; + return (res - y) != x; + } + + template + inline bool subOverflow(T x, T y, T & res) + { + return __builtin_sub_overflow(x, y, &res); + } + + template <> + inline bool subOverflow(Int32 x, Int32 y, Int32 & res) + { + return __builtin_ssub_overflow(x, y, &res); + } + + template <> + inline bool subOverflow(long x, long y, long & res) + { + return __builtin_ssubl_overflow(x, y, &res); + } + + template <> + inline bool subOverflow(long long x, long long y, long long & res) + { + return __builtin_ssubll_overflow(x, y, &res); + } + + template <> + inline bool subOverflow(__int128 x, __int128 y, __int128 & res) + { + res = x - y; + return (res + y) != x; + } + + template + inline bool mulOverflow(T x, T y, T & res) + { + return __builtin_mul_overflow(x, y, &res); + } + + template <> + inline bool mulOverflow(Int32 x, Int32 y, Int32 & res) + { + return __builtin_smul_overflow(x, y, &res); + } + + template <> + inline bool mulOverflow(long x, long y, long & res) + { + return __builtin_smull_overflow(x, y, &res); + } + + template <> + inline bool mulOverflow(long long x, long long y, long long & res) + { + return __builtin_smulll_overflow(x, y, &res); + } + + template <> + inline bool mulOverflow(__int128 x, __int128 y, __int128 & res) + { + res = x * y; + return (res / y) != x; + } +} namespace DB { @@ -94,5 +189,60 @@ Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & ar /// Similar function as above. Additionally transform the result type if needed. Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & args, size_t result); +template +bool callByTypeAndNumber(UInt8 number, F && f) +{ + switch (number) + { + case TypeId::value: f(T(), UInt8()); break; + case TypeId::value: f(T(), UInt16()); break; + case TypeId::value: f(T(), UInt32()); break; + case TypeId::value: f(T(), UInt64()); break; + //case TypeId::value: f(T(), UInt128()); break; + + case TypeId::value: f(T(), Int8()); break; + case TypeId::value: f(T(), Int16()); break; + case TypeId::value: f(T(), Int32()); break; + case TypeId::value: f(T(), Int64()); break; + case TypeId::value: f(T(), Int128()); break; + + case TypeId::value: f(T(), Decimal32()); break; + case TypeId::value: f(T(), Decimal64()); break; + case TypeId::value: f(T(), Decimal128()); break; + default: + return false; + } + + return true; +} + +/// Unroll template using TypeNumber +template +inline bool callByNumbers(UInt8 type_num1, UInt8 type_num2, F && f) +{ + switch (type_num1) + { + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + //case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + case TypeId::value: return callByTypeAndNumber(type_num2, std::forward(f)); + + default: + break; + }; + + return false; +} } diff --git a/dbms/src/Functions/FunctionsArithmetic.h b/dbms/src/Functions/FunctionsArithmetic.h index d6f4cd3a0f2..8e5743665ac 100644 --- a/dbms/src/Functions/FunctionsArithmetic.h +++ b/dbms/src/Functions/FunctionsArithmetic.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -37,6 +38,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION; + extern const int DECIMAL_OVERFLOW; } @@ -88,8 +90,10 @@ template struct UnaryOperationImpl { using ResultType = typename Op::ResultType; + using ArrayA = typename ColumnVector::Container; + using ArrayC = typename ColumnVector::Container; - static void NO_INLINE vector(const PaddedPODArray & a, PaddedPODArray & c) + static void NO_INLINE vector(const ArrayA & a, ArrayC & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) @@ -107,6 +111,7 @@ template struct PlusImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; + static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) @@ -115,6 +120,12 @@ struct PlusImpl return static_cast(a) + b; } + template + static inline bool apply(A a, B b, Result & c) + { + return common::addOverflow(static_cast(a), b, c); + } + #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; @@ -130,6 +141,7 @@ template struct MultiplyImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; + static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) @@ -137,6 +149,12 @@ struct MultiplyImpl return static_cast(a) * b; } + template + static inline bool apply(A a, B b, Result & c) + { + return common::mulOverflow(static_cast(a), b, c); + } + #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; @@ -151,6 +169,7 @@ template struct MinusImpl { using ResultType = typename NumberTraits::ResultOfSubtraction::Type; + static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) @@ -158,6 +177,12 @@ struct MinusImpl return static_cast(a) - b; } + template + static inline bool apply(A a, B b, Result & c) + { + return common::subOverflow(static_cast(a), b, c); + } + #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; @@ -172,6 +197,7 @@ template struct DivideFloatingImpl { using ResultType = typename NumberTraits::ResultOfFloatingPointDivision::Type; + static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) @@ -551,7 +577,7 @@ using GreatestImpl = std::conditional_t struct NegateImpl { - using ResultType = typename NumberTraits::ResultOfNegate::Type; + using ResultType = std::conditional_t, A, typename NumberTraits::ResultOfNegate::Type>; static inline ResultType apply(A a) { @@ -593,11 +619,13 @@ struct BitNotImpl template struct AbsImpl { - using ResultType = typename NumberTraits::ResultOfAbs::Type; + using ResultType = std::conditional_t, A, typename NumberTraits::ResultOfAbs::Type>; static inline ResultType apply(A a) { - if constexpr (std::is_integral_v && std::is_signed_v) + if constexpr (IsDecimalNumber) + return a < 0 ? A(-a) : a; + else if constexpr (std::is_integral_v && std::is_signed_v) return a < 0 ? static_cast(~a) + 1 : a; else if constexpr (std::is_integral_v && std::is_unsigned_v) return static_cast(a); @@ -687,6 +715,199 @@ struct IntExp10Impl #endif }; + +template struct NativeType { using Type = T; }; +template <> struct NativeType { using Type = Int32; }; +template <> struct NativeType { using Type = Int64; }; +template <> struct NativeType { using Type = Int128; }; + +/// Binary operations for Decimals need scale args +/// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2); +/// * no agrs scale. ScaleR = Scale1 + Scale2; +/// / first arg scale. ScaleR = Scale1 (scale_a = DecimalType::getScale()). +template typename Operation, typename ResultType_> +struct DecimalBinaryOperation +{ + using ResultType = ResultType_; + using NativeResultType = typename NativeType::Type; + using Op = Operation; + using ArrayA = typename ColumnVector::Container; + using ArrayB = typename ColumnVector::Container; + using ArrayC = typename ColumnVector::Container; + + static constexpr bool is_plus_minus = std::is_same_v, PlusImpl> || + std::is_same_v, MinusImpl>; + static constexpr bool is_multiply = std::is_same_v, MultiplyImpl>; + static constexpr bool is_division = std::is_same_v, DivideFloatingImpl>; + static constexpr bool is_compare = std::is_same_v, LeastBaseImpl> || + std::is_same_v, GreatestBaseImpl>; + static constexpr bool is_plus_minus_compare = is_plus_minus || is_compare; + static constexpr bool can_overflow = is_plus_minus || is_multiply; + + static void NO_INLINE vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const ArrayA & a, B b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = a.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a[i], b, scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a[i], b, scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a[i], b); + } + + static void NO_INLINE constant_vector(A a, const ArrayB & b, ArrayC & c, + ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + size_t size = b.size(); + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_a); + return; + } + else if (scale_b != 1) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaled(a, b[i], scale_b); + return; + } + } + else if constexpr (is_division && IsDecimalNumber) + { + for (size_t i = 0; i < size; ++i) + c[i] = applyScaledDiv(a, b[i], scale_a); + return; + } + + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) + c[i] = apply(a, b[i]); + } + + static ResultType constant_constant(A a, B b, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) + { + if constexpr (is_plus_minus_compare) + { + if (scale_a != 1) + return applyScaled(a, b, scale_a); + else if (scale_b != 1) + return applyScaled(a, b, scale_b); + } + else if constexpr (is_division && IsDecimalNumber) + return applyScaledDiv(a, b, scale_a); + return apply(a, b); + } + +private: + /// there's implicit type convertion here + static NativeResultType apply(NativeResultType a, NativeResultType b) + { + if constexpr (can_overflow) + { + NativeResultType res; + if (Op::template apply(a, b, res)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + return res; + } + else + return Op::template apply(a, b); + } + + template + static NativeResultType applyScaled(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_plus_minus_compare) + { + NativeResultType res; + + bool overflow = false; + if constexpr (scale_left) + overflow |= common::mulOverflow(a, scale, a); + else + overflow |= common::mulOverflow(b, scale, b); + + if constexpr (can_overflow) + overflow |= Op::template apply(a, b, res); + else + res = Op::template apply(a, b); + + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + + return res; + } + } + + static NativeResultType applyScaledDiv(NativeResultType a, NativeResultType b, NativeResultType scale) + { + if constexpr (is_division) + { + bool overflow = false; + if constexpr (!IsDecimalNumber) + overflow |= common::mulOverflow(scale, scale, scale); + overflow |= common::mulOverflow(a, scale, a); + if (overflow) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); + + return Op::template apply(a, b); + } + } +}; + + /// Used to indicate undefined operation struct InvalidType; @@ -709,6 +930,16 @@ template constexpr bool IsDateOrDateTime = false; template <> constexpr bool IsDateOrDateTime = true; template <> constexpr bool IsDateOrDateTime = true; +template constexpr bool IsDecimal = false; +template <> constexpr bool IsDecimal> = true; +template <> constexpr bool IsDecimal> = true; +template <> constexpr bool IsDecimal> = true; + +template constexpr bool UseLeftDecimal = false; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; +template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; + template using DataTypeFromFieldType = std::conditional_t, InvalidType, DataTypeNumber>; template