diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md index 93c681b0336..5c7dc6cef09 100644 --- a/CHANGELOG.draft.md +++ b/CHANGELOG.draft.md @@ -1 +1,31 @@ ## RU + +## ClickHouse release 18.10.3, 2018-08-13 + +### Новые возможности: +* поддержка межсерверной репликации по HTTPS +* MurmurHash +* ODBCDriver2 с поддержкой NULL-ов +* поддержка UUID в ключевых колонках (экспериментально) + +### Улучшения: +* добавлена поддержка SETTINGS для движка Kafka +* поддежка пустых кусков после мержей в движках Summing, Collapsing and VersionedCollapsing +* удаление старых записей о полностью выполнившихся мутациях +* исправлена логика REPLACE PARTITION для движка RplicatedMergeTree +* добавлена системная таблица system.merge_tree_settings +* в системную таблицу system.tables добавлены столбцы зависимостей: dependencies_database и dependencies_table +* заменен аллокатор, теперь используется jemalloc вместо tcmalloc +* улучшена валидация connection string ODBC +* удалена поддержка CHECK TABLE для распределенных таблиц +* добавлены stateful тесты (пока без данных) +* добавлена опция конфига max_partition_size_to_drop +* добавлена настройка output_format_json_escape_slashes +* добавлена настройка max_fetch_partition_retries_count +* добавлена настройка prefer_localhost_replica +* добавлены libressl, unixodbc и mariadb-connector-c как сабмодули + +### Исправление ошибок: +* #2786 +* #2777 +* #2795 diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fff4641e24..2ae51712f6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ project (ClickHouse) -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 3.3) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/") @@ -218,7 +218,7 @@ else () set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc") endif () -option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF) +option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) set(NOT_UNBUNDLED 0) else () diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 46d2e9f3ed6..abc30d99e32 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -7,9 +7,9 @@ endif () if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386") set (ARCH_I386 1) endif () -if ( ( ARCH_ARM AND NOT ARCH_AARCH64 ) OR ARCH_I386) +if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386) set (ARCH_32 1) - message (WARNING "Support for 32bit platforms is highly experimental") + message (FATAL_ERROR "32bit platforms are not supported") endif () if (CMAKE_SYSTEM MATCHES "Linux") @@ -24,7 +24,3 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set (COMPILER_CLANG 1) endif () - -if (OS_LINUX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") - set (OS_LINUX_X86_64 1) -endif () diff --git a/cmake/find_rdkafka.cmake b/cmake/find_rdkafka.cmake index 447e93e28a6..dc8e9913bc7 100644 --- a/cmake/find_rdkafka.cmake +++ b/cmake/find_rdkafka.cmake @@ -2,7 +2,7 @@ option (ENABLE_RDKAFKA "Enable kafka" ON) if (ENABLE_RDKAFKA) -if (OS_LINUX_X86_64) +if (OS_LINUX) option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) endif () diff --git a/cmake/find_zlib.cmake b/cmake/find_zlib.cmake index 501a50f688b..f1f7c128d7b 100644 --- a/cmake/find_zlib.cmake +++ b/cmake/find_zlib.cmake @@ -1,4 +1,4 @@ -if (NOT OS_FREEBSD) +if (NOT OS_FREEBSD AND NOT APPLE) option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED}) endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 6faa507a356..ed394df3728 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,10 +1,10 @@ # Third-party libraries may have substandard code. if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z") endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 788ce74618a..38c7dbc1022 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,11 +1,11 @@ # This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54405 CACHE STRING "") +set(VERSION_REVISION 54406 CACHE STRING "") set(VERSION_MAJOR 18 CACHE STRING "") -set(VERSION_MINOR 10 CACHE STRING "") -set(VERSION_PATCH 3 CACHE STRING "") -set(VERSION_GITHASH 1fa1b34f1ab01ea2e1a833eebd36a4806e529f52 CACHE STRING "") -set(VERSION_DESCRIBE v18.10.3-testing CACHE STRING "") -set(VERSION_STRING 18.10.3 CACHE STRING "") +set(VERSION_MINOR 11 CACHE STRING "") +set(VERSION_PATCH 0 CACHE STRING "") +set(VERSION_GITHASH 76af46ed5d223b3a7af92e31eae291174da16355 CACHE STRING "") +set(VERSION_DESCRIBE v18.11.0-testing CACHE STRING "") +set(VERSION_STRING 18.11.0 CACHE STRING "") # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index d58925bef3c..afb13d6341f 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1,3 +1,5 @@ +#include "TestHint.h" + #include #include #include @@ -20,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -90,102 +93,6 @@ namespace ErrorCodes } -/// Checks expected server and client error codes in testmode. -/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }". -class TestHint -{ -public: - TestHint(bool enabled_, const String & query) - : enabled(enabled_), - server_error(0), - client_error(0) - { - if (!enabled_) - return; - - size_t pos = query.find("--"); - if (pos != String::npos && query.find("--", pos + 2) != String::npos) - return; /// It's not last comment. Hint belongs to commented query. - - if (pos != String::npos) - { - pos = query.find('{', pos + 2); - if (pos != String::npos) - { - String hint = query.substr(pos + 1); - pos = hint.find('}'); - hint.resize(pos); - parse(hint); - } - } - } - - /// @returns true if it's possible to continue without reconnect - bool checkActual(int & actual_server_error, int & actual_client_error, - bool & got_exception, std::unique_ptr & last_exception) const - { - if (!enabled) - return true; - - if (allErrorsExpected(actual_server_error, actual_client_error)) - { - got_exception = false; - last_exception.reset(); - actual_server_error = 0; - actual_client_error = 0; - return false; - } - - if (lostExpectedError(actual_server_error, actual_client_error)) - { - std::cerr << "Success when error expected. It expects server error " - << server_error << ", client error " << client_error << "." << std::endl; - got_exception = true; - last_exception = std::make_unique("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS - return false; - } - - return true; - } - - int serverError() const { return server_error; } - int clientError() const { return client_error; } - -private: - bool enabled; - int server_error; - int client_error; - - void parse(const String & hint) - { - std::stringstream ss; - ss << hint; - while (!ss.eof()) - { - String item; - ss >> item; - if (item.empty()) - break; - - if (item == "serverError") - ss >> server_error; - else if (item == "clientError") - ss >> client_error; - } - } - - bool allErrorsExpected(int actual_server_error, int actual_client_error) const - { - return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); - } - - bool lostExpectedError(int actual_server_error, int actual_client_error) const - { - return (server_error && !actual_server_error) || (client_error && !actual_client_error); - } -}; - - class Client : public Poco::Util::Application { public: @@ -235,6 +142,11 @@ private: std::optional out_file_buf; BlockOutputStreamPtr block_out_stream; + /// The user could specify special file for server logs (stderr by default) + std::unique_ptr out_logs_buf; + String server_logs_file; + BlockOutputStreamPtr logs_out_stream; + String home_path; String current_profile; @@ -408,20 +320,10 @@ private: /// If exception code isn't zero, we should return non-zero return code anyway. return e.code() ? e.code() : -1; } - catch (const Poco::Exception & e) - { - std::cerr << "Poco::Exception: " << e.displayText() << std::endl; - return ErrorCodes::POCO_EXCEPTION; - } - catch (const std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return ErrorCodes::STD_EXCEPTION; - } catch (...) { - std::cerr << "Unknown exception" << std::endl; - return ErrorCodes::UNKNOWN_EXCEPTION; + std::cerr << getCurrentExceptionMessage(false) << std::endl; + return getCurrentExceptionCode(); } } @@ -469,7 +371,12 @@ private: format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size); insert_format = "Values"; - insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); + + /// Setting value from cmd arg overrides one from config + if (context.getSettingsRef().max_insert_block_size.changed) + insert_format_max_block_size = context.getSettingsRef().max_insert_block_size; + else + insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); if (!is_interactive) { @@ -782,6 +689,7 @@ private: { const char * pos = begin; ASTPtr ast = parseQuery(pos, end, true); + if (!ast) { if (ignore_error) @@ -797,7 +705,7 @@ private: return true; } - ASTInsertQuery * insert = typeid_cast(&*ast); + ASTInsertQuery * insert = typeid_cast(ast.get()); if (insert && insert->data) { @@ -990,7 +898,7 @@ private: /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. sendData(sample); - receivePacket(); + receiveEndOfQuery(); } } @@ -1072,6 +980,11 @@ private: connection->sendData(block); processed_rows += block.rows(); + /// Check if server send Log packet + auto packet_type = connection->checkPacket(); + if (packet_type && *packet_type == Protocol::Server::Log) + receiveAndProcessPacket(); + if (!block) break; } @@ -1083,18 +996,28 @@ private: /// Flush all buffers. void resetOutput() { - block_out_stream = nullptr; + block_out_stream.reset(); + logs_out_stream.reset(); + if (pager_cmd) { pager_cmd->in.close(); pager_cmd->wait(); } pager_cmd = nullptr; + if (out_file_buf) { out_file_buf->next(); out_file_buf.reset(); } + + if (out_logs_buf) + { + out_logs_buf->next(); + out_logs_buf.reset(); + } + std_out.next(); } @@ -1127,7 +1050,7 @@ private: continue; /// If there is no new data, continue checking whether the query was cancelled after a timeout. } - if (!receivePacket()) + if (!receiveAndProcessPacket()) break; } @@ -1138,7 +1061,7 @@ private: /// Receive a part of the result, or progress info or an exception and process it. /// Returns true if one should continue receiving packets. - bool receivePacket() + bool receiveAndProcessPacket() { Connection::Packet packet = connection->receivePacket(); @@ -1169,6 +1092,10 @@ private: last_exception = std::move(packet.exception); return false; + case Protocol::Server::Log: + onLogData(packet.block); + return true; + case Protocol::Server::EndOfStream: onEndOfStream(); return false; @@ -1182,22 +1109,59 @@ private: /// Receive the block that serves as an example of the structure of table where data will be inserted. bool receiveSampleBlock(Block & out) { - Connection::Packet packet = connection->receivePacket(); - - switch (packet.type) + while (true) { - case Protocol::Server::Data: - out = packet.block; - return true; + Connection::Packet packet = connection->receivePacket(); - case Protocol::Server::Exception: - onException(*packet.exception); - last_exception = std::move(packet.exception); - return false; + switch (packet.type) + { + case Protocol::Server::Data: + out = packet.block; + return true; - default: - throw NetException("Unexpected packet from server (expected Data, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Data, Exception or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } + } + } + + + /// Process Log packets, exit when recieve Exception or EndOfStream + bool receiveEndOfQuery() + { + while (true) + { + Connection::Packet packet = connection->receivePacket(); + + switch (packet.type) + { + case Protocol::Server::EndOfStream: + onEndOfStream(); + return true; + + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Exception, EndOfStream or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } } } @@ -1253,6 +1217,38 @@ private: } + void initLogsOutputStream() + { + if (!logs_out_stream) + { + WriteBuffer * wb = out_logs_buf.get(); + + if (!out_logs_buf) + { + if (server_logs_file.empty()) + { + /// Use stderr by default + out_logs_buf = std::make_unique(STDERR_FILENO); + wb = out_logs_buf.get(); + } + else if (server_logs_file == "-") + { + /// Use stdout if --server_logs_file=- specified + wb = &std_out; + } + else + { + out_logs_buf = std::make_unique(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); + wb = out_logs_buf.get(); + } + } + + logs_out_stream = std::make_shared(*wb); + logs_out_stream->writePrefix(); + } + } + + void onData(Block & block) { if (written_progress_chars) @@ -1276,6 +1272,14 @@ private: } + void onLogData(Block & block) + { + initLogsOutputStream(); + logs_out_stream->write(block); + logs_out_stream->flush(); + } + + void onTotals(Block & block) { initBlockOutputStream(block); @@ -1436,6 +1440,9 @@ private: if (block_out_stream) block_out_stream->writeSuffix(); + if (logs_out_stream) + logs_out_stream->writeSuffix(); + resetOutput(); if (is_interactive && !written_first_block) @@ -1511,7 +1518,9 @@ public: ioctl(0, TIOCGWINSZ, &terminal_size); - unsigned line_length = boost::program_options::options_description::m_default_line_length; + namespace po = boost::program_options; + + unsigned line_length = po::options_description::m_default_line_length; unsigned min_description_length = line_length / 2; if (!stdin_is_not_tty) { @@ -1519,28 +1528,28 @@ public: min_description_length = std::min(min_description_length, line_length - 2); } -#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value (), DESCRIPTION) +#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value (), DESCRIPTION) /// Main commandline options related to client functionality and all parameters from Settings. - boost::program_options::options_description main_description("Main options", line_length, min_description_length); + po::options_description main_description("Main options", line_length, min_description_length); main_description.add_options() ("help", "produce help message") - ("config-file,c", boost::program_options::value(), "config-file path") - ("host,h", boost::program_options::value()->default_value("localhost"), "server host") - ("port", boost::program_options::value()->default_value(9000), "server port") + ("config-file,c", po::value(), "config-file path") + ("host,h", po::value()->default_value("localhost"), "server host") + ("port", po::value()->default_value(9000), "server port") ("secure,s", "secure") - ("user,u", boost::program_options::value()->default_value("default"), "user") - ("password", boost::program_options::value(), "password") + ("user,u", po::value()->default_value("default"), "user") + ("password", po::value(), "password") ("ask-password", "ask-password") - ("query_id", boost::program_options::value(), "query_id") - ("query,q", boost::program_options::value(), "query") - ("database,d", boost::program_options::value(), "database") - ("pager", boost::program_options::value(), "pager") + ("query_id", po::value(), "query_id") + ("query,q", po::value(), "query") + ("database,d", po::value(), "database") + ("pager", po::value(), "pager") ("multiline,m", "multiline") ("multiquery,n", "multiquery") + ("format,f", po::value(), "default output format") ("testmode,T", "enable test hints in comments") ("ignore-error", "do not stop processing in multiquery mode") - ("format,f", boost::program_options::value(), "default output format") ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command") ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)") ("stacktrace", "print stack traces of exceptions") @@ -1548,27 +1557,29 @@ public: ("version,V", "print version information and exit") ("version-clean", "print version in machine-readable format and exit") ("echo", "in batch mode, print query before execution") - ("max_client_network_bandwidth", boost::program_options::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") - ("compression", boost::program_options::value(), "enable or disable compression") + ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") + ("compression", po::value(), "enable or disable compression") + ("log-level", po::value(), "client log level") + ("server_logs_file", po::value(), "put server logs into specified file") APPLY_FOR_SETTINGS(DECLARE_SETTING) ; #undef DECLARE_SETTING /// Commandline options related to external tables. - boost::program_options::options_description external_description("External tables options"); + po::options_description external_description("External tables options"); external_description.add_options() - ("file", boost::program_options::value(), "data file or - for stdin") - ("name", boost::program_options::value()->default_value("_data"), "name of the table") - ("format", boost::program_options::value()->default_value("TabSeparated"), "data format") - ("structure", boost::program_options::value(), "structure") - ("types", boost::program_options::value(), "types") + ("file", po::value(), "data file or - for stdin") + ("name", po::value()->default_value("_data"), "name of the table") + ("format", po::value()->default_value("TabSeparated"), "data format") + ("structure", po::value(), "structure") + ("types", po::value(), "types") ; /// Parse main commandline options. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( common_arguments.size(), common_arguments.data()).options(main_description).run(); - boost::program_options::variables_map options; - boost::program_options::store(parsed, options); + po::variables_map options; + po::store(parsed, options); if (options.count("version") || options.count("V")) { @@ -1591,14 +1602,17 @@ public: exit(0); } + if (options.count("log-level")) + Poco::Logger::root().setLevel(options["log-level"].as()); + size_t number_of_external_tables_with_stdin_source = 0; for (size_t i = 0; i < external_tables_arguments.size(); ++i) { /// Parse commandline options related to external tables. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run(); - boost::program_options::variables_map external_options; - boost::program_options::store(parsed, external_options); + po::variables_map external_options; + po::store(parsed, external_options); try { @@ -1672,6 +1686,8 @@ public: max_client_network_bandwidth = options["max_client_network_bandwidth"].as(); if (options.count("compression")) config().setBool("compression", options["compression"].as()); + if (options.count("server_logs_file")) + server_logs_file = options["server_logs_file"].as(); } }; @@ -1691,6 +1707,11 @@ int mainEntryClickHouseClient(int argc, char ** argv) std::cerr << "Bad arguments: " << e.what() << std::endl; return 1; } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; + return 1; + } return client.run(); } diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h new file mode 100644 index 00000000000..790e58ee7fe --- /dev/null +++ b/dbms/programs/client/TestHint.h @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +/// Checks expected server and client error codes in testmode. +/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }". +class TestHint +{ +public: + TestHint(bool enabled_, const String & query) + : enabled(enabled_) + { + if (!enabled_) + return; + + /// TODO: This is absolutely wrong. Fragment may be contained inside string literal. + size_t pos = query.find("--"); + + if (pos != String::npos && query.find("--", pos + 2) != String::npos) + return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query. + + if (pos != String::npos) + { + /// TODO: This is also wrong. Comment may already have ended by line break. + pos = query.find('{', pos + 2); + + if (pos != String::npos) + { + String hint = query.substr(pos + 1); + + /// TODO: And this is wrong for the same reason. + pos = hint.find('}'); + hint.resize(pos); + parse(hint); + } + } + } + + /// @returns true if it's possible to continue without reconnect + bool checkActual(int & actual_server_error, int & actual_client_error, + bool & got_exception, std::unique_ptr & last_exception) const + { + if (!enabled) + return true; + + if (allErrorsExpected(actual_server_error, actual_client_error)) + { + got_exception = false; + last_exception.reset(); + actual_server_error = 0; + actual_client_error = 0; + return false; + } + + if (lostExpectedError(actual_server_error, actual_client_error)) + { + std::cerr << "Success when error expected. It expects server error " + << server_error << ", client error " << client_error << "." << std::endl; + got_exception = true; + last_exception = std::make_unique("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS + return false; + } + + return true; + } + + int serverError() const { return server_error; } + int clientError() const { return client_error; } + +private: + bool enabled = false; + int server_error = 0; + int client_error = 0; + + void parse(const String & hint) + { + std::stringstream ss; + ss << hint; + while (!ss.eof()) + { + String item; + ss >> item; + if (item.empty()) + break; + + if (item == "serverError") + ss >> server_error; + else if (item == "clientError") + ss >> client_error; + } + } + + bool allErrorsExpected(int actual_server_error, int actual_client_error) const + { + return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); + } + + bool lostExpectedError(int actual_server_error, int actual_client_error) const + { + return (server_error && !actual_server_error) || (client_error && !actual_client_error); + } +}; + +} diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index a000401f02c..fab1d5af4c5 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -2143,6 +2144,9 @@ void ClusterCopierApp::mainImpl() context->addDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); + /// Initialize query scope just in case. + CurrentThread::QueryScope query_scope(*context); + auto copier = std::make_unique(task_path, host_id, default_database, *context); copier->setSafeMode(is_safe_mode); copier->setCopyFaultProbability(copy_fault_probability); diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index ae9e9b7b720..4528ad40128 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -270,6 +271,9 @@ void LocalServer::processQueries() context->setCurrentQueryId(""); applyCmdSettings(*context); + /// Use the same query_id (and thread group) for all queries + CurrentThread::QueryScope query_scope_holder(*context); + bool echo_query = config().hasOption("echo") || config().hasOption("verbose"); std::exception_ptr exception; diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt index dcdfa7009d4..df68a8c546c 100644 --- a/dbms/programs/odbc-bridge/CMakeLists.txt +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -1,12 +1,30 @@ add_library (clickhouse-odbc-bridge-lib - Handlers.cpp + PingHandler.cpp + MainHandler.cpp + ColumnInfoHandler.cpp HandlerFactory.cpp ODBCBridge.cpp - ) + validateODBCConnectionString.cpp +) -target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon) +target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms) target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) +if (USE_POCO_SQLODBC) + target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS}) +endif () + +if (USE_POCO_DATAODBC) + target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) +endif() + + +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () + if (CLICKHOUSE_SPLIT_BINARY) add_executable (clickhouse-odbc-bridge odbc-bridge.cpp) target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib) diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp new file mode 100644 index 00000000000..3b4c7f42cea --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -0,0 +1,123 @@ +#include "ColumnInfoHandler.h" +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "validateODBCConnectionString.h" + +namespace DB +{ +namespace +{ + DataTypePtr getDataType(SQLSMALLINT type) + { + const auto & factory = DataTypeFactory::instance(); + + switch (type) + { + case SQL_INTEGER: + return factory.get("Int32"); + case SQL_SMALLINT: + return factory.get("Int16"); + case SQL_FLOAT: + return factory.get("Float32"); + case SQL_REAL: + return factory.get("Float32"); + case SQL_DOUBLE: + return factory.get("Float64"); + case SQL_DATETIME: + return factory.get("DateTime"); + case SQL_TYPE_TIMESTAMP: + return factory.get("DateTime"); + case SQL_TYPE_DATE: + return factory.get("Date"); + default: + return factory.get("String"); + } + } +} + +void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("table")) + { + process_error("No 'table' param in request URL"); + return; + } + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + std::string table_name = params.get("table"); + std::string connection_string = params.get("connection_string"); + LOG_TRACE(log, "Will fetch info for table '" << table_name << "'"); + LOG_TRACE(log, "Got connection str '" << connection_string << "'"); + + try + { + Poco::Data::ODBC::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + SQLHDBC hdbc = session.dbc().handle(); + + SQLHSTMT hstmt = nullptr; + + if (Poco::Data::ODBC::Utility::isError(SQLAllocStmt(hdbc, &hstmt))) + throw Poco::Data::ODBC::ODBCException("Could not allocate connection handle."); + + SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP)); + + /// TODO Why not do SQLColumns instead? + std::string query = "SELECT * FROM " + table_name + " WHERE 1 = 0"; + if (Poco::Data::ODBC::Utility::isError(Poco::Data::ODBC::SQLPrepare(hstmt, reinterpret_cast(&query[0]), query.size()))) + throw Poco::Data::ODBC::DescriptorException(session.dbc()); + + if (Poco::Data::ODBC::Utility::isError(SQLExecute(hstmt))) + throw Poco::Data::ODBC::StatementException(hstmt); + + SQLSMALLINT cols = 0; + if (Poco::Data::ODBC::Utility::isError(SQLNumResultCols(hstmt, &cols))) + throw Poco::Data::ODBC::StatementException(hstmt); + + /// TODO cols not checked + + NamesAndTypesList columns; + for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol) + { + SQLSMALLINT type = 0; + /// TODO Why 301? + SQLCHAR column_name[301]; + /// TODO Result is not checked. + Poco::Data::ODBC::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), NULL, &type, NULL, NULL, NULL); + columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); + } + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + writeStringBinary(columns.toString(), out); + } + catch (...) + { + process_error("Error getting columns from ODBC '" + getCurrentExceptionMessage(false) + "'"); + tryLogCurrentException(log); + } +} +} +#endif diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.h b/dbms/programs/odbc-bridge/ColumnInfoHandler.h new file mode 100644 index 00000000000..426cea15b34 --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include +#include + +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +/** The structure of the table is taken from the query "SELECT * FROM table WHERE 1=0". + * TODO: It would be much better to utilize ODBC methods dedicated for columns description. + * If there is no such table, an exception is thrown. + */ +namespace DB +{ +class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler +{ +public: + ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + size_t keep_alive_timeout; + std::shared_ptr context; +}; +} +#endif diff --git a/dbms/programs/odbc-bridge/HandlerFactory.cpp b/dbms/programs/odbc-bridge/HandlerFactory.cpp index 9a3824cef79..f552203174e 100644 --- a/dbms/programs/odbc-bridge/HandlerFactory.cpp +++ b/dbms/programs/odbc-bridge/HandlerFactory.cpp @@ -1,4 +1,6 @@ #include "HandlerFactory.h" +#include "PingHandler.h" +#include "ColumnInfoHandler.h" #include #include @@ -9,15 +11,24 @@ namespace DB { Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) { - const auto & uri = request.getURI(); - LOG_TRACE(log, "Request URI: " + uri); + Poco::URI uri{request.getURI()}; + LOG_TRACE(log, "Request URI: " + uri.toString()); - if (uri == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) + if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) return new PingHandler(keep_alive_timeout); if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - return new ODBCHandler(pool_map, keep_alive_timeout, context); + { + if (uri.getPath() == "/columns_info") +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + return new ODBCColumnsInfoHandler(keep_alive_timeout, context); +#else + return nullptr; +#endif + else + return new ODBCHandler(pool_map, keep_alive_timeout, context); + } return nullptr; } } diff --git a/dbms/programs/odbc-bridge/HandlerFactory.h b/dbms/programs/odbc-bridge/HandlerFactory.h index 92a0267a16c..4fe00ffca98 100644 --- a/dbms/programs/odbc-bridge/HandlerFactory.h +++ b/dbms/programs/odbc-bridge/HandlerFactory.h @@ -3,7 +3,8 @@ #include #include #include -#include "Handlers.h" +#include "MainHandler.h" +#include "ColumnInfoHandler.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -13,7 +14,7 @@ namespace DB { -/** Factory for '/ping' and '/' handlers. +/** Factory for '/ping', '/' and '/columns_info' handlers. * Also stores Session pools for ODBC connections */ class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory diff --git a/dbms/programs/odbc-bridge/Handlers.cpp b/dbms/programs/odbc-bridge/MainHandler.cpp similarity index 86% rename from dbms/programs/odbc-bridge/Handlers.cpp rename to dbms/programs/odbc-bridge/MainHandler.cpp index 9e8cb5dee1a..e7764907dbe 100644 --- a/dbms/programs/odbc-bridge/Handlers.cpp +++ b/dbms/programs/odbc-bridge/MainHandler.cpp @@ -1,30 +1,24 @@ -#include "Handlers.h" -#include +#include "MainHandler.h" + +#include "validateODBCConnectionString.h" #include -#include #include #include #include #include #include -#include #include #include #include -#include -#include #include #include #include +#include #include + namespace DB { -namespace ErrorCodes -{ - extern const int BAD_REQUEST_PARAMETER; -} - namespace { std::unique_ptr parseColumns(std::string && column_string) @@ -44,7 +38,7 @@ ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str) if (!pool_map->count(connection_str)) { pool_map->emplace(connection_str, createAndCheckResizePocoSessionPool([connection_str] { - return std::make_shared("ODBC", connection_str); + return std::make_shared("ODBC", validateODBCConnectionString(connection_str)); })); } return pool_map->at(connection_str); @@ -129,18 +123,4 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne tryLogCurrentException(log); } } - -void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response) -{ - try - { - setResponseDefaultHeaders(response, keep_alive_timeout); - const char * data = "Ok.\n"; - response.sendBuffer(data, strlen(data)); - } - catch (...) - { - tryLogCurrentException("PingHandler"); - } -} } diff --git a/dbms/programs/odbc-bridge/Handlers.h b/dbms/programs/odbc-bridge/MainHandler.h similarity index 77% rename from dbms/programs/odbc-bridge/Handlers.h rename to dbms/programs/odbc-bridge/MainHandler.h index a8cb65015d7..ae139f393f8 100644 --- a/dbms/programs/odbc-bridge/Handlers.h +++ b/dbms/programs/odbc-bridge/MainHandler.h @@ -46,15 +46,4 @@ private: PoolPtr getPool(const std::string & connection_str); }; -/** Simple ping handler, answers "Ok." to GET request - */ -class PingHandler : public Poco::Net::HTTPRequestHandler -{ -public: - PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {} - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - size_t keep_alive_timeout; -}; } diff --git a/dbms/programs/odbc-bridge/PingHandler.cpp b/dbms/programs/odbc-bridge/PingHandler.cpp new file mode 100644 index 00000000000..b0313e46bf3 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.cpp @@ -0,0 +1,22 @@ +#include "PingHandler.h" +#include +#include +#include +#include + +namespace DB +{ +void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response) +{ + try + { + setResponseDefaultHeaders(response, keep_alive_timeout); + const char * data = "Ok.\n"; + response.sendBuffer(data, strlen(data)); + } + catch (...) + { + tryLogCurrentException("PingHandler"); + } +} +} diff --git a/dbms/programs/odbc-bridge/PingHandler.h b/dbms/programs/odbc-bridge/PingHandler.h new file mode 100644 index 00000000000..d8109a50bb6 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace DB +{ +/** Simple ping handler, answers "Ok." to GET request + */ +class PingHandler : public Poco::Net::HTTPRequestHandler +{ +public: + PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {} + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + size_t keep_alive_timeout; +}; +} diff --git a/dbms/programs/odbc-bridge/tests/CMakeLists.txt b/dbms/programs/odbc-bridge/tests/CMakeLists.txt new file mode 100644 index 00000000000..5240a917429 --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) +target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib) diff --git a/dbms/src/Common/tests/validate-odbc-connection-string.cpp b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp similarity index 89% rename from dbms/src/Common/tests/validate-odbc-connection-string.cpp rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp index f668cd07fcd..c4558811f77 100644 --- a/dbms/src/Common/tests/validate-odbc-connection-string.cpp +++ b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include "../validateODBCConnectionString.h" using namespace DB; diff --git a/dbms/src/Common/tests/validate-odbc-connection-string.reference b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference similarity index 100% rename from dbms/src/Common/tests/validate-odbc-connection-string.reference rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference diff --git a/dbms/src/Common/tests/validate-odbc-connection-string.sh b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh similarity index 100% rename from dbms/src/Common/tests/validate-odbc-connection-string.sh rename to dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh diff --git a/dbms/src/Common/validateODBCConnectionString.cpp b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp similarity index 99% rename from dbms/src/Common/validateODBCConnectionString.cpp rename to dbms/programs/odbc-bridge/validateODBCConnectionString.cpp index 313de4885af..a817a01c288 100644 --- a/dbms/src/Common/validateODBCConnectionString.cpp +++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include "validateODBCConnectionString.h" namespace DB diff --git a/dbms/src/Common/validateODBCConnectionString.h b/dbms/programs/odbc-bridge/validateODBCConnectionString.h similarity index 100% rename from dbms/src/Common/validateODBCConnectionString.h rename to dbms/programs/odbc-bridge/validateODBCConnectionString.h diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index f2a114c4702..0db338e0fbf 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -9,10 +9,11 @@ #include -#include +#include #include #include #include +#include #include #include #include @@ -208,6 +209,13 @@ void HTTPHandler::processQuery( Poco::Net::HTTPServerResponse & response, Output & used_output) { + Context context = server.context(); + context.setGlobalContext(server.context()); + + /// It will forcibly detach query even if unexpected error ocurred and detachQuery() was not called + /// Normal detaching is happen in BlockIO callbacks + CurrentThread::QueryScope query_scope_holder(context); + LOG_TRACE(log, "Request URI: " << request.getURI()); std::istream & istr = request.stream(); @@ -257,14 +265,9 @@ void HTTPHandler::processQuery( } std::string query_id = params.get("query_id", ""); - - const auto & config = server.config(); - - Context context = server.context(); - context.setGlobalContext(server.context()); - context.setUser(user, password, request.clientAddress(), quota_key); context.setCurrentQueryId(query_id); + CurrentThread::attachQueryContext(context); /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. @@ -273,6 +276,7 @@ void HTTPHandler::processQuery( String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); + const auto & config = server.config(); if (session_is_set) { @@ -421,34 +425,45 @@ void HTTPHandler::processQuery( std::unique_ptr in; - // Used in case of POST request with form-data, but it not to be expectd to be deleted after that scope + static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", + "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"}; + + Names reserved_param_suffixes; + + auto param_could_be_skipped = [&] (const String & name) + { + if (reserved_param_names.count(name)) + return true; + + for (const String & suffix : reserved_param_suffixes) + { + if (endsWith(name, suffix)) + return true; + } + + return false; + }; + + /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope. std::string full_query; /// Support for "external data for query processing". if (startsWith(request.getContentType().data(), "multipart/form-data")) { ExternalTablesHandler handler(context, params); - - /// Params are of both form params POST and uri (GET params) params.load(request, istr, handler); - for (const auto & it : params) - { - if (it.first == "query") - { - full_query += it.second; - } - } - in = std::make_unique(full_query); + /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters. + reserved_param_suffixes.emplace_back("_format"); + reserved_param_suffixes.emplace_back("_types"); + reserved_param_suffixes.emplace_back("_structure"); - /// Erase unneeded parameters to avoid confusing them later with context settings or query - /// parameters. - for (const auto & it : handler.names) - { - params.erase(it + "_format"); - params.erase(it + "_types"); - params.erase(it + "_structure"); - } + /// Params are of both form params POST and uri (GET params) + for (const auto & it : params) + if (it.first == "query") + full_query += it.second; + + in = std::make_unique(full_query); } else in = std::make_unique(*in_param, *in_post_maybe_compressed); @@ -475,11 +490,6 @@ void HTTPHandler::processQuery( auto readonly_before_query = settings.readonly; - NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", - "session_id", "session_timeout", "session_check" - }; - for (auto it = params.begin(); it != params.end(); ++it) { if (it->first == "database") @@ -490,7 +500,7 @@ void HTTPHandler::processQuery( { context.setDefaultFormat(it->second); } - else if (reserved_param_names.find(it->first) != reserved_param_names.end()) + else if (param_could_be_skipped(it->first)) { } else diff --git a/dbms/programs/server/MetricsTransmitter.cpp b/dbms/programs/server/MetricsTransmitter.cpp index 278347a6774..ed87eb5cf9f 100644 --- a/dbms/programs/server/MetricsTransmitter.cpp +++ b/dbms/programs/server/MetricsTransmitter.cpp @@ -81,7 +81,7 @@ void MetricsTransmitter::transmit(std::vector & prev_count { for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) { - const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed); + const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed); const auto counter_increment = counter - prev_counters[i]; prev_counters[i] = counter; diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 153f48c9aef..3a55add70af 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -365,6 +366,13 @@ int Server::main(const std::vector & /*args*/) dns_cache_updater = std::make_unique(*global_context); } + if (!TaskStatsInfoGetter::checkProcessHasRequiredPermissions()) + { + LOG_INFO(log, "It looks like the process has not CAP_NET_ADMIN capability, some performance statistics will be disabled." + " It could happen due to incorrect clickhouse package installation." + " You could resolve the problem manually calling 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'"); + } + { Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 4f75f7dd6a7..c2f8d34b5d2 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -1,7 +1,15 @@ -#include "TCPHandler.h" - #include +#include #include +#include + +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -15,14 +23,13 @@ #include #include #include +#include #include #include -#include -#include -#include -#include -#include -#include +#include + +#include "TCPHandler.h" + namespace DB { @@ -140,13 +147,29 @@ void TCPHandler::runImpl() if (!receivePacket()) continue; - /// Get blocks of temporary tables - readData(global_settings); + CurrentThread::initializeQuery(); - /// Reset the input stream, as we received an empty block while receiving external table data. - /// So, the stream has been marked as cancelled and we can't read from it anymore. - state.block_in.reset(); - state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + /// Should we send internal logs to client? + if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS + && query_context.getSettingsRef().send_logs_level.value != "none") + { + state.logs_queue = std::make_shared(); + state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context.getSettingsRef().send_logs_level.value); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue); + } + + query_context.setExternalTablesInitializer([&global_settings, this] (Context & context) { + if (&context != &query_context) + throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); + + /// Get blocks of temporary tables + readData(global_settings); + + /// Reset the input stream, as we received an empty block while receiving external table data. + /// So, the stream has been marked as cancelled and we can't read from it anymore. + state.block_in.reset(); + state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + }); /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage); @@ -163,8 +186,9 @@ void TCPHandler::runImpl() else processOrdinaryQuery(); - sendEndOfStream(); + sendLogs(); + sendEndOfStream(); state.reset(); } catch (const Exception & e) @@ -209,7 +233,20 @@ void TCPHandler::runImpl() try { if (exception) + { + try + { + /// Try to send logs to client, but it could be risky too + /// Assume that we can't break output here + sendLogs(); + } + catch (...) + { + tryLogCurrentException(log, "Can't send logs to client"); + } + sendException(*exception); + } } catch (...) { @@ -220,6 +257,9 @@ void TCPHandler::runImpl() try { + /// It will forcibly detach query even if unexpected error ocсurred and detachQuery() was not called + CurrentThread::detachQueryIfNotDetached(); + state.reset(); } catch (...) @@ -252,12 +292,14 @@ void TCPHandler::readData(const Settings & global_settings) constexpr size_t min_poll_interval = 5000; // 5 ms size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); - while (1) + sendLogs(); + + while (true) { Stopwatch watch(CLOCK_MONOTONIC_COARSE); /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. - while (1) + while (true) { if (static_cast(*in).poll(poll_interval)) break; @@ -289,6 +331,8 @@ void TCPHandler::readData(const Settings & global_settings) /// We accept and process data. And if they are over, then we leave. if (!receivePacket()) break; + + sendLogs(); } } @@ -346,6 +390,8 @@ void TCPHandler::processOrdinaryQuery() sendProgress(); } + sendLogs(); + if (async_in.poll(query_context.getSettingsRef().interactive_delay / 1000)) { /// There is the following result block. @@ -368,6 +414,7 @@ void TCPHandler::processOrdinaryQuery() sendExtremes(); sendProfileInfo(); sendProgress(); + sendLogs(); } sendData(block); @@ -692,11 +739,14 @@ void TCPHandler::initBlockOutput(const Block & block) { if (!state.block_out) { - if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_out = std::make_shared( - *out, CompressionSettings(query_context.getSettingsRef())); - else - state.maybe_compressed_out = out; + if (!state.maybe_compressed_out) + { + if (state.compression == Protocol::Compression::Enable) + state.maybe_compressed_out = std::make_shared( + *out, CompressionSettings(query_context.getSettingsRef())); + else + state.maybe_compressed_out = out; + } state.block_out = std::make_shared( *state.maybe_compressed_out, @@ -705,6 +755,18 @@ void TCPHandler::initBlockOutput(const Block & block) } } +void TCPHandler::initLogsBlockOutput(const Block & block) +{ + if (!state.logs_block_out) + { + /// Use uncompressed stream since log blocks usually contain only one row + state.logs_block_out = std::make_shared( + *out, + client_revision, + block.cloneEmpty()); + } +} + bool TCPHandler::isQueryCancelled() { @@ -745,6 +807,7 @@ void TCPHandler::sendData(const Block & block) initBlockOutput(block); writeVarUInt(Protocol::Server::Data, *out); + /// Send external table name (empty name is the main table) writeStringBinary("", *out); state.block_out->write(block); @@ -753,6 +816,19 @@ void TCPHandler::sendData(const Block & block) } +void TCPHandler::sendLogData(const Block & block) +{ + initLogsBlockOutput(block); + + writeVarUInt(Protocol::Server::Log, *out); + /// Send log tag (empty tag is the default tag) + writeStringBinary("", *out); + + state.logs_block_out->write(block); + out->next(); +} + + void TCPHandler::sendException(const Exception & e) { writeVarUInt(Protocol::Server::Exception, *out); @@ -784,6 +860,37 @@ void TCPHandler::sendProgress() } +void TCPHandler::sendLogs() +{ + if (!state.logs_queue) + return; + + MutableColumns logs_columns; + MutableColumns curr_logs_columns; + size_t rows = 0; + + for (; state.logs_queue->tryPop(curr_logs_columns); ++rows) + { + if (rows == 0) + { + logs_columns = std::move(curr_logs_columns); + } + else + { + for (size_t j = 0; j < logs_columns.size(); ++j) + logs_columns[j]->insertRangeFrom(*curr_logs_columns[j], 0, curr_logs_columns[j]->size()); + } + } + + if (rows > 0) + { + Block block = InternalTextLogsQueue::getSampleBlock(); + block.setColumns(std::move(logs_columns)); + sendLogData(block); + } +} + + void TCPHandler::run() { try diff --git a/dbms/programs/server/TCPHandler.h b/dbms/programs/server/TCPHandler.h index af122513cf7..1969d02b48b 100644 --- a/dbms/programs/server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -5,16 +5,18 @@ #include #include #include -#include #include #include -#include +#include #include #include +#include +#include #include #include "IServer.h" + namespace CurrentMetrics { extern const Metric TCPConnection; @@ -63,6 +65,9 @@ struct QueryState /// Timeouts setter for current query std::unique_ptr timeout_setter; + /// A queue with internal logs that will be passed to client + InternalTextLogsQueuePtr logs_queue; + BlockOutputStreamPtr logs_block_out; void reset() { @@ -140,8 +145,10 @@ private: void sendHello(); void sendData(const Block & block); /// Write a block to the network. + void sendLogData(const Block & block); void sendException(const Exception & e); void sendProgress(); + void sendLogs(); void sendEndOfStream(); void sendProfileInfo(); void sendTotals(); @@ -150,6 +157,7 @@ private: /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled. void initBlockInput(); void initBlockOutput(const Block & block); + void initLogsBlockOutput(const Block & block); bool isQueryCancelled(); diff --git a/dbms/programs/server/config.d/listen.xml b/dbms/programs/server/config.d/listen.xml new file mode 100644 index 00000000000..24c64bbb60a --- /dev/null +++ b/dbms/programs/server/config.d/listen.xml @@ -0,0 +1 @@ +0.0.0.0 \ No newline at end of file diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp b/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp new file mode 100644 index 00000000000..e5cf72590f0 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, const DataTypes & arguments, const Array & params) +{ + assertNoParameters(name, params); + + if (arguments.size() > AggregateFunctionRetentionData::max_events ) + throw Exception("Too many event arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + return std::make_shared(arguments); +} + +} + +void registerAggregateFunctionRetention(AggregateFunctionFactory & factory) +{ + factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h new file mode 100644 index 00000000000..be5e0810d13 --- /dev/null +++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h @@ -0,0 +1,150 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; +} + +struct AggregateFunctionRetentionData +{ + static constexpr auto max_events = 32; + + using Events = std::bitset; + + Events events; + + void add(UInt8 event) + { + events.set(event); + } + + void merge(const AggregateFunctionRetentionData & other) + { + events |= other.events; + } + + void serialize(WriteBuffer & buf) const + { + UInt32 event_value = events.to_ulong(); + writeBinary(event_value, buf); + } + + void deserialize(ReadBuffer & buf) + { + UInt32 event_value; + readBinary(event_value, buf); + events = event_value; + } +}; + +/** + * The max size of events is 32, that's enough for retention analytics + * + * Usage: + * - retention(cond1, cond2, cond3, ....) + * - returns [cond1_flag, cond1_flag && cond2_flag, cond1_flag && cond3_flag, ...] + */ +class AggregateFunctionRetention final + : public IAggregateFunctionDataHelper +{ +private: + UInt8 events_size; + +public: + String getName() const override + { + return "retention"; + } + + AggregateFunctionRetention(const DataTypes & arguments) + { + for (const auto i : ext::range(0, arguments.size())) + { + auto cond_arg = arguments[i].get(); + if (!typeid_cast(cond_arg)) + throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i) + " of aggregate function " + + getName() + ", must be UInt8", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + events_size = arguments.size(); + } + + + DataTypePtr getReturnType() const override + { + return std::make_shared(std::make_shared()); + } + + void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override + { + for (const auto i : ext::range(0, events_size)) + { + auto event = static_cast *>(columns[i])->getData()[row_num]; + if (event) + { + this->data(place).add(i); + break; + } + } + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + this->data(place).serialize(buf); + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override + { + auto & data_to = static_cast(to).getData(); + auto & offsets_to = static_cast(to).getOffsets(); + + const bool first_flag = this->data(place).events.test(0); + data_to.insert(first_flag ? Field(static_cast(1)) : Field(static_cast(0))); + for (const auto i : ext::range(1, events_size)) + { + if (first_flag && this->data(place).events.test(i)) + data_to.insert(Field(static_cast(1))); + else + data_to.insert(Field(static_cast(0))); + } + offsets_to.push_back(offsets_to.size() == 0 ? events_size : offsets_to.back() + events_size); + } + + const char * getHeaderFilePath() const override + { + return __FILE__; + } +}; + +} diff --git a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp index 5efb1e5bf1b..3517ad57a73 100644 --- a/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/dbms/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -34,6 +34,7 @@ void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory &); void registerAggregateFunctionHistogram(AggregateFunctionFactory & factory); +void registerAggregateFunctionRetention(AggregateFunctionFactory & factory); void registerAggregateFunctions() { @@ -59,6 +60,7 @@ void registerAggregateFunctions() registerAggregateFunctionsBitwise(factory); registerAggregateFunctionsMaxIntersections(factory); registerAggregateFunctionHistogram(factory); + registerAggregateFunctionRetention(factory); } { diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index affd89b1c28..1ed186085f7 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -114,6 +114,7 @@ void Connection::disconnect() //LOG_TRACE(log_wrapper.get(), "Disconnecting"); in = nullptr; + last_input_packet_type.reset(); out = nullptr; // can write to socket if (socket) socket->close(); @@ -379,6 +380,7 @@ void Connection::sendQuery( maybe_compressed_in.reset(); maybe_compressed_out.reset(); block_in.reset(); + block_logs_in.reset(); block_out.reset(); /// Send empty block which means end of data. @@ -506,20 +508,50 @@ bool Connection::poll(size_t timeout_microseconds) } -bool Connection::hasReadBufferPendingData() const +bool Connection::hasReadPendingData() const { - return static_cast(*in).hasPendingData(); + return last_input_packet_type.has_value() || static_cast(*in).hasPendingData(); +} + + +std::optional Connection::checkPacket(size_t timeout_microseconds) +{ + if (last_input_packet_type.has_value()) + return last_input_packet_type; + + if (hasReadPendingData() || poll(timeout_microseconds)) + { + // LOG_TRACE(log_wrapper.get(), "Receiving packet type"); + UInt64 packet_type; + readVarUInt(packet_type, *in); + + last_input_packet_type.emplace(packet_type); + return last_input_packet_type; + } + + return {}; } Connection::Packet Connection::receivePacket() { - //LOG_TRACE(log_wrapper.get(), "Receiving packet"); - try { Packet res; - readVarUInt(res.type, *in); + + /// Have we already read packet type? + if (last_input_packet_type) + { + res.type = *last_input_packet_type; + last_input_packet_type.reset(); + } + else + { + //LOG_TRACE(log_wrapper.get(), "Receiving packet type"); + readVarUInt(res.type, *in); + } + + //LOG_TRACE(log_wrapper.get(), "Receiving packet " << res.type << " " << Protocol::Server::toString(res.type)); switch (res.type) { @@ -549,6 +581,10 @@ Connection::Packet Connection::receivePacket() res.block = receiveData(); return res; + case Protocol::Server::Log: + res.block = receiveLogData(); + return res; + case Protocol::Server::EndOfStream: return res; @@ -576,14 +612,26 @@ Block Connection::receiveData() //LOG_TRACE(log_wrapper.get(), "Receiving data"); initBlockInput(); + return receiveDataImpl(block_in); +} + +Block Connection::receiveLogData() +{ + initBlockLogsInput(); + return receiveDataImpl(block_logs_in); +} + + +Block Connection::receiveDataImpl(BlockInputStreamPtr & stream) +{ String external_table_name; readStringBinary(external_table_name, *in); size_t prev_bytes = in->count(); /// Read one block from network. - Block res = block_in->read(); + Block res = stream->read(); if (throttler) throttler->add(in->count() - prev_bytes); @@ -592,20 +640,39 @@ Block Connection::receiveData() } +void Connection::initInputBuffers() +{ + +} + + void Connection::initBlockInput() { if (!block_in) { - if (compression == Protocol::Compression::Enable) - maybe_compressed_in = std::make_shared(*in); - else - maybe_compressed_in = in; + if (!maybe_compressed_in) + { + if (compression == Protocol::Compression::Enable) + maybe_compressed_in = std::make_shared(*in); + else + maybe_compressed_in = in; + } block_in = std::make_shared(*maybe_compressed_in, server_revision); } } +void Connection::initBlockLogsInput() +{ + if (!block_logs_in) + { + /// Have to return superset of SystemLogsQueue::getSampleBlock() columns + block_logs_in = std::make_shared(*in, server_revision); + } +} + + void Connection::setDescription() { auto resolved_address = getResolvedAddress(); diff --git a/dbms/src/Client/Connection.h b/dbms/src/Client/Connection.h index dabb50b53a9..ad98df3cc8f 100644 --- a/dbms/src/Client/Connection.h +++ b/dbms/src/Client/Connection.h @@ -23,6 +23,7 @@ #include #include +#include namespace DB @@ -138,7 +139,10 @@ public: bool poll(size_t timeout_microseconds = 0); /// Check, if has data in read buffer. - bool hasReadBufferPendingData() const; + bool hasReadPendingData() const; + + /// Checks if there is input data in connection and reads packet ID. + std::optional checkPacket(size_t timeout_microseconds = 0); /// Receive packet from server. Packet receivePacket(); @@ -195,6 +199,7 @@ private: std::unique_ptr socket; std::shared_ptr in; std::shared_ptr out; + std::optional last_input_packet_type; String query_id; Protocol::Compression compression; /// Enable data compression for communication. @@ -214,6 +219,7 @@ private: /// From where to read query execution result. std::shared_ptr maybe_compressed_in; BlockInputStreamPtr block_in; + BlockInputStreamPtr block_logs_in; /// Where to write data for INSERT. std::shared_ptr maybe_compressed_out; @@ -249,11 +255,16 @@ private: bool ping(); Block receiveData(); + Block receiveLogData(); + Block receiveDataImpl(BlockInputStreamPtr & stream); + std::unique_ptr receiveException(); Progress receiveProgress(); BlockStreamProfileInfo receiveProfileInfo(); + void initInputBuffers(); void initBlockInput(); + void initBlockLogsInput(); void throwUnexpectedPacket(UInt64 packet_type, const char * expected) const; }; diff --git a/dbms/src/Client/MultiplexedConnections.cpp b/dbms/src/Client/MultiplexedConnections.cpp index 8fe27ecf7fa..3e88a20caa3 100644 --- a/dbms/src/Client/MultiplexedConnections.cpp +++ b/dbms/src/Client/MultiplexedConnections.cpp @@ -247,6 +247,7 @@ Connection::Packet MultiplexedConnections::receivePacketUnlocked() case Protocol::Server::ProfileInfo: case Protocol::Server::Totals: case Protocol::Server::Extremes: + case Protocol::Server::Log: break; case Protocol::Server::EndOfStream: @@ -276,7 +277,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead for (const ReplicaState & state : replica_states) { Connection * connection = state.connection; - if ((connection != nullptr) && connection->hasReadBufferPendingData()) + if ((connection != nullptr) && connection->hasReadPendingData()) read_list.push_back(*connection->socket); } diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 5c2c4142d99..8fe2f9134e3 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -118,7 +118,7 @@ MutableColumnPtr ColumnVector::cloneResized(size_t size) const memcpy(&new_col.data[0], &data[0], count * sizeof(data[0])); if (size > count) - memset(&new_col.data[count], static_cast(value_type()), (size - count) * sizeof(value_type)); + memset(static_cast(&new_col.data[count]), static_cast(value_type()), (size - count) * sizeof(value_type)); } return std::move(res); diff --git a/dbms/src/Common/ActionBlocker.h b/dbms/src/Common/ActionBlocker.h index 2ddea2d4359..055d266c72e 100644 --- a/dbms/src/Common/ActionBlocker.h +++ b/dbms/src/Common/ActionBlocker.h @@ -1,14 +1,15 @@ #pragma once - #include #include #include + namespace DB { -/// An atomic variable that is used to block and interrupt certain actions -/// If it is not zero then actions related with it should be considered as interrupted +/// An atomic variable that is used to block and interrupt certain actions. +/// If it is not zero then actions related with it should be considered as interrupted. +/// Uses shared_ptr and the lock uses weak_ptr to be able to "hold" a lock when an object with blocker has already died. class ActionBlocker { public: @@ -33,4 +34,5 @@ private: CounterPtr counter; }; + } diff --git a/dbms/src/Common/ActionLock.h b/dbms/src/Common/ActionLock.h index 3d6bfc8ada7..1167a23b8dd 100644 --- a/dbms/src/Common/ActionLock.h +++ b/dbms/src/Common/ActionLock.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include + namespace DB { diff --git a/dbms/src/Common/BackgroundSchedulePool.cpp b/dbms/src/Common/BackgroundSchedulePool.cpp index 9556c9a037b..d4b3714df51 100644 --- a/dbms/src/Common/BackgroundSchedulePool.cpp +++ b/dbms/src/Common/BackgroundSchedulePool.cpp @@ -4,8 +4,11 @@ #include #include #include +#include #include #include +#include + namespace CurrentMetrics { @@ -140,6 +143,12 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size) { LOG_INFO(&Logger::get("BackgroundSchedulePool"), "Create BackgroundSchedulePool with " << size << " threads"); + /// Put all threads of both thread pools to one thread group + /// The master thread exits immediately + CurrentThread::initializeQuery(); + thread_group = CurrentThread::getGroup(); + CurrentThread::detachQuery(); + threads.resize(size); for (auto & thread : threads) thread = std::thread([this] { threadFunction(); }); @@ -212,9 +221,11 @@ void BackgroundSchedulePool::threadFunction() { setThreadName("BackgrSchedPool"); - MemoryTracker memory_tracker; - memory_tracker.setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); - current_memory_tracker = &memory_tracker; + /// Put all threads to one thread pool + CurrentThread::attachTo(thread_group); + SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); + + CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool); while (!shutdown) { @@ -224,8 +235,6 @@ void BackgroundSchedulePool::threadFunction() task_notification.execute(); } } - - current_memory_tracker = nullptr; } @@ -233,6 +242,10 @@ void BackgroundSchedulePool::delayExecutionThreadFunction() { setThreadName("BckSchPoolDelay"); + /// Put all threads to one thread pool + CurrentThread::attachTo(thread_group); + SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); + while (!shutdown) { TaskInfoPtr task; diff --git a/dbms/src/Common/BackgroundSchedulePool.h b/dbms/src/Common/BackgroundSchedulePool.h index 0ebcc207b2c..f55cc95dbbc 100644 --- a/dbms/src/Common/BackgroundSchedulePool.h +++ b/dbms/src/Common/BackgroundSchedulePool.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB { @@ -138,6 +139,9 @@ private: std::thread delayed_thread; /// Tasks ordered by scheduled time. DelayedTasks delayed_tasks; + + /// Thread group used for profiling purposes + ThreadGroupStatusPtr thread_group; }; using BackgroundSchedulePoolPtr = std::shared_ptr; diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp new file mode 100644 index 00000000000..38a42cb65c8 --- /dev/null +++ b/dbms/src/Common/CurrentThread.cpp @@ -0,0 +1,80 @@ +#include "CurrentThread.h" +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +void CurrentThread::updatePerformanceCounters() +{ + get()->updatePerformanceCounters(); +} + +ThreadStatusPtr CurrentThread::get() +{ +#ifndef NDEBUG + if (!current_thread || current_thread.use_count() <= 0) + throw Exception("Thread #" + std::to_string(Poco::ThreadNumber::get()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR); + + if (Poco::ThreadNumber::get() != current_thread->thread_number) + throw Exception("Current thread has different thread number", ErrorCodes::LOGICAL_ERROR); +#endif + + return current_thread; +} + +ProfileEvents::Counters & CurrentThread::getProfileEvents() +{ + return current_thread->performance_counters; +} + +MemoryTracker & CurrentThread::getMemoryTracker() +{ + return current_thread->memory_tracker; +} + +void CurrentThread::updateProgressIn(const Progress & value) +{ + current_thread->progress_in.incrementPiecewiseAtomically(value); +} + +void CurrentThread::updateProgressOut(const Progress & value) +{ + current_thread->progress_out.incrementPiecewiseAtomically(value); +} + +void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue) +{ + get()->attachInternalTextLogsQueue(logs_queue); +} + +std::shared_ptr CurrentThread::getInternalTextLogsQueue() +{ + /// NOTE: this method could be called at early server startup stage + /// NOTE: this method could be called in ThreadStatus destructor, therefore we make use_count() check just in case + + if (!current_thread || current_thread.use_count() <= 0) + return nullptr; + + if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died) + return nullptr; + + return current_thread->getInternalTextLogsQueue(); +} + +ThreadGroupStatusPtr CurrentThread::getGroup() +{ + return get()->getThreadGroup(); +} + +} diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h new file mode 100644 index 00000000000..afde7ad8bf2 --- /dev/null +++ b/dbms/src/Common/CurrentThread.h @@ -0,0 +1,83 @@ +#pragma once +#include +#include + + +namespace ProfileEvents +{ +class Counters; +} + +class MemoryTracker; + + +namespace DB +{ + +class Context; +class QueryStatus; +class ThreadStatus; +struct Progress; +using ThreadStatusPtr = std::shared_ptr; +class InternalTextLogsQueue; +class ThreadGroupStatus; +using ThreadGroupStatusPtr = std::shared_ptr; + + +class CurrentThread +{ +public: + + /// Handler to current thread + static ThreadStatusPtr get(); + /// Group to which belongs current thread + static ThreadGroupStatusPtr getGroup(); + + /// A logs queue used by TCPHandler to pass logs to a client + static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue); + static std::shared_ptr getInternalTextLogsQueue(); + + /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats + static void updatePerformanceCounters(); + + static ProfileEvents::Counters & getProfileEvents(); + static MemoryTracker & getMemoryTracker(); + + /// Update read and write rows (bytes) statistics (used in system.query_thread_log) + static void updateProgressIn(const Progress & value); + static void updateProgressOut(const Progress & value); + + /// Query management: + + /// Call from master thread as soon as possible (e.g. when thread accepted connection) + static void initializeQuery(); + + /// Sets query_context for current thread group + static void attachQueryContext(Context & query_context); + + /// You must call one of these methods when create a query child thread: + /// Add current thread to a group associated with the thread group + static void attachTo(const ThreadGroupStatusPtr & thread_group); + /// Is useful for a ThreadPool tasks + static void attachToIfDetached(const ThreadGroupStatusPtr & thread_group); + + /// Update ProfileEvents and dumps info to system.query_thread_log + static void finalizePerformanceCounters(); + + /// Returns a non-empty string if the thread is attached to a query + static std::string getCurrentQueryID(); + + /// Non-master threads call this method in destructor automatically + static void detachQuery(); + static void detachQueryIfNotDetached(); + + /// Initializes query with current thread as master thread in constructor, and detaches it in desstructor + struct QueryScope + { + explicit QueryScope(Context & query_context); + ~QueryScope(); + }; +}; + +} + diff --git a/dbms/src/Common/DNSResolver.cpp b/dbms/src/Common/DNSResolver.cpp index e3d9442deea..c82f302d7b2 100644 --- a/dbms/src/Common/DNSResolver.cpp +++ b/dbms/src/Common/DNSResolver.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -79,6 +80,10 @@ struct DNSResolver::Impl { SimpleCache cache_host; + /// Cached server host name + std::mutex mutex; + std::optional host_name; + /// If disabled, will not make cache lookups, will resolve addresses manually on each call std::atomic disable_cache{false}; }; @@ -108,6 +113,9 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U void DNSResolver::dropCache() { impl->cache_host.drop(); + + std::unique_lock lock(impl->mutex); + impl->host_name.reset(); } void DNSResolver::setDisableCacheFlag(bool is_disabled) @@ -115,6 +123,19 @@ void DNSResolver::setDisableCacheFlag(bool is_disabled) impl->disable_cache = is_disabled; } +String DNSResolver::getHostName() +{ + if (impl->disable_cache) + return Poco::Net::DNS::hostName(); + + std::unique_lock lock(impl->mutex); + + if (!impl->host_name.has_value()) + impl->host_name.emplace(Poco::Net::DNS::hostName()); + + return *impl->host_name; +} + DNSResolver::~DNSResolver() = default; diff --git a/dbms/src/Common/DNSResolver.h b/dbms/src/Common/DNSResolver.h index fb3892e101f..097e646fa65 100644 --- a/dbms/src/Common/DNSResolver.h +++ b/dbms/src/Common/DNSResolver.h @@ -25,6 +25,9 @@ public: Poco::Net::SocketAddress resolveAddress(const std::string & host, UInt16 port); + /// Get this server host name + String getHostName(); + /// Disables caching void setDisableCacheFlag(bool is_disabled = true); diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index 4a2c47528b6..c0ae5f247bf 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -385,6 +385,8 @@ namespace ErrorCodes extern const int BAD_REQUEST_PARAMETER = 408; extern const int EXTERNAL_EXECUTABLE_NOT_FOUND = 409; extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING = 410; + extern const int PTHREAD_ERROR = 411; + extern const int NETLINK_ERROR = 412; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h deleted file mode 100644 index 6331ecba5ca..00000000000 --- a/dbms/src/Common/ExternalTable.h +++ /dev/null @@ -1,225 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - - -/// The base class containing the basic information about external table and -/// basic functions for extracting this information from text fields. -class BaseExternalTable -{ -public: - std::string file; /// File with data or '-' if stdin - std::string name; /// The name of the table - std::string format; /// Name of the data storage format - - /// Description of the table structure: (column name, data type name) - std::vector> structure; - - std::unique_ptr read_buffer; - Block sample_block; - - virtual ~BaseExternalTable() {} - - /// Initialize read_buffer, depending on the data source. By default, does nothing. - virtual void initReadBuffer() {} - - /// Get the table data - a pair (a thread with the contents of the table, the name of the table) - ExternalTableData getData(const Context & context) - { - initReadBuffer(); - initSampleBlock(); - ExternalTableData res = std::make_pair(std::make_shared(context.getInputFormat( - format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE)), name); - return res; - } - -protected: - /// Clear all accumulated information - void clean() - { - name = ""; - file = ""; - format = ""; - structure.clear(); - sample_block = Block(); - read_buffer.reset(); - } - - /// Function for debugging information output - void write() - { - std::cerr << "file " << file << std::endl; - std::cerr << "name " << name << std::endl; - std::cerr << "format " << format << std::endl; - std::cerr << "structure: \n"; - for (size_t i = 0; i < structure.size(); ++i) - std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl; - } - - static std::vector split(const std::string & s, const std::string & d) - { - std::vector res; - boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on); - return res; - } - - /// Construct the `structure` vector from the text field `structure` - virtual void parseStructureFromStructureField(const std::string & argument) - { - std::vector vals = split(argument, " ,"); - - if (vals.size() & 1) - throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS); - - for (size_t i = 0; i < vals.size(); i += 2) - structure.emplace_back(vals[i], vals[i + 1]); - } - - /// Construct the `structure` vector from the text field `types` - virtual void parseStructureFromTypesField(const std::string & argument) - { - std::vector vals = split(argument, " ,"); - - for (size_t i = 0; i < vals.size(); ++i) - structure.emplace_back("_" + toString(i + 1), vals[i]); - } - -private: - /// Initialize sample_block according to the structure of the table stored in the `structure` - void initSampleBlock() - { - const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - - for (size_t i = 0; i < structure.size(); ++i) - { - ColumnWithTypeAndName column; - column.name = structure[i].first; - column.type = data_type_factory.get(structure[i].second); - column.column = column.type->createColumn(); - sample_block.insert(std::move(column)); - } - } -}; - - -/// Parsing of external table used in the tcp client. -class ExternalTable : public BaseExternalTable -{ -public: - void initReadBuffer() override - { - if (file == "-") - read_buffer = std::make_unique(STDIN_FILENO); - else - read_buffer = std::make_unique(file); - } - - /// Extract parameters from variables_map, which is built on the client command line - ExternalTable(const boost::program_options::variables_map & external_options) - { - if (external_options.count("file")) - file = external_options["file"].as(); - else - throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("name")) - name = external_options["name"].as(); - else - throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("format")) - format = external_options["format"].as(); - else - throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - - if (external_options.count("structure")) - parseStructureFromStructureField(external_options["structure"].as()); - else if (external_options.count("types")) - parseStructureFromTypesField(external_options["types"].as()); - else - throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); - } -}; - -/// Parsing of external table used when sending tables via http -/// The `handlePart` function will be called for each table passed, -/// so it's also necessary to call `clean` at the end of the `handlePart`. -class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable -{ -public: - std::vector names; - - ExternalTablesHandler(Context & context_, Poco::Net::NameValueCollection params_) : context(context_), params(params_) { } - - void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) - { - /// The buffer is initialized here, not in the virtual function initReadBuffer - read_buffer = std::make_unique(stream); - - /// Retrieve a collection of parameters from MessageHeader - Poco::Net::NameValueCollection content; - std::string label; - Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content); - - /// Get parameters - name = content.get("name", "_data"); - format = params.get(name + "_format", "TabSeparated"); - - if (params.has(name + "_structure")) - parseStructureFromStructureField(params.get(name + "_structure")); - else if (params.has(name + "_types")) - parseStructureFromTypesField(params.get(name + "_types")); - else - throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS); - - ExternalTableData data = getData(context); - - /// Create table - NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); - storage->startup(); - context.addExternalTable(data.second, storage); - BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef()); - - /// Write data - data.first->readPrefix(); - output->writePrefix(); - while(Block block = data.first->read()) - output->write(block); - data.first->readSuffix(); - output->writeSuffix(); - - names.push_back(name); - /// We are ready to receive the next file, for this we clear all the information received - clean(); - } - -private: - Context & context; - Poco::Net::NameValueCollection params; -}; - - -} diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 2c4d781056d..7e957ae1ae4 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -1,11 +1,10 @@ +#include "MemoryTracker.h" #include #include #include #include +#include #include -#include - -#include namespace DB @@ -19,7 +18,7 @@ namespace DB MemoryTracker::~MemoryTracker() { - if (peak) + if (static_cast(level) < static_cast(VariableContext::Process) && peak) { try { @@ -56,13 +55,16 @@ void MemoryTracker::logPeakMemoryUsage() const void MemoryTracker::alloc(Int64 size) { + if (blocker.isCancelled()) + return; + /** Using memory_order_relaxed means that if allocations are done simultaneously, * we allow exception about memory limit exceeded to be thrown only on next allocation. * So, we allow over-allocations. */ Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed); - if (!next.load(std::memory_order_relaxed)) + if (!parent.load(std::memory_order_relaxed)) CurrentMetrics::add(metric, size); Int64 current_limit = limit.load(std::memory_order_relaxed); @@ -102,45 +104,62 @@ void MemoryTracker::alloc(Int64 size) if (will_be > peak.load(std::memory_order_relaxed)) /// Races doesn't matter. Could rewrite with CAS, but not worth. peak.store(will_be, std::memory_order_relaxed); - if (auto loaded_next = next.load(std::memory_order_relaxed)) + if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->alloc(size); } void MemoryTracker::free(Int64 size) { - Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size; + if (blocker.isCancelled()) + return; - /** Sometimes, query could free some data, that was allocated outside of query context. - * Example: cache eviction. - * To avoid negative memory usage, we "saturate" amount. - * Memory usage will be calculated with some error. - * NOTE The code is not atomic. Not worth to fix. - */ - if (new_amount < 0) + if (level == VariableContext::Thread) { - amount.fetch_sub(new_amount); - size += new_amount; + /// Could become negative if memory allocated in this thread is freed in another one + amount.fetch_sub(size, std::memory_order_relaxed); + } + else + { + Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size; + + /** Sometimes, query could free some data, that was allocated outside of query context. + * Example: cache eviction. + * To avoid negative memory usage, we "saturate" amount. + * Memory usage will be calculated with some error. + * NOTE: The code is not atomic. Not worth to fix. + */ + if (unlikely(new_amount < 0)) + { + amount.fetch_sub(new_amount); + size += new_amount; + } } - if (auto loaded_next = next.load(std::memory_order_relaxed)) + if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); else CurrentMetrics::sub(metric, size); } -void MemoryTracker::reset() +void MemoryTracker::resetCounters() { - if (!next.load(std::memory_order_relaxed)) - CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed)); - amount.store(0, std::memory_order_relaxed); peak.store(0, std::memory_order_relaxed); limit.store(0, std::memory_order_relaxed); } +void MemoryTracker::reset() +{ + if (!parent.load(std::memory_order_relaxed)) + CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed)); + + resetCounters(); +} + + void MemoryTracker::setOrRaiseLimit(Int64 value) { /// This is just atomic set to maximum. @@ -149,29 +168,26 @@ void MemoryTracker::setOrRaiseLimit(Int64 value) ; } -#if __APPLE__ && __clang__ -__thread MemoryTracker * current_memory_tracker = nullptr; -#else -thread_local MemoryTracker * current_memory_tracker = nullptr; -#endif namespace CurrentMemoryTracker { void alloc(Int64 size) { - if (current_memory_tracker) - current_memory_tracker->alloc(size); + DB::CurrentThread::getMemoryTracker().alloc(size); } void realloc(Int64 old_size, Int64 new_size) { - if (current_memory_tracker) - current_memory_tracker->alloc(new_size - old_size); + DB::CurrentThread::getMemoryTracker().alloc(new_size - old_size); } void free(Int64 size) { - if (current_memory_tracker) - current_memory_tracker->free(size); + DB::CurrentThread::getMemoryTracker().free(size); } } + +DB::SimpleActionLock getCurrentMemoryTrackerActionLock() +{ + return DB::CurrentThread::getMemoryTracker().blocker.cancel(); +} diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 32a93978fcd..68c145393fe 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace CurrentMetrics @@ -26,7 +28,7 @@ class MemoryTracker /// Singly-linked list. All information will be passed to subsequent memory trackers also (it allows to implement trackers hierarchy). /// In terms of tree nodes it is the list of parents. Lifetime of these trackers should "include" lifetime of current tracker. - std::atomic next {}; + std::atomic parent {}; /// You could specify custom metric to track memory usage. CurrentMetrics::Metric metric = CurrentMetrics::MemoryTracking; @@ -35,11 +37,14 @@ class MemoryTracker const char * description = nullptr; public: - MemoryTracker() {} - MemoryTracker(Int64 limit_) : limit(limit_) {} + MemoryTracker(VariableContext level = VariableContext::Thread) : level(level) {} + MemoryTracker(Int64 limit_, VariableContext level = VariableContext::Thread) : limit(limit_), level(level) {} + MemoryTracker(MemoryTracker * parent_, VariableContext level = VariableContext::Thread) : parent(parent_), level(level) {} ~MemoryTracker(); + VariableContext level; + /** Call the following functions before calling of corresponding operations with memory allocators. */ void alloc(Int64 size); @@ -79,9 +84,15 @@ public: } /// next should be changed only once: from nullptr to some value. - void setNext(MemoryTracker * elem) + /// NOTE: It is not true in MergeListElement + void setParent(MemoryTracker * elem) { - next.store(elem, std::memory_order_relaxed); + parent.store(elem, std::memory_order_relaxed); + } + + MemoryTracker * getParent() + { + return parent.load(std::memory_order_relaxed); } /// The memory consumption could be shown in realtime via CurrentMetrics counter @@ -95,26 +106,21 @@ public: description = description_; } - /// Reset the accumulated data. + /// Reset the accumulated data + void resetCounters(); + + /// Reset the accumulated data and the parent. void reset(); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage() const; + + /// To be able to temporarily stop memory tracker + DB::SimpleActionBlocker blocker; }; -/** The MemoryTracker object is quite difficult to pass to all places where significant amounts of memory are allocated. - * Therefore, a thread-local pointer to used MemoryTracker is set, or nullptr if MemoryTracker does not need to be used. - * This pointer is set when memory consumption is monitored in current thread. - * So, you just need to pass it to all the threads that handle one request. - */ -#if defined(__APPLE__) && defined(__clang__) -extern __thread MemoryTracker * current_memory_tracker; -#else -extern thread_local MemoryTracker * current_memory_tracker; -#endif - -/// Convenience methods, that use current_memory_tracker if it is available. +/// Convenience methods, that use current thread's memory_tracker if it is available. namespace CurrentMemoryTracker { void alloc(Int64 size); @@ -123,20 +129,4 @@ namespace CurrentMemoryTracker } -#include - -struct TemporarilyDisableMemoryTracker : private boost::noncopyable -{ - MemoryTracker * memory_tracker; - - TemporarilyDisableMemoryTracker() - { - memory_tracker = current_memory_tracker; - current_memory_tracker = nullptr; - } - - ~TemporarilyDisableMemoryTracker() - { - current_memory_tracker = memory_tracker; - } -}; +DB::SimpleActionLock getCurrentMemoryTrackerActionLock(); diff --git a/dbms/src/Common/NaNUtils.h b/dbms/src/Common/NaNUtils.h index 8397dd56abb..af0b26266a7 100644 --- a/dbms/src/Common/NaNUtils.h +++ b/dbms/src/Common/NaNUtils.h @@ -50,7 +50,7 @@ std::enable_if_t, T> NaNOrZero() #if 1 /// __int128 template -std::enable_if_t, __int128> NaNOrZero() +std::enable_if_t && !std::numeric_limits::is_integer, __int128> NaNOrZero() { return __int128(0); } diff --git a/dbms/src/Common/ODBCBridgeHelper.cpp b/dbms/src/Common/ODBCBridgeHelper.cpp index 87c7e0a3b30..785c457062d 100644 --- a/dbms/src/Common/ODBCBridgeHelper.cpp +++ b/dbms/src/Common/ODBCBridgeHelper.cpp @@ -1,25 +1,26 @@ #include #include -#include #include #include +#include #include #include #include #include #include + namespace DB { namespace ErrorCodes { extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING; } -ODBCBridgeHelper::ODBCBridgeHelper(const Context & context_global_, const std::string & connection_string_) - : context_global(context_global_), connection_string(validateODBCConnectionString(connection_string_)) +ODBCBridgeHelper::ODBCBridgeHelper( + const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_) + : config(config_), http_timeout(http_timeout_), connection_string(connection_string_) { - const auto & config = context_global.getConfigRef(); size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); @@ -30,8 +31,6 @@ ODBCBridgeHelper::ODBCBridgeHelper(const Context & context_global_, const std::s } void ODBCBridgeHelper::startODBCBridge() const { - const auto & config = context_global.getConfigRef(); - const auto & settings = context_global.getSettingsRef(); Poco::Path path{config.getString("application.dir", "")}; path.setFileName("clickhouse-odbc-bridge"); @@ -42,7 +41,7 @@ void ODBCBridgeHelper::startODBCBridge() const command << path.toString() << ' '; command << "--http-port " << config.getUInt("odbc_bridge.port", DEFAULT_PORT) << ' '; command << "--listen-host " << config.getString("odbc_bridge.listen_host", DEFAULT_HOST) << ' '; - command << "--http-timeout " << settings.http_receive_timeout.value.totalSeconds() << ' '; + command << "--http-timeout " << http_timeout.totalMicroseconds() << ' '; if (config.has("logger.odbc_bridge_log")) command << "--log-path " << config.getString("logger.odbc_bridge_log") << ' '; if (config.has("logger.odbc_bridge_errlog")) @@ -58,12 +57,12 @@ void ODBCBridgeHelper::startODBCBridge() const cmd->wait(); } -std::vector> ODBCBridgeHelper::getURLParams(const NamesAndTypesList & cols, size_t max_block_size) const +std::vector> ODBCBridgeHelper::getURLParams(const std::string & cols, size_t max_block_size) const { std::vector> result; result.emplace_back("connection_string", connection_string); /// already validated - result.emplace_back("columns", cols.toString()); + result.emplace_back("columns", cols); result.emplace_back("max_block_size", std::to_string(max_block_size)); return result; @@ -73,7 +72,7 @@ bool ODBCBridgeHelper::checkODBCBridgeIsRunning() const { try { - ReadWriteBufferFromHTTP buf(ping_url, ODBCBridgeHelper::PING_METHOD, nullptr); + ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr); return checkString(ODBCBridgeHelper::PING_OK_ANSWER, buf); } catch (...) @@ -103,4 +102,30 @@ void ODBCBridgeHelper::startODBCBridgeSync() const throw Exception("ODBCBridgeHelper: clickhouse-odbc-bridge is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING); } } + +Poco::URI ODBCBridgeHelper::getMainURI() const +{ + size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); + std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); + + Poco::URI main_uri; + main_uri.setHost(bridge_host); + main_uri.setPort(bridge_port); + main_uri.setScheme("http"); + main_uri.setPath(MAIN_HANDLER); + return main_uri; +} + +Poco::URI ODBCBridgeHelper::getColumnsInfoURI() const +{ + size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT); + std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST); + + Poco::URI columns_info_uri; + columns_info_uri.setHost(bridge_host); + columns_info_uri.setPort(bridge_port); + columns_info_uri.setScheme("http"); + columns_info_uri.setPath(COL_INFO_HANDLER); + return columns_info_uri; +} } diff --git a/dbms/src/Common/ODBCBridgeHelper.h b/dbms/src/Common/ODBCBridgeHelper.h index 4ef0cc871dd..807782d73eb 100644 --- a/dbms/src/Common/ODBCBridgeHelper.h +++ b/dbms/src/Common/ODBCBridgeHelper.h @@ -2,8 +2,8 @@ #include #include -#include #include +#include namespace DB { @@ -16,7 +16,11 @@ namespace ErrorCodes class ODBCBridgeHelper { private: - const Context & context_global; + + using Configuration = Poco::Util::AbstractConfiguration; + + const Configuration & config; + Poco::Timespan http_timeout; std::string connection_string; @@ -31,17 +35,18 @@ public: static constexpr inline auto DEFAULT_FORMAT = "RowBinary"; static constexpr inline auto PING_HANDLER = "/ping"; static constexpr inline auto MAIN_HANDLER = "/"; + static constexpr inline auto COL_INFO_HANDLER = "/columns_info"; static constexpr inline auto PING_OK_ANSWER = "Ok."; - static const inline std::string PING_METHOD = Poco::Net::HTTPRequest::HTTP_GET; - static const inline std::string MAIN_METHOD = Poco::Net::HTTPRequest::HTTP_POST; + ODBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_); - ODBCBridgeHelper(const Context & context_global_, const std::string & connection_string_); - - std::vector> getURLParams(const NamesAndTypesList & cols, size_t max_block_size) const; + std::vector> getURLParams(const std::string & cols, size_t max_block_size) const; bool checkODBCBridgeIsRunning() const; void startODBCBridge() const; void startODBCBridgeSync() const; + + Poco::URI getMainURI() const; + Poco::URI getColumnsInfoURI() const; }; } diff --git a/dbms/src/Common/ProfileEvents.cpp b/dbms/src/Common/ProfileEvents.cpp index adef2331609..8a1f98bbdf0 100644 --- a/dbms/src/Common/ProfileEvents.cpp +++ b/dbms/src/Common/ProfileEvents.cpp @@ -1,5 +1,7 @@ #include - +#include +#include +#include /// Available events. Add something here as you wish. #define APPLY_FOR_EVENTS(M) \ @@ -37,6 +39,11 @@ M(CreatedReadBufferAIO) \ M(CreatedWriteBufferOrdinary) \ M(CreatedWriteBufferAIO) \ + M(DiskReadElapsedMicroseconds) \ + M(DiskWriteElapsedMicroseconds) \ + M(NetworkReceiveElapsedMicroseconds) \ + M(NetworkSendElapsedMicroseconds) \ + M(ThrottlerSleepMicroseconds) \ \ M(ReplicatedPartFetches) \ M(ReplicatedPartFailedFetches) \ @@ -143,31 +150,93 @@ M(RWLockAcquiredWriteLocks) \ M(RWLockReadersWaitMilliseconds) \ M(RWLockWritersWaitMilliseconds) \ + M(NetworkErrors) \ \ - M(NetworkErrors) + M(RealTimeMicroseconds) \ + M(UserTimeMicroseconds) \ + M(SystemTimeMicroseconds) \ + M(SoftPageFaults) \ + M(HardPageFaults) \ + M(VoluntaryContextSwitches) \ + M(InvoluntaryContextSwitches) \ + \ + M(OSIOWaitMicroseconds) \ + M(OSCPUWaitMicroseconds) \ + M(OSCPUVirtualTimeMicroseconds) \ + M(OSReadBytes) \ + M(OSWriteBytes) \ + M(OSReadChars) \ + M(OSWriteChars) \ + namespace ProfileEvents { - #define M(NAME) extern const Event NAME = __COUNTER__; + +#define M(NAME) extern const Event NAME = __COUNTER__; + APPLY_FOR_EVENTS(M) +#undef M +constexpr Event END = __COUNTER__; + +/// Global variable, initialized by zeros. +Counter global_counters_array[END] {}; +/// Initialize global counters statically +Counters global_counters(global_counters_array); + +const Event Counters::num_counters = END; + + +Counters::Counters(VariableContext level, Counters * parent) + : counters_holder(new Counter[num_counters] {}), + parent(parent), + level(level) +{ + counters = counters_holder.get(); +} + +void Counters::resetCounters() +{ + if (counters) + { + for (Event i = 0; i < num_counters; ++i) + counters[i].store(0, std::memory_order_relaxed); + } +} + +void Counters::reset() +{ + parent = nullptr; + resetCounters(); +} + +Counters Counters::getPartiallyAtomicSnapshot() const +{ + Counters res(VariableContext::Snapshot, nullptr); + for (Event i = 0; i < num_counters; ++i) + res.counters[i].store(counters[i].load(std::memory_order_relaxed), std::memory_order_relaxed); + return res; +} + +const char * getDescription(Event event) +{ + static const char * descriptions[] = + { + #define M(NAME) #NAME, APPLY_FOR_EVENTS(M) #undef M - constexpr Event END = __COUNTER__; + }; - std::atomic counters[END] {}; /// Global variable, initialized by zeros. + return descriptions[event]; +} - const char * getDescription(Event event) - { - static const char * descriptions[] = - { - #define M(NAME) #NAME, - APPLY_FOR_EVENTS(M) - #undef M - }; - return descriptions[event]; - } +Event end() { return END; } + + +void increment(Event event, Count amount) +{ + DB::CurrentThread::getProfileEvents().increment(event, amount); +} - Event end() { return END; } } #undef APPLY_FOR_EVENTS diff --git a/dbms/src/Common/ProfileEvents.h b/dbms/src/Common/ProfileEvents.h index 0cb88f0ceed..38d8a9df7b9 100644 --- a/dbms/src/Common/ProfileEvents.h +++ b/dbms/src/Common/ProfileEvents.h @@ -1,8 +1,9 @@ #pragma once -#include +#include #include - +#include +#include /** Implements global counters for various events happening in the application * - for high level profiling. @@ -14,19 +15,80 @@ namespace ProfileEvents /// Event identifier (index in array). using Event = size_t; using Count = size_t; + using Counter = std::atomic; + class Counters; + + /// Counters - how many times each event happened + extern Counters global_counters; + + class Counters + { + Counter * counters = nullptr; + std::unique_ptr counters_holder; + /// Used to propagate increments + Counters * parent = nullptr; + + public: + + VariableContext level = VariableContext::Thread; + + /// By default, any instance have to increment global counters + Counters(VariableContext level = VariableContext::Thread, Counters * parent = &global_counters); + + /// Global level static initializer + Counters(Counter * allocated_counters) + : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {} + + Counter & operator[] (Event event) + { + return counters[event]; + } + + const Counter & operator[] (Event event) const + { + return counters[event]; + } + + inline void increment(Event event, Count amount = 1) + { + Counters * current = this; + do + { + current->counters[event].fetch_add(amount, std::memory_order_relaxed); + current = current->parent; + } while (current != nullptr); + } + + /// Every single value is fetched atomically, but not all values as a whole. + Counters getPartiallyAtomicSnapshot() const; + + /// Reset all counters to zero and reset parent. + void reset(); + + /// Get parent (thread unsafe) + Counters * getParent() + { + return parent; + } + + /// Set parent (thread unsafe) + void setParent(Counters * parent_) + { + parent = parent_; + } + + /// Set all counters to zero + void resetCounters(); + + static const Event num_counters; + }; + + /// Increment a counter for event. Thread-safe. + void increment(Event event, Count amount = 1); /// Get text description of event by identifier. Returns statically allocated string. const char * getDescription(Event event); - /// Counters - how many times each event happened. - extern std::atomic counters[]; - - /// Increment a counter for event. Thread-safe. - inline void increment(Event event, Count amount = 1) - { - counters[event].fetch_add(amount, std::memory_order_relaxed); - } - /// Get index just after last event identifier. Event end(); } diff --git a/dbms/src/Common/RWLockFIFO.cpp b/dbms/src/Common/RWLockFIFO.cpp index 51a2f756475..a1211a0bb9d 100644 --- a/dbms/src/Common/RWLockFIFO.cpp +++ b/dbms/src/Common/RWLockFIFO.cpp @@ -86,7 +86,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C handler_ptr->it_client->info += "; " + client.info; - return handler_ptr; + return handler_ptr; } if (type == Type::Write || queue.empty() || queue.back().type == Type::Write) diff --git a/dbms/src/Common/SimpleActionBlocker.h b/dbms/src/Common/SimpleActionBlocker.h new file mode 100644 index 00000000000..4a96db0e09d --- /dev/null +++ b/dbms/src/Common/SimpleActionBlocker.h @@ -0,0 +1,79 @@ +#pragma once +#include + + +namespace DB +{ + +class SimpleActionLock; + + +/// Similar to ActionBlocker, but without weak_ptr magic +class SimpleActionBlocker +{ + using Counter = std::atomic; + Counter counter = 0; + +public: + + SimpleActionBlocker() = default; + + bool isCancelled() const { return counter > 0; } + + /// Temporarily blocks corresponding actions (while the returned object is alive) + friend class SimpleActionLock; + inline SimpleActionLock cancel(); + + /// Cancel the actions forever. + void cancelForever() { ++counter; } +}; + + +/// Blocks related action while a SimpleActionLock instance exists +class SimpleActionLock +{ + SimpleActionBlocker * block = nullptr; + +public: + + SimpleActionLock() = default; + + explicit SimpleActionLock(SimpleActionBlocker & block_) : block(&block_) + { + ++block->counter; + } + + SimpleActionLock(const SimpleActionLock &) = delete; + + SimpleActionLock(SimpleActionLock && rhs) noexcept + { + *this = std::move(rhs); + } + + SimpleActionLock & operator=(const SimpleActionLock &) = delete; + + SimpleActionLock & operator=(SimpleActionLock && rhs) noexcept + { + if (block) + --block->counter; + + block = rhs.block; + rhs.block = nullptr; + + return *this; + } + + ~SimpleActionLock() + { + if (block) + --block->counter; + } +}; + + +SimpleActionLock SimpleActionBlocker::cancel() +{ + return SimpleActionLock(*this); +} + +} diff --git a/dbms/src/Common/StatusFile.cpp b/dbms/src/Common/StatusFile.cpp index 463d04b62e7..4da9ea48aa7 100644 --- a/dbms/src/Common/StatusFile.cpp +++ b/dbms/src/Common/StatusFile.cpp @@ -30,7 +30,7 @@ StatusFile::StatusFile(const std::string & path_) std::string contents; { ReadBufferFromFile in(path, 1024); - LimitReadBuffer limit_in(in, 1024); + LimitReadBuffer limit_in(in, 1024, false); readStringUntilEOF(contents, limit_in); } diff --git a/dbms/src/Common/Stopwatch.cpp b/dbms/src/Common/Stopwatch.cpp new file mode 100644 index 00000000000..b6d7092c054 --- /dev/null +++ b/dbms/src/Common/Stopwatch.cpp @@ -0,0 +1,14 @@ +#include +#include "Stopwatch.h" + +StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current() +{ + StopwatchRUsage::Timestamp res; + + ::rusage rusage; + ::getrusage(RUSAGE_THREAD, &rusage); + + res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL; + res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL; + return res; +} diff --git a/dbms/src/Common/Stopwatch.h b/dbms/src/Common/Stopwatch.h index 334e1574dde..c1ec623e100 100644 --- a/dbms/src/Common/Stopwatch.h +++ b/dbms/src/Common/Stopwatch.h @@ -32,9 +32,11 @@ public: void stop() { stop_ns = nanoseconds(); is_running = false; } void reset() { start_ns = 0; stop_ns = 0; is_running = false; } void restart() { start(); } - UInt64 elapsed() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } - UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; } - double elapsedSeconds() const { return static_cast(elapsed()) / 1000000000ULL; } + UInt64 elapsed() const { return elapsedNanoseconds(); } + UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; } + UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; } + UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; } + double elapsedSeconds() const { return static_cast(elapsedNanoseconds()) / 1000000000ULL; } private: UInt64 start_ns = 0; @@ -131,3 +133,59 @@ private: /// Most significant bit is a lock. When it is set, compareAndRestartDeferred method will return false. UInt64 nanoseconds() const { return StopWatchDetail::nanoseconds(clock_type) & 0x7FFFFFFFFFFFFFFFULL; } }; + + +/// Like ordinary StopWatch, but uses getrusage() system call +struct StopwatchRUsage +{ + StopwatchRUsage() = default; + + void start() { start_ts = Timestamp::current(); is_running = true; } + void stop() { stop_ts = Timestamp::current(); is_running = false; } + void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; } + void restart() { start(); } + + UInt64 elapsed(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys); + } + + UInt64 elapsedNanoseconds(bool count_user = true, bool count_sys = true) const + { + return (is_running ? Timestamp::current() : stop_ts).nanoseconds(count_user, count_sys) - start_ts.nanoseconds(count_user, count_sys); + } + + UInt64 elapsedMicroseconds(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys) / 1000UL; + } + + UInt64 elapsedMilliseconds(bool count_user = true, bool count_sys = true) const + { + return elapsedNanoseconds(count_user, count_sys) / 1000000UL; + } + + double elapsedSeconds(bool count_user = true, bool count_sys = true) const + { + return static_cast(elapsedNanoseconds(count_user, count_sys)) / 1000000000.0; + } + +private: + + struct Timestamp + { + UInt64 user_ns = 0; + UInt64 sys_ns = 0; + + static Timestamp current(); + + UInt64 nanoseconds(bool count_user = true, bool count_sys = true) const + { + return (count_user ? user_ns : 0) + (count_sys ? sys_ns : 0); + } + }; + + Timestamp start_ts; + Timestamp stop_ts; + bool is_running = false; +}; diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp new file mode 100644 index 00000000000..8f4d1c6c4ee --- /dev/null +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -0,0 +1,254 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// Basic idea is motivated by "iotop" tool. +/// More info: https://www.kernel.org/doc/Documentation/accounting/taskstats.txt + +#define GENLMSG_DATA(glh) ((void *)((char*)NLMSG_DATA(glh) + GENL_HDRLEN)) +#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN) +#define NLA_DATA(na) ((void *)((char*)(na) + NLA_HDRLEN)) +#define NLA_PAYLOAD(len) (len - NLA_HDRLEN) + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NETLINK_ERROR; +} + + +namespace +{ + +static size_t constexpr MAX_MSG_SIZE = 1024; + + +struct NetlinkMessage +{ + ::nlmsghdr n; + ::genlmsghdr g; + char buf[MAX_MSG_SIZE]; +}; + + +int sendCommand( + int sock_fd, + UInt16 nlmsg_type, + UInt32 nlmsg_pid, + UInt8 genl_cmd, + UInt16 nla_type, + void * nla_data, + int nla_len) noexcept +{ + NetlinkMessage msg{}; + + msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + msg.n.nlmsg_type = nlmsg_type; + msg.n.nlmsg_flags = NLM_F_REQUEST; + msg.n.nlmsg_seq = 0; + msg.n.nlmsg_pid = nlmsg_pid; + msg.g.cmd = genl_cmd; + msg.g.version = 1; + + ::nlattr * attr = static_cast<::nlattr *>(GENLMSG_DATA(&msg)); + attr->nla_type = nla_type; + attr->nla_len = nla_len + 1 + NLA_HDRLEN; + + memcpy(NLA_DATA(attr), nla_data, nla_len); + msg.n.nlmsg_len += NLMSG_ALIGN(attr->nla_len); + + char * buf = reinterpret_cast(&msg); + ssize_t buflen = msg.n.nlmsg_len; + + ::sockaddr_nl nladdr{}; + nladdr.nl_family = AF_NETLINK; + + while (true) + { + ssize_t r = ::sendto(sock_fd, buf, buflen, 0, reinterpret_cast(&nladdr), sizeof(nladdr)); + + if (r >= buflen) + break; + + if (r > 0) + { + buf += r; + buflen -= r; + } + else if (errno != EAGAIN) + return -1; + } + + return 0; +} + + +UInt16 getFamilyId(int nl_sock_fd) noexcept +{ + struct + { + ::nlmsghdr header; + ::genlmsghdr ge_header; + char buf[256]; + } answer; + + static char name[] = TASKSTATS_GENL_NAME; + + if (sendCommand( + nl_sock_fd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY, + CTRL_ATTR_FAMILY_NAME, (void *) name, + strlen(TASKSTATS_GENL_NAME) + 1)) + return 0; + + UInt16 id = 0; + ssize_t rep_len = ::recv(nl_sock_fd, &answer, sizeof(answer), 0); + if (answer.header.nlmsg_type == NLMSG_ERROR || (rep_len < 0) || !NLMSG_OK((&answer.header), rep_len)) + return 0; + + const ::nlattr * attr; + attr = static_cast(GENLMSG_DATA(&answer)); + attr = reinterpret_cast(reinterpret_cast(attr) + NLA_ALIGN(attr->nla_len)); + if (attr->nla_type == CTRL_ATTR_FAMILY_ID) + id = *static_cast(NLA_DATA(attr)); + + return id; +} + +} + + +TaskStatsInfoGetter::TaskStatsInfoGetter() = default; + +void TaskStatsInfoGetter::init() +{ + if (netlink_socket_fd >= 0) + return; + + netlink_socket_fd = ::socket(PF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + if (netlink_socket_fd < 0) + throwFromErrno("Can't create PF_NETLINK socket"); + + ::sockaddr_nl addr{}; + addr.nl_family = AF_NETLINK; + + if (::bind(netlink_socket_fd, reinterpret_cast(&addr), sizeof(addr)) < 0) + throwFromErrno("Can't bind PF_NETLINK socket"); + + netlink_family_id = getFamilyId(netlink_socket_fd); +} + +bool TaskStatsInfoGetter::getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error) +{ + init(); + + if (sendCommand(netlink_socket_fd, netlink_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(pid_t))) + throwFromErrno("Can't send a Netlink command"); + + NetlinkMessage msg; + ssize_t rv = ::recv(netlink_socket_fd, &msg, sizeof(msg), 0); + + if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), rv)) + { + const ::nlmsgerr * err = static_cast(NLMSG_DATA(&msg)); + if (throw_on_error) + throw Exception("Can't get Netlink response, error: " + std::to_string(err->error), ErrorCodes::NETLINK_ERROR); + else + return false; + } + + rv = GENLMSG_PAYLOAD(&msg.n); + + const ::nlattr * attr = static_cast(GENLMSG_DATA(&msg)); + ssize_t len = 0; + + while (len < rv) + { + len += NLA_ALIGN(attr->nla_len); + + if (attr->nla_type == TASKSTATS_TYPE_AGGR_TGID || attr->nla_type == TASKSTATS_TYPE_AGGR_PID) + { + int aggr_len = NLA_PAYLOAD(attr->nla_len); + int len2 = 0; + + attr = static_cast(NLA_DATA(attr)); + while (len2 < aggr_len) + { + if (attr->nla_type == TASKSTATS_TYPE_STATS) + { + const ::taskstats * ts = static_cast(NLA_DATA(attr)); + out_stats = *ts; + } + + len2 += NLA_ALIGN(attr->nla_len); + attr = reinterpret_cast(reinterpret_cast(attr) + len2); + } + } + + attr = reinterpret_cast(reinterpret_cast(GENLMSG_DATA(&msg)) + len); + } + + return true; +} + +void TaskStatsInfoGetter::getStat(::taskstats & stat, int tid) +{ + tid = tid < 0 ? getDefaultTID() : tid; + getStatImpl(tid, stat, true); +} + +bool TaskStatsInfoGetter::tryGetStat(::taskstats & stat, int tid) +{ + tid = tid < 0 ? getDefaultTID() : tid; + return getStatImpl(tid, stat, false); +} + +TaskStatsInfoGetter::~TaskStatsInfoGetter() +{ + if (netlink_socket_fd >= 0) + close(netlink_socket_fd); +} + +int TaskStatsInfoGetter::getCurrentTID() +{ + /// This call is always successful. - man gettid + return static_cast(syscall(SYS_gettid)); +} + +int TaskStatsInfoGetter::getDefaultTID() +{ + if (default_tid < 0) + default_tid = getCurrentTID(); + + return default_tid; +} + +static bool tryGetTaskStats() +{ + TaskStatsInfoGetter getter; + ::taskstats stat; + return getter.tryGetStat(stat); +} + +bool TaskStatsInfoGetter::checkProcessHasRequiredPermissions() +{ + /// It is thread- and exception- safe since C++11 + static bool res = tryGetTaskStats(); + return res; +} + +} diff --git a/dbms/src/Common/TaskStatsInfoGetter.h b/dbms/src/Common/TaskStatsInfoGetter.h new file mode 100644 index 00000000000..c89194cf88a --- /dev/null +++ b/dbms/src/Common/TaskStatsInfoGetter.h @@ -0,0 +1,43 @@ +#pragma once +#include + +struct taskstats; + + +namespace DB +{ + +class Exception; + + +/// Get taskstat info from OS kernel via Netlink protocol. +class TaskStatsInfoGetter +{ +public: + TaskStatsInfoGetter(); + TaskStatsInfoGetter(const TaskStatsInfoGetter &) = delete; + + void getStat(::taskstats & stat, int tid = -1); + bool tryGetStat(::taskstats & stat, int tid = -1); + + ~TaskStatsInfoGetter(); + + /// Make a syscall and returns Linux thread id + static int getCurrentTID(); + + /// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info + static bool checkProcessHasRequiredPermissions(); + +private: + /// Caches current thread tid to avoid extra sys calls + int getDefaultTID(); + int default_tid = -1; + + bool getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error = false); + void init(); + + int netlink_socket_fd = -1; + UInt16 netlink_family_id = 0; +}; + +} diff --git a/dbms/src/Common/ThreadProfileEvents.h b/dbms/src/Common/ThreadProfileEvents.h new file mode 100644 index 00000000000..3a780f509a7 --- /dev/null +++ b/dbms/src/Common/ThreadProfileEvents.h @@ -0,0 +1,144 @@ +#pragma once +#include +#include + +#include +#include +#include +#include + + +namespace ProfileEvents +{ + extern const Event RealTimeMicroseconds; + extern const Event UserTimeMicroseconds; + extern const Event SystemTimeMicroseconds; + extern const Event SoftPageFaults; + extern const Event HardPageFaults; + extern const Event VoluntaryContextSwitches; + extern const Event InvoluntaryContextSwitches; + + extern const Event OSIOWaitMicroseconds; + extern const Event OSCPUWaitMicroseconds; + extern const Event OSCPUVirtualTimeMicroseconds; + extern const Event OSReadChars; + extern const Event OSWriteChars; + extern const Event OSReadBytes; + extern const Event OSWriteBytes; +} + + +namespace DB +{ + +/// Handles overflow +template +inline TUInt safeDiff(TUInt prev, TUInt curr) +{ + return curr >= prev ? curr - prev : 0; +} + + +inline UInt64 getCurrentTimeNanoseconds(clockid_t clock_type = CLOCK_MONOTONIC) +{ + struct timespec ts; + clock_gettime(clock_type, &ts); + return ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + + +struct RUsageCounters +{ + /// In nanoseconds + UInt64 real_time = 0; + UInt64 user_time = 0; + UInt64 sys_time = 0; + + UInt64 soft_page_faults = 0; + UInt64 hard_page_faults = 0; + + RUsageCounters() = default; + RUsageCounters(const ::rusage & rusage_, UInt64 real_time_) + { + set(rusage_, real_time_); + } + + void set(const ::rusage & rusage, UInt64 real_time_) + { + real_time = real_time_; + user_time = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL; + sys_time = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL; + + soft_page_faults = static_cast(rusage.ru_minflt); + hard_page_faults = static_cast(rusage.ru_majflt); + } + + static RUsageCounters zeros(UInt64 real_time_ = getCurrentTimeNanoseconds()) + { + RUsageCounters res; + res.real_time = real_time_; + return res; + } + + static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds()) + { + ::rusage rusage; + ::getrusage(RUSAGE_THREAD, &rusage); + return RUsageCounters(rusage, real_time_); + } + + static void incrementProfileEvents(const RUsageCounters & prev, const RUsageCounters & curr, ProfileEvents::Counters & profile_events) + { + profile_events.increment(ProfileEvents::RealTimeMicroseconds, (curr.real_time - prev.real_time) / 1000U); + profile_events.increment(ProfileEvents::UserTimeMicroseconds, (curr.user_time - prev.user_time) / 1000U); + profile_events.increment(ProfileEvents::SystemTimeMicroseconds, (curr.sys_time - prev.sys_time) / 1000U); + + profile_events.increment(ProfileEvents::SoftPageFaults, curr.soft_page_faults - prev.soft_page_faults); + profile_events.increment(ProfileEvents::HardPageFaults, curr.hard_page_faults - prev.hard_page_faults); + } + + static void updateProfileEvents(RUsageCounters & last_counters, ProfileEvents::Counters & profile_events) + { + auto current_counters = current(); + incrementProfileEvents(last_counters, current_counters, profile_events); + last_counters = current_counters; + } +}; + + +struct TasksStatsCounters +{ + ::taskstats stat; + + TasksStatsCounters() = default; + + static TasksStatsCounters current(); + + static void incrementProfileEvents(const TasksStatsCounters & prev, const TasksStatsCounters & curr, ProfileEvents::Counters & profile_events) + { + profile_events.increment(ProfileEvents::OSCPUWaitMicroseconds, + safeDiff(prev.stat.cpu_delay_total, curr.stat.cpu_delay_total) / 1000U); + profile_events.increment(ProfileEvents::OSIOWaitMicroseconds, + safeDiff(prev.stat.blkio_delay_total, curr.stat.blkio_delay_total) / 1000U); + profile_events.increment(ProfileEvents::OSCPUVirtualTimeMicroseconds, + safeDiff(prev.stat.cpu_run_virtual_total, curr.stat.cpu_run_virtual_total) / 1000U); + + /// Too old struct version, do not read new fields + if (curr.stat.version < TASKSTATS_VERSION) + return; + + profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char)); + profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char)); + profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes)); + profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes)); + } + + static void updateProfileEvents(TasksStatsCounters & last_counters, ProfileEvents::Counters & profile_events) + { + auto current_counters = current(); + incrementProfileEvents(last_counters, current_counters, profile_events); + last_counters = current_counters; + } +}; + +} diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp new file mode 100644 index 00000000000..d4cca1b326c --- /dev/null +++ b/dbms/src/Common/ThreadStatus.cpp @@ -0,0 +1,118 @@ +#include "ThreadStatus.h" +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int PTHREAD_ERROR; +} + + +/// Order of current_thread and current_thread_scope matters +thread_local ThreadStatusPtr current_thread = ThreadStatus::create(); +thread_local ThreadStatus::CurrentThreadScope current_thread_scope; + + +TasksStatsCounters TasksStatsCounters::current() +{ + TasksStatsCounters res; + current_thread->taskstats_getter->getStat(res.stat, current_thread->os_thread_id); + return res; +} + + +ThreadStatus::ThreadStatus() +{ + thread_number = Poco::ThreadNumber::get(); + os_thread_id = TaskStatsInfoGetter::getCurrentTID(); + + last_rusage = std::make_unique(); + last_taskstats = std::make_unique(); + taskstats_getter = std::make_unique(); + + memory_tracker.setDescription("(for thread)"); + log = &Poco::Logger::get("ThreadStatus"); + + /// NOTE: It is important not to do any non-trivial actions (like updating ProfileEvents or logging) before ThreadStatus is created + /// Otherwise it could lead to SIGSEGV due to current_thread dereferencing +} + +ThreadStatusPtr ThreadStatus::create() +{ + return ThreadStatusPtr(new ThreadStatus); +} + +ThreadStatus::~ThreadStatus() = default; + +void ThreadStatus::initPerformanceCounters() +{ + performance_counters_finalized = false; + + /// Clear stats from previous query if a new query is started + /// TODO: make separate query_thread_performance_counters and thread_performance_counters + performance_counters.resetCounters(); + memory_tracker.resetCounters(); + memory_tracker.setDescription("(for thread)"); + + query_start_time_nanoseconds = getCurrentTimeNanoseconds(); + query_start_time = time(nullptr); + ++queries_started; + + *last_rusage = RUsageCounters::current(query_start_time_nanoseconds); + has_permissions_for_taskstats = TaskStatsInfoGetter::checkProcessHasRequiredPermissions(); + if (has_permissions_for_taskstats) + *last_taskstats = TasksStatsCounters::current(); +} + +void ThreadStatus::updatePerformanceCounters() +{ + try + { + RUsageCounters::updateProfileEvents(*last_rusage, performance_counters); + if (has_permissions_for_taskstats) + TasksStatsCounters::updateProfileEvents(*last_taskstats, performance_counters); + } + catch (...) + { + tryLogCurrentException(log); + } +} + +void ThreadStatus::assertState(const std::initializer_list & permitted_states, const char * description) +{ + for (auto permitted_state : permitted_states) + { + if (getCurrentState() == permitted_state) + return; + } + + std::stringstream ss; + ss << "Unexpected thread state " << getCurrentState(); + if (description) + ss << ": " << description; + throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR); +} + +void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue) +{ + logs_queue_ptr = logs_queue; + + if (!thread_group) + return; + + std::unique_lock lock(thread_group->mutex); + thread_group->logs_queue_ptr = logs_queue; +} + +} diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h new file mode 100644 index 00000000000..b708b3dce03 --- /dev/null +++ b/dbms/src/Common/ThreadStatus.h @@ -0,0 +1,197 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + + +namespace Poco +{ + class Logger; +} + + +namespace DB +{ + +class Context; +class QueryStatus; +class ThreadStatus; +using ThreadStatusPtr = std::shared_ptr; +class QueryThreadLog; +struct TasksStatsCounters; +struct RUsageCounters; +class TaskStatsInfoGetter; +class InternalTextLogsQueue; +using InternalTextLogsQueuePtr = std::shared_ptr; +using InternalTextLogsQueueWeakPtr = std::weak_ptr; + + +class ThreadGroupStatus +{ +public: + + mutable std::shared_mutex mutex; + + ProfileEvents::Counters performance_counters{VariableContext::Process}; + MemoryTracker memory_tracker{VariableContext::Process}; + + Context * query_context = nullptr; + Context * global_context = nullptr; + + InternalTextLogsQueueWeakPtr logs_queue_ptr; + + /// Key is Poco's thread_id + using QueryThreadStatuses = std::map; + QueryThreadStatuses thread_statuses; + + /// The first thread created this thread group + ThreadStatusPtr master_thread; + + String query; +}; + +using ThreadGroupStatusPtr = std::shared_ptr; + + +class ThreadStatus : public std::enable_shared_from_this +{ +public: + + /// Poco's thread number (the same number is used in logs) + UInt32 thread_number = 0; + /// Linux's PID (or TGID) (the same id is shown by ps util) + Int32 os_thread_id = -1; + + /// TODO: merge them into common entity + ProfileEvents::Counters performance_counters{VariableContext::Thread}; + MemoryTracker memory_tracker{VariableContext::Thread}; + + /// Statistics of read and write rows/bytes + Progress progress_in; + Progress progress_out; + +public: + + static ThreadStatusPtr create(); + + ThreadGroupStatusPtr getThreadGroup() const + { + return thread_group; + } + + enum ThreadState + { + DetachedFromQuery = 0, /// We just created thread or it is a background thread + AttachedToQuery, /// Thread executes enqueued query + Died, /// Thread does not exist + }; + + int getCurrentState() const + { + return thread_state.load(std::memory_order_relaxed); + } + + String getQueryID(); + + /// Starts new query and create new thread group for it, current thread becomes master thread of the query + void initializeQuery(); + + /// Attaches slave thread to existing thread group + void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true); + + InternalTextLogsQueuePtr getInternalTextLogsQueue() const + { + return thread_state == Died ? nullptr : logs_queue_ptr.lock(); + } + + void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue); + + /// Sets query context for current thread and its thread group + /// NOTE: query_context have to be alive until detachQuery() is called + void attachQueryContext(Context & query_context); + + /// Update several ProfileEvents counters + void updatePerformanceCounters(); + + /// Update ProfileEvents and dumps info to system.query_thread_log + void finalizePerformanceCounters(); + + /// Detaches thread from the thread group and the query, dumps performance counters if they have not been dumped + void detachQuery(bool exit_if_already_detached = false, bool thread_exits = false); + + ~ThreadStatus(); + +protected: + + ThreadStatus(); + + void initPerformanceCounters(); + + void logToQueryThreadLog(QueryThreadLog & thread_log); + + void assertState(const std::initializer_list & permitted_states, const char * description = nullptr); + + ThreadGroupStatusPtr thread_group; + + std::atomic thread_state{ThreadState::DetachedFromQuery}; + + /// Is set once + Context * global_context = nullptr; + /// Use it only from current thread + Context * query_context = nullptr; + + /// A logs queue used by TCPHandler to pass logs to a client + InternalTextLogsQueueWeakPtr logs_queue_ptr; + + bool performance_counters_finalized = false; + UInt64 query_start_time_nanoseconds = 0; + time_t query_start_time = 0; + size_t queries_started = 0; + + Poco::Logger * log = nullptr; + + friend class CurrentThread; + friend struct TasksStatsCounters; + + /// Use ptr not to add extra dependencies in the header + std::unique_ptr last_rusage; + std::unique_ptr last_taskstats; + std::unique_ptr taskstats_getter; + bool has_permissions_for_taskstats = false; + +public: + + /// Implicitly finalizes current thread in the destructor + class CurrentThreadScope + { + public: + void (*deleter)() = nullptr; + + CurrentThreadScope() = default; + ~CurrentThreadScope() + { + try + { + if (deleter) + deleter(); + } + catch (...) + { + std::terminate(); + } + } + }; + +private: + static void defaultThreadDeleter(); +}; + + +extern thread_local ThreadStatusPtr current_thread; +extern thread_local ThreadStatus::CurrentThreadScope current_thread_scope; + +} diff --git a/dbms/src/Common/Throttler.h b/dbms/src/Common/Throttler.h index abc87ffae50..9c7b446dd96 100644 --- a/dbms/src/Common/Throttler.h +++ b/dbms/src/Common/Throttler.h @@ -5,10 +5,17 @@ #include #include #include +#include #include #include +namespace ProfileEvents +{ + extern const Event ThrottlerSleepMicroseconds; +} + + namespace DB { @@ -69,10 +76,14 @@ public: if (desired_ns > elapsed_ns) { UInt64 sleep_ns = desired_ns - elapsed_ns; - timespec sleep_ts; + ::timespec sleep_ts; sleep_ts.tv_sec = sleep_ns / 1000000000; sleep_ts.tv_nsec = sleep_ns % 1000000000; - nanosleep(&sleep_ts, nullptr); /// NOTE Returns early in case of a signal. This is considered normal. + + /// NOTE: Returns early in case of a signal. This is considered normal. + ::nanosleep(&sleep_ts, nullptr); + + ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_ns / 1000UL); } } diff --git a/dbms/src/Common/UInt128.h b/dbms/src/Common/UInt128.h index b27e6b068e2..9a4f4b570f6 100644 --- a/dbms/src/Common/UInt128.h +++ b/dbms/src/Common/UInt128.h @@ -28,9 +28,17 @@ struct UInt128 UInt64 high; UInt128() = default; - explicit UInt128(const UInt64 rhs) : low(rhs), high() {} explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) {} +#if 1 + explicit UInt128(const unsigned __int128 rhs) + : low(rhs & 0xffffffffffffffffll), + high(rhs >> 64) + {} +#else + explicit UInt128(const UInt64 rhs) : low(rhs), high() {} +#endif + auto tuple() const { return std::tie(high, low); } bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); } diff --git a/dbms/src/Common/VariableContext.h b/dbms/src/Common/VariableContext.h new file mode 100644 index 00000000000..2fe4ffb565a --- /dev/null +++ b/dbms/src/Common/VariableContext.h @@ -0,0 +1,12 @@ +#pragma once + +/// Used in ProfileEvents and MemoryTracker to determine their hierarchy level +/// The less value the higher level (zero level is the root) +enum class VariableContext +{ + Global = 0, + User, /// Group of processes + Process, /// For example, a query or a merge + Thread, /// A thread of a process + Snapshot /// Does not belong to anybody +}; diff --git a/dbms/src/Common/setThreadName.cpp b/dbms/src/Common/setThreadName.cpp index 1a33c330027..d92b2e84715 100644 --- a/dbms/src/Common/setThreadName.cpp +++ b/dbms/src/Common/setThreadName.cpp @@ -6,10 +6,20 @@ #else #include #endif +#include +#include #include #include +namespace DB +{ +namespace ErrorCodes +{ + extern const int PTHREAD_ERROR; +} +} + void setThreadName(const char * name) { @@ -22,5 +32,21 @@ void setThreadName(const char * name) #else if (0 != prctl(PR_SET_NAME, name, 0, 0, 0)) #endif - DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME...)"); + DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)"); +} + +std::string getThreadName() +{ + std::string name(16, '\0'); + +#if defined(__FreeBSD__) || defined(__APPLE__) + if (pthread_get_name_np(pthread_self(), name.data(), name.size()); + throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR); +#else + if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0)) +#endif + DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)"); + + name.resize(std::strlen(name.data())); + return name; } diff --git a/dbms/src/Common/setThreadName.h b/dbms/src/Common/setThreadName.h index dc6af7336e0..cdcb6b46914 100644 --- a/dbms/src/Common/setThreadName.h +++ b/dbms/src/Common/setThreadName.h @@ -1,7 +1,10 @@ #pragma once +#include /** Sets the thread name (maximum length is 15 bytes), * which will be visible in ps, gdb, /proc, * for convenience of observation and debugging. */ void setThreadName(const char * name); + +std::string getThreadName(); diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index 1c4330b97ce..f4d01e85bd2 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -71,6 +71,3 @@ target_link_libraries (cow_columns clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) target_link_libraries (stopwatch clickhouse_common_io) - -add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) -target_link_libraries (validate-odbc-connection-string dbms) diff --git a/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp b/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp index 9d1eda9178c..15b01367cac 100644 --- a/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp +++ b/dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp @@ -32,7 +32,7 @@ TEST(Common, RWLockFIFO_1) auto func = [&] (size_t threads, int round) { - for (int i = 0; i < cycles; ++i) + for (int i = 0; i < cycles; ++i) { auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockFIFO::Read : RWLockFIFO::Write; auto sleep_for = std::chrono::duration(std::uniform_int_distribution<>(1, 100)(gen)); diff --git a/dbms/src/Core/AccurateComparison.h b/dbms/src/Core/AccurateComparison.h index 10109d92e8a..23876d8e306 100644 --- a/dbms/src/Core/AccurateComparison.h +++ b/dbms/src/Core/AccurateComparison.h @@ -332,10 +332,32 @@ inline bool equalsOp(DB::Float32 f, DB::UInt128 u) return equalsOp(static_cast(f), u); } -inline bool greaterOp(DB::Int128 i, DB::Float64 f) { return static_cast(f) < i; } -inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return static_cast(f) < i; } -inline bool greaterOp(DB::Float64 f, DB::Int128 i) { return static_cast(f) > i; } -inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return static_cast(f) > i; } +inline bool greaterOp(DB::Int128 i, DB::Float64 f) +{ + static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; + static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return static_cast(i) > f; + + return (f < static_cast(min_int128)) + || (f < static_cast(max_int128) && i > static_cast(f)); +} + +inline bool greaterOp(DB::Float64 f, DB::Int128 i) +{ + static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; + static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; + + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return f > static_cast(i); + + return (f >= static_cast(max_int128)) + || (f > static_cast(min_int128) && static_cast(f) > i); +} + +inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return greaterOp(i, static_cast(f)); } +inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return greaterOp(static_cast(f), i); } inline bool equalsOp(DB::Int128 i, DB::Float64 f) { return i == static_cast(f) && static_cast(i) == f; } inline bool equalsOp(DB::Int128 i, DB::Float32 f) { return i == static_cast(f) && static_cast(i) == f; } diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 331e226103d..d7a2ca419e3 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -46,6 +46,7 @@ #define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 #define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372 #define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401 +#define DBMS_MIN_REVISION_WITH_SERVER_LOGS 54406 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. #define DBMS_TCP_PROTOCOL_VERSION 54226 diff --git a/dbms/src/Core/ExternalTable.cpp b/dbms/src/Core/ExternalTable.cpp new file mode 100644 index 00000000000..3d8d8077e97 --- /dev/null +++ b/dbms/src/Core/ExternalTable.cpp @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + + +ExternalTableData BaseExternalTable::getData(const Context & context) +{ + initReadBuffer(); + initSampleBlock(); + auto input = context.getInputFormat(format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE); + return std::make_pair(std::make_shared(input), name); +} + +void BaseExternalTable::clean() +{ + name = ""; + file = ""; + format = ""; + structure.clear(); + sample_block = Block(); + read_buffer.reset(); +} + +/// Function for debugging information output +void BaseExternalTable::write() +{ + std::cerr << "file " << file << std::endl; + std::cerr << "name " << name << std::endl; + std::cerr << "format " << format << std::endl; + std::cerr << "structure: \n"; + for (size_t i = 0; i < structure.size(); ++i) + std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl; +} + +std::vector BaseExternalTable::split(const std::string & s, const std::string & d) +{ + std::vector res; + boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on); + return res; +} + +void BaseExternalTable::parseStructureFromStructureField(const std::string & argument) +{ + std::vector vals = split(argument, " ,"); + + if (vals.size() & 1) + throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS); + + for (size_t i = 0; i < vals.size(); i += 2) + structure.emplace_back(vals[i], vals[i + 1]); +} + +void BaseExternalTable::parseStructureFromTypesField(const std::string & argument) +{ + std::vector vals = split(argument, " ,"); + + for (size_t i = 0; i < vals.size(); ++i) + structure.emplace_back("_" + toString(i + 1), vals[i]); +} + +void BaseExternalTable::initSampleBlock() +{ + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + + for (size_t i = 0; i < structure.size(); ++i) + { + ColumnWithTypeAndName column; + column.name = structure[i].first; + column.type = data_type_factory.get(structure[i].second); + column.column = column.type->createColumn(); + sample_block.insert(std::move(column)); + } +} + + +void ExternalTable::initReadBuffer() +{ + if (file == "-") + read_buffer = std::make_unique(STDIN_FILENO); + else + read_buffer = std::make_unique(file); +} + +ExternalTable::ExternalTable(const boost::program_options::variables_map & external_options) +{ + if (external_options.count("file")) + file = external_options["file"].as(); + else + throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("name")) + name = external_options["name"].as(); + else + throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("format")) + format = external_options["format"].as(); + else + throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); + + if (external_options.count("structure")) + parseStructureFromStructureField(external_options["structure"].as()); + else if (external_options.count("types")) + parseStructureFromTypesField(external_options["types"].as()); + else + throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS); +} + + +void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) +{ + const Settings & settings = context.getSettingsRef(); + + /// The buffer is initialized here, not in the virtual function initReadBuffer + read_buffer_impl = std::make_unique(stream); + + if (settings.http_max_multipart_form_data_size) + read_buffer = std::make_unique( + *read_buffer_impl, settings.http_max_multipart_form_data_size, + true, "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting"); + else + read_buffer = std::move(read_buffer_impl); + + /// Retrieve a collection of parameters from MessageHeader + Poco::Net::NameValueCollection content; + std::string label; + Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content); + + /// Get parameters + name = content.get("name", "_data"); + format = params.get(name + "_format", "TabSeparated"); + + if (params.has(name + "_structure")) + parseStructureFromStructureField(params.get(name + "_structure")); + else if (params.has(name + "_types")) + parseStructureFromTypesField(params.get(name + "_types")); + else + throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS); + + ExternalTableData data = getData(context); + + /// Create table + NamesAndTypesList columns = sample_block.getNamesAndTypesList(); + StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); + storage->startup(); + context.addExternalTable(data.second, storage); + BlockOutputStreamPtr output = storage->write(ASTPtr(), settings); + + /// Write data + data.first->readPrefix(); + output->writePrefix(); + while(Block block = data.first->read()) + output->write(block); + data.first->readSuffix(); + output->writeSuffix(); + + /// We are ready to receive the next file, for this we clear all the information received + clean(); +} + +} diff --git a/dbms/src/Core/ExternalTable.h b/dbms/src/Core/ExternalTable.h new file mode 100644 index 00000000000..6927f19b524 --- /dev/null +++ b/dbms/src/Core/ExternalTable.h @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + + +namespace Poco +{ + namespace Net + { + class NameValueCollection; + class MessageHeader; + } +} + +namespace boost +{ + namespace program_options + { + class variables_map; + } +} + + +namespace DB +{ + +class Context; + + +/// The base class containing the basic information about external table and +/// basic functions for extracting this information from text fields. +class BaseExternalTable +{ +public: + std::string file; /// File with data or '-' if stdin + std::string name; /// The name of the table + std::string format; /// Name of the data storage format + + /// Description of the table structure: (column name, data type name) + std::vector> structure; + + std::unique_ptr read_buffer; + Block sample_block; + + virtual ~BaseExternalTable() {} + + /// Initialize read_buffer, depending on the data source. By default, does nothing. + virtual void initReadBuffer() {} + + /// Get the table data - a pair (a stream with the contents of the table, the name of the table) + ExternalTableData getData(const Context & context); + +protected: + /// Clear all accumulated information + void clean(); + + /// Function for debugging information output + void write(); + + static std::vector split(const std::string & s, const std::string & d); + + /// Construct the `structure` vector from the text field `structure` + virtual void parseStructureFromStructureField(const std::string & argument); + + /// Construct the `structure` vector from the text field `types` + virtual void parseStructureFromTypesField(const std::string & argument); + +private: + /// Initialize sample_block according to the structure of the table stored in the `structure` + void initSampleBlock(); +}; + + +/// Parsing of external table used in the tcp client. +class ExternalTable : public BaseExternalTable +{ +public: + void initReadBuffer() override; + + /// Extract parameters from variables_map, which is built on the client command line + ExternalTable(const boost::program_options::variables_map & external_options); +}; + + +/// Parsing of external table used when sending tables via http +/// The `handlePart` function will be called for each table passed, +/// so it's also necessary to call `clean` at the end of the `handlePart`. +class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable +{ +public: + ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {} + + void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream); + +private: + Context & context; + const Poco::Net::NameValueCollection & params; + std::unique_ptr read_buffer_impl; +}; + + +} diff --git a/dbms/src/Core/Protocol.h b/dbms/src/Core/Protocol.h index cd5456cca34..5451e1550f6 100644 --- a/dbms/src/Core/Protocol.h +++ b/dbms/src/Core/Protocol.h @@ -69,6 +69,7 @@ namespace Protocol Totals = 7, /// A block with totals (compressed or not). Extremes = 8, /// A block with minimums and maximums (compressed or not). TablesStatusResponse = 9, /// A response to TablesStatus request. + Log = 10 /// System logs of the query execution }; /// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10 @@ -77,8 +78,8 @@ namespace Protocol /// See https://www.securecoding.cert.org/confluence/display/cplusplus/INT36-CPP.+Do+not+use+out-of-range+enumeration+values inline const char * toString(UInt64 packet) { - static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse" }; - return packet < 10 + static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse", "Log" }; + return packet < 11 ? data[packet] : "Unknown packet"; } @@ -97,6 +98,7 @@ namespace Protocol Cancel = 3, /// Cancel the query execution. Ping = 4, /// Check that connection to the server is alive. TablesStatusRequest = 5, /// Check status of tables on the server. + KeepAlive = 6 /// Keep the connection alive }; inline const char * toString(UInt64 packet) diff --git a/dbms/src/Core/Types.h b/dbms/src/Core/Types.h index 178d3ab1ca1..2eb74e630a8 100644 --- a/dbms/src/Core/Types.h +++ b/dbms/src/Core/Types.h @@ -120,9 +120,8 @@ namespace DB { /// Own FieldType for Decimal template - class Dec + struct Dec { - public: using NativeType = T; Dec() = default; @@ -149,7 +148,6 @@ namespace DB const Dec & operator /= (const T & x) { value /= x; return *this; } const Dec & operator %= (const T & x) { value %= x; return *this; } - private: T value; }; diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp b/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp new file mode 100644 index 00000000000..ba31b45bfd2 --- /dev/null +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.cpp @@ -0,0 +1,84 @@ +#include "AsynchronousBlockInputStream.h" +#include + + +namespace DB +{ + +Block AsynchronousBlockInputStream::readImpl() +{ + /// If there were no calculations yet, calculate the first block synchronously + if (!started) + { + calculate(); + started = true; + } + else /// If the calculations are already in progress - wait for the result + pool.wait(); + + if (exception) + std::rethrow_exception(exception); + + Block res = block; + if (!res) + return res; + + /// Start the next block calculation + block.clear(); + next(); + + return res; +} + + +void AsynchronousBlockInputStream::next() +{ + ready.reset(); + + pool.schedule([this, thread_group=CurrentThread::getGroup()] () + { + CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; + + try + { + if (first) + setThreadName("AsyncBlockInput"); + + /// AsynchronousBlockInputStream is used in Client which does not create queries and thread groups + if (thread_group) + CurrentThread::attachToIfDetached(thread_group); + } + catch (...) + { + exception = std::current_exception(); + ready.set(); + return; + } + + calculate(); + }); +} + + +void AsynchronousBlockInputStream::calculate() +{ + try + { + if (first) + { + first = false; + children.back()->readPrefix(); + } + + block = children.back()->read(); + } + catch (...) + { + exception = std::current_exception(); + } + + ready.set(); +} + +} + diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.h b/dbms/src/DataStreams/AsynchronousBlockInputStream.h index 0a80628cf2a..c790deb49c2 100644 --- a/dbms/src/DataStreams/AsynchronousBlockInputStream.h +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -91,64 +92,12 @@ protected: Block block; std::exception_ptr exception; + Block readImpl() override; - Block readImpl() override - { - /// If there were no calculations yet, calculate the first block synchronously - if (!started) - { - calculate(current_memory_tracker); - started = true; - } - else /// If the calculations are already in progress - wait for the result - pool.wait(); - - if (exception) - std::rethrow_exception(exception); - - Block res = block; - if (!res) - return res; - - /// Start the next block calculation - block.clear(); - next(); - - return res; - } - - - void next() - { - ready.reset(); - pool.schedule(std::bind(&AsynchronousBlockInputStream::calculate, this, current_memory_tracker)); - } - + void next(); /// Calculations that can be performed in a separate thread - void calculate(MemoryTracker * memory_tracker) - { - CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; - - try - { - if (first) - { - first = false; - setThreadName("AsyncBlockInput"); - current_memory_tracker = memory_tracker; - children.back()->readPrefix(); - } - - block = children.back()->read(); - } - catch (...) - { - exception = std::current_exception(); - } - - ready.set(); - } + void calculate(); }; } diff --git a/dbms/src/DataStreams/BlockIO.h b/dbms/src/DataStreams/BlockIO.h index 6d97e30e510..9f69f834e5f 100644 --- a/dbms/src/DataStreams/BlockIO.h +++ b/dbms/src/DataStreams/BlockIO.h @@ -12,8 +12,7 @@ class ProcessListEntry; struct BlockIO { /** process_list_entry should be destroyed after in and after out, - * since in and out contain pointer to an object inside process_list_entry - * (MemoryTracker * current_memory_tracker), + * since in and out contain pointer to objects inside process_list_entry (query-level MemoryTracker for example), * which could be used before destroying of in and out. */ std::shared_ptr process_list_entry; @@ -38,12 +37,17 @@ struct BlockIO exception_callback(); } + /// We provide the correct order of destruction. + void reset() + { + out.reset(); + in.reset(); + process_list_entry.reset(); + } + BlockIO & operator= (const BlockIO & rhs) { - /// We provide the correct order of destruction. - out = nullptr; - in = nullptr; - process_list_entry = nullptr; + reset(); process_list_entry = rhs.process_list_entry; in = rhs.in; diff --git a/dbms/src/DataStreams/CountingBlockOutputStream.h b/dbms/src/DataStreams/CountingBlockOutputStream.h index 12732473ac3..ea1b5ec037d 100644 --- a/dbms/src/DataStreams/CountingBlockOutputStream.h +++ b/dbms/src/DataStreams/CountingBlockOutputStream.h @@ -20,7 +20,7 @@ public: progress_callback = callback; } - void setProcessListElement(ProcessListElement * elem) + void setProcessListElement(QueryStatus * elem) { process_elem = elem; } @@ -43,7 +43,7 @@ protected: BlockOutputStreamPtr stream; Progress progress; ProgressCallback progress_callback; - ProcessListElement * process_elem = nullptr; + QueryStatus * process_elem = nullptr; }; } diff --git a/dbms/src/DataStreams/FilterBlockInputStream.cpp b/dbms/src/DataStreams/FilterBlockInputStream.cpp index bd75dda1293..39d0d0c9615 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.cpp +++ b/dbms/src/DataStreams/FilterBlockInputStream.cpp @@ -17,8 +17,9 @@ namespace ErrorCodes } -FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name) - : expression(expression_) +FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, + const String & filter_column_name, bool remove_filter) + : remove_filter(remove_filter), expression(expression_) { children.push_back(input); @@ -40,6 +41,9 @@ FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input FilterDescription filter_description_check(*column_elem.column); column_elem.column = column_elem.type->createColumnConst(header.rows(), UInt64(1)); } + + if (remove_filter) + header.erase(filter_column_name); } @@ -69,7 +73,7 @@ Block FilterBlockInputStream::readImpl() Block res; if (constant_filter_description.always_false) - return res; + return removeFilterIfNeed(std::move(res)); /// Until non-empty block after filtering or end of stream. while (1) @@ -81,7 +85,7 @@ Block FilterBlockInputStream::readImpl() expression->execute(res); if (constant_filter_description.always_true) - return res; + return removeFilterIfNeed(std::move(res)); size_t columns = res.columns(); ColumnPtr column = res.safeGetByPosition(filter_column).column; @@ -100,7 +104,7 @@ Block FilterBlockInputStream::readImpl() } if (constant_filter_description.always_true) - return res; + return removeFilterIfNeed(std::move(res)); FilterDescription filter_and_holder(*column); @@ -142,7 +146,7 @@ Block FilterBlockInputStream::readImpl() /// Replace the column with the filter by a constant. res.safeGetByPosition(filter_column).column = res.safeGetByPosition(filter_column).type->createColumnConst(filtered_rows, UInt64(1)); /// No need to touch the rest of the columns. - return res; + return removeFilterIfNeed(std::move(res)); } /// Filter the rest of the columns. @@ -170,9 +174,18 @@ Block FilterBlockInputStream::readImpl() current_column.column = current_column.column->filter(*filter_and_holder.data, -1); } - return res; + return removeFilterIfNeed(std::move(res)); } } +Block FilterBlockInputStream::removeFilterIfNeed(Block && block) +{ + if (block && remove_filter) + block.erase(static_cast(filter_column)); + + return std::move(block); +} + + } diff --git a/dbms/src/DataStreams/FilterBlockInputStream.h b/dbms/src/DataStreams/FilterBlockInputStream.h index 8089cf87420..ca63b34f45c 100644 --- a/dbms/src/DataStreams/FilterBlockInputStream.h +++ b/dbms/src/DataStreams/FilterBlockInputStream.h @@ -20,7 +20,8 @@ private: using ExpressionActionsPtr = std::shared_ptr; public: - FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name_); + FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, + const String & filter_column_name_, bool remove_filter = false); String getName() const override; Block getTotals() override; @@ -29,12 +30,16 @@ public: protected: Block readImpl() override; + bool remove_filter; + private: ExpressionActionsPtr expression; Block header; ssize_t filter_column; ConstantFilterDescription constant_filter_description; + + Block removeFilterIfNeed(Block && block); }; } diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index 5c4cf2aa219..998ea2b42db 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -1,6 +1,13 @@ #include #include #include +#include + + +namespace ProfileEvents +{ + extern const Event ThrottlerSleepMicroseconds; +} namespace DB @@ -286,21 +293,34 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value) size_t total_rows = progress.total_rows; - if (limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0)) - { - double total_elapsed = info.total_stopwatch.elapsedSeconds(); + constexpr UInt64 profile_events_update_period_microseconds = 10 * 1000; // 10 milliseconds + UInt64 total_elapsed_microseconds = info.total_stopwatch.elapsedMicroseconds(); - if (total_elapsed > limits.timeout_before_checking_execution_speed.totalMicroseconds() / 1000000.0) + if (last_profile_events_update_time + profile_events_update_period_microseconds < total_elapsed_microseconds) + { + CurrentThread::updatePerformanceCounters(); + last_profile_events_update_time = total_elapsed_microseconds; + } + + if ((limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0)) + && (static_cast(total_elapsed_microseconds) > limits.timeout_before_checking_execution_speed.totalMicroseconds())) + { + /// Do not count sleeps in throttlers + UInt64 throttler_sleep_microseconds = CurrentThread::getProfileEvents()[ProfileEvents::ThrottlerSleepMicroseconds]; + double elapsed_seconds = (throttler_sleep_microseconds > total_elapsed_microseconds) + ? 0.0 : (total_elapsed_microseconds - throttler_sleep_microseconds) / 1000000.0; + + if (elapsed_seconds > 0) { - if (limits.min_execution_speed && progress.rows / total_elapsed < limits.min_execution_speed) - throw Exception("Query is executing too slow: " + toString(progress.rows / total_elapsed) + if (limits.min_execution_speed && progress.rows / elapsed_seconds < limits.min_execution_speed) + throw Exception("Query is executing too slow: " + toString(progress.rows / elapsed_seconds) + " rows/sec., minimum: " + toString(limits.min_execution_speed), ErrorCodes::TOO_SLOW); /// If the predicted execution time is longer than `max_execution_time`. if (limits.max_execution_time != 0 && total_rows) { - double estimated_execution_time_seconds = total_elapsed * (static_cast(total_rows) / progress.rows); + double estimated_execution_time_seconds = elapsed_seconds * (static_cast(total_rows) / progress.rows); if (estimated_execution_time_seconds > limits.max_execution_time.totalSeconds()) throw Exception("Estimated query execution time (" + toString(estimated_execution_time_seconds) + " seconds)" @@ -363,7 +383,7 @@ void IProfilingBlockInputStream::setProgressCallback(const ProgressCallback & ca } -void IProfilingBlockInputStream::setProcessListElement(ProcessListElement * elem) +void IProfilingBlockInputStream::setProcessListElement(QueryStatus * elem) { process_list_elem = elem; diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.h b/dbms/src/DataStreams/IProfilingBlockInputStream.h index 5febcb18c56..ab0db8dd99d 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.h +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.h @@ -20,6 +20,7 @@ namespace ErrorCodes } class QuotaForIntervals; +class QueryStatus; class ProcessListElement; class IProfilingBlockInputStream; @@ -103,7 +104,7 @@ public: * Based on this information, the quota and some restrictions will be checked. * This information will also be available in the SHOW PROCESSLIST request. */ - void setProcessListElement(ProcessListElement * elem); + void setProcessListElement(QueryStatus * elem); /** Set the approximate total number of rows to read. */ @@ -178,7 +179,9 @@ protected: std::atomic is_cancelled{false}; std::atomic is_killed{false}; ProgressCallback progress_callback; - ProcessListElement * process_list_elem = nullptr; + QueryStatus * process_list_elem = nullptr; + /// According to total_stopwatch in microseconds + UInt64 last_profile_events_update_time = 0; /// Additional information that can be generated during the work process. diff --git a/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp new file mode 100644 index 00000000000..02d2f8cf440 --- /dev/null +++ b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp @@ -0,0 +1,82 @@ +#include "InternalTextLogsRowOutputStream.h" +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +Block InternalTextLogsRowOutputStream::getHeader() const +{ + return InternalTextLogsQueue::getSampleBlock(); +} + +void InternalTextLogsRowOutputStream::write(const Block & block) +{ + auto & array_event_time = typeid_cast(*block.getByName("event_time").column).getData(); + auto & array_microseconds = typeid_cast(*block.getByName("event_time_microseconds").column).getData(); + + auto & column_host_name = typeid_cast(*block.getByName("host_name").column); + auto & column_query_id = typeid_cast(*block.getByName("query_id").column); + + auto & array_thread_number = typeid_cast(*block.getByName("thread_number").column).getData(); + auto & array_priority = typeid_cast(*block.getByName("priority").column).getData(); + auto & column_source = typeid_cast(*block.getByName("source").column); + auto & column_text = typeid_cast(*block.getByName("text").column); + + for (size_t row_num = 0; row_num < block.rows(); ++row_num) + { + auto host_name = column_host_name.getDataAt(row_num); + if (host_name.size) + { + writeCString("[", wb); + writeString(host_name, wb); + writeCString("] ", wb); + } + + auto event_time = array_event_time[row_num]; + writeDateTimeText<'.', ':'>(event_time, wb); + + auto microseconds = array_microseconds[row_num]; + writeChar('.', wb); + writeChar('0' + ((microseconds / 100000) % 10), wb); + writeChar('0' + ((microseconds / 10000) % 10), wb); + writeChar('0' + ((microseconds / 1000) % 10), wb); + writeChar('0' + ((microseconds / 100) % 10), wb); + writeChar('0' + ((microseconds / 10) % 10), wb); + writeChar('0' + ((microseconds / 1) % 10), wb); + + auto query_id = column_query_id.getDataAt(row_num); + if (query_id.size) + { + writeCString(" {", wb); + writeString(query_id, wb); + writeCString("}", wb); + } + + UInt32 thread_number = array_thread_number[row_num]; + writeCString(" [ ", wb); + writeIntText(thread_number, wb); + writeCString(" ] <", wb); + + Int8 priority = array_priority[row_num]; + writeString(InternalTextLogsQueue::getPriorityName(priority), wb); + writeCString("> ", wb); + + auto source = column_source.getDataAt(row_num); + writeString(source, wb); + writeCString(": ", wb); + + auto text = column_text.getDataAt(row_num); + writeString(text, wb); + + writeChar('\n', wb); + } +} + +} diff --git a/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h new file mode 100644 index 00000000000..3f54a00e633 --- /dev/null +++ b/dbms/src/DataStreams/InternalTextLogsRowOutputStream.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + + +namespace DB +{ + +/// Prints internal server logs +/// Input blocks have to have the same structure as SystemLogsQueue::getSampleBlock() +/// NOTE: IRowOutputStream does not suite well for this case +class InternalTextLogsRowOutputStream : public IBlockOutputStream +{ +public: + + InternalTextLogsRowOutputStream(WriteBuffer & buf_out) : wb(buf_out) {} + + Block getHeader() const override; + + void write(const Block & block) override; + + void flush() override + { + wb.next(); + } + +private: + + WriteBuffer & wb; +}; + +} diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp index 334c65d0c40..022366cbc04 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -175,10 +176,10 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start() { auto & child = children[i]; - auto memory_tracker = current_memory_tracker; - reading_pool->schedule([&child, memory_tracker] + auto thread_group = CurrentThread::getGroup(); + reading_pool->schedule([&child, thread_group] { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); setThreadName("MergeAggReadThr"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; child->readPrefix(); @@ -196,8 +197,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start() */ for (size_t i = 0; i < merging_threads; ++i) - pool.schedule(std::bind(&MergingAggregatedMemoryEfficientBlockInputStream::mergeThread, - this, current_memory_tracker)); + pool.schedule([this, thread_group=CurrentThread::getGroup()] () { mergeThread(thread_group); } ); } } @@ -293,14 +293,16 @@ void MergingAggregatedMemoryEfficientBlockInputStream::finalize() } -void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker * memory_tracker) +void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupStatusPtr thread_group) { - setThreadName("MergeAggMergThr"); - current_memory_tracker = memory_tracker; CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; try { + if (thread_group) + CurrentThread::attachTo(thread_group); + setThreadName("MergeAggMergThr"); + while (!parallel_merge_data->finish) { /** Receiving next blocks is processing by one thread pool, and merge is in another. @@ -480,10 +482,10 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate { if (need_that_input(input)) { - auto memory_tracker = current_memory_tracker; - reading_pool->schedule([&input, &read_from_input, memory_tracker] + auto thread_group = CurrentThread::getGroup(); + reading_pool->schedule([&input, &read_from_input, thread_group] { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); setThreadName("MergeAggReadThr"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; read_from_input(input); diff --git a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h index 837c10869cf..bdabd8cc1f8 100644 --- a/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h +++ b/dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -151,7 +152,7 @@ private: std::unique_ptr parallel_merge_data; - void mergeThread(MemoryTracker * memory_tracker); + void mergeThread(ThreadGroupStatusPtr main_thread); void finalize(); }; diff --git a/dbms/src/DataStreams/ParallelInputsProcessor.h b/dbms/src/DataStreams/ParallelInputsProcessor.h index 115dc6e1a3a..a1c4e2ac480 100644 --- a/dbms/src/DataStreams/ParallelInputsProcessor.h +++ b/dbms/src/DataStreams/ParallelInputsProcessor.h @@ -12,6 +12,7 @@ #include #include #include +#include /** Allows to process multiple block input streams (sources) in parallel, using specified number of threads. @@ -105,8 +106,9 @@ public: { active_threads = max_threads; threads.reserve(max_threads); + auto thread_group = CurrentThread::getGroup(); for (size_t i = 0; i < max_threads; ++i) - threads.emplace_back(std::bind(&ParallelInputsProcessor::thread, this, current_memory_tracker, i)); + threads.emplace_back([=] () { thread(thread_group, i); } ); } /// Ask all sources to stop earlier than they run out. @@ -174,16 +176,16 @@ private: } } - void thread(MemoryTracker * memory_tracker, size_t thread_num) + void thread(ThreadGroupStatusPtr thread_group, size_t thread_num) { - current_memory_tracker = memory_tracker; std::exception_ptr exception; - - setThreadName("ParalInputsProc"); CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; try { + setThreadName("ParalInputsProc"); + CurrentThread::attachTo(thread_group); + while (!finish) { InputData unprepared_input; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 349c74732d6..670a70ad6bb 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include #include @@ -137,7 +139,7 @@ void RemoteBlockInputStream::sendExternalTables() for (const auto & table : external_tables) { StoragePtr cur = table.second; - QueryProcessingStage::Enum read_from_table_stage = QueryProcessingStage::Complete; + QueryProcessingStage::Enum read_from_table_stage = cur->getQueryProcessingStage(context); BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context, read_from_table_stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) @@ -232,6 +234,12 @@ Block RemoteBlockInputStream::readImpl() extremes = packet.block; break; + case Protocol::Server::Log: + /// Pass logs from remote server to client + if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) + log_queue->pushBlock(std::move(packet.block)); + break; + default: got_unknown_packet_from_replica = true; throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); diff --git a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp index d9095ec91b9..f1e68a6a0c1 100644 --- a/dbms/src/DataStreams/RemoteBlockOutputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockOutputStream.cpp @@ -4,6 +4,8 @@ #include #include +#include +#include namespace DB @@ -24,23 +26,33 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const */ connection.sendQuery(query, "", QueryProcessingStage::Complete, settings, nullptr); - Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::Data == packet.type) + while (true) { - header = packet.block; + Connection::Packet packet = connection.receivePacket(); - if (!header) - throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); + if (Protocol::Server::Data == packet.type) + { + header = packet.block; + + if (!header) + throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR); + break; + } + else if (Protocol::Server::Exception == packet.type) + { + packet.exception->rethrow(); + break; + } + else if (Protocol::Server::Log == packet.type) + { + /// Pass logs from remote server to client + if (auto log_queue = CurrentThread::getInternalTextLogsQueue()) + log_queue->pushBlock(std::move(packet.block)); + } + else + throw NetException("Unexpected packet from server (expected Data or Exception, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } - else if (Protocol::Server::Exception == packet.type) - { - packet.exception->rethrow(); - return; - } - else - throw NetException("Unexpected packet from server (expected Data or Exception, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } @@ -55,15 +67,11 @@ void RemoteBlockOutputStream::write(const Block & block) catch (const NetException &) { /// Try to get more detailed exception from server - if (connection.poll(0)) + auto packet_type = connection.checkPacket(); + if (packet_type && *packet_type == Protocol::Server::Exception) { Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::Exception == packet.type) - { - packet.exception->rethrow(); - return; - } + packet.exception->rethrow(); } throw; @@ -83,18 +91,23 @@ void RemoteBlockOutputStream::writeSuffix() /// Empty block means end of data. connection.sendData(Block()); - /// Receive EndOfStream packet. - Connection::Packet packet = connection.receivePacket(); - - if (Protocol::Server::EndOfStream == packet.type) + /// Wait for EndOfStream or Exception packet, skip Log packets. + while (true) { - /// Do nothing. + Connection::Packet packet = connection.receivePacket(); + + if (Protocol::Server::EndOfStream == packet.type) + break; + else if (Protocol::Server::Exception == packet.type) + packet.exception->rethrow(); + else if (Protocol::Server::Log == packet.type) + { + // Do nothing + } + else + throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } - else if (Protocol::Server::Exception == packet.type) - packet.exception->rethrow(); - else - throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); finished = true; } diff --git a/dbms/src/DataStreams/tests/expression_stream.cpp b/dbms/src/DataStreams/tests/expression_stream.cpp index 445c2a4b0df..3556c06b026 100644 --- a/dbms/src/DataStreams/tests/expression_stream.cpp +++ b/dbms/src/DataStreams/tests/expression_stream.cpp @@ -47,7 +47,7 @@ try Names column_names; column_names.push_back("number"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreamPtr in; in = table->read(column_names, {}, context, stage, 8192, 1)[0]; diff --git a/dbms/src/DataStreams/tests/filter_stream.cpp b/dbms/src/DataStreams/tests/filter_stream.cpp index 5e634fcf9a8..f1e9494e874 100644 --- a/dbms/src/DataStreams/tests/filter_stream.cpp +++ b/dbms/src/DataStreams/tests/filter_stream.cpp @@ -52,7 +52,7 @@ try Names column_names; column_names.push_back("number"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0]; in = std::make_shared(in, expression, "equals(modulo(number, 3), 1)"); diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/DataStreams/tests/union_stream2.cpp b/dbms/src/DataStreams/tests/union_stream2.cpp index fe0b225c376..377dd76bf2e 100644 --- a/dbms/src/DataStreams/tests/union_stream2.cpp +++ b/dbms/src/DataStreams/tests/union_stream2.cpp @@ -34,7 +34,7 @@ try StoragePtr table = context.getTable("default", "hits6"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context); BlockInputStreams streams = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads); for (size_t i = 0, size = streams.size(); i < size; ++i) diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h index c70bfd3fca7..22ae7c4ee10 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.h +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -88,15 +88,6 @@ public: static constexpr bool is_parametric = true; -#if 1 /// TODO: remove this ctor - DataTypeDecimal() - : precision(0), - scale(0) - { - throw Exception("Ctor for success build", ErrorCodes::LOGICAL_ERROR); - } -#endif - DataTypeDecimal(UInt32 precision_, UInt32 scale_) : precision(precision_), scale(scale_) diff --git a/dbms/src/Dictionaries/ODBCDictionarySource.cpp b/dbms/src/Dictionaries/ODBCDictionarySource.cpp index b4e88ff60e1..4813c1a80ba 100644 --- a/dbms/src/Dictionaries/ODBCDictionarySource.cpp +++ b/dbms/src/Dictionaries/ODBCDictionarySource.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,7 @@ namespace size_t max_block_size, const ConnectionTimeouts & timeouts) { - read_buf = std::make_unique(uri, ODBCBridgeHelper::MAIN_METHOD, callback, timeouts); + read_buf = std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, callback, timeouts); reader = FormatFactory::instance().getInput(ODBCBridgeHelper::DEFAULT_FORMAT, *read_buf, sample_block, context, max_block_size); } @@ -51,7 +52,6 @@ namespace std::unique_ptr read_buf; BlockInputStreamPtr reader; }; - } static const size_t max_block_size = 8192; @@ -71,19 +71,13 @@ ODBCDictionarySource::ODBCDictionarySource(const DictionaryStructure & dict_stru query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::None}, /// NOTE Better to obtain quoting style via ODBC interface. load_all_query{query_builder.composeLoadAllQuery()}, invalidate_query{config.getString(config_prefix + ".invalidate_query", "")}, - odbc_bridge_helper{context, config.getString(config_prefix + ".connection_string")}, + odbc_bridge_helper{context.getConfigRef(), context.getSettingsRef().http_receive_timeout.value, config.getString(config_prefix + ".connection_string")}, timeouts{ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef())}, global_context(context) { - const auto & global_config = context.getConfigRef(); - size_t bridge_port = global_config.getUInt("odbc_bridge.port", ODBCBridgeHelper::DEFAULT_PORT); - std::string bridge_host = global_config.getString("odbc_bridge.host", ODBCBridgeHelper::DEFAULT_HOST); + bridge_url = odbc_bridge_helper.getMainURI(); - bridge_url.setHost(bridge_host); - bridge_url.setPort(bridge_port); - bridge_url.setScheme("http"); - - auto url_params = odbc_bridge_helper.getURLParams(sample_block.getNamesAndTypesList(), max_block_size); + auto url_params = odbc_bridge_helper.getURLParams(sample_block.getNamesAndTypesList().toString(), max_block_size); for (const auto & [name, value] : url_params) bridge_url.addQueryParameter(name, value); } diff --git a/dbms/src/Dictionaries/tests/CMakeLists.txt b/dbms/src/Dictionaries/tests/CMakeLists.txt index f0a4cf4ab68..e69de29bb2d 100644 --- a/dbms/src/Dictionaries/tests/CMakeLists.txt +++ b/dbms/src/Dictionaries/tests/CMakeLists.txt @@ -1,2 +0,0 @@ -add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) -target_link_libraries (validate-odbc-connection-string dbms) diff --git a/dbms/src/Functions/FunctionsArithmetic.h b/dbms/src/Functions/FunctionsArithmetic.h index bbb4bd1369c..a68bb657944 100644 --- a/dbms/src/Functions/FunctionsArithmetic.h +++ b/dbms/src/Functions/FunctionsArithmetic.h @@ -1290,7 +1290,7 @@ public: using RightDataType = std::decay_t; using ResultDataType = typename BinaryOperationTraits::ResultDataType; using OpSpec = Op; - return !std::is_same_v && OpSpec::compilable; + return !std::is_same_v && !IsDecimal && OpSpec::compilable; }); } @@ -1303,7 +1303,7 @@ public: using RightDataType = std::decay_t; using ResultDataType = typename BinaryOperationTraits::ResultDataType; using OpSpec = Op; - if constexpr (!std::is_same_v && OpSpec::compilable) + if constexpr (!std::is_same_v && !IsDecimal && OpSpec::compilable) { auto & b = static_cast &>(builder); auto type = std::make_shared(); @@ -1435,7 +1435,8 @@ public: { return castType(arguments[0].get(), [&](const auto & type) { - return Op::FieldType>::compilable; + using DataType = std::decay_t; + return !IsDecimal && Op::compilable; }); } @@ -1444,9 +1445,10 @@ public: llvm::Value * result = nullptr; castType(types[0].get(), [&](const auto & type) { - using T0 = typename std::decay_t::FieldType; + using DataType = std::decay_t; + using T0 = typename DataType::FieldType; using T1 = typename Op::ResultType; - if constexpr (Op::compilable) + if constexpr (!std::is_same_v && !IsDecimal && Op::compilable) { auto & b = static_cast &>(builder); auto * v = nativeCast(b, types[0], values[0](), std::make_shared>()); diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index a2f0c06b7a4..7bb35e3a478 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ #include #include #include -#include +#include namespace DB @@ -160,7 +161,7 @@ public: void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override { block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst( - input_rows_count, Poco::Net::DNS::hostName())->convertToFullColumnIfConst(); + input_rows_count, DNSResolver::instance().getHostName())->convertToFullColumnIfConst(); } }; diff --git a/dbms/src/IO/LimitReadBuffer.cpp b/dbms/src/IO/LimitReadBuffer.cpp index 82f9fee4855..d5830458baa 100644 --- a/dbms/src/IO/LimitReadBuffer.cpp +++ b/dbms/src/IO/LimitReadBuffer.cpp @@ -1,15 +1,30 @@ #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LIMIT_EXCEEDED; +} + + bool LimitReadBuffer::nextImpl() { /// Let underlying buffer calculate read bytes in `next()` call. in.position() = position(); - if (bytes >= limit || !in.next()) + if (bytes >= limit) + { + if (throw_exception) + throw Exception("Limit for LimitReadBuffer exceeded: " + exception_message, ErrorCodes::LIMIT_EXCEEDED); + else + return false; + } + + if (!in.next()) return false; working_buffer = in.buffer(); @@ -21,8 +36,8 @@ bool LimitReadBuffer::nextImpl() } -LimitReadBuffer::LimitReadBuffer(ReadBuffer & in_, size_t limit_) - : ReadBuffer(in_.position(), 0), in(in_), limit(limit_) +LimitReadBuffer::LimitReadBuffer(ReadBuffer & in, size_t limit, bool throw_exception, std::string exception_message) + : ReadBuffer(in.position(), 0), in(in), limit(limit), throw_exception(throw_exception), exception_message(std::move(exception_message)) { size_t remaining_bytes_in_buffer = in.buffer().end() - in.position(); if (remaining_bytes_in_buffer > limit) diff --git a/dbms/src/IO/LimitReadBuffer.h b/dbms/src/IO/LimitReadBuffer.h index 655cd810a44..31ec037ff73 100644 --- a/dbms/src/IO/LimitReadBuffer.h +++ b/dbms/src/IO/LimitReadBuffer.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include @@ -13,12 +13,14 @@ class LimitReadBuffer : public ReadBuffer { private: ReadBuffer & in; - size_t limit; + UInt64 limit; + bool throw_exception; + std::string exception_message; bool nextImpl() override; public: - LimitReadBuffer(ReadBuffer & in_, size_t limit_); + LimitReadBuffer(ReadBuffer & in, UInt64 limit, bool throw_exception, std::string exception_message = {}); ~LimitReadBuffer() override; }; diff --git a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp index b9ad6be79a8..9bacf699cc8 100644 --- a/dbms/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/dbms/src/IO/ReadBufferFromFileDescriptor.cpp @@ -14,6 +14,7 @@ namespace ProfileEvents extern const Event ReadBufferFromFileDescriptorRead; extern const Event ReadBufferFromFileDescriptorReadFailed; extern const Event ReadBufferFromFileDescriptorReadBytes; + extern const Event DiskReadElapsedMicroseconds; extern const Event Seek; } @@ -47,9 +48,7 @@ bool ReadBufferFromFileDescriptor::nextImpl() { ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorRead); - std::optional watch; - if (profile_callback) - watch.emplace(clock_type); + Stopwatch watch(profile_callback ? clock_type : CLOCK_MONOTONIC); ssize_t res = 0; { @@ -68,12 +67,17 @@ bool ReadBufferFromFileDescriptor::nextImpl() if (res > 0) bytes_read += res; + /// NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one and we will count cpu time of other thread + /// It is better to use taskstats::blkio_delay_total, but it is quite expensive to get it (TaskStatsInfoGetter has about 500K RPS) + watch.stop(); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + if (profile_callback) { ProfileInfo info; info.bytes_requested = internal_buffer.size(); info.bytes_read = res; - info.nanoseconds = watch->elapsed(); + info.nanoseconds = watch.elapsed(); profile_callback(info); } } @@ -114,12 +118,17 @@ off_t ReadBufferFromFileDescriptor::doSeek(off_t offset, int whence) else { ProfileEvents::increment(ProfileEvents::Seek); + Stopwatch watch(profile_callback ? clock_type : CLOCK_MONOTONIC); pos = working_buffer.end(); - off_t res = lseek(fd, new_pos, SEEK_SET); + off_t res = ::lseek(fd, new_pos, SEEK_SET); if (-1 == res) throwFromErrno("Cannot seek through file " + getFileName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE); pos_in_file = new_pos; + + watch.stop(); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + return res; } } diff --git a/dbms/src/IO/ReadBufferFromPocoSocket.cpp b/dbms/src/IO/ReadBufferFromPocoSocket.cpp index 697fc16f637..9dc21c19714 100644 --- a/dbms/src/IO/ReadBufferFromPocoSocket.cpp +++ b/dbms/src/IO/ReadBufferFromPocoSocket.cpp @@ -4,6 +4,13 @@ #include #include +#include + + +namespace ProfileEvents +{ + extern const Event NetworkReceiveElapsedMicroseconds; +} namespace DB @@ -20,6 +27,7 @@ namespace ErrorCodes bool ReadBufferFromPocoSocket::nextImpl() { ssize_t bytes_read = 0; + Stopwatch watch; /// Add more details to exceptions. try @@ -42,6 +50,9 @@ bool ReadBufferFromPocoSocket::nextImpl() if (bytes_read < 0) throw NetException("Cannot read from socket (" + peer_address.toString() + ")", ErrorCodes::CANNOT_READ_FROM_SOCKET); + /// NOTE: it is quite inaccurate on high loads since the thread could be replaced by another one + ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); + if (bytes_read) working_buffer.resize(bytes_read); else diff --git a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp index d22fe7229b8..928b338ac31 100644 --- a/dbms/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/dbms/src/IO/WriteBufferFromFileDescriptor.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace ProfileEvents @@ -14,6 +15,7 @@ namespace ProfileEvents extern const Event WriteBufferFromFileDescriptorWrite; extern const Event WriteBufferFromFileDescriptorWriteFailed; extern const Event WriteBufferFromFileDescriptorWriteBytes; + extern const Event DiskWriteElapsedMicroseconds; } namespace CurrentMetrics @@ -38,6 +40,8 @@ void WriteBufferFromFileDescriptor::nextImpl() if (!offset()) return; + Stopwatch watch; + size_t bytes_written = 0; while (bytes_written != offset()) { @@ -59,6 +63,7 @@ void WriteBufferFromFileDescriptor::nextImpl() bytes_written += res; } + ProfileEvents::increment(ProfileEvents::DiskWriteElapsedMicroseconds, watch.elapsedMicroseconds()); ProfileEvents::increment(ProfileEvents::WriteBufferFromFileDescriptorWriteBytes, bytes_written); } diff --git a/dbms/src/IO/WriteBufferFromPocoSocket.cpp b/dbms/src/IO/WriteBufferFromPocoSocket.cpp index 8e2d01cf6cc..0712b2c0603 100644 --- a/dbms/src/IO/WriteBufferFromPocoSocket.cpp +++ b/dbms/src/IO/WriteBufferFromPocoSocket.cpp @@ -4,6 +4,13 @@ #include #include +#include + + +namespace ProfileEvents +{ + extern const Event NetworkSendElapsedMicroseconds; +} namespace DB @@ -22,6 +29,8 @@ void WriteBufferFromPocoSocket::nextImpl() if (!offset()) return; + Stopwatch watch; + size_t bytes_written = 0; while (bytes_written < offset()) { @@ -47,8 +56,11 @@ void WriteBufferFromPocoSocket::nextImpl() if (res < 0) throw NetException("Cannot write to socket (" + peer_address.toString() + ")", ErrorCodes::CANNOT_WRITE_TO_SOCKET); + bytes_written += res; } + + ProfileEvents::increment(ProfileEvents::NetworkSendElapsedMicroseconds, watch.elapsedMicroseconds()); } WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) diff --git a/dbms/src/IO/tests/limit_read_buffer.cpp b/dbms/src/IO/tests/limit_read_buffer.cpp index 078c4db7c66..7b9b4744a28 100644 --- a/dbms/src/IO/tests/limit_read_buffer.cpp +++ b/dbms/src/IO/tests/limit_read_buffer.cpp @@ -24,13 +24,13 @@ int main(int argc, char ** argv) writeCString("--- first ---\n", out); { - LimitReadBuffer limit_in(in, limit); + LimitReadBuffer limit_in(in, limit, false); copyData(limit_in, out); } writeCString("\n--- second ---\n", out); { - LimitReadBuffer limit_in(in, limit); + LimitReadBuffer limit_in(in, limit, false); copyData(limit_in, out); } diff --git a/dbms/src/IO/tests/limit_read_buffer2.cpp b/dbms/src/IO/tests/limit_read_buffer2.cpp index d2c73d4cf74..93145ddcc8a 100644 --- a/dbms/src/IO/tests/limit_read_buffer2.cpp +++ b/dbms/src/IO/tests/limit_read_buffer2.cpp @@ -20,7 +20,7 @@ try ReadBuffer in(&src[0], src.size(), 0); - LimitReadBuffer limit_in(in, 1); + LimitReadBuffer limit_in(in, 1, false); { WriteBufferFromString out(dst); @@ -57,7 +57,7 @@ try char x; readChar(x, in); - LimitReadBuffer limit_in(in, 1); + LimitReadBuffer limit_in(in, 1, false); copyData(limit_in, out); @@ -102,7 +102,7 @@ try ReadBuffer in(&src[0], src.size(), 0); { - LimitReadBuffer limit_in(in, 1); + LimitReadBuffer limit_in(in, 1, false); char x; readChar(x, limit_in); diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 343115cb975..dc9dde6acb8 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #if __has_include() @@ -139,8 +140,9 @@ Aggregator::Aggregator(const Params & params_) : params(params_), isCancelled([]() { return false; }) { - if (current_memory_tracker) - memory_usage_before_aggregation = current_memory_tracker->get(); + /// Use query-level memory tracker + if (auto memory_tracker = CurrentThread::getMemoryTracker().getParent()) + memory_usage_before_aggregation = memory_tracker->get(); aggregate_functions.resize(params.aggregates_size); for (size_t i = 0; i < params.aggregates_size; ++i) @@ -804,8 +806,8 @@ bool Aggregator::executeOnBlock(const Block & block, AggregatedDataVariants & re size_t result_size = result.sizeWithoutOverflowRow(); Int64 current_memory_usage = 0; - if (current_memory_tracker) - current_memory_usage = current_memory_tracker->get(); + if (auto memory_tracker = CurrentThread::getMemoryTracker().getParent()) + current_memory_usage = memory_tracker->get(); auto result_size_bytes = current_memory_usage - memory_usage_before_aggregation; /// Here all the results in the sum are taken into account, from different threads. @@ -1278,9 +1280,9 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( bool final, ThreadPool * thread_pool) const { - auto converter = [&](size_t bucket, MemoryTracker * memory_tracker) + auto converter = [&](size_t bucket, ThreadGroupStatusPtr thread_group) { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); return convertOneBucketToBlock(data_variants, method, final, bucket); }; @@ -1295,7 +1297,7 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( if (method.data.impls[bucket].empty()) continue; - tasks[bucket] = std::packaged_task(std::bind(converter, bucket, current_memory_tracker)); + tasks[bucket] = std::packaged_task(std::bind(converter, bucket, CurrentThread::getGroup())); if (thread_pool) thread_pool->schedule([bucket, &tasks] { tasks[bucket](); }); @@ -1725,17 +1727,17 @@ private: return; parallel_merge_data->pool.schedule(std::bind(&MergingAndConvertingBlockInputStream::thread, this, - max_scheduled_bucket_num, current_memory_tracker)); + max_scheduled_bucket_num, CurrentThread::getGroup())); } - void thread(Int32 bucket_num, MemoryTracker * memory_tracker) + void thread(Int32 bucket_num, ThreadGroupStatusPtr thread_group) { - current_memory_tracker = memory_tracker; - setThreadName("MergingAggregtd"); - CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; - try { + setThreadName("MergingAggregtd"); + CurrentThread::attachToIfDetached(thread_group); + CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread}; + /// TODO: add no_more_keys support maybe auto & merged_data = *data[0]; @@ -2035,9 +2037,9 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV LOG_TRACE(log, "Merging partially aggregated two-level data."); - auto merge_bucket = [&bucket_to_blocks, &result, this](Int32 bucket, Arena * aggregates_pool, MemoryTracker * memory_tracker) + auto merge_bucket = [&bucket_to_blocks, &result, this](Int32 bucket, Arena * aggregates_pool, ThreadGroupStatusPtr thread_group) { - current_memory_tracker = memory_tracker; + CurrentThread::attachToIfDetached(thread_group); for (Block & block : bucket_to_blocks[bucket]) { @@ -2070,7 +2072,7 @@ void Aggregator::mergeStream(const BlockInputStreamPtr & stream, AggregatedDataV result.aggregates_pools.push_back(std::make_shared()); Arena * aggregates_pool = result.aggregates_pools.back().get(); - auto task = std::bind(merge_bucket, bucket, aggregates_pool, current_memory_tracker); + auto task = std::bind(merge_bucket, bucket, aggregates_pool, CurrentThread::getGroup()); if (thread_pool) thread_pool->schedule(task); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d59e01526b0..1b45665e0e1 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -1529,7 +1530,7 @@ void Context::initializeSystemLogs() } -QueryLog * Context::getQueryLog() +QueryLog * Context::getQueryLog(bool create_if_not_exists) { auto lock = getLock(); @@ -1538,29 +1539,49 @@ QueryLog * Context::getQueryLog() if (!system_logs->query_log) { + if (!create_if_not_exists) + return nullptr; + if (shared->shutdown_called) throw Exception("Logical error: query log should be destroyed before tables shutdown", ErrorCodes::LOGICAL_ERROR); if (!global_context) throw Exception("Logical error: no global context for query log", ErrorCodes::LOGICAL_ERROR); - auto & config = getConfigRef(); - - String database = config.getString("query_log.database", "system"); - String table = config.getString("query_log.table", "query_log"); - String partition_by = config.getString("query_log.partition_by", "toYYYYMM(event_date)"); - size_t flush_interval_milliseconds = config.getUInt64("query_log.flush_interval_milliseconds", DEFAULT_QUERY_LOG_FLUSH_INTERVAL_MILLISECONDS); - - String engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024"; - - system_logs->query_log = std::make_unique(*global_context, database, table, engine, flush_interval_milliseconds); + system_logs->query_log = createDefaultSystemLog(*global_context, "system", "query_log", getConfigRef(), "query_log"); } return system_logs->query_log.get(); } -PartLog * Context::getPartLog(const String & part_database) +QueryThreadLog * Context::getQueryThreadLog(bool create_if_not_exists) +{ + auto lock = getLock(); + + if (!system_logs) + return nullptr; + + if (!system_logs->query_thread_log) + { + if (!create_if_not_exists) + return nullptr; + + if (shared->shutdown_called) + throw Exception("Logical error: query log should be destroyed before tables shutdown", ErrorCodes::LOGICAL_ERROR); + + if (!global_context) + throw Exception("Logical error: no global context for query thread log", ErrorCodes::LOGICAL_ERROR); + + system_logs->query_thread_log = createDefaultSystemLog( + *global_context, "system", "query_thread_log", getConfigRef(), "query_thread_log"); + } + + return system_logs->query_thread_log.get(); +} + + +PartLog * Context::getPartLog(const String & part_database, bool create_if_not_exists) { auto lock = getLock(); @@ -1577,24 +1598,21 @@ PartLog * Context::getPartLog(const String & part_database) /// Will not log operations on system tables (including part_log itself). /// It doesn't make sense and not allow to destruct PartLog correctly due to infinite logging and flushing, /// and also make troubles on startup. - if (part_database == database) + if (!part_database.empty() && part_database == database) return nullptr; if (!system_logs->part_log) { + if (!create_if_not_exists) + return nullptr; + if (shared->shutdown_called) throw Exception("Logical error: part log should be destroyed before tables shutdown", ErrorCodes::LOGICAL_ERROR); if (!global_context) throw Exception("Logical error: no global context for part log", ErrorCodes::LOGICAL_ERROR); - String table = config.getString("part_log.table", "part_log"); - String partition_by = config.getString("query_log.partition_by", "toYYYYMM(event_date)"); - size_t flush_interval_milliseconds = config.getUInt64("part_log.flush_interval_milliseconds", DEFAULT_QUERY_LOG_FLUSH_INTERVAL_MILLISECONDS); - - String engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024"; - - system_logs->part_log = std::make_unique(*global_context, database, table, engine, flush_interval_milliseconds); + system_logs->part_log = createDefaultSystemLog(*global_context, "system", "part_log", getConfigRef(), "part_log"); } return system_logs->part_log.get(); @@ -1801,6 +1819,25 @@ std::shared_ptr Context::getActionLocksManager() } +void Context::setExternalTablesInitializer(ExternalTablesInitializer && initializer) +{ + if (external_tables_initializer_callback) + throw Exception("External tables initializer is already set", ErrorCodes::LOGICAL_ERROR); + + external_tables_initializer_callback = std::move(initializer); +} + +void Context::initializeExternalTablesIfSet() +{ + if (external_tables_initializer_callback) + { + external_tables_initializer_callback(*this); + /// Reset callback + external_tables_initializer_callback = {}; + } +} + + SessionCleaner::~SessionCleaner() { try diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 9248eb31fdc..99fa7bf565b 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -35,6 +35,7 @@ namespace DB { struct ContextShared; +class Context; class IRuntimeComponentsFactory; class QuotaForIntervals; class EmbeddedDictionaries; @@ -49,11 +50,12 @@ class Compiler; class MarkCache; class UncompressedCache; class ProcessList; -class ProcessListElement; +class QueryStatus; class Macros; struct Progress; class Clusters; class QueryLog; +class QueryThreadLog; class PartLog; struct MergeTreeSettings; class IDatabase; @@ -86,6 +88,9 @@ using Dependencies = std::vector; using TableAndCreateAST = std::pair; using TableAndCreateASTs = std::map; +/// Callback for external tables initializer +using ExternalTablesInitializer = std::function; + /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) * and copied part (which can be its own for each session or query). @@ -101,13 +106,14 @@ private: std::shared_ptr runtime_components_factory; ClientInfo client_info; + ExternalTablesInitializer external_tables_initializer_callback; std::shared_ptr quota; /// Current quota. By default - empty quota, that have no limits. String current_database; Settings settings; /// Setting for query execution. using ProgressCallback = std::function; ProgressCallback progress_callback; /// Callback for tracking progress of query execution. - ProcessListElement * process_list_elem = nullptr; /// For tracking total resource usage for query. + QueryStatus * process_list_elem = nullptr; /// For tracking total resource usage for query. String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. @@ -166,6 +172,11 @@ public: /// Compute and set actual user settings, client_info.current_user should be set void calculateUserSettings(); + /// We have to copy external tables inside executeQuery() to track limits. Therefore, set callback for it. Must set once. + void setExternalTablesInitializer(ExternalTablesInitializer && initializer); + /// This method is called in executeQuery() and will call the external tables initializer. + void initializeExternalTablesIfSet(); + ClientInfo & getClientInfo() { return client_info; } const ClientInfo & getClientInfo() const { return client_info; } @@ -311,9 +322,9 @@ public: /** Set in executeQuery and InterpreterSelectQuery. Then it is used in IProfilingBlockInputStream, * to update and monitor information about the total number of resources spent for the query. */ - void setProcessListElement(ProcessListElement * elem); + void setProcessListElement(QueryStatus * elem); /// Can return nullptr if the query was not inserted into the ProcessList. - ProcessListElement * getProcessListElement() const; + QueryStatus * getProcessListElement() const; /// List all queries. ProcessList & getProcessList(); @@ -366,11 +377,12 @@ public: void initializeSystemLogs(); /// Nullptr if the query log is not ready for this moment. - QueryLog * getQueryLog(); + QueryLog * getQueryLog(bool create_if_not_exists = true); + QueryThreadLog * getQueryThreadLog(bool create_if_not_exists = true); /// Returns an object used to log opertaions with parts if it possible. /// Provide table name to make required cheks. - PartLog * getPartLog(const String & part_database); + PartLog * getPartLog(const String & part_database, bool create_if_not_exists = true); const MergeTreeSettings & getMergeTreeSettings() const; diff --git a/dbms/src/Interpreters/DNSCacheUpdater.cpp b/dbms/src/Interpreters/DNSCacheUpdater.cpp index a884422640b..a4cc8a19dad 100644 --- a/dbms/src/Interpreters/DNSCacheUpdater.cpp +++ b/dbms/src/Interpreters/DNSCacheUpdater.cpp @@ -64,7 +64,7 @@ DNSCacheUpdater::DNSCacheUpdater(Context & context_) bool DNSCacheUpdater::run() { /// TODO: Ensusre that we get global counter (not thread local) - auto num_current_network_exceptions = ProfileEvents::counters[ProfileEvents::NetworkErrors].load(std::memory_order_relaxed); + auto num_current_network_exceptions = ProfileEvents::global_counters[ProfileEvents::NetworkErrors].load(std::memory_order_relaxed); if (num_current_network_exceptions >= last_num_network_erros + min_errors_to_update_cache && time(nullptr) > last_update_time + min_update_period_seconds) diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index 1a00f5c43d7..e6864b15ef1 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -1079,10 +1079,25 @@ void ExpressionActionsChain::finalize() for (int i = static_cast(steps.size()) - 1; i >= 0; --i) { Names required_output = steps[i].required_output; + std::unordered_map required_output_indexes; + for (size_t j = 0; j < required_output.size(); ++j) + required_output_indexes[required_output[j]] = j; + auto & can_remove_required_output = steps[i].can_remove_required_output; + if (i + 1 < static_cast(steps.size())) { + const NameSet & additional_input = steps[i + 1].additional_input; for (const auto & it : steps[i + 1].actions->getRequiredColumnsWithTypes()) - required_output.push_back(it.name); + { + if (additional_input.count(it.name) == 0) + { + auto iter = required_output_indexes.find(it.name); + if (iter == required_output_indexes.end()) + required_output.push_back(it.name); + else if (!can_remove_required_output.empty()) + can_remove_required_output[iter->second] = false; + } + } } steps[i].actions->finalize(required_output); } diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 993ba772d75..bf80cf51eab 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -241,7 +241,14 @@ struct ExpressionActionsChain struct Step { ExpressionActionsPtr actions; + /// Columns were added to the block before current step in addition to prev step output. + NameSet additional_input; + /// Columns which are required in the result of current step. Names required_output; + /// True if column from required_output is needed only for current step and not used in next actions + /// (and can be removed from block). Example: filter column for where actions. + /// If not empty, has the same size with required_output; is filled in finalize(). + std::vector can_remove_required_output; Step(const ExpressionActionsPtr & actions_ = nullptr, const Names & required_output_ = Names()) : actions(actions_), required_output(required_output_) {} diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 2bf8a45abfb..889524a3fa5 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -2733,6 +2733,67 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty return true; } +bool ExpressionAnalyzer::appendPrewhere(ExpressionActionsChain & chain, bool only_types) +{ + assertSelect(); + + if (!select_query->prewhere_expression) + return false; + + initChain(chain, source_columns); + auto & step = chain.getLastStep(); + getRootActions(select_query->prewhere_expression, only_types, false, step.actions); + String prewhere_column_name = select_query->prewhere_expression->getColumnName(); + step.required_output.push_back(prewhere_column_name); + step.can_remove_required_output.push_back(true); + + { + /// Remove unused source_columns from prewhere actions. + auto tmp_actions = std::make_shared(source_columns, settings); + getRootActions(select_query->prewhere_expression, only_types, false, tmp_actions); + tmp_actions->finalize({prewhere_column_name}); + auto required_columns = tmp_actions->getRequiredColumns(); + NameSet required_source_columns(required_columns.begin(), required_columns.end()); + + auto names = step.actions->getSampleBlock().getNames(); + NameSet name_set(names.begin(), names.end()); + + for (const auto & column : source_columns) + if (required_source_columns.count(column.name) == 0) + name_set.erase(column.name); + + Names required_output(name_set.begin(), name_set.end()); + step.actions->finalize(required_output); + } + + { + /// Add empty action with input = {prewhere actions output} + {unused source columns} + /// Reasons: + /// 1. Remove remove source columns which are used only in prewhere actions during prewhere actions execution. + /// Example: select A prewhere B > 0. B can be removed at prewhere step. + /// 2. Store side columns which were calculated during prewhere actions execution if they are used. + /// Example: select F(A) prewhere F(A) > 0. F(A) can be saved from prewhere step. + /// 3. Check if we can remove filter column at prewhere step. If we can, action will store single REMOVE_COLUMN. + ColumnsWithTypeAndName columns = step.actions->getSampleBlock().getColumnsWithTypeAndName(); + auto required_columns = step.actions->getRequiredColumns(); + NameSet prewhere_input_names(required_columns.begin(), required_columns.end()); + NameSet unused_source_columns; + + for (const auto & column : source_columns) + { + if (prewhere_input_names.count(column.name) == 0) + { + columns.emplace_back(column.type, column.name); + unused_source_columns.emplace(column.name); + } + } + + chain.steps.emplace_back(std::make_shared(std::move(columns), settings)); + chain.steps.back().additional_input = std::move(unused_source_columns); + } + + return true; +} bool ExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, bool only_types) { @@ -2745,6 +2806,8 @@ bool ExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, bool only_t ExpressionActionsChain::Step & step = chain.steps.back(); step.required_output.push_back(select_query->where_expression->getColumnName()); + step.can_remove_required_output = {true}; + getRootActions(select_query->where_expression, only_types, false, step.actions); return true; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index e44853c67ad..28d4ca7c10f 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -152,6 +152,8 @@ public: /// Before aggregation: bool appendArrayJoin(ExpressionActionsChain & chain, bool only_types); bool appendJoin(ExpressionActionsChain & chain, bool only_types); + /// remove_filter is set in ExpressionActionsChain::finalize(); + bool appendPrewhere(ExpressionActionsChain & chain, bool only_types); bool appendWhere(ExpressionActionsChain & chain, bool only_types); bool appendGroupBy(ExpressionActionsChain & chain, bool only_types); void appendAggregateFunctionsArguments(ExpressionActionsChain & chain, bool only_types); diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 816d4798e30..71223c7c81b 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -66,7 +66,7 @@ void ExternalLoader::init(bool throw_on_error) { /// During synchronous loading of external dictionaries at moment of query execution, /// we should not use per query memory limit. - TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); reloadAndUpdate(throw_on_error); } diff --git a/dbms/src/Interpreters/InternalTextLogsQueue.cpp b/dbms/src/Interpreters/InternalTextLogsQueue.cpp new file mode 100644 index 00000000000..a27838a4203 --- /dev/null +++ b/dbms/src/Interpreters/InternalTextLogsQueue.cpp @@ -0,0 +1,68 @@ +#include "InternalTextLogsQueue.h" +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +InternalTextLogsQueue::InternalTextLogsQueue() + : ConcurrentBoundedQueue(std::numeric_limits::max()), + max_priority(Poco::Message::Priority::PRIO_INFORMATION) {} + + +Block InternalTextLogsQueue::getSampleBlock() +{ + return Block { + {std::make_shared(), "event_time"}, + {std::make_shared(), "event_time_microseconds"}, + {std::make_shared(), "host_name"}, + {std::make_shared(), "query_id"}, + {std::make_shared(), "thread_number"}, + {std::make_shared(), "priority"}, + {std::make_shared(), "source"}, + {std::make_shared(), "text"} + }; +} + +MutableColumns InternalTextLogsQueue::getSampleColumns() +{ + static Block sample_block = getSampleBlock(); + return sample_block.cloneEmptyColumns(); +} + +void InternalTextLogsQueue::pushBlock(Block && log_block) +{ + static Block sample_block = getSampleBlock(); + + if (blocksHaveEqualStructure(sample_block, log_block)) + emplace(log_block.mutateColumns()); + else + LOG_WARNING(&Poco::Logger::get("InternalTextLogsQueue"), "Log block have different structure"); +} + +const char * InternalTextLogsQueue::getPriorityName(int priority) +{ + /// See Poco::Message::Priority + + static const char * PRIORITIES [] = { + "Unknown", + "Fatal", + "Critical", + "Error", + "Warning", + "Notice", + "Information", + "Debug", + "Trace" + }; + + return (priority >= 1 && priority <= 8) ? PRIORITIES[priority] : PRIORITIES[0]; +} + +} diff --git a/dbms/src/Interpreters/InternalTextLogsQueue.h b/dbms/src/Interpreters/InternalTextLogsQueue.h new file mode 100644 index 00000000000..42711181a1a --- /dev/null +++ b/dbms/src/Interpreters/InternalTextLogsQueue.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + + +namespace DB +{ + +class InternalTextLogsQueue : public ConcurrentBoundedQueue +{ +public: + /// You should not push logs in the queue if their priority greater max_priority + int max_priority; + + InternalTextLogsQueue(); + + static Block getSampleBlock(); + static MutableColumns getSampleColumns(); + + /// Is used to pass block from remote server to the client + void pushBlock(Block && log_block); + + /// Converts priority from Poco::Message::Priority to a string + static const char * getPriorityName(int priority); +}; + +using InternalTextLogsQueuePtr = std::shared_ptr; + +} + + + diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 2de851da8b0..926a0be7b5f 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -293,9 +293,37 @@ InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpression * throw out unnecessary columns based on the entire query. In unnecessary parts of the query, we will not execute subqueries. */ + bool has_prewhere = false; + bool has_where = false; + size_t where_step_num; + + auto finalizeChain = [&](ExpressionActionsChain & chain) + { + chain.finalize(); + + if (has_prewhere) + res.prewhere_info->remove_prewhere_column = chain.steps.at(0).can_remove_required_output.at(0); + if (has_where) + res.remove_where_filter = chain.steps.at(where_step_num).can_remove_required_output.at(0); + + has_prewhere = has_where = false; + + chain.clear(); + }; + { ExpressionActionsChain chain; + if (query_analyzer->appendPrewhere(chain, !res.first_stage)) + { + has_prewhere = true; + + res.prewhere_info = std::make_shared( + chain.steps.front().actions, query.prewhere_expression->getColumnName()); + + chain.addStep(); + } + res.need_aggregate = query_analyzer->hasAggregation(); query_analyzer->appendArrayJoin(chain, dry_run || !res.first_stage); @@ -309,7 +337,8 @@ InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpression if (query_analyzer->appendWhere(chain, dry_run || !res.first_stage)) { - res.has_where = true; + where_step_num = chain.steps.size() - 1; + has_where = res.has_where = true; res.before_where = chain.getLastActions(); chain.addStep(); } @@ -320,8 +349,7 @@ InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpression query_analyzer->appendAggregateFunctionsArguments(chain, dry_run || !res.first_stage); res.before_aggregation = chain.getLastActions(); - chain.finalize(); - chain.clear(); + finalizeChain(chain); if (query_analyzer->appendHaving(chain, dry_run || !res.second_stage)) { @@ -348,8 +376,7 @@ InterpreterSelectQuery::AnalysisResult InterpreterSelectQuery::analyzeExpression query_analyzer->appendProjectResult(chain); res.final_projection = chain.getLastActions(); - chain.finalize(); - chain.clear(); + finalizeChain(chain); } /// Before executing WHERE and HAVING, remove the extra columns from the block (mostly the aggregation keys). @@ -376,31 +403,65 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt * then perform the remaining operations with one resulting stream. */ + const Settings & settings = context.getSettingsRef(); + + QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; + + /// PREWHERE optimization + if (storage) + { + if (!dry_run) + from_stage = storage->getQueryProcessingStage(context); + + query_analyzer->makeSetsForIndex(); + + auto optimize_prewhere = [&](auto & merge_tree) + { + SelectQueryInfo query_info; + query_info.query = query_ptr; + query_info.sets = query_analyzer->getPreparedSets(); + + /// Try transferring some condition from WHERE to PREWHERE if enabled and viable + if (settings.optimize_move_to_prewhere && query.where_expression && !query.prewhere_expression && !query.final()) + MergeTreeWhereOptimizer{query_info, context, merge_tree.getData(), query_analyzer->getRequiredSourceColumns(), log}; + }; + + if (const StorageMergeTree * merge_tree = dynamic_cast(storage.get())) + optimize_prewhere(*merge_tree); + else if (const StorageReplicatedMergeTree * merge_tree = dynamic_cast(storage.get())) + optimize_prewhere(*merge_tree); + } + AnalysisResult expressions; if (dry_run) { pipeline.streams.emplace_back(std::make_shared(source_header)); expressions = analyzeExpressions(QueryProcessingStage::FetchColumns, true); + + if (expressions.prewhere_info) + pipeline.streams.back() = std::make_shared( + pipeline.streams.back(), expressions.prewhere_info->prewhere_actions, + expressions.prewhere_info->prewhere_column_name, expressions.prewhere_info->remove_prewhere_column + ); } else { if (input) pipeline.streams.push_back(input); - /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */ - QueryProcessingStage::Enum from_stage = executeFetchColumns(pipeline); + expressions = analyzeExpressions(from_stage, false); - if (from_stage == QueryProcessingStage::WithMergeableState && to_stage == QueryProcessingStage::WithMergeableState) + if (from_stage == QueryProcessingStage::WithMergeableState && + to_stage == QueryProcessingStage::WithMergeableState) throw Exception("Distributed on Distributed is not supported", ErrorCodes::NOT_IMPLEMENTED); + /** Read the data from Storage. from_stage - to what stage the request was completed in Storage. */ + executeFetchColumns(from_stage, pipeline, expressions.prewhere_info); + LOG_TRACE(log, QueryProcessingStage::toString(from_stage) << " -> " << QueryProcessingStage::toString(to_stage)); - - expressions = analyzeExpressions(from_stage, false); } - const Settings & settings = context.getSettingsRef(); - if (to_stage > QueryProcessingStage::FetchColumns) { /// Now we will compose block streams that perform the necessary actions. @@ -433,7 +494,7 @@ void InterpreterSelectQuery::executeImpl(Pipeline & pipeline, const BlockInputSt } if (expressions.has_where) - executeWhere(pipeline, expressions.before_where); + executeWhere(pipeline, expressions.before_where, expressions.remove_where_filter); if (expressions.need_aggregate) executeAggregation(pipeline, expressions.before_aggregation, aggregate_overflow_row, aggregate_final); @@ -562,7 +623,8 @@ static void getLimitLengthAndOffset(ASTSelectQuery & query, size_t & length, siz } } -QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline & pipeline) +void InterpreterSelectQuery::executeFetchColumns( + QueryProcessingStage::Enum processing_stage, Pipeline & pipeline, const PrewhereInfoPtr & prewhere_info) { /// Actions to calculate ALIAS if required. ExpressionActionsPtr alias_actions; @@ -652,8 +714,6 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline max_streams = 1; } - QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; - /// Initialize the initial data streams to which the query transforms are superimposed. Table or subquery or prepared input? if (!pipeline.streams.empty()) { @@ -685,32 +745,24 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline if (max_streams > 1 && !is_remote) max_streams *= settings.max_streams_to_max_threads_ratio; - query_analyzer->makeSetsForIndex(); - SelectQueryInfo query_info; query_info.query = query_ptr; query_info.sets = query_analyzer->getPreparedSets(); + query_info.prewhere_info = prewhere_info; - /// PREWHERE optimization - { - auto optimize_prewhere = [&](auto & merge_tree) - { - /// Try transferring some condition from WHERE to PREWHERE if enabled and viable - if (settings.optimize_move_to_prewhere && query.where_expression && !query.prewhere_expression && !query.final()) - MergeTreeWhereOptimizer{query_info, context, merge_tree.getData(), required_columns, log}; - }; - - if (const StorageMergeTree * merge_tree = dynamic_cast(storage.get())) - optimize_prewhere(*merge_tree); - else if (const StorageReplicatedMergeTree * merge_tree = dynamic_cast(storage.get())) - optimize_prewhere(*merge_tree); - } - - pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams); + pipeline.streams = storage->read(required_columns, query_info, context, processing_stage, max_block_size, max_streams); if (pipeline.streams.empty()) + { pipeline.streams.emplace_back(std::make_shared(storage->getSampleBlockForColumns(required_columns))); + if (query_info.prewhere_info) + pipeline.streams.back() = std::make_shared( + pipeline.streams.back(), prewhere_info->prewhere_actions, + prewhere_info->prewhere_column_name, prewhere_info->remove_prewhere_column + ); + } + pipeline.transform([&](auto & stream) { stream->addTableLock(table_lock); @@ -755,23 +807,21 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); /// Aliases in table declaration. - if (from_stage == QueryProcessingStage::FetchColumns && alias_actions) + if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) { pipeline.transform([&](auto & stream) { stream = std::make_shared(stream, alias_actions); }); } - - return from_stage; } -void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression) +void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_fiter) { pipeline.transform([&](auto & stream) { - stream = std::make_shared(stream, expression, query.where_expression->getColumnName()); + stream = std::make_shared(stream, expression, query.where_expression->getColumnName(), remove_fiter); }); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.h b/dbms/src/Interpreters/InterpreterSelectQuery.h index 26f6c2f15ab..29b3c88d3a6 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.h +++ b/dbms/src/Interpreters/InterpreterSelectQuery.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace Poco { class Logger; } @@ -137,6 +138,8 @@ private: bool has_order_by = false; bool has_limit_by = false; + bool remove_where_filter = false; + ExpressionActionsPtr before_join; /// including JOIN ExpressionActionsPtr before_where; ExpressionActionsPtr before_aggregation; @@ -154,6 +157,7 @@ private: bool second_stage = false; SubqueriesForSets subqueries_for_sets; + PrewhereInfoPtr prewhere_info; }; AnalysisResult analyzeExpressions(QueryProcessingStage::Enum from_stage, bool dry_run); @@ -168,10 +172,9 @@ private: /// dry_run - don't read from table, use empty header block instead. void executeWithMultipleStreamsImpl(Pipeline & pipeline, const BlockInputStreamPtr & input, bool dry_run); - /// Fetch data from the table. Returns the stage to which the query was processed in Storage. - QueryProcessingStage::Enum executeFetchColumns(Pipeline & pipeline); + void executeFetchColumns(QueryProcessingStage::Enum processing_stage, Pipeline & pipeline, const PrewhereInfoPtr & prewhere_info); - void executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression); + void executeWhere(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool remove_filter); void executeAggregation(Pipeline & pipeline, const ExpressionActionsPtr & expression, bool overflow_row, bool final); void executeMergeAggregated(Pipeline & pipeline, bool overflow_row, bool final); void executeTotalsAndHaving(Pipeline & pipeline, bool has_having, const ExpressionActionsPtr & expression, bool overflow_row); diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 0e272a7a4bd..ce85b76c29a 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -10,6 +10,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -48,7 +51,7 @@ ExecutionStatus getOverallExecutionStatusOfCommands() return ExecutionStatus(0); } -/// Consequently execute all commands and genreates final exception message for failed commands +/// Consequently tries to execute all commands and genreates final exception message for failed commands template ExecutionStatus getOverallExecutionStatusOfCommands(Callable && command, Callables && ... commands) { @@ -70,6 +73,16 @@ ExecutionStatus getOverallExecutionStatusOfCommands(Callable && command, Callabl return ExecutionStatus(res_status, res_message); } +/// Consequently tries to execute all commands and throws exception with info about failed commands +template +void executeCommandsAndThrowIfError(Callables && ... commands) +{ + auto status = getOverallExecutionStatusOfCommands(std::forward(commands)...); + if (status.code != 0) + throw Exception(status.message, status.code); +} + + /// Implements SYSTEM [START|STOP] void startStopAction(Context & context, ASTSystemQuery & query, StorageActionBlockType action_type, bool start) { @@ -139,15 +152,11 @@ BlockIO InterpreterSystemQuery::execute() system_context.getExternalDictionaries().reloadDictionary(query.target_dictionary); break; case Type::RELOAD_DICTIONARIES: - { - auto status = getOverallExecutionStatusOfCommands( - [&] { system_context.getExternalDictionaries().reload(); }, - [&] { system_context.getEmbeddedDictionaries().reload(); } + executeCommandsAndThrowIfError( + [&] () { system_context.getExternalDictionaries().reload(); }, + [&] () { system_context.getEmbeddedDictionaries().reload(); } ); - if (status.code != 0) - throw Exception(status.message, status.code); break; - } case Type::RELOAD_EMBEDDED_DICTIONARIES: system_context.getEmbeddedDictionaries().reload(); break; @@ -189,6 +198,13 @@ BlockIO InterpreterSystemQuery::execute() throw Exception("There is no " + query.target_database + "." + query.target_table + " replicated table", ErrorCodes::BAD_ARGUMENTS); break; + case Type::FLUSH_SYSTEM_TABLES: + executeCommandsAndThrowIfError( + [&] () { if (auto query_log = context.getQueryLog(false)) query_log->flush(); }, + [&] () { if (auto part_log = context.getPartLog("", false)) part_log->flush(); }, + [&] () { if (auto query_thread_log = context.getQueryThreadLog(false)) query_thread_log->flush(); } + ); + break; case Type::STOP_LISTEN_QUERIES: case Type::START_LISTEN_QUERIES: throw Exception(String(ASTSystemQuery::typeToString(query.type)) + " is not supported yet", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index 10b24fe4773..4388bca73e3 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -1,18 +1,20 @@ #include #include +#include #include #include #include #include +#include #include +#include #include #include -#include #include - #include + namespace DB { @@ -72,11 +74,13 @@ static bool isUnlimitedQuery(const IAST * ast) } -ProcessList::EntryPtr ProcessList::insert( - const String & query_, const IAST * ast, const ClientInfo & client_info, const Settings & settings) +ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * ast, Context & query_context) { EntryPtr res; + const ClientInfo & client_info = query_context.getClientInfo(); + const Settings & settings = query_context.getSettingsRef(); + if (client_info.current_query_id.empty()) throw Exception("Query id cannot be empty", ErrorCodes::LOGICAL_ERROR); @@ -85,11 +89,11 @@ ProcessList::EntryPtr ProcessList::insert( { std::unique_lock lock(mutex); - if (!is_unlimited_query && max_size && cur_size >= max_size) + if (!is_unlimited_query && max_size && processes.size() >= max_size) { auto max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); - if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]{ return cur_size < max_size; })) + if (!max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(max_wait_ms), [&]{ return processes.size() < max_size; })) throw Exception("Too many simultaneous queries. Maximum: " + toString(max_size), ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES); } @@ -129,50 +133,69 @@ ProcessList::EntryPtr ProcessList::insert( } } - ++cur_size; + auto process_it = processes.emplace(processes.end(), + query_, client_info, settings.max_memory_usage, settings.memory_tracker_fault_probability, priorities.insert(settings.priority)); - res = std::make_shared(*this, cont.emplace(cont.end(), - query_, client_info, - settings.max_memory_usage, settings.memory_tracker_fault_probability, - priorities.insert(settings.priority))); + res = std::make_shared(*this, process_it); - ProcessListForUser & user_process_list = user_to_queries[client_info.current_user]; - user_process_list.queries.emplace(client_info.current_query_id, &res->get()); + process_it->query_context = &query_context; - if (current_memory_tracker) + if (!client_info.current_query_id.empty()) { + ProcessListForUser & user_process_list = user_to_queries[client_info.current_user]; + user_process_list.queries.emplace(client_info.current_query_id, &res->get()); + + process_it->setUserProcessList(&user_process_list); + /// Limits are only raised (to be more relaxed) or set to something instead of zero, /// because settings for different queries will interfere each other: /// setting from one query effectively sets values for all other queries. - /// Track memory usage for all simultaneously running queries from single user. - user_process_list.user_memory_tracker.setOrRaiseLimit(settings.max_memory_usage_for_user); - user_process_list.user_memory_tracker.setDescription("(for user)"); - current_memory_tracker->setNext(&user_process_list.user_memory_tracker); - /// Track memory usage for all simultaneously running queries. /// You should specify this value in configuration for default profile, /// not for specific users, sessions or queries, /// because this setting is effectively global. total_memory_tracker.setOrRaiseLimit(settings.max_memory_usage_for_all_queries); total_memory_tracker.setDescription("(total)"); - user_process_list.user_memory_tracker.setNext(&total_memory_tracker); + + /// Track memory usage for all simultaneously running queries from single user. + user_process_list.user_memory_tracker.setParent(&total_memory_tracker); + user_process_list.user_memory_tracker.setOrRaiseLimit(settings.max_memory_usage_for_user); + user_process_list.user_memory_tracker.setDescription("(for user)"); + + /// Actualize thread group info + if (auto thread_group = CurrentThread::getGroup()) + { + std::unique_lock lock_thread_group(thread_group->mutex); + thread_group->performance_counters.setParent(&user_process_list.user_performance_counters); + thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker); + thread_group->query = process_it->query; + + /// Set query-level memory trackers + thread_group->memory_tracker.setOrRaiseLimit(process_it->max_memory_usage); + thread_group->memory_tracker.setDescription("(for query)"); + if (process_it->memory_tracker_fault_probability) + thread_group->memory_tracker.setFaultProbability(process_it->memory_tracker_fault_probability); + + /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values + /// since allocation and deallocation could happen in different threads + + process_it->thread_group = std::move(thread_group); + } + + if (!user_process_list.user_throttler) + { + if (settings.max_network_bandwidth_for_user) + user_process_list.user_throttler = std::make_shared(settings.max_network_bandwidth_for_user, total_network_throttler); + else if (settings.max_network_bandwidth_for_all_users) + user_process_list.user_throttler = total_network_throttler; + } } if (!total_network_throttler && settings.max_network_bandwidth_for_all_users) { total_network_throttler = std::make_shared(settings.max_network_bandwidth_for_all_users); } - - if (!user_process_list.user_throttler) - { - if (settings.max_network_bandwidth_for_user) - user_process_list.user_throttler = std::make_shared(settings.max_network_bandwidth_for_user, total_network_throttler); - else if (settings.max_network_bandwidth_for_all_users) - user_process_list.user_throttler = total_network_throttler; - } - - res->get().user_process_list = &user_process_list; } return res; @@ -189,10 +212,10 @@ ProcessListEntry::~ProcessListEntry() String user = it->getClientInfo().current_user; String query_id = it->getClientInfo().current_query_id; - const ProcessListElement * process_list_element_ptr = &*it; + const QueryStatus * process_list_element_ptr = &*it; - /// This removes the memory_tracker of one query. - parent.cont.erase(it); + /// This removes the memory_tracker of one request. + parent.processes.erase(it); auto user_process_list_it = parent.user_to_queries.find(user); if (user_process_list_it == parent.user_to_queries.end()) @@ -225,15 +248,14 @@ ProcessListEntry::~ProcessListEntry() std::terminate(); } - /// If there are no more queries for the user, then we will reset memory tracker and network throttler. - if (user_process_list.queries.empty()) - user_process_list.reset(); - - --parent.cur_size; parent.have_space.notify_one(); + /// If there are no more queries for the user, then we will reset memory tracker and network throttler. + if (user_process_list.queries.empty()) + user_process_list.resetTrackers(); + /// This removes memory_tracker for all requests. At this time, no other memory_trackers live. - if (parent.cur_size == 0) + if (parent.processes.size() == 0) { /// Reset MemoryTracker, similarly (see above). parent.total_memory_tracker.logPeakMemoryUsage(); @@ -243,7 +265,25 @@ ProcessListEntry::~ProcessListEntry() } -void ProcessListElement::setQueryStreams(const BlockIO & io) +QueryStatus::QueryStatus( + const String & query_, + const ClientInfo & client_info_, + size_t max_memory_usage_, + double memory_tracker_fault_probability_, + QueryPriorities::Handle && priority_handle_) + : + query(query_), + client_info(client_info_), + priority_handle(std::move(priority_handle_)), + num_queries_increment{CurrentMetrics::Query}, + max_memory_usage(max_memory_usage_), + memory_tracker_fault_probability(memory_tracker_fault_probability_) +{ +} + +QueryStatus::~QueryStatus() = default; + +void QueryStatus::setQueryStreams(const BlockIO & io) { std::lock_guard lock(query_streams_mutex); @@ -252,7 +292,7 @@ void ProcessListElement::setQueryStreams(const BlockIO & io) query_streams_status = QueryStreamsStatus::Initialized; } -void ProcessListElement::releaseQueryStreams() +void QueryStatus::releaseQueryStreams() { BlockInputStreamPtr in; BlockOutputStreamPtr out; @@ -268,14 +308,14 @@ void ProcessListElement::releaseQueryStreams() /// Destroy streams outside the mutex lock } -bool ProcessListElement::streamsAreReleased() +bool QueryStatus::streamsAreReleased() { std::lock_guard lock(query_streams_mutex); return query_streams_status == QueryStreamsStatus::Released; } -bool ProcessListElement::tryGetQueryStreams(BlockInputStreamPtr & in, BlockOutputStreamPtr & out) const +bool QueryStatus::tryGetQueryStreams(BlockInputStreamPtr & in, BlockOutputStreamPtr & out) const { std::lock_guard lock(query_streams_mutex); @@ -288,7 +328,13 @@ bool ProcessListElement::tryGetQueryStreams(BlockInputStreamPtr & in, BlockOutpu } -ThrottlerPtr ProcessListElement::getUserNetworkThrottler() +void QueryStatus::setUserProcessList(ProcessListForUser * user_process_list_) +{ + user_process_list = user_process_list_; +} + + +ThrottlerPtr QueryStatus::getUserNetworkThrottler() { if (!user_process_list) return {}; @@ -296,7 +342,7 @@ ThrottlerPtr ProcessListElement::getUserNetworkThrottler() } -ProcessListElement * ProcessList::tryGetProcessListElement(const String & current_query_id, const String & current_user) +QueryStatus * ProcessList::tryGetProcessListElement(const String & current_query_id, const String & current_user) { auto user_it = user_to_queries.find(current_user); if (user_it != user_to_queries.end()) @@ -316,7 +362,7 @@ ProcessList::CancellationCode ProcessList::sendCancelToQuery(const String & curr { std::lock_guard lock(mutex); - ProcessListElement * elem = tryGetProcessListElement(current_query_id, current_user); + QueryStatus * elem = tryGetProcessListElement(current_query_id, current_user); if (!elem) return CancellationCode::NotFound; @@ -342,4 +388,61 @@ ProcessList::CancellationCode ProcessList::sendCancelToQuery(const String & curr return CancellationCode::QueryIsNotInitializedYet; } + +QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_events, bool get_settings) const +{ + QueryStatusInfo res; + + res.query = query; + res.client_info = client_info; + res.elapsed_seconds = watch.elapsedSeconds(); + res.is_cancelled = is_killed.load(std::memory_order_relaxed); + res.read_rows = progress_in.rows; + res.read_bytes = progress_in.bytes; + res.total_rows = progress_in.total_rows; + res.written_rows = progress_out.rows; + res.written_bytes = progress_out.bytes; + + if (thread_group) + { + res.memory_usage = thread_group->memory_tracker.get(); + res.peak_memory_usage = thread_group->memory_tracker.getPeak(); + + if (get_thread_list) + { + std::shared_lock lock(thread_group->mutex); + res.thread_numbers.reserve(thread_group->thread_statuses.size()); + + for (auto & thread_status_elem : thread_group->thread_statuses) + res.thread_numbers.emplace_back(thread_status_elem.second->thread_number); + } + + if (get_profile_events) + res.profile_counters = std::make_shared(thread_group->performance_counters.getPartiallyAtomicSnapshot()); + } + + if (get_settings && query_context) + res.query_settings = std::make_shared(query_context->getSettingsRef()); + + return res; +} + + +ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_events, bool get_settings) const +{ + Info per_query_infos; + + std::lock_guard lock(mutex); + + per_query_infos.reserve(processes.size()); + for (const auto & process : processes) + per_query_infos.emplace_back(process.getInfo(get_thread_list, get_profile_events, get_settings)); + + return per_query_infos; +} + + +ProcessListForUser::ProcessListForUser() = default; + + } diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h index 19cf01732d7..b2a800c77bd 100644 --- a/dbms/src/Interpreters/ProcessList.h +++ b/dbms/src/Interpreters/ProcessList.h @@ -6,15 +6,21 @@ #include #include #include +#include +#include #include #include #include +#include #include +#include +#include +#include +#include #include #include -#include +#include #include -#include namespace CurrentMetrics @@ -28,9 +34,14 @@ namespace DB class IStorage; using StoragePtr = std::shared_ptr; using Tables = std::map; +class Context; struct Settings; class IAST; + struct ProcessListForUser; +class QueryStatus; +class ThreadStatus; +class ProcessListEntry; /** List of currently executing queries. @@ -40,7 +51,7 @@ struct ProcessListForUser; /** Information of process list element. * To output in SHOW PROCESSLIST query. Does not contain any complex objects, that do something on copy or destructor. */ -struct ProcessInfo +struct QueryStatusInfo { String query; double elapsed_seconds; @@ -53,18 +64,32 @@ struct ProcessInfo Int64 peak_memory_usage; ClientInfo client_info; bool is_cancelled; + + /// Optional fields, filled by request + std::vector thread_numbers; + std::shared_ptr profile_counters; + std::shared_ptr query_settings; }; /// Query and information about its execution. -class ProcessListElement +class QueryStatus { +protected: friend class ProcessList; + friend class ThreadStatus; + friend class CurrentThread; + friend class ProcessListEntry; -private: String query; ClientInfo client_info; + /// Is set once when init + Context * query_context = nullptr; + + /// Info about all threads involved in query execution + ThreadGroupStatusPtr thread_group; + Stopwatch watch; /// Progress of input stream @@ -72,16 +97,18 @@ private: /// Progress of output stream Progress progress_out; - MemoryTracker memory_tracker; - QueryPriorities::Handle priority_handle; - CurrentMetrics::Increment num_queries {CurrentMetrics::Query}; + CurrentMetrics::Increment num_queries_increment{CurrentMetrics::Query}; + + size_t max_memory_usage = 0; + double memory_tracker_fault_probability = 0.0; std::atomic is_killed { false }; - /// Be careful using it. For example, queries field could be modified concurrently. - const ProcessListForUser * user_process_list = nullptr; + void setUserProcessList(ProcessListForUser * user_process_list_); + /// Be careful using it. For example, queries field of ProcessListForUser could be modified concurrently. + const ProcessListForUser * getUserProcessList() const { return user_process_list; } mutable std::mutex query_streams_mutex; @@ -100,27 +127,18 @@ private: QueryStreamsStatus query_streams_status{NotInitialized}; + ProcessListForUser * user_process_list = nullptr; + public: - ProcessListElement( + + QueryStatus( const String & query_, const ClientInfo & client_info_, size_t max_memory_usage, double memory_tracker_fault_probability, - QueryPriorities::Handle && priority_handle_) - : query(query_), client_info(client_info_), memory_tracker(max_memory_usage), - priority_handle(std::move(priority_handle_)) - { - memory_tracker.setDescription("(for query)"); - current_memory_tracker = &memory_tracker; + QueryPriorities::Handle && priority_handle_); - if (memory_tracker_fault_probability) - memory_tracker.setFaultProbability(memory_tracker_fault_probability); - } - - ~ProcessListElement() - { - current_memory_tracker = nullptr; - } + ~QueryStatus(); const ClientInfo & getClientInfo() const { @@ -141,6 +159,7 @@ public: bool updateProgressIn(const Progress & value) { + CurrentThread::updateProgressIn(value); progress_in.incrementPiecewiseAtomically(value); if (priority_handle) @@ -151,29 +170,16 @@ public: bool updateProgressOut(const Progress & value) { + CurrentThread::updateProgressOut(value); progress_out.incrementPiecewiseAtomically(value); + return !is_killed.load(std::memory_order_relaxed); } + QueryStatusInfo getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; - ProcessInfo getInfo() const - { - ProcessInfo res; - - res.query = query; - res.client_info = client_info; - res.elapsed_seconds = watch.elapsedSeconds(); - res.is_cancelled = is_killed.load(std::memory_order_relaxed); - res.read_rows = progress_in.rows; - res.read_bytes = progress_in.bytes; - res.total_rows = progress_in.total_rows; - res.written_rows = progress_out.rows; - res.written_bytes = progress_out.bytes; - res.memory_usage = memory_tracker.get(); - res.peak_memory_usage = memory_tracker.getPeak(); - - return res; - } + Context * tryGetQueryContext() { return query_context; } + const Context * tryGetQueryContext() const { return query_context; } /// Copies pointers to in/out streams void setQueryStreams(const BlockIO & io); @@ -192,12 +198,15 @@ public: /// Data about queries for one user. struct ProcessListForUser { + ProcessListForUser(); + /// query_id -> ProcessListElement(s). There can be multiple queries with the same query_id as long as all queries except one are cancelled. - using QueryToElement = std::unordered_multimap; + using QueryToElement = std::unordered_multimap; QueryToElement queries; + ProfileEvents::Counters user_performance_counters{VariableContext::User, &ProfileEvents::global_counters}; /// Limit and counter for memory of all simultaneously running queries of single user. - MemoryTracker user_memory_tracker; + MemoryTracker user_memory_tracker{VariableContext::User}; /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; @@ -206,7 +215,7 @@ struct ProcessListForUser /// Sometimes it is important to reset the MemoryTracker, because it may accumulate skew /// due to the fact that there are cases when memory can be allocated while processing the query, but released later. /// Clears network bandwidth Throttler, so it will not count periods of inactivity. - void reset() + void resetTrackers() { user_memory_tracker.reset(); if (user_throttler) @@ -222,44 +231,45 @@ class ProcessList; class ProcessListEntry { private: - using Container = std::list; + using Container = std::list; ProcessList & parent; Container::iterator it; + public: ProcessListEntry(ProcessList & parent_, Container::iterator it_) : parent(parent_), it(it_) {} ~ProcessListEntry(); - ProcessListElement * operator->() { return &*it; } - const ProcessListElement * operator->() const { return &*it; } + QueryStatus * operator->() { return &*it; } + const QueryStatus * operator->() const { return &*it; } - ProcessListElement & get() { return *it; } - const ProcessListElement & get() const { return *it; } + QueryStatus & get() { return *it; } + const QueryStatus & get() const { return *it; } }; class ProcessList { - friend class ProcessListEntry; public: - using Element = ProcessListElement; + using Element = QueryStatus; using Entry = ProcessListEntry; /// list, for iterators not to invalidate. NOTE: could replace with cyclic buffer, but not worth. using Container = std::list; - using Info = std::vector; + using Info = std::vector; /// User -> queries using UserToQueries = std::unordered_map; -private: +protected: + friend class ProcessListEntry; + mutable std::mutex mutex; mutable std::condition_variable have_space; /// Number of currently running queries has become less than maximum. /// List of queries - Container cont; - size_t cur_size; /// In C++03 or C++11 and old ABI, std::list::size is not O(1). + Container processes; size_t max_size; /// 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. /// Stores per-user info: queries, statistics and limits @@ -269,16 +279,16 @@ private: QueryPriorities priorities; /// Limit and counter for memory of all simultaneously running queries. - MemoryTracker total_memory_tracker; + MemoryTracker total_memory_tracker{VariableContext::Global}; /// Limit network bandwidth for all users ThrottlerPtr total_network_throttler; /// Call under lock. Finds process with specified current_user and current_query_id. - ProcessListElement * tryGetProcessListElement(const String & current_query_id, const String & current_user); + QueryStatus * tryGetProcessListElement(const String & current_query_id, const String & current_user); public: - ProcessList(size_t max_size_ = 0) : cur_size(0), max_size(max_size_) {} + ProcessList(size_t max_size_ = 0) : max_size(max_size_) {} using EntryPtr = std::shared_ptr; @@ -287,23 +297,13 @@ public: * If timeout is passed - throw an exception. * Don't count KILL QUERY queries. */ - EntryPtr insert(const String & query_, const IAST * ast, const ClientInfo & client_info, const Settings & settings); + EntryPtr insert(const String & query_, const IAST * ast, Context & query_context); /// Number of currently executing queries. - size_t size() const { return cur_size; } + size_t size() const { return processes.size(); } /// Get current state of process list. - Info getInfo() const - { - std::lock_guard lock(mutex); - - Info res; - res.reserve(cur_size); - for (const auto & elem : cont) - res.emplace_back(elem.getInfo()); - - return res; - } + Info getInfo(bool get_thread_list = false, bool get_profile_events = false, bool get_settings = false) const; void setMaxSize(size_t max_size_) { diff --git a/dbms/src/Interpreters/ProfileEventsExt.cpp b/dbms/src/Interpreters/ProfileEventsExt.cpp new file mode 100644 index 00000000000..8344ac6e40c --- /dev/null +++ b/dbms/src/Interpreters/ProfileEventsExt.cpp @@ -0,0 +1,59 @@ +#include "ProfileEventsExt.h" +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + +/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io +void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, DB::IColumn * column_values_, bool nonzero_only) +{ + /// Convert ptr and make simple check + auto column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; + auto column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; + + size_t size = 0; + + for (Event event = 0; event < Counters::num_counters; ++event) + { + UInt64 value = counters[event].load(std::memory_order_relaxed); + + if (nonzero_only && 0 == value) + continue; + + ++size; + + if (column_names) + { + const char * desc = ProfileEvents::getDescription(event); + column_names->getData().insertData(desc, strlen(desc)); + } + + if (column_values) + column_values->getData().insert(value); + } + + if (column_names) + { + auto & offsets = column_names->getOffsets(); + offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + } + + if (column_values) + { + /// Nested columns case + bool the_same_offsets = column_names && column_names->getOffsetsPtr().get() == column_values->getOffsetsPtr().get(); + if (!the_same_offsets) + { + auto & offsets = column_values->getOffsets(); + offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + } + } +} + +} diff --git a/dbms/src/Interpreters/ProfileEventsExt.h b/dbms/src/Interpreters/ProfileEventsExt.h new file mode 100644 index 00000000000..4883505ee64 --- /dev/null +++ b/dbms/src/Interpreters/ProfileEventsExt.h @@ -0,0 +1,12 @@ +#pragma once +#include +#include + + +namespace ProfileEvents +{ + +/// Dumps profile events to two column Array(String) and Array(UInt64) +void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names, DB::IColumn * column_value, bool nonzero_only = true); + +} diff --git a/dbms/src/Interpreters/QueryLog.cpp b/dbms/src/Interpreters/QueryLog.cpp index 633d594cb8a..a005b63a6c3 100644 --- a/dbms/src/Interpreters/QueryLog.cpp +++ b/dbms/src/Interpreters/QueryLog.cpp @@ -1,12 +1,16 @@ +#include #include #include #include +#include #include #include #include #include #include +#include #include +#include #include #include #include @@ -20,55 +24,52 @@ Block QueryLogElement::createBlock() { return { - {ColumnUInt8::create(), std::make_shared(), "type"}, - {ColumnUInt16::create(), std::make_shared(), "event_date"}, - {ColumnUInt32::create(), std::make_shared(), "event_time"}, - {ColumnUInt32::create(), std::make_shared(), "query_start_time"}, - {ColumnUInt64::create(), std::make_shared(), "query_duration_ms"}, + {std::make_shared(), "type"}, + {std::make_shared(), "event_date"}, + {std::make_shared(), "event_time"}, + {std::make_shared(), "query_start_time"}, + {std::make_shared(), "query_duration_ms"}, - {ColumnUInt64::create(), std::make_shared(), "read_rows"}, - {ColumnUInt64::create(), std::make_shared(), "read_bytes"}, + {std::make_shared(), "read_rows"}, + {std::make_shared(), "read_bytes"}, + {std::make_shared(), "written_rows"}, + {std::make_shared(), "written_bytes"}, + {std::make_shared(), "result_rows"}, + {std::make_shared(), "result_bytes"}, + {std::make_shared(), "memory_usage"}, - {ColumnUInt64::create(), std::make_shared(), "written_rows"}, - {ColumnUInt64::create(), std::make_shared(), "written_bytes"}, + {std::make_shared(), "query"}, + {std::make_shared(), "exception"}, + {std::make_shared(), "stack_trace"}, - {ColumnUInt64::create(), std::make_shared(), "result_rows"}, - {ColumnUInt64::create(), std::make_shared(), "result_bytes"}, + {std::make_shared(), "is_initial_query"}, + {std::make_shared(), "user"}, + {std::make_shared(), "query_id"}, + {std::make_shared(16), "address"}, + {std::make_shared(), "port"}, + {std::make_shared(), "initial_user"}, + {std::make_shared(), "initial_query_id"}, + {std::make_shared(16), "initial_address"}, + {std::make_shared(), "initial_port"}, + {std::make_shared(), "interface"}, + {std::make_shared(), "os_user"}, + {std::make_shared(), "client_hostname"}, + {std::make_shared(), "client_name"}, + {std::make_shared(), "client_revision"}, + {std::make_shared(), "client_version_major"}, + {std::make_shared(), "client_version_minor"}, + {std::make_shared(), "client_version_patch"}, + {std::make_shared(), "http_method"}, + {std::make_shared(), "http_user_agent"}, + {std::make_shared(), "quota_key"}, - {ColumnUInt64::create(), std::make_shared(), "memory_usage"}, + {std::make_shared(), "revision"}, - {ColumnString::create(), std::make_shared(), "query"}, - {ColumnString::create(), std::make_shared(), "exception"}, - {ColumnString::create(), std::make_shared(), "stack_trace"}, - - {ColumnUInt8::create(), std::make_shared(), "is_initial_query"}, - - {ColumnString::create(), std::make_shared(), "user"}, - {ColumnString::create(), std::make_shared(), "query_id"}, - {ColumnFixedString::create(16), std::make_shared(16), "address"}, - {ColumnUInt16::create(), std::make_shared(), "port"}, - - {ColumnString::create(), std::make_shared(), "initial_user"}, - {ColumnString::create(), std::make_shared(), "initial_query_id"}, - {ColumnFixedString::create(16), std::make_shared(16), "initial_address"}, - {ColumnUInt16::create(), std::make_shared(), "initial_port"}, - - {ColumnUInt8::create(), std::make_shared(), "interface"}, - - {ColumnString::create(), std::make_shared(), "os_user"}, - {ColumnString::create(), std::make_shared(), "client_hostname"}, - {ColumnString::create(), std::make_shared(), "client_name"}, - {ColumnUInt32::create(), std::make_shared(), "client_revision"}, - {ColumnUInt32::create(), std::make_shared(), "client_version_major"}, - {ColumnUInt32::create(), std::make_shared(), "client_version_minor"}, - {ColumnUInt32::create(), std::make_shared(), "client_version_patch"}, - - {ColumnUInt8::create(), std::make_shared(), "http_method"}, - {ColumnString::create(), std::make_shared(), "http_user_agent"}, - - {ColumnString::create(), std::make_shared(), "quota_key"}, - - {ColumnUInt32::create(), std::make_shared(), "revision"}, + {std::make_shared(std::make_shared()), "thread_numbers"}, + {std::make_shared(std::make_shared()), "ProfileEvents.Names"}, + {std::make_shared(std::make_shared()), "ProfileEvents.Values"}, + {std::make_shared(std::make_shared()), "Settings.Names"}, + {std::make_shared(std::make_shared()), "Settings.Values"} }; } @@ -110,10 +111,8 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insert(UInt64(read_rows)); columns[i++]->insert(UInt64(read_bytes)); - columns[i++]->insert(UInt64(written_rows)); columns[i++]->insert(UInt64(written_bytes)); - columns[i++]->insert(UInt64(result_rows)); columns[i++]->insert(UInt64(result_bytes)); @@ -123,6 +122,47 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insertData(exception.data(), exception.size()); columns[i++]->insertData(stack_trace.data(), stack_trace.size()); + appendClientInfo(client_info, columns, i); + + columns[i++]->insert(UInt64(ClickHouseRevision::get())); + + { + Array threads_array; + threads_array.reserve(thread_numbers.size()); + for (const UInt32 thread_number : thread_numbers) + threads_array.emplace_back(UInt64(thread_number)); + columns[i++]->insert(threads_array); + } + + if (profile_counters) + { + auto column_names = columns[i++].get(); + auto column_values = columns[i++].get(); + ProfileEvents::dumpToArrayColumns(*profile_counters, column_names, column_values, true); + } + else + { + columns[i++]->insertDefault(); + columns[i++]->insertDefault(); + } + + if (query_settings) + { + auto column_names = columns[i++].get(); + auto column_values = columns[i++].get(); + query_settings->dumpToArrayColumns(column_names, column_values, true); + } + else + { + columns[i++]->insertDefault(); + columns[i++]->insertDefault(); + } + + block.setColumns(std::move(columns)); +} + +void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableColumns & columns, size_t & i) +{ columns[i++]->insert(UInt64(client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY)); columns[i++]->insert(client_info.current_user); @@ -149,10 +189,5 @@ void QueryLogElement::appendToBlock(Block & block) const columns[i++]->insert(client_info.http_user_agent); columns[i++]->insert(client_info.quota_key); - - columns[i++]->insert(UInt64(ClickHouseRevision::get())); - - block.setColumns(std::move(columns)); } - } diff --git a/dbms/src/Interpreters/QueryLog.h b/dbms/src/Interpreters/QueryLog.h index 8b4c7734ebe..95d563fd21e 100644 --- a/dbms/src/Interpreters/QueryLog.h +++ b/dbms/src/Interpreters/QueryLog.h @@ -3,6 +3,12 @@ #include +namespace ProfileEvents +{ + class Counters; +} + + namespace DB { @@ -53,10 +59,16 @@ struct QueryLogElement ClientInfo client_info; + std::vector thread_numbers; + std::shared_ptr profile_counters; + std::shared_ptr query_settings; + static std::string name() { return "QueryLog"; } static Block createBlock(); void appendToBlock(Block & block) const; + + static void appendClientInfo(const ClientInfo & client_info, MutableColumns & columns, size_t & i); }; diff --git a/dbms/src/Interpreters/QueryThreadLog.cpp b/dbms/src/Interpreters/QueryThreadLog.cpp new file mode 100644 index 00000000000..8a84d814388 --- /dev/null +++ b/dbms/src/Interpreters/QueryThreadLog.cpp @@ -0,0 +1,116 @@ +#include "QueryThreadLog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +Block QueryThreadLogElement::createBlock() +{ + return + { + {std::make_shared(), "event_date"}, + {std::make_shared(), "event_time"}, + {std::make_shared(), "query_start_time"}, + {std::make_shared(), "query_duration_ms"}, + + {std::make_shared(), "read_rows"}, + {std::make_shared(), "read_bytes"}, + {std::make_shared(), "written_rows"}, + {std::make_shared(), "written_bytes"}, + {std::make_shared(), "memory_usage"}, + {std::make_shared(), "peak_memory_usage"}, + + {std::make_shared(), "thread_name"}, + {std::make_shared(), "thread_number"}, + {std::make_shared(), "os_thread_id"}, + {std::make_shared(), "master_thread_number"}, + {std::make_shared(), "master_os_thread_id"}, + {std::make_shared(), "query"}, + + {std::make_shared(), "is_initial_query"}, + {std::make_shared(), "user"}, + {std::make_shared(), "query_id"}, + {std::make_shared(16), "address"}, + {std::make_shared(), "port"}, + {std::make_shared(), "initial_user"}, + {std::make_shared(), "initial_query_id"}, + {std::make_shared(16), "initial_address"}, + {std::make_shared(), "initial_port"}, + {std::make_shared(), "interface"}, + {std::make_shared(), "os_user"}, + {std::make_shared(), "client_hostname"}, + {std::make_shared(), "client_name"}, + {std::make_shared(), "client_revision"}, + {std::make_shared(), "client_version_major"}, + {std::make_shared(), "client_version_minor"}, + {std::make_shared(), "client_version_patch"}, + {std::make_shared(), "http_method"}, + {std::make_shared(), "http_user_agent"}, + {std::make_shared(), "quota_key"}, + + {std::make_shared(), "revision"}, + + {std::make_shared(std::make_shared()), "ProfileEvents.Names"}, + {std::make_shared(std::make_shared()), "ProfileEvents.Values"} + }; +} + +void QueryThreadLogElement::appendToBlock(Block & block) const +{ + MutableColumns columns = block.mutateColumns(); + + size_t i = 0; + + columns[i++]->insert(UInt64(DateLUT::instance().toDayNum(event_time))); + columns[i++]->insert(UInt64(event_time)); + columns[i++]->insert(UInt64(query_start_time)); + columns[i++]->insert(UInt64(query_duration_ms)); + + columns[i++]->insert(UInt64(read_rows)); + columns[i++]->insert(UInt64(read_bytes)); + columns[i++]->insert(UInt64(written_rows)); + columns[i++]->insert(UInt64(written_bytes)); + + columns[i++]->insert(Int64(memory_usage)); + columns[i++]->insert(Int64(peak_memory_usage)); + + columns[i++]->insertData(thread_name.data(), thread_name.size()); + columns[i++]->insert(UInt64(thread_number)); + columns[i++]->insert(Int64(os_thread_id)); + columns[i++]->insert(UInt64(master_thread_number)); + columns[i++]->insert(Int64(master_os_thread_id)); + + columns[i++]->insertData(query.data(), query.size()); + + QueryLogElement::appendClientInfo(client_info, columns, i); + + columns[i++]->insert(UInt64(ClickHouseRevision::get())); + + if (profile_counters) + { + auto column_names = columns[i++].get(); + auto column_values = columns[i++].get(); + dumpToArrayColumns(*profile_counters, column_names, column_values, true); + } + else + { + columns[i++]->insertDefault(); + columns[i++]->insertDefault(); + } +} + +} diff --git a/dbms/src/Interpreters/QueryThreadLog.h b/dbms/src/Interpreters/QueryThreadLog.h new file mode 100644 index 00000000000..36b8dacd1e6 --- /dev/null +++ b/dbms/src/Interpreters/QueryThreadLog.h @@ -0,0 +1,61 @@ +#pragma once + +#include + + +namespace ProfileEvents +{ + class Counters; +} + + +namespace DB +{ + +struct QueryThreadLogElement +{ + time_t event_time{}; + /// When query was attached to current thread + time_t query_start_time{}; + /// Real time spent by the thread to execute the query + UInt64 query_duration_ms{}; + + /// The data fetched from DB in current thread to execute the query + UInt64 read_rows{}; + UInt64 read_bytes{}; + + /// The data written to DB + UInt64 written_rows{}; + UInt64 written_bytes{}; + + Int64 memory_usage{}; + Int64 peak_memory_usage{}; + + String thread_name; + UInt32 thread_number{}; + Int32 os_thread_id{}; + UInt32 master_thread_number{}; + Int32 master_os_thread_id{}; + + String query; + ClientInfo client_info; + + std::shared_ptr profile_counters; + + static std::string name() { return "QueryThreadLog"; } + + static Block createBlock(); + void appendToBlock(Block & block) const; +}; + + +class QueryThreadLog : public SystemLog +{ + using SystemLog::SystemLog; +}; + + +} + + + diff --git a/dbms/src/Interpreters/Settings.cpp b/dbms/src/Interpreters/Settings.cpp index 03701dc1334..c27047e3cc2 100644 --- a/dbms/src/Interpreters/Settings.cpp +++ b/dbms/src/Interpreters/Settings.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -176,4 +179,40 @@ void Settings::serialize(WriteBuffer & buf) const #undef WRITE } +void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_values_, bool changed_only) +{ + /// Convert ptr and make simple check + auto column_names = (column_names_) ? &typeid_cast(*column_names_) : nullptr; + auto column_values = (column_values_) ? &typeid_cast(*column_values_) : nullptr; + + size_t size = 0; + +#define ADD_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \ + if (!changed_only || NAME.changed) \ + { \ + if (column_names) \ + column_names->getData().insertData(#NAME, strlen(#NAME)); \ + if (column_values) \ + column_values->getData().insert(NAME.toString()); \ + ++size; \ + } + APPLY_FOR_SETTINGS(ADD_SETTING) +#undef ADD_SETTING + + if (column_names) + { + auto & offsets = column_names->getOffsets(); + offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + } + + /// Nested columns case + bool the_same_offsets = column_names && column_values && column_names->getOffsetsPtr() == column_values->getOffsetsPtr(); + + if (column_values && !the_same_offsets) + { + auto & offsets = column_values->getOffsets(); + offsets.push_back((offsets.empty() ? 0 : offsets.back()) + size); + } +} + } diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 24a673eb0df..e1e02af96e4 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -15,6 +15,7 @@ namespace Poco namespace DB { +class IColumn; class Field; /** Settings of query execution. @@ -266,9 +267,12 @@ struct Settings M(SettingBool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.") \ \ M(SettingUInt64, enable_conditional_computation, 0, "Enable conditional computations") \ - \ M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.") \ - M(SettingBool, enable_optimize_predicate_expression, 0, "") \ + M(SettingBool, log_profile_events, true, "Log query performance statistics into the query_log and query_thread_log.") \ + M(SettingBool, log_query_settings, true, "Log query settings into the query_log.") \ + M(SettingBool, log_query_threads, true, "Log query threads into system.query_thread_log table.") \ + M(SettingString, send_logs_level, "none", "Send server text logs with specified minumum level to client. Valid values: 'trace', 'debug', 'info', 'warning', 'error', 'none'") \ + M(SettingBool, enable_optimize_predicate_expression, 0, "If it is set to true, optimize predicates to subqueries.") \ \ M(SettingUInt64, low_cardinality_max_dictionary_size, 8192, "Maximum size (in rows) of shared global dictionary for LowCardinality type.") \ M(SettingBool, low_cardinality_use_single_dictionary_for_part, false, "LowCardinality type serialization setting. If is true, than will use additional keys when global dictionary overflows. Otherwise, will create several shared dictionaries.") \ @@ -278,6 +282,7 @@ struct Settings M(SettingBool, prefer_localhost_replica, 1, "1 - always send query to local replica, if it exists. 0 - choose replica to send query between local and remote ones according to load_balancing") \ M(SettingUInt64, max_fetch_partition_retries_count, 5, "Amount of retries while fetching partition from another host.") \ M(SettingBool, asterisk_left_columns_only, 0, "If it is set to true, the asterisk only return left of join query.") \ + M(SettingUInt64, http_max_multipart_form_data_size, 1024 * 1024 * 1024, "Limit on size of multipart/form-data content. This setting cannot be parsed from URL parameters and should be set in user profile. Note that content is parsed and external tables are created in memory before start of query execution. And this is the only limit that has effect on that stage (limits on max memory usage and max execution time have no effect while reading HTTP form data).") \ #define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \ @@ -318,6 +323,9 @@ struct Settings /// Write changed settings to buffer. (For example, to be sent to remote server.) void serialize(WriteBuffer & buf) const; + + /// Dumps profile events to two column Array(String) and Array(UInt64) + void dumpToArrayColumns(IColumn * column_names, IColumn * column_values, bool changed_only = true); }; diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index a68a55c79b2..bc5edaba14a 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -1,5 +1,6 @@ #include #include +#include #include diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index dcd7d37da02..5bf3dcfd200 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace DB @@ -54,17 +55,19 @@ namespace DB class Context; class QueryLog; +class QueryThreadLog; class PartLog; -/// System logs should be destroyed in destructor of last Context and before tables, +/// System logs should be destroyed in destructor of the last Context and before tables, /// because SystemLog destruction makes insert query while flushing data into underlying tables struct SystemLogs { ~SystemLogs(); - std::unique_ptr query_log; /// Used to log queries. - std::unique_ptr part_log; /// Used to log operations with parts + std::unique_ptr query_log; /// Used to log queries. + std::unique_ptr query_thread_log; /// Used to log query threads. + std::unique_ptr part_log; /// Used to log operations with parts }; @@ -73,6 +76,8 @@ class SystemLog : private boost::noncopyable { public: + using Self = SystemLog; + /** Parameter: table name where to write log. * If table is not exists, then it get created with specified engine. * If it already exists, then its structure is checked to be compatible with structure of log record. @@ -100,6 +105,9 @@ public: LOG_ERROR(log, "SystemLog queue is full"); } + /// Flush data in the buffer to disk + void flush(bool quiet = false); + protected: Context & context; const String database_name; @@ -118,6 +126,7 @@ protected: * than accumulation of large amount of log records (for example, for query log - processing of large amount of queries). */ std::vector data; + std::mutex data_mutex; Logger * log; @@ -126,7 +135,6 @@ protected: std::thread saving_thread; void threadFunction(); - void flush(); /** Creates new table if it does not exist. * Renames old table if its structure is not suitable. @@ -184,7 +192,16 @@ void SystemLog::threadFunction() QueueItem element; bool has_element = false; - if (data.empty()) + bool is_empty; + { + std::unique_lock lock(data_mutex); + is_empty = data.empty(); + } + + /// data.size() is increased only in this function + /// TODO: get rid of data and queue duality + + if (is_empty) { queue.pop(element); has_element = true; @@ -206,14 +223,17 @@ void SystemLog::threadFunction() break; } else + { + std::unique_lock lock(data_mutex); data.push_back(element.second); + } } size_t milliseconds_elapsed = time_after_last_write.elapsed() / 1000000; if (milliseconds_elapsed >= flush_interval_milliseconds) { /// Write data to a table. - flush(); + flush(true); time_after_last_write.restart(); } } @@ -228,10 +248,15 @@ void SystemLog::threadFunction() template -void SystemLog::flush() +void SystemLog::flush(bool quiet) { + std::unique_lock lock(data_mutex); + try { + if (quiet && data.empty()) + return; + LOG_TRACE(log, "Flushing system log"); /// We check for existence of the table and create it as needed at every flush. @@ -347,5 +372,24 @@ void SystemLog::prepareTable() is_prepared = true; } +/// Creates a system log with MergeTree engines using parameters from config +template +std::unique_ptr createDefaultSystemLog( + Context & context_, + const String & default_database_name, + const String & default_table_name, + Poco::Util::AbstractConfiguration & config, + const String & config_prefix) +{ + String database = config.getString(config_prefix + ".database", default_database_name); + String table = config.getString(config_prefix + ".table", default_table_name); + String partition_by = config.getString(config_prefix + ".partition_by", "toYYYYMM(event_date)"); + String engine = "ENGINE = MergeTree PARTITION BY (" + partition_by + ") ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024"; + + size_t flush_interval_milliseconds = config.getUInt64("query_log.flush_interval_milliseconds", DEFAULT_QUERY_LOG_FLUSH_INTERVAL_MILLISECONDS); + + return std::make_unique(context_, database, table, engine, flush_interval_milliseconds); +} + } diff --git a/dbms/src/Interpreters/ThreadStatusExt.cpp b/dbms/src/Interpreters/ThreadStatusExt.cpp new file mode 100644 index 00000000000..ffd24a7e906 --- /dev/null +++ b/dbms/src/Interpreters/ThreadStatusExt.cpp @@ -0,0 +1,256 @@ +#include +#include +#include +#include +#include +#include +#include + + +/// Implement some methods of ThreadStatus and CurrentThread here to avoid extra linking dependencies in clickhouse_common_io +namespace DB +{ + +void ThreadStatus::attachQueryContext(Context & query_context_) +{ + query_context = &query_context_; + if (!global_context) + global_context = &query_context->getGlobalContext(); + + if (!thread_group) + return; + + std::unique_lock lock(thread_group->mutex); + thread_group->query_context = query_context; + if (!thread_group->global_context) + thread_group->global_context = global_context; +} + +String ThreadStatus::getQueryID() +{ + if (query_context) + return query_context->getClientInfo().current_query_id; + + return {}; +} + +void ThreadStatus::defaultThreadDeleter() +{ + ThreadStatus & thread = *CurrentThread::get(); + LOG_TRACE(thread.log, "Thread " << thread.thread_number << " exited"); + thread.detachQuery(true, true); +} + +void ThreadStatus::initializeQuery() +{ + assertState({ThreadState::DetachedFromQuery}, __PRETTY_FUNCTION__); + + thread_group = std::make_shared(); + + performance_counters.setParent(&thread_group->performance_counters); + memory_tracker.setParent(&thread_group->memory_tracker); + thread_group->memory_tracker.setDescription("(for query)"); + + thread_group->master_thread = shared_from_this(); + thread_group->thread_statuses.emplace(thread_number, shared_from_this()); + + initPerformanceCounters(); + thread_state = ThreadState::AttachedToQuery; + current_thread_scope.deleter = ThreadStatus::defaultThreadDeleter; +} + +void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached) +{ + if (thread_state == ThreadState::AttachedToQuery) + { + if (check_detached) + throw Exception("Can't attach query to the thread, it is already attached", ErrorCodes::LOGICAL_ERROR); + return; + } + + assertState({ThreadState::DetachedFromQuery}, __PRETTY_FUNCTION__); + + if (!thread_group_) + throw Exception("Attempt to attach to nullptr thread group", ErrorCodes::LOGICAL_ERROR); + + /// Attach current thread to thread group and copy useful information from it + thread_group = thread_group_; + + performance_counters.setParent(&thread_group->performance_counters); + memory_tracker.setParent(&thread_group->memory_tracker); + + { + std::unique_lock lock(thread_group->mutex); + + logs_queue_ptr = thread_group->logs_queue_ptr; + query_context = thread_group->query_context; + + if (!global_context) + global_context = thread_group->global_context; + + if (!thread_group->thread_statuses.emplace(thread_number, shared_from_this()).second) + throw Exception("Thread " + std::to_string(thread_number) + " is attached twice", ErrorCodes::LOGICAL_ERROR); + } + + initPerformanceCounters(); + thread_state = ThreadState::AttachedToQuery; + current_thread_scope.deleter = ThreadStatus::defaultThreadDeleter; +} + +void ThreadStatus::finalizePerformanceCounters() +{ + if (performance_counters_finalized) + return; + + performance_counters_finalized = true; + updatePerformanceCounters(); + + try + { + bool log_to_query_thread_log = global_context && query_context && query_context->getSettingsRef().log_query_threads.value != 0; + if (log_to_query_thread_log) + if (auto thread_log = global_context->getQueryThreadLog()) + logToQueryThreadLog(*thread_log); + } + catch (...) + { + tryLogCurrentException(log); + } +} + +void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) +{ + if (exit_if_already_detached && thread_state == ThreadState::DetachedFromQuery) + { + thread_state = thread_exits ? ThreadState::Died : ThreadState::DetachedFromQuery; + return; + } + + assertState({ThreadState::AttachedToQuery}, __PRETTY_FUNCTION__); + finalizePerformanceCounters(); + + /// For better logging ({query_id} will be shown here) + if (thread_group && thread_group.use_count() == 1) + thread_group->memory_tracker.logPeakMemoryUsage(); + + /// Detach from thread group + performance_counters.setParent(&ProfileEvents::global_counters); + memory_tracker.setParent(nullptr); + query_context = nullptr; + thread_group.reset(); + + thread_state = thread_exits ? ThreadState::Died : ThreadState::DetachedFromQuery; +} + +void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) +{ + QueryThreadLogElement elem; + + elem.event_time = time(nullptr); + elem.query_start_time = query_start_time; + elem.query_duration_ms = (getCurrentTimeNanoseconds() - query_start_time_nanoseconds) / 1000000U; + + elem.read_rows = progress_in.rows.load(std::memory_order_relaxed); + elem.read_bytes = progress_in.bytes.load(std::memory_order_relaxed); + elem.written_rows = progress_out.rows.load(std::memory_order_relaxed); + elem.written_bytes = progress_out.bytes.load(std::memory_order_relaxed); + elem.memory_usage = memory_tracker.get(); + elem.peak_memory_usage = memory_tracker.getPeak(); + + elem.thread_name = getThreadName(); + elem.thread_number = thread_number; + elem.os_thread_id = os_thread_id; + + if (thread_group) + { + { + std::shared_lock lock(thread_group->mutex); + + if (thread_group->master_thread) + { + elem.master_thread_number = thread_group->master_thread->thread_number; + elem.master_os_thread_id = thread_group->master_thread->os_thread_id; + } + + elem.query = thread_group->query; + } + } + + if (query_context) + { + elem.client_info = query_context->getClientInfo(); + + if (query_context->getSettingsRef().log_profile_events.value != 0) + { + /// NOTE: Here we are in the same thread, so we can make memcpy() + elem.profile_counters = std::make_shared(performance_counters.getPartiallyAtomicSnapshot()); + } + } + + thread_log.add(elem); +} + + +void CurrentThread::initializeQuery() +{ + get()->initializeQuery(); +} + +void CurrentThread::attachTo(const ThreadGroupStatusPtr & thread_group) +{ + get()->attachQuery(thread_group, true); +} + +void CurrentThread::attachToIfDetached(const ThreadGroupStatusPtr & thread_group) +{ + get()->attachQuery(thread_group, false); +} + +std::string CurrentThread::getCurrentQueryID() +{ + if (!current_thread || current_thread.use_count() <= 0) + return {}; + + return current_thread->getQueryID(); +} + +void CurrentThread::attachQueryContext(Context & query_context) +{ + return get()->attachQueryContext(query_context); +} + +void CurrentThread::finalizePerformanceCounters() +{ + get()->finalizePerformanceCounters(); +} + +void CurrentThread::detachQuery() +{ + get()->detachQuery(false); +} + +void CurrentThread::detachQueryIfNotDetached() +{ + get()->detachQuery(true); +} + + +CurrentThread::QueryScope::QueryScope(Context & query_context) +{ + CurrentThread::initializeQuery(); + CurrentThread::attachQueryContext(query_context); +} + +CurrentThread::QueryScope::~QueryScope() +{ + try + { + CurrentThread::detachQueryIfNotDetached(); + } + catch (...) + { + tryLogCurrentException("CurrentThread", __PRETTY_FUNCTION__); + } +} + +} diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index de36a84fd26..8de5fa6737c 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -62,7 +62,6 @@ static void logQuery(const String & query, const Context & context) LOG_DEBUG(&Logger::get("executeQuery"), "(from " << context.getClientInfo().current_address.toString() << (current_user != "default" ? ", user: " + context.getClientInfo().current_user : "") - << ", query_id: " << current_query_id << (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()) << ") " << joinLines(query) @@ -118,6 +117,9 @@ static void onExceptionBeforeStart(const String & query, Context & context, time setExceptionStackTrace(elem); logException(context, elem); + /// Update performance counters before logging to query_log + CurrentThread::finalizePerformanceCounters(); + if (log_queries) if (auto query_log = context.getQueryLog()) query_log->add(elem); @@ -134,6 +136,7 @@ static std::tuple executeQueryImpl( time_t current_time = time(nullptr); context.setQueryContext(context); + CurrentThread::attachQueryContext(context); const Settings & settings = context.getSettingsRef(); @@ -189,15 +192,13 @@ static std::tuple executeQueryImpl( ProcessList::EntryPtr process_list_entry; if (!internal && nullptr == typeid_cast(&*ast)) { - process_list_entry = context.getProcessList().insert( - query, - ast.get(), - context.getClientInfo(), - settings); - + process_list_entry = context.getProcessList().insert(query, ast.get(), context); context.setProcessListElement(&process_list_entry->get()); } + /// Load external tables if they were provided + context.initializeExternalTablesIfSet(); + auto interpreter = InterpreterFactory::get(ast, context, stage); res = interpreter->execute(); @@ -255,6 +256,9 @@ static std::tuple executeQueryImpl( /// Log into system table start of query execution, if need. if (log_queries) { + if (settings.log_query_settings) + elem.query_settings = std::make_shared(context.getSettingsRef()); + if (auto query_log = context.getQueryLog()) query_log->add(elem); } @@ -262,12 +266,16 @@ static std::tuple executeQueryImpl( /// Also make possible for caller to log successful query finish and exception during execution. res.finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable { - ProcessListElement * process_list_elem = context.getProcessListElement(); + QueryStatus * process_list_elem = context.getProcessListElement(); + const Settings & settings = context.getSettingsRef(); if (!process_list_elem) return; - ProcessInfo info = process_list_elem->getInfo(); + /// Update performance counters before logging to query_log + CurrentThread::finalizePerformanceCounters(); + + QueryStatusInfo info = process_list_elem->getInfo(true, settings.log_profile_events); double elapsed_seconds = info.elapsed_seconds; @@ -288,18 +296,18 @@ static std::tuple executeQueryImpl( { if (auto profiling_stream = dynamic_cast(stream_in)) { - const BlockStreamProfileInfo & info = profiling_stream->getProfileInfo(); + const BlockStreamProfileInfo & stream_in_info = profiling_stream->getProfileInfo(); /// NOTE: INSERT SELECT query contains zero metrics - elem.result_rows = info.rows; - elem.result_bytes = info.bytes; + elem.result_rows = stream_in_info.rows; + elem.result_bytes = stream_in_info.bytes; } } else if (stream_out) /// will be used only for ordinary INSERT queries { if (auto counting_stream = dynamic_cast(stream_out)) { - /// NOTE: Redundancy. The same values could be extracted from process_list_elem->progress_out. + /// NOTE: Redundancy. The same values coulld be extracted from process_list_elem->progress_out.query_settings = process_list_elem->progress_in elem.result_rows = counting_stream->getProgress().rows; elem.result_bytes = counting_stream->getProgress().bytes; } @@ -314,6 +322,9 @@ static std::tuple executeQueryImpl( << formatReadableSizeWithBinarySuffix(elem.read_bytes / elapsed_seconds) << "/sec."); } + elem.thread_numbers = std::move(info.thread_numbers); + elem.profile_counters = std::move(info.profile_counters); + if (log_queries) { if (auto query_log = context.getQueryLog()) @@ -331,11 +342,15 @@ static std::tuple executeQueryImpl( elem.query_duration_ms = 1000 * (elem.event_time - elem.query_start_time); elem.exception = getCurrentExceptionMessage(false); - ProcessListElement * process_list_elem = context.getProcessListElement(); + QueryStatus * process_list_elem = context.getProcessListElement(); + const Settings & settings = context.getSettingsRef(); + + /// Update performance counters before logging to query_log + CurrentThread::finalizePerformanceCounters(); if (process_list_elem) { - ProcessInfo info = process_list_elem->getInfo(); + QueryStatusInfo info = process_list_elem->getInfo(true, settings.log_profile_events, false); elem.query_duration_ms = info.elapsed_seconds * 1000; @@ -343,11 +358,15 @@ static std::tuple executeQueryImpl( elem.read_bytes = info.read_bytes; elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0; + + elem.thread_numbers = std::move(info.thread_numbers); + elem.profile_counters = std::move(info.profile_counters); } setExceptionStackTrace(elem); logException(context, elem); + /// In case of exception we log internal queries also if (log_queries) { if (auto query_log = context.getQueryLog()) diff --git a/dbms/src/Interpreters/tests/CMakeLists.txt b/dbms/src/Interpreters/tests/CMakeLists.txt index 6b7a9b59feb..04808feb926 100644 --- a/dbms/src/Interpreters/tests/CMakeLists.txt +++ b/dbms/src/Interpreters/tests/CMakeLists.txt @@ -46,4 +46,7 @@ target_link_libraries (in_join_subqueries_preprocessor dbms) add_check(in_join_subqueries_preprocessor) add_executable (users users.cpp) -target_link_libraries (users dbms ${Boost_FILESYSTEM_LIBRARY} dbms) +target_link_libraries (users dbms ${Boost_FILESYSTEM_LIBRARY}) + +add_executable (internal_iotop internal_iotop.cpp) +target_link_libraries (internal_iotop dbms) diff --git a/dbms/src/Interpreters/tests/internal_iotop.cpp b/dbms/src/Interpreters/tests/internal_iotop.cpp new file mode 100644 index 00000000000..536a2b3d0ae --- /dev/null +++ b/dbms/src/Interpreters/tests/internal_iotop.cpp @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + + +std::mutex mutex; + + +std::ostream & operator << (std::ostream & stream, const ::taskstats & stat) +{ +#define PRINT(field) (stream << #field << " " << stat.field) + + PRINT(ac_pid) << ", "; + + PRINT(read_bytes) << ", "; + PRINT(write_bytes) << ", "; + + PRINT(read_char) << ", "; + PRINT(write_char) << ", "; + + PRINT(swapin_delay_total) << ", "; + PRINT(blkio_delay_total) << ", "; + PRINT(cpu_delay_total) << ", "; + + PRINT(ac_pid) << ", "; + + PRINT(ac_utime) << ", "; + PRINT(ac_stime) << ", "; + +#undef PRINT + + return stream; +} + +using namespace DB; + + +void do_io(size_t id) +{ + ::taskstats stat; + int tid = TaskStatsInfoGetter::getCurrentTID(); + TaskStatsInfoGetter get_info; + + if (!get_info.tryGetStat(stat, tid)) + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ". Can't get stat\n"; + } + else + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ", intitial\n" << stat << "\n"; + } + + size_t copy_size = 1048576 * (1 + id); + std::string path_dst = "test_out_" + std::to_string(id); + + { + ReadBufferFromFile rb("/dev/urandom"); + WriteBufferFromFile wb(path_dst, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096); + copyData(rb, wb, copy_size); + wb.close(); + } + + get_info.getStat(stat, tid); + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ", step1\n" << stat << "\n"; + } + + { + ReadBufferFromFile rb(path_dst); + WriteBufferFromOwnString wb; + copyData(rb, wb, copy_size); + } + + get_info.getStat(stat, tid); + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ", step2\n" << stat << "\n"; + } + + { + ReadBufferFromFile rb(path_dst); + WriteBufferFromOwnString wb; + copyData(rb, wb, copy_size); + } + + get_info.getStat(stat, tid); + { + std::lock_guard lock(mutex); + std::cerr << "#" << id << ", tid " << tid << ", step3\n" << stat << "\n"; + } + + Poco::File(path_dst).remove(false); +} + +void test_perf() +{ + + ::taskstats stat; + int tid = TaskStatsInfoGetter::getCurrentTID(); + TaskStatsInfoGetter get_info; + + rusage rusage; + + constexpr size_t num_samples = 1000000; + { + Stopwatch watch; + for (size_t i = 0; i < num_samples; ++i) + getrusage(RUSAGE_THREAD, &rusage); + + auto ms = watch.elapsedMilliseconds(); + if (ms > 0) + std::cerr << "RUsage: " << double(ms) / num_samples << " ms per call, " << 1000 * num_samples / ms << " calls per second\n"; + } + + { + Stopwatch watch; + for (size_t i = 0; i < num_samples; ++i) + get_info.getStat(stat, tid); + + auto ms = watch.elapsedMilliseconds(); + if (ms > 0) + std::cerr << "Netlink: " << double(ms) / num_samples << " ms per call, " << 1000 * num_samples / ms << " calls per second\n"; + } + + std::cerr << stat << "\n"; +} + +int main() +try +{ + std::cerr << "pid " << getpid() << "\n"; + + size_t num_threads = 2; + ThreadPool pool(num_threads); + for (size_t i = 0; i < num_threads; ++i) + pool.schedule([i]() { do_io(i); }); + pool.wait(); + + test_perf(); + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(true); + return -1; +} + diff --git a/dbms/src/Parsers/ASTSystemQuery.cpp b/dbms/src/Parsers/ASTSystemQuery.cpp index 01f43e3d909..51bd2c8c26a 100644 --- a/dbms/src/Parsers/ASTSystemQuery.cpp +++ b/dbms/src/Parsers/ASTSystemQuery.cpp @@ -61,6 +61,8 @@ const char * ASTSystemQuery::typeToString(Type type) return "STOP REPLICATION QUEUES"; case Type::START_REPLICATION_QUEUES: return "START REPLICATION QUEUES"; + case Type::FLUSH_SYSTEM_TABLES: + return "FLUSH SYSTEM TABLES"; default: throw Exception("Unknown SYSTEM query command", ErrorCodes::BAD_TYPE_OF_FIELD); } diff --git a/dbms/src/Parsers/ASTSystemQuery.h b/dbms/src/Parsers/ASTSystemQuery.h index 0964e50c64e..98d8975d545 100644 --- a/dbms/src/Parsers/ASTSystemQuery.h +++ b/dbms/src/Parsers/ASTSystemQuery.h @@ -35,6 +35,7 @@ public: START_REPLICATEDS_SENDS, STOP_REPLICATION_QUEUES, START_REPLICATION_QUEUES, + FLUSH_SYSTEM_TABLES, END }; diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 87b3866e061..888bd64f03e 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -3,6 +3,8 @@ #include #include #include +#include + namespace DB diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 0482efa4cd2..39e78ffd363 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include + namespace CurrentMetrics { extern const Metric DistributedSend; @@ -191,9 +193,12 @@ void DistributedBlockOutputStream::waitForJobs() ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutputStream::JobReplica & job, const Block & current_block) { - auto memory_tracker = current_memory_tracker; - return [this, memory_tracker, &job, ¤t_block]() + auto thread_group = CurrentThread::getGroup(); + return [this, thread_group, &job, ¤t_block]() { + CurrentThread::attachToIfDetached(thread_group); + setThreadName("DistrOutStrProc"); + ++job.blocks_started; SCOPE_EXIT({ @@ -204,12 +209,6 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp job.max_elapsed_time_for_block_ms = std::max(job.max_elapsed_time_for_block_ms, elapsed_time_for_block_ms); }); - if (!current_memory_tracker) - { - current_memory_tracker = memory_tracker; - setThreadName("DistrOutStrProc"); - } - const auto & shard_info = cluster->getShardsInfo()[job.shard_index]; size_t num_shards = cluster->getShardsInfo().size(); auto & shard_job = per_shard_jobs[job.shard_index]; diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index 58ac6302ff4..2d1092d04b8 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -157,6 +157,11 @@ public: return res; } + /** Returns stage to which query is going to be processed in read() function. + * (Normally, the function only reads the columns from the list, but in other cases, + * for example, the request can be partially processed on a remote server.) + */ + virtual QueryProcessingStage::Enum getQueryProcessingStage(const Context &) const { return QueryProcessingStage::FetchColumns; } /** Read a set of columns from the table. * Accepts a list of columns to read, as well as a description of the query, @@ -164,9 +169,7 @@ public: * (indexes, locks, etc.) * Returns a stream with which you can read data sequentially * or multiple streams for parallel data reading. - * The `processed_stage` info is also written to what stage the request was processed. - * (Normally, the function only reads the columns from the list, but in other cases, - * for example, the request can be partially processed on a remote server.) + * The `processed_stage` must be the result of getQueryProcessingStage() function. * * context contains settings for one query. * Usually Storage does not care about these settings, since they are used in the interpreter. @@ -181,7 +184,7 @@ public: const Names & /*column_names*/, const SelectQueryInfo & /*query_info*/, const Context & /*context*/, - QueryProcessingStage::Enum & /*processed_stage*/, + QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned /*num_streams*/) { @@ -344,6 +347,20 @@ protected: using ITableDeclaration::ITableDeclaration; using std::enable_shared_from_this::shared_from_this; + void checkQueryProcessingStage(QueryProcessingStage::Enum processed_stage, const Context & context) + { + auto expected_stage = getQueryProcessingStage(context); + checkQueryProcessingStage(processed_stage, expected_stage); + } + + void checkQueryProcessingStage(QueryProcessingStage::Enum processed_stage, QueryProcessingStage::Enum expected_stage) + { + if (processed_stage != expected_stage) + throw Exception("Unexpected query processing stage for storage " + getName() + + ": expected " + QueryProcessingStage::toString(expected_stage) + + ", got " + QueryProcessingStage::toString(processed_stage), ErrorCodes::LOGICAL_ERROR); + } + private: friend class TableStructureReadLock; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 80cb5ff2bad..d7794ffe295 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -269,12 +269,12 @@ BlockInputStreams StorageKafka::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { + checkQueryProcessingStage(processed_stage, context); check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; if (num_consumers == 0) return BlockInputStreams(); diff --git a/dbms/src/Storages/Kafka/StorageKafka.h b/dbms/src/Storages/Kafka/StorageKafka.h index 9652d1d6a46..5b806d71d8f 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.h +++ b/dbms/src/Storages/Kafka/StorageKafka.h @@ -39,7 +39,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp index bd4a64371c3..c7e49f59e7a 100644 --- a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp +++ b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.cpp @@ -6,8 +6,10 @@ #include #include #include +#include #include +#include #include #include @@ -56,6 +58,12 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_) : size(size_) { LOG_INFO(&Logger::get("BackgroundProcessingPool"), "Create BackgroundProcessingPool with " << size << " threads"); + /// Put all threads to one thread group + /// The master thread exits immediately + CurrentThread::initializeQuery(); + thread_group = CurrentThread::getGroup(); + CurrentThread::detachQuery(); + threads.resize(size); for (auto & thread : threads) thread = std::thread([this] { threadFunction(); }); @@ -114,9 +122,11 @@ void BackgroundProcessingPool::threadFunction() { setThreadName("BackgrProcPool"); - MemoryTracker memory_tracker; - memory_tracker.setMetric(CurrentMetrics::MemoryTrackingInBackgroundProcessingPool); - current_memory_tracker = &memory_tracker; + /// Put all threads to one thread pool + CurrentThread::attachTo(thread_group); + SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); + + CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundProcessingPool); pcg64 rng(randomSeed()); std::this_thread::sleep_for(std::chrono::duration(std::uniform_real_distribution(0, sleep_seconds_random_part)(rng))); @@ -201,8 +211,6 @@ void BackgroundProcessingPool::threadFunction() task->iterator = tasks.emplace(next_time_to_execute, task); } } - - current_memory_tracker = nullptr; } } diff --git a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h index 0495b5e8c9d..2b61413c5a8 100644 --- a/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h +++ b/dbms/src/Storages/MergeTree/BackgroundProcessingPool.h @@ -12,6 +12,8 @@ #include #include #include +#include + namespace DB { @@ -64,6 +66,8 @@ protected: std::atomic shutdown {false}; std::condition_variable wake_event; + /// Thread group used for profiling purposes + ThreadGroupStatusPtr thread_group; void threadFunction(); }; diff --git a/dbms/src/Storages/MergeTree/MergeList.cpp b/dbms/src/Storages/MergeTree/MergeList.cpp index 9714d1b80da..12f3fa08202 100644 --- a/dbms/src/Storages/MergeTree/MergeList.cpp +++ b/dbms/src/Storages/MergeTree/MergeList.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace CurrentMetrics { @@ -20,11 +22,12 @@ MergeListElement::MergeListElement(const std::string & database, const std::stri source_part_names.emplace_back(source_part->name); /// Each merge is executed into separate background processing pool thread - background_pool_task_memory_tracker = current_memory_tracker; - if (background_pool_task_memory_tracker) + background_thread_memory_tracker = &CurrentThread::getMemoryTracker(); + if (background_thread_memory_tracker) { memory_tracker.setMetric(CurrentMetrics::MemoryTrackingForMerges); - background_pool_task_memory_tracker->setNext(&memory_tracker); + background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); + background_thread_memory_tracker->setParent(&memory_tracker); } } @@ -56,8 +59,8 @@ MergeInfo MergeListElement::getInfo() const MergeListElement::~MergeListElement() { /// Unplug memory_tracker from current background processing pool thread - if (background_pool_task_memory_tracker) - background_pool_task_memory_tracker->setNext(nullptr); + if (background_thread_memory_tracker) + background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); } } diff --git a/dbms/src/Storages/MergeTree/MergeList.h b/dbms/src/Storages/MergeTree/MergeList.h index d948cafc2ac..bc61b96f18b 100644 --- a/dbms/src/Storages/MergeTree/MergeList.h +++ b/dbms/src/Storages/MergeTree/MergeList.h @@ -65,8 +65,9 @@ struct MergeListElement : boost::noncopyable /// Updated only for Vertical algorithm std::atomic columns_written{}; - MemoryTracker memory_tracker; - MemoryTracker * background_pool_task_memory_tracker; + MemoryTracker memory_tracker{VariableContext::Process}; + MemoryTracker * background_thread_memory_tracker; + MemoryTracker * background_thread_memory_tracker_prev_parent = nullptr; /// Poco thread number used in logs UInt32 thread_number; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp index 12e6ead5325..8e765babc40 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -20,8 +21,7 @@ namespace ErrorCodes MergeTreeBaseBlockInputStream::MergeTreeBaseBlockInputStream( MergeTreeData & storage, - const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column_name, + const PrewhereInfoPtr & prewhere_info, UInt64 max_block_size_rows, UInt64 preferred_block_size_bytes, UInt64 preferred_max_column_in_block_size_bytes, @@ -32,8 +32,7 @@ MergeTreeBaseBlockInputStream::MergeTreeBaseBlockInputStream( const Names & virt_column_names) : storage(storage), - prewhere_actions(prewhere_actions), - prewhere_column_name(prewhere_column_name), + prewhere_info(prewhere_info), max_block_size_rows(max_block_size_rows), preferred_block_size_bytes(preferred_block_size_bytes), preferred_max_column_in_block_size_bytes(preferred_max_column_in_block_size_bytes), @@ -117,20 +116,20 @@ Block MergeTreeBaseBlockInputStream::readFromPart() if (!task->range_reader.isInitialized()) { - if (prewhere_actions) + if (prewhere_info) { if (reader->getColumns().empty()) { task->range_reader = MergeTreeRangeReader( - pre_reader.get(), index_granularity, nullptr, prewhere_actions, - &prewhere_column_name, &task->ordered_names, + pre_reader.get(), index_granularity, nullptr, prewhere_info->prewhere_actions, + &prewhere_info->prewhere_column_name, &task->ordered_names, task->should_reorder, task->remove_prewhere_column, true); } else { task->pre_range_reader = MergeTreeRangeReader( - pre_reader.get(), index_granularity, nullptr, prewhere_actions, - &prewhere_column_name, &task->ordered_names, + pre_reader.get(), index_granularity, nullptr, prewhere_info->prewhere_actions, + &prewhere_info->prewhere_column_name, &task->ordered_names, task->should_reorder, task->remove_prewhere_column, false); task->range_reader = MergeTreeRangeReader( @@ -141,7 +140,7 @@ Block MergeTreeBaseBlockInputStream::readFromPart() else { task->range_reader = MergeTreeRangeReader( - reader.get(), index_granularity, nullptr, prewhere_actions, + reader.get(), index_granularity, nullptr, nullptr, nullptr, &task->ordered_names, task->should_reorder, false, true); } } @@ -167,10 +166,10 @@ Block MergeTreeBaseBlockInputStream::readFromPart() task->size_predictor->update(read_result.block); } - if (read_result.block && prewhere_actions && !task->remove_prewhere_column) + if (read_result.block && prewhere_info && !task->remove_prewhere_column) { /// Convert const column to full here because it's cheaper to filter const column than full. - auto & column = read_result.block.getByName(prewhere_column_name); + auto & column = read_result.block.getByName(prewhere_info->prewhere_column_name); column.column = column.column->convertToFullColumnIfConst(); } @@ -215,6 +214,20 @@ void MergeTreeBaseBlockInputStream::injectVirtualColumns(Block & block) const } +void MergeTreeBaseBlockInputStream::executePrewhereActions(Block & block, const PrewhereInfoPtr & prewhere_info) +{ + if (prewhere_info) + { + prewhere_info->prewhere_actions->execute(block); + if (prewhere_info->remove_prewhere_column) + block.erase(prewhere_info->prewhere_column_name); + + if (!block) + block.insert({nullptr, std::make_shared(), "_nothing"}); + } +} + + MergeTreeBaseBlockInputStream::~MergeTreeBaseBlockInputStream() = default; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h index f5e8e586e5d..e2763b08e19 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseBlockInputStream.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -18,8 +19,7 @@ class MergeTreeBaseBlockInputStream : public IProfilingBlockInputStream public: MergeTreeBaseBlockInputStream( MergeTreeData & storage, - const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, UInt64 max_block_size_rows, UInt64 preferred_block_size_bytes, UInt64 preferred_max_column_in_block_size_bytes, @@ -31,8 +31,10 @@ public: ~MergeTreeBaseBlockInputStream() override; + static void executePrewhereActions(Block & block, const PrewhereInfoPtr & prewhere_info); + protected: - Block readImpl() override final; + Block readImpl() final; /// Creates new this->task, and initilizes readers virtual bool getNewTask() = 0; @@ -47,8 +49,7 @@ protected: protected: MergeTreeData & storage; - ExpressionActionsPtr prewhere_actions; - String prewhere_column_name; + PrewhereInfoPtr prewhere_info; UInt64 max_block_size_rows; UInt64 preferred_block_size_bytes; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp index c1abe931d0f..e0d5215b9c4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp @@ -24,8 +24,7 @@ MergeTreeBlockInputStream::MergeTreeBlockInputStream( Names column_names, const MarkRanges & mark_ranges_, bool use_uncompressed_cache_, - ExpressionActionsPtr prewhere_actions_, - String prewhere_column_, + const PrewhereInfoPtr & prewhere_info, bool check_columns, size_t min_bytes_to_use_direct_io_, size_t max_read_buffer_size_, @@ -34,10 +33,10 @@ MergeTreeBlockInputStream::MergeTreeBlockInputStream( size_t part_index_in_query_, bool quiet) : - MergeTreeBaseBlockInputStream{storage_, prewhere_actions_, prewhere_column_, max_block_size_rows_, + MergeTreeBaseBlockInputStream{storage_, prewhere_info, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names}, - ordered_names{column_names}, + required_columns{column_names}, data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), all_mark_ranges(mark_ranges_), @@ -61,7 +60,7 @@ MergeTreeBlockInputStream::MergeTreeBlockInputStream( addTotalRowsApprox(total_rows); - header = storage.getSampleBlockForColumns(ordered_names); + header = storage.getSampleBlockForColumns(required_columns); /// Types may be different during ALTER (when this stream is used to perform an ALTER). /// NOTE: We may use similar code to implement non blocking ALTERs. @@ -79,6 +78,9 @@ MergeTreeBlockInputStream::MergeTreeBlockInputStream( } injectVirtualColumns(header); + executePrewhereActions(header, prewhere_info); + + ordered_names = getHeader().getNames(); } @@ -99,15 +101,15 @@ try } is_first_task = false; - Names pre_column_names, column_names = ordered_names; - bool remove_prewhere_column = false; + Names pre_column_names; + Names column_names = required_columns; /// inject columns required for defaults evaluation bool should_reorder = !injectRequiredColumns(storage, data_part, column_names).empty(); - if (prewhere_actions) + if (prewhere_info) { - pre_column_names = prewhere_actions->getRequiredColumns(); + pre_column_names = prewhere_info->prewhere_actions->getRequiredColumns(); if (pre_column_names.empty()) pre_column_names.push_back(column_names[0]); @@ -117,9 +119,6 @@ try should_reorder = true; const NameSet pre_name_set(pre_column_names.begin(), pre_column_names.end()); - /// If the expression in PREWHERE is not a column of the table, you do not need to output a column with it - /// (from storage expect to receive only the columns of the table). - remove_prewhere_column = !pre_name_set.count(prewhere_column_name); Names post_column_names; for (const auto & name : column_names) @@ -159,9 +158,9 @@ try auto size_predictor = (preferred_block_size_bytes == 0) ? nullptr : std::make_unique(data_part, ordered_names, data_part->storage.getSampleBlock()); - task = std::make_unique(data_part, remaining_mark_ranges, part_index_in_query, ordered_names, - column_name_set, columns, pre_columns, remove_prewhere_column, should_reorder, - std::move(size_predictor)); + task = std::make_unique( + data_part, remaining_mark_ranges, part_index_in_query, ordered_names, column_name_set, columns, pre_columns, + prewhere_info && prewhere_info->remove_prewhere_column, should_reorder, std::move(size_predictor)); if (!reader) { @@ -175,7 +174,7 @@ try owned_mark_cache.get(), save_marks_in_cache, storage, all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); - if (prewhere_actions) + if (prewhere_info) pre_reader = std::make_unique( path, data_part, pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.h index d4c410287fd..df63eaea5dd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -24,8 +25,7 @@ public: Names column_names, const MarkRanges & mark_ranges, bool use_uncompressed_cache, - ExpressionActionsPtr prewhere_actions, - String prewhere_column, + const PrewhereInfoPtr & prewhere_info, bool check_columns, size_t min_bytes_to_use_direct_io, size_t max_read_buffer_size, @@ -51,6 +51,8 @@ private: Block header; /// Used by Task + Names required_columns; + /// Names from header. Used in order to order columns in read blocks. Names ordered_names; NameSet column_name_set; NamesAndTypesList columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index e7447f07b09..b945ff6dfad 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1208,7 +1208,7 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart( MarkRanges ranges{MarkRange(0, part->marks_count)}; BlockInputStreamPtr part_in = std::make_shared( *this, part, DEFAULT_MERGE_BLOCK_SIZE, 0, 0, expression->getRequiredColumns(), ranges, - false, nullptr, "", false, 0, DBMS_DEFAULT_BUFFER_SIZE, false); + false, nullptr, false, 0, DBMS_DEFAULT_BUFFER_SIZE, false); auto compression_settings = this->context.chooseCompressionSettings( part->bytes_on_disk, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b318ac92ce7..0953e1f845f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -603,7 +603,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { auto input = std::make_unique( data, part, DEFAULT_MERGE_BLOCK_SIZE, 0, 0, merging_column_names, MarkRanges(1, MarkRange(0, part->marks_count)), - false, nullptr, "", true, aio_threshold, DBMS_DEFAULT_BUFFER_SIZE, false); + false, nullptr, true, aio_threshold, DBMS_DEFAULT_BUFFER_SIZE, false); input->setProgressCallback(MergeProgressCallback( merge_entry, sum_input_rows_upper_bound, column_sizes, watch_prev_elapsed, merge_alg)); @@ -747,7 +747,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { auto column_part_stream = std::make_shared( data, parts[part_num], DEFAULT_MERGE_BLOCK_SIZE, 0, 0, column_name_, MarkRanges{MarkRange(0, parts[part_num]->marks_count)}, - false, nullptr, "", true, aio_threshold, DBMS_DEFAULT_BUFFER_SIZE, false, Names{}, 0, true); + false, nullptr, true, aio_threshold, DBMS_DEFAULT_BUFFER_SIZE, false, Names{}, 0, true); column_part_stream->setProgressCallback(MergeProgressCallbackVerticalStep( merge_entry, sum_input_rows_exact, column_sizes, column_name, watch_prev_elapsed)); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 93ee9220387..956a267789a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -135,13 +135,12 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( const Names & column_names_to_return, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, const unsigned num_streams, Int64 max_block_number_to_read) const { return readFromParts( - data.getDataPartsVector(), column_names_to_return, query_info, context, processed_stage, + data.getDataPartsVector(), column_names_to_return, query_info, context, max_block_size, num_streams, max_block_number_to_read); } @@ -150,7 +149,6 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( const Names & column_names_to_return, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, const size_t max_block_size, const unsigned num_streams, Int64 max_block_number_to_read) const @@ -207,7 +205,6 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( std::multiset part_values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_part"); data.check(real_column_names); - processed_stage = QueryProcessingStage::FetchColumns; const Settings & settings = context.getSettingsRef(); Names primary_sort_columns = data.getPrimarySortColumns(); @@ -510,23 +507,9 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( LOG_DEBUG(log, "MinMax index condition: " << minmax_idx_condition->toString()); /// PREWHERE - ExpressionActionsPtr prewhere_actions; String prewhere_column; if (select.prewhere_expression) - { - ExpressionAnalyzer analyzer(select.prewhere_expression, context, nullptr, available_real_columns); - prewhere_actions = analyzer.getActions(false); prewhere_column = select.prewhere_expression->getColumnName(); - SubqueriesForSets prewhere_subqueries = analyzer.getSubqueriesForSets(); - - /** Compute the subqueries right now. - * NOTE Disadvantage - these calculations do not fit into the query execution pipeline. - * They are done before the execution of the pipeline; they can not be interrupted; during the computation, packets of progress are not sent. - */ - if (!prewhere_subqueries.empty()) - CreatingSetsBlockInputStream(std::make_shared(Block()), prewhere_subqueries, - SizeLimits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode)).read(); - } RangesInDataParts parts_with_ranges; @@ -583,8 +566,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( column_names_to_read, max_block_size, settings.use_uncompressed_cache, - prewhere_actions, - prewhere_column, + query_info.prewhere_info, virt_column_names, settings); } @@ -596,8 +578,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( column_names_to_read, max_block_size, settings.use_uncompressed_cache, - prewhere_actions, - prewhere_column, + query_info.prewhere_info, virt_column_names, settings); } @@ -622,8 +603,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( const Names & column_names, size_t max_block_size, bool use_uncompressed_cache, - ExpressionActionsPtr prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Names & virt_columns, const Settings & settings) const { @@ -658,7 +638,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( num_streams = std::max((sum_marks + min_marks_for_concurrent_read - 1) / min_marks_for_concurrent_read, parts.size()); MergeTreeReadPoolPtr pool = std::make_shared( - num_streams, sum_marks, min_marks_for_concurrent_read, parts, data, prewhere_actions, prewhere_column, true, + num_streams, sum_marks, min_marks_for_concurrent_read, parts, data, prewhere_info, true, column_names, MergeTreeReadPool::BackoffSettings(settings), settings.preferred_block_size_bytes, false); /// Let's estimate total number of rows for progress bar. @@ -670,7 +650,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( res.emplace_back(std::make_shared( i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, data, use_uncompressed_cache, - prewhere_actions, prewhere_column, settings, virt_columns)); + prewhere_info, settings, virt_columns)); if (i == 0) { @@ -744,7 +724,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( BlockInputStreamPtr source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, - use_uncompressed_cache, prewhere_actions, prewhere_column, true, settings.min_bytes_to_use_direct_io, + use_uncompressed_cache, prewhere_info, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, true, virt_columns, part.part_index_in_query); res.push_back(source_stream); @@ -763,8 +743,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal const Names & column_names, size_t max_block_size, bool use_uncompressed_cache, - ExpressionActionsPtr prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Names & virt_columns, const Settings & settings) const { @@ -790,7 +769,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal BlockInputStreamPtr source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, - prewhere_actions, prewhere_column, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, true, + prewhere_info, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, true, virt_columns, part.part_index_in_query); to_merge.emplace_back(std::make_shared(source_stream, data.getPrimaryExpression())); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 96788cea015..f854480b356 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -26,7 +26,6 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, size_t max_block_size, unsigned num_streams, Int64 max_block_number_to_read) const; @@ -36,7 +35,6 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, size_t max_block_size, unsigned num_streams, Int64 max_block_number_to_read) const; @@ -52,8 +50,7 @@ private: const Names & column_names, size_t max_block_size, bool use_uncompressed_cache, - ExpressionActionsPtr prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Names & virt_columns, const Settings & settings) const; @@ -62,8 +59,7 @@ private: const Names & column_names, size_t max_block_size, bool use_uncompressed_cache, - ExpressionActionsPtr prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Names & virt_columns, const Settings & settings) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 0a1e2395696..db0d9f9076a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1,10 +1,12 @@ #include #include -#include #include +#include +#include #if __SSE2__ #include +#include #endif namespace DB @@ -436,7 +438,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar merge_tree_reader->evaluateMissingDefaults(read_result.block); if (should_reorder || always_reorder || block.columns()) - merge_tree_reader->reorderColumns(read_result.block, *ordered_names); + merge_tree_reader->reorderColumns(read_result.block, *ordered_names, prewhere_column_name); } } else @@ -452,7 +454,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar merge_tree_reader->evaluateMissingDefaults(read_result.block); if (should_reorder || always_reorder) - merge_tree_reader->reorderColumns(read_result.block, *ordered_names); + merge_tree_reader->reorderColumns(read_result.block, *ordered_names, prewhere_column_name); } } @@ -611,23 +613,25 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r if (!result.block) return; + auto getNumRows = [&]() + { + /// If block has single column, it's filter. We need to count bytes in it in order to get the number of rows. + if (result.block.columns() > 1) + return result.block.rows(); + else if (result.getFilter()) + return countBytesInFilter(result.getFilter()->getData()); + else + return prev_rows; + }; + if (remove_prewhere_column) result.block.erase(*prewhere_column_name); else - { - /// Calculate the number of rows in block in order to create const column. - size_t rows = result.block.rows(); - /// If block has single column, it's filter. We need to count bytes in it in order to get the number of rows. - if (result.block.columns() == 1) - { - if (result.getFilter()) - rows = countBytesInFilter(result.getFilter()->getData()); - else - rows = prev_rows; - } + prewhere_column.column = prewhere_column.type->createColumnConst(getNumRows(), UInt64(1)); - prewhere_column.column = prewhere_column.type->createColumnConst(rows, UInt64(1)); - } + /// If block is empty, create column in order to store rows number. + if (last_reader_in_chain && result.block.columns() == 0) + result.block.insert({ColumnNothing::create(getNumRows()), std::make_shared(), "_nothing"}); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 370797968b3..55ba286dc08 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace ProfileEvents @@ -15,19 +16,20 @@ namespace DB MergeTreeReadPool::MergeTreeReadPool( const size_t threads, const size_t sum_marks, const size_t min_marks_for_concurrent_read, - RangesInDataParts parts, MergeTreeData & data, const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column_name, const bool check_columns, const Names & column_names, + RangesInDataParts parts, MergeTreeData & data, const PrewhereInfoPtr & prewhere_info, + const bool check_columns, const Names & column_names, const BackoffSettings & backoff_settings, size_t preferred_block_size_bytes, const bool do_not_steal_tasks) : backoff_settings{backoff_settings}, backoff_state{threads}, data{data}, - column_names{column_names}, do_not_steal_tasks{do_not_steal_tasks}, predict_block_size_bytes{preferred_block_size_bytes > 0} + column_names{column_names}, do_not_steal_tasks{do_not_steal_tasks}, + predict_block_size_bytes{preferred_block_size_bytes > 0}, prewhere_info{prewhere_info} { - const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_actions, prewhere_column_name, check_columns); + const auto per_part_sum_marks = fillPerPartInfo(parts, prewhere_info, check_columns); fillPerThreadInfo(threads, sum_marks, per_part_sum_marks, parts, min_marks_for_concurrent_read); } -MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, const size_t thread) +MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, const size_t thread, const Names & ordered_names) { const std::lock_guard lock{mutex}; @@ -111,9 +113,9 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, : std::make_unique(*per_part_size_predictor[part_idx]); /// make a copy return std::make_unique( - part.data_part, ranges_to_get_from_part, part.part_index_in_query, column_names, + part.data_part, ranges_to_get_from_part, part.part_index_in_query, ordered_names, per_part_column_name_set[part_idx], per_part_columns[part_idx], per_part_pre_columns[part_idx], - per_part_remove_prewhere_column[part_idx], per_part_should_reorder[part_idx], std::move(curr_task_size_predictor)); + prewhere_info && prewhere_info->remove_prewhere_column, per_part_should_reorder[part_idx], std::move(curr_task_size_predictor)); } @@ -122,7 +124,6 @@ Block MergeTreeReadPool::getHeader() const return data.getSampleBlockForColumns(column_names); } - void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInfo info) { if (backoff_settings.min_read_latency_ms == 0 || do_not_steal_tasks) @@ -165,8 +166,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf std::vector MergeTreeReadPool::fillPerPartInfo( - RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name, - const bool check_columns) + RangesInDataParts & parts, const PrewhereInfoPtr & prewhere_info, const bool check_columns) { std::vector per_part_sum_marks; Block sample_block = data.getSampleBlock(); @@ -193,10 +193,10 @@ std::vector MergeTreeReadPool::fillPerPartInfo( Names required_pre_column_names; - if (prewhere_actions) + if (prewhere_info) { /// collect columns required for PREWHERE evaluation - required_pre_column_names = prewhere_actions->getRequiredColumns(); + required_pre_column_names = prewhere_info->prewhere_actions->getRequiredColumns(); /// there must be at least one column required for PREWHERE if (required_pre_column_names.empty()) @@ -208,13 +208,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo( should_reoder = true; /// will be used to distinguish between PREWHERE and WHERE columns when applying filter - const NameSet pre_name_set{ - std::begin(required_pre_column_names), std::end(required_pre_column_names) - }; - /** If expression in PREWHERE is not table column, then no need to return column with it to caller - * (because storage is expected only to read table columns). - */ - per_part_remove_prewhere_column.push_back(0 == pre_name_set.count(prewhere_column_name)); + const NameSet pre_name_set(required_pre_column_names.begin(), required_pre_column_names.end()); Names post_column_names; for (const auto & name : required_column_names) @@ -223,8 +217,6 @@ std::vector MergeTreeReadPool::fillPerPartInfo( required_column_names = post_column_names; } - else - per_part_remove_prewhere_column.push_back(false); per_part_column_name_set.emplace_back(std::begin(required_column_names), std::end(required_column_names)); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h index cf7a9c71ef1..8b19c6b2a1a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -66,12 +67,12 @@ private: public: MergeTreeReadPool( const size_t threads, const size_t sum_marks, const size_t min_marks_for_concurrent_read, - RangesInDataParts parts, MergeTreeData & data, const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column_name, const bool check_columns, const Names & column_names, + RangesInDataParts parts, MergeTreeData & data, const PrewhereInfoPtr & prewhere_info, + const bool check_columns, const Names & column_names, const BackoffSettings & backoff_settings, size_t preferred_block_size_bytes, const bool do_not_steal_tasks = false); - MergeTreeReadTaskPtr getTask(const size_t min_marks_to_read, const size_t thread); + MergeTreeReadTaskPtr getTask(const size_t min_marks_to_read, const size_t thread, const Names & ordered_names); /** Each worker could call this method and pass information about read performance. * If read performance is too low, pool could decide to lower number of threads: do not assign more tasks to several threads. @@ -83,8 +84,7 @@ public: private: std::vector fillPerPartInfo( - RangesInDataParts & parts, const ExpressionActionsPtr & prewhere_actions, const String & prewhere_column_name, - const bool check_columns); + RangesInDataParts & parts, const PrewhereInfoPtr & prewhere_info, const bool check_columns); void fillPerThreadInfo( const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, @@ -93,15 +93,15 @@ private: std::vector> per_part_columns_lock; MergeTreeData & data; Names column_names; + Names ordered_names; bool do_not_steal_tasks; bool predict_block_size_bytes; std::vector per_part_column_name_set; std::vector per_part_columns; std::vector per_part_pre_columns; - /// @todo actually all of these values are either true or false for the whole query, thus no vector required - std::vector per_part_remove_prewhere_column; std::vector per_part_should_reorder; std::vector per_part_size_predictor; + PrewhereInfoPtr prewhere_info; struct Part { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 122e0557e90..7207d81cce1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -267,7 +267,7 @@ void MergeTreeReader::Stream::loadMarks() auto load = [&]() -> MarkCache::MappedPtr { /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. - TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); size_t file_size = Poco::File(path).getSize(); size_t expected_file_size = sizeof(MarkInCompressedFile) * marks_count; @@ -542,7 +542,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, boo } } -void MergeTreeReader::reorderColumns(Block & res, const Names & ordered_names) +void MergeTreeReader::reorderColumns(Block & res, const Names & ordered_names, const String * filter_name) { try { @@ -552,6 +552,9 @@ void MergeTreeReader::reorderColumns(Block & res, const Names & ordered_names) if (res.has(name)) ordered_block.insert(res.getByName(name)); + if (filter_name && !ordered_block.has(*filter_name) && res.has(*filter_name)) + ordered_block.insert(res.getByName(*filter_name)); + std::swap(res, ordered_block); } catch (Exception & e) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index a389918fdc4..0866c75f82a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -45,7 +45,8 @@ public: /// If at least one column was added, reorders all columns in the block according to ordered_names. void fillMissingColumns(Block & res, bool & should_reorder, bool & should_evaluate_missing_defaults); /// Sort columns to ensure consistent order among all blocks. - void reorderColumns(Block & res, const Names & ordered_names); + /// If filter_name is not nullptr and block has filter column, move it to the end of block. + void reorderColumns(Block & res, const Names & ordered_names, const String * filter_name); /// Evaluate defaulted columns if necessary. void evaluateMissingDefaults(Block & res); diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp index 7e0b9af4bbe..f1a23933e65 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp @@ -16,12 +16,11 @@ MergeTreeThreadBlockInputStream::MergeTreeThreadBlockInputStream( size_t preferred_max_column_in_block_size_bytes, MergeTreeData & storage, const bool use_uncompressed_cache, - const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Settings & settings, const Names & virt_column_names) : - MergeTreeBaseBlockInputStream{storage, prewhere_actions, prewhere_column, max_block_size_rows, + MergeTreeBaseBlockInputStream{storage, prewhere_info, max_block_size_rows, preferred_block_size_bytes, preferred_max_column_in_block_size_bytes, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, use_uncompressed_cache, true, virt_column_names}, thread{thread}, @@ -35,6 +34,8 @@ MergeTreeThreadBlockInputStream::MergeTreeThreadBlockInputStream( } else min_marks_to_read = min_marks_to_read_; + + ordered_names = getHeader().getNames(); } @@ -42,6 +43,7 @@ Block MergeTreeThreadBlockInputStream::getHeader() const { auto res = pool->getHeader(); injectVirtualColumns(res); + executePrewhereActions(res, prewhere_info); return res; } @@ -49,7 +51,7 @@ Block MergeTreeThreadBlockInputStream::getHeader() const /// Requests read task from MergeTreeReadPool and signals whether it got one bool MergeTreeThreadBlockInputStream::getNewTask() { - task = pool->getTask(min_marks_to_read, thread); + task = pool->getTask(min_marks_to_read, thread, ordered_names); if (!task) { @@ -78,7 +80,7 @@ bool MergeTreeThreadBlockInputStream::getNewTask() path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); - if (prewhere_actions) + if (prewhere_info) pre_reader = std::make_unique( path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, task->mark_ranges, min_bytes_to_use_direct_io, @@ -92,7 +94,7 @@ bool MergeTreeThreadBlockInputStream::getNewTask() storage, task->mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, reader->getAvgValueSizeHints(), profile_callback); - if (prewhere_actions) + if (prewhere_info) pre_reader = std::make_unique( path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, storage, task->mark_ranges, min_bytes_to_use_direct_io, diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h index 1a9c9a11bef..7972c93e5a2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadBlockInputStream.h @@ -23,8 +23,7 @@ public: size_t preferred_max_column_in_block_size_bytes, MergeTreeData & storage, const bool use_uncompressed_cache, - const ExpressionActionsPtr & prewhere_actions, - const String & prewhere_column, + const PrewhereInfoPtr & prewhere_info, const Settings & settings, const Names & virt_column_names); @@ -38,11 +37,15 @@ protected: /// Requests read task from MergeTreeReadPool and signals whether it got one bool getNewTask() override; +private: /// "thread" index (there are N threads and each thread is assigned index in interval [0..N-1]) size_t thread; std::shared_ptr pool; size_t min_marks_to_read; + + /// Names from header. Used in order to order columns in read blocks. + Names ordered_names; }; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index e43aac81945..fd2fdb50897 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -471,7 +471,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm * And otherwise it will look like excessively growing memory consumption in context of query. * (observed in long INSERT SELECTs) */ - TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); /// Write index. The index contains Primary Key value for each `index_granularity` row. for (size_t i = index_offset; i < rows; i += storage.index_granularity) diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index d7fffa2ac90..60331a4049c 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -22,12 +22,12 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, unsigned num_streams) override { return MergeTreeDataSelectExecutor(part->storage).readFromParts( - {part}, column_names, query_info, context, processed_stage, max_block_size, num_streams, 0); + {part}, column_names, query_info, context, max_block_size, num_streams, 0); } protected: diff --git a/dbms/src/Storages/SelectQueryInfo.h b/dbms/src/Storages/SelectQueryInfo.h index f3577cf920c..a448f2dc0bb 100644 --- a/dbms/src/Storages/SelectQueryInfo.h +++ b/dbms/src/Storages/SelectQueryInfo.h @@ -10,12 +10,29 @@ namespace DB class IAST; using ASTPtr = std::shared_ptr; +class ExpressionActions; +using ExpressionActionsPtr = std::shared_ptr; + class Set; using SetPtr = std::shared_ptr; /// Information about calculated sets in right hand side of IN. using PreparedSets = std::unordered_map; +struct PrewhereInfo +{ + /// Actions which are executed on block in order to get filter column for prewhere step. + ExpressionActionsPtr prewhere_actions; + String prewhere_column_name; + bool remove_prewhere_column = false; + + PrewhereInfo() = default; + explicit PrewhereInfo(ExpressionActionsPtr prewhere_actions_, String prewhere_column_name_) + : prewhere_actions(std::move(prewhere_actions_)), prewhere_column_name(std::move(prewhere_column_name_)) {} +}; + +using PrewhereInfoPtr = std::shared_ptr; + /** Query along with some additional data, * that can be used during query processing @@ -25,6 +42,8 @@ struct SelectQueryInfo { ASTPtr query; + PrewhereInfoPtr prewhere_info; + /// Prepared sets are used for indices by storage engine. /// Example: x IN (1, 2, 3) PreparedSets sets; diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index c1e783c4c7a..f2072f1bc1f 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -103,15 +103,30 @@ private: }; +QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(const Context & context) const +{ + if (!no_destination) + { + auto destination = context.getTable(destination_database, destination_table); + + if (destination.get() == this) + throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); + + return destination->getQueryProcessingStage(context); + } + + return QueryProcessingStage::FetchColumns; +} + BlockInputStreams StorageBuffer::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); BlockInputStreams streams_from_dst; @@ -178,7 +193,7 @@ static void appendBlock(const Block & from, Block & to) try { /// Avoid "memory limit exceeded" exceptions during rollback. - TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker; + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index ea8a0562628..2d4cb37876b 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -53,11 +53,13 @@ public: std::string getName() const override { return "Buffer"; } std::string getTableName() const override { return name; } + QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; + BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp index 06ff93cf4e2..b0bc0be9dd8 100644 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ b/dbms/src/Storages/StorageCatBoostPool.cpp @@ -57,10 +57,9 @@ public: reader->readSuffix(); } - Block getHeader() const override { return sample_block; } + Block getHeader() const override { return reader->getHeader(); } private: - Block sample_block; std::unique_ptr read_buf; BlockInputStreamPtr reader; std::string file_name; @@ -78,8 +77,8 @@ static std::string resolvePath(const boost::filesystem::path & base_path, std::s { boost::filesystem::path resolved_path(path); if (!resolved_path.is_absolute()) - return (base_path / resolved_path).string(); - return resolved_path.string(); + return boost::filesystem::canonical(resolved_path, base_path).string(); + return boost::filesystem::canonical(resolved_path).string(); } static void checkCreationIsAllowed(const String & base_path, const String & path) @@ -262,10 +261,12 @@ void StorageCatBoostPool::createSampleBlockAndColumns() BlockInputStreams StorageCatBoostPool::read(const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & /*processed_stage*/, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned /*threads*/) { + checkQueryProcessingStage(processed_stage, context); + auto stream = std::make_shared( data_description_file_name, "TSV", sample_block, context, max_block_size); diff --git a/dbms/src/Storages/StorageCatBoostPool.h b/dbms/src/Storages/StorageCatBoostPool.h index 5d035ea81dc..54a7bf1b655 100644 --- a/dbms/src/Storages/StorageCatBoostPool.h +++ b/dbms/src/Storages/StorageCatBoostPool.h @@ -17,7 +17,7 @@ public: BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned threads) override; diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index e4f2a154474..26fdccbc3b8 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -38,11 +38,11 @@ BlockInputStreams StorageDictionary::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned /*threads*/) { - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); auto dictionary = context.getExternalDictionaries().getDictionary(dictionary_name); return BlockInputStreams{dictionary->getBlockInputStream(column_names, max_block_size)}; } diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 2e47c5fbe0d..cffaf8879cd 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -27,7 +27,7 @@ public: BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size = DEFAULT_BLOCK_SIZE, unsigned threads = 1) override; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 8d1887bb1bc..f561727566d 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -220,17 +220,14 @@ StoragePtr StorageDistributed::createWithOwnCluster( return res; } - -BlockInputStreams StorageDistributed::read( - const Names & /*column_names*/, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - const size_t /*max_block_size*/, - const unsigned /*num_streams*/) +QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Context & context) const { auto cluster = getCluster(); + return getQueryProcessingStage(context, cluster); +} +QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Context & context, const ClusterPtr & cluster) const +{ const Settings & settings = context.getSettingsRef(); size_t num_local_shards = cluster->getLocalShardCount(); @@ -238,11 +235,24 @@ BlockInputStreams StorageDistributed::read( size_t result_size = (num_remote_shards * settings.max_parallel_replicas) + num_local_shards; if (settings.distributed_group_by_no_merge) - processed_stage = QueryProcessingStage::Complete; + return QueryProcessingStage::Complete; else /// Normal mode. - processed_stage = result_size == 1 - ? QueryProcessingStage::Complete - : QueryProcessingStage::WithMergeableState; + return result_size == 1 ? QueryProcessingStage::Complete + : QueryProcessingStage::WithMergeableState; +} + +BlockInputStreams StorageDistributed::read( + const Names & /*column_names*/, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + const size_t /*max_block_size*/, + const unsigned /*num_streams*/) +{ + auto cluster = getCluster(); + checkQueryProcessingStage(processed_stage, getQueryProcessingStage(context, cluster)); + + const Settings & settings = context.getSettingsRef(); const auto & modified_query_ast = rewriteSelectQuery( query_info.query, remote_database, remote_table, remote_table_function_ptr); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index fdb08c31c00..1ae53f5637c 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -60,11 +60,14 @@ public: bool isRemote() const override { return true; } + QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; + QueryProcessingStage::Enum getQueryProcessingStage(const Context & context, const ClusterPtr & cluster) const; + BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index e81f982ca89..eee89f4397a 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -190,10 +190,11 @@ BlockInputStreams StorageFile::read( const Names & /*column_names*/, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & /*processed_stage*/, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned /*num_streams*/) { + checkQueryProcessingStage(processed_stage, context); return BlockInputStreams(1, std::make_shared(*this, context, max_block_size)); } diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 4292ea6d7f9..6716dc306a4 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -35,7 +35,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 3205646e152..b90b275b7be 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -572,12 +572,12 @@ BlockInputStreams StorageLog::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { + checkQueryProcessingStage(processed_stage, context); check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; loadMarks(); NamesAndTypesList all_columns = Nested::collect(getColumns().getAllPhysical().addTypes(column_names)); diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index 7c671ea0567..91a03670e59 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -30,7 +30,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 4e9fb726324..e88feb9a31a 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -168,11 +168,16 @@ bool StorageMaterializedView::hasColumn(const String & column_name) const return getTargetTable()->hasColumn(column_name); } +QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(const Context & context) const +{ + return getTargetTable()->getQueryProcessingStage(context); +} + BlockInputStreams StorageMaterializedView::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 8ef047dba67..f4196de8538 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -45,11 +45,13 @@ public: void checkTableCanBeDropped() const override; void checkPartitionCanBeDropped(const ASTPtr & partition) override; + QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; + BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index bf7e39f759b..b141e9a4017 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -83,13 +83,13 @@ StorageMemory::StorageMemory(String table_name_, ColumnsDescription columns_desc BlockInputStreams StorageMemory::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, - const Context & /*context*/, - QueryProcessingStage::Enum & processed_stage, + const Context & context, + QueryProcessingStage::Enum processed_stage, size_t /*max_block_size*/, unsigned num_streams) { + checkQueryProcessingStage(processed_stage, context); check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; std::lock_guard lock(mutex); diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 6a894bf3204..27c948bf87f 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -32,7 +32,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 0a001fd9c2f..6b8392a0942 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -134,11 +134,46 @@ bool StorageMerge::mayBenefitFromIndexForIn(const ASTPtr & left_in_operand) cons } +QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage(const Context & context) const +{ + auto stage_in_source_tables = QueryProcessingStage::FetchColumns; + + auto database = context.getDatabase(source_database); + auto iterator = database->getIterator(context); + + bool first = true; + + while (iterator->isValid()) + { + if (table_name_regexp.match(iterator->name())) + { + auto & table = iterator->table(); + if (table.get() != this) + { + auto stage = table->getQueryProcessingStage(context); + + if (first) + stage_in_source_tables = stage; + else if (stage != stage_in_source_tables) + throw Exception("Source tables for Merge table are processing data up to different stages", + ErrorCodes::INCOMPATIBLE_SOURCE_TABLES); + + first = false; + } + } + + iterator->next(); + } + + return stage_in_source_tables; +} + + BlockInputStreams StorageMerge::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { @@ -158,8 +193,6 @@ BlockInputStreams StorageMerge::read( real_column_names.push_back(name); } - std::optional processed_stage_in_source_tables; - /** First we make list of selected tables to find out its size. * This is necessary to correctly pass the recommended number of threads to each table. */ @@ -167,11 +200,20 @@ BlockInputStreams StorageMerge::read( const ASTPtr & query = query_info.query; - /// If PREWHERE is used in query, you need to make sure that all tables support this. - if (typeid_cast(*query).prewhere_expression) - for (const auto & elem : selected_tables) + for (const auto & elem : selected_tables) + { + /// Check processing stage again in case new table was added after getQueryProcessingStage call. + auto stage = elem.first->getQueryProcessingStage(context); + if (stage != processed_stage) + throw Exception("Source tables for Merge table are processing data up to different stages", + ErrorCodes::INCOMPATIBLE_SOURCE_TABLES); + + /// If PREWHERE is used in query, you need to make sure that all tables support this. + if (typeid_cast(*query).prewhere_expression) if (!elem.first->supportsPrewhere()) - throw Exception("Storage " + elem.first->getName() + " doesn't support PREWHERE.", ErrorCodes::ILLEGAL_PREWHERE); + throw Exception("Storage " + elem.first->getName() + " doesn't support PREWHERE.", + ErrorCodes::ILLEGAL_PREWHERE); + } Block virtual_columns_block = getBlockWithVirtualColumns(selected_tables); @@ -212,30 +254,24 @@ BlockInputStreams StorageMerge::read( SelectQueryInfo modified_query_info; modified_query_info.query = modified_query_ast; + modified_query_info.prewhere_info = query_info.prewhere_info; modified_query_info.sets = query_info.sets; BlockInputStreams source_streams; if (curr_table_number < num_streams) { - QueryProcessingStage::Enum processed_stage_in_source_table = processed_stage; source_streams = table->read( real_column_names, modified_query_info, modified_context, - processed_stage_in_source_table, + processed_stage, max_block_size, tables_count >= num_streams ? 1 : (num_streams / tables_count)); - if (!processed_stage_in_source_tables) - processed_stage_in_source_tables.emplace(processed_stage_in_source_table); - else if (processed_stage_in_source_table != *processed_stage_in_source_tables) - throw Exception("Source tables for Merge table are processing data up to different stages", - ErrorCodes::INCOMPATIBLE_SOURCE_TABLES); - if (!header) { - switch (processed_stage_in_source_table) + switch (processed_stage) { case QueryProcessingStage::FetchColumns: header = getSampleBlockForColumns(column_names); @@ -244,7 +280,7 @@ BlockInputStreams StorageMerge::read( case QueryProcessingStage::Complete: header = materializeBlock(InterpreterSelectQuery( query_info.query, context, std::make_shared(getSampleBlockForColumns(column_names)), - processed_stage_in_source_table, true).getSampleBlock()); + processed_stage, true).getSampleBlock()); break; } } @@ -261,25 +297,17 @@ BlockInputStreams StorageMerge::read( } else { - if (!processed_stage_in_source_tables) - throw Exception("Logical error: unknown processed stage in source tables", ErrorCodes::LOGICAL_ERROR); - /// If many streams, initialize it lazily, to avoid long delay before start of query processing. source_streams.emplace_back(std::make_shared(header, [=]() -> BlockInputStreamPtr { - QueryProcessingStage::Enum processed_stage_in_source_table = processed_stage; BlockInputStreams streams = table->read( real_column_names, modified_query_info, modified_context, - processed_stage_in_source_table, + processed_stage, max_block_size, 1); - if (processed_stage_in_source_table != *processed_stage_in_source_tables) - throw Exception("Source tables for Merge table are processing data up to different stages", - ErrorCodes::INCOMPATIBLE_SOURCE_TABLES); - if (streams.empty()) { return std::make_shared(header); @@ -303,9 +331,6 @@ BlockInputStreams StorageMerge::read( res.insert(res.end(), source_streams.begin(), source_streams.end()); } - if (processed_stage_in_source_tables) - processed_stage = *processed_stage_in_source_tables; - if (res.empty()) return res; diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index abb8114a50c..7ebc59947c6 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -29,11 +29,13 @@ public: NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; + QueryProcessingStage::Enum getQueryProcessingStage(const Context &) const override; + BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 00e9f4c8697..7c96915652e 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -110,11 +110,12 @@ BlockInputStreams StorageMergeTree::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { - return reader.read(column_names, query_info, context, processed_stage, max_block_size, num_streams, 0); + checkQueryProcessingStage(processed_stage, context); + return reader.read(column_names, query_info, context, max_block_size, num_streams, 0); } BlockOutputStreamPtr StorageMergeTree::write(const ASTPtr & /*query*/, const Settings & /*settings*/) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index 762640a605b..d85de91ab30 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -56,7 +56,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 25657b583a7..e2d5e07b161 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -51,12 +51,12 @@ BlockInputStreams StorageMySQL::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned) { check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); String query = transformQueryForExternalDatabase( *query_info.query, getColumns().ordinary, IdentifierQuotingStyle::Backticks, remote_database_name, remote_table_name, context); diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 52197d54ae0..574f019db4a 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -36,7 +36,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index cdde480a951..21d35535cc8 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -23,11 +23,12 @@ public: BlockInputStreams read( const Names & column_names, const SelectQueryInfo &, - const Context &, - QueryProcessingStage::Enum &, + const Context & context, + QueryProcessingStage::Enum processing_stage, size_t, unsigned) override { + checkQueryProcessingStage(processing_stage, context); return { std::make_shared(getSampleBlockForColumns(column_names)) }; } diff --git a/dbms/src/Storages/StorageODBC.cpp b/dbms/src/Storages/StorageODBC.cpp index 657456e3f6f..b2474896b62 100644 --- a/dbms/src/Storages/StorageODBC.cpp +++ b/dbms/src/Storages/StorageODBC.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -33,23 +32,17 @@ StorageODBC::StorageODBC(const std::string & table_name_, const ColumnsDescription & columns_, const Context & context_) : IStorageURLBase(Poco::URI(), context_, table_name_, ODBCBridgeHelper::DEFAULT_FORMAT, columns_) - , odbc_bridge_helper(context_, connection_string) + , odbc_bridge_helper(context_global.getConfigRef(), context_global.getSettingsRef().http_receive_timeout.value, connection_string) , remote_database_name(remote_database_name_) , remote_table_name(remote_table_name_) , log(&Poco::Logger::get("StorageODBC")) { - const auto & config = context_global.getConfigRef(); - size_t bridge_port = config.getUInt("odbc_bridge.port", ODBCBridgeHelper::DEFAULT_PORT); - std::string bridge_host = config.getString("odbc_bridge.host", ODBCBridgeHelper::DEFAULT_HOST); - - uri.setHost(bridge_host); - uri.setPort(bridge_port); - uri.setScheme("http"); + uri = odbc_bridge_helper.getMainURI(); } std::string StorageODBC::getReadMethod() const { - return ODBCBridgeHelper::MAIN_METHOD; + return Poco::Net::HTTPRequest::HTTP_POST; } std::vector> StorageODBC::getReadURIParams(const Names & column_names, @@ -64,7 +57,7 @@ std::vector> StorageODBC::getReadURIParams(c auto column_data = getColumn(name); cols.emplace_back(column_data.name, column_data.type); } - return odbc_bridge_helper.getURLParams(cols, max_block_size); + return odbc_bridge_helper.getURLParams(cols.toString(), max_block_size); } std::function StorageODBC::getReadPOSTDataCallback(const Names & /*column_names*/, @@ -82,10 +75,12 @@ std::function StorageODBC::getReadPOSTDataCallback(const N BlockInputStreams StorageODBC::read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { + check(column_names); + checkQueryProcessingStage(processed_stage, context); odbc_bridge_helper.startODBCBridgeSync(); return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams); diff --git a/dbms/src/Storages/StorageODBC.h b/dbms/src/Storages/StorageODBC.h index ae33dedf827..c5208ce682d 100644 --- a/dbms/src/Storages/StorageODBC.h +++ b/dbms/src/Storages/StorageODBC.h @@ -22,7 +22,7 @@ public: BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 3718d56fcdb..7bf69ee2961 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2063,7 +2063,9 @@ void StorageReplicatedMergeTree::queueUpdatingTask() if (e.code == ZooKeeperImpl::ZooKeeper::ZSESSIONEXPIRED) { - restarting_thread->wakeup(); + /// Can be called before starting restarting_thread + if (restarting_thread) + restarting_thread->wakeup(); return; } @@ -2824,10 +2826,11 @@ BlockInputStreams StorageReplicatedMergeTree::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned num_streams) { + checkQueryProcessingStage(processed_stage, context); const Settings & settings = context.getSettingsRef(); /** The `select_sequential_consistency` setting has two meanings: @@ -2865,8 +2868,7 @@ BlockInputStreams StorageReplicatedMergeTree::read( } } - return reader.read( - column_names, query_info, context, processed_stage, max_block_size, num_streams, max_block_number_to_read); + return reader.read(column_names, query_info, context, max_block_size, num_streams, max_block_number_to_read); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 61dceb0e408..2d146c8f5d2 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -106,7 +106,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index 090f17921ff..203a9961fd1 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -235,14 +235,14 @@ BlockInputStreams StorageStripeLog::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, unsigned num_streams) { + checkQueryProcessingStage(processed_stage, context); std::shared_lock lock(rwlock); check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; NameSet column_names_set(column_names.begin(), column_names.end()); diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index c3fc98ccf00..22010b39b63 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -32,7 +32,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 737ef3ca430..8ff3eb71869 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -384,12 +384,12 @@ BlockInputStreams StorageTinyLog::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t max_block_size, const unsigned /*num_streams*/) { + checkQueryProcessingStage(processed_stage, context); check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; return BlockInputStreams(1, std::make_shared( max_block_size, Nested::collect(getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index c0f63ef977e..c33c2c87d9a 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -31,7 +31,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageURL.cpp b/dbms/src/Storages/StorageURL.cpp index 94d2e4aaf72..10816537fb9 100644 --- a/dbms/src/Storages/StorageURL.cpp +++ b/dbms/src/Storages/StorageURL.cpp @@ -156,10 +156,12 @@ std::function IStorageURLBase::getReadPOSTDataCallback(con BlockInputStreams IStorageURLBase::read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned /*num_streams*/) { + checkQueryProcessingStage(processed_stage, context); + auto request_uri = uri; auto params = getReadURIParams(column_names, query_info, context, processed_stage, max_block_size); for (const auto & [param, value] : params) diff --git a/dbms/src/Storages/StorageURL.h b/dbms/src/Storages/StorageURL.h index b375f800b4a..e731713e243 100644 --- a/dbms/src/Storages/StorageURL.h +++ b/dbms/src/Storages/StorageURL.h @@ -24,7 +24,7 @@ public: BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 06809ffb4e3..17628681847 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -33,11 +33,11 @@ BlockInputStreams StorageView::read( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); BlockInputStreams res = InterpreterSelectWithUnionQuery(inner_query, context, column_names).executeWithMultipleStreams(); /// It's expected that the columns read from storage are not constant. diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index 20a8f76189c..12f666a0648 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -25,7 +25,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/System/IStorageSystemOneBlock.h b/dbms/src/Storages/System/IStorageSystemOneBlock.h index 96286f56eee..76d62a6e2dc 100644 --- a/dbms/src/Storages/System/IStorageSystemOneBlock.h +++ b/dbms/src/Storages/System/IStorageSystemOneBlock.h @@ -33,12 +33,12 @@ public: BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t /*max_block_size*/, unsigned /*num_streams*/) override { check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); Block sample_block = getSampleBlock(); MutableColumns res_columns = sample_block.cloneEmptyColumns(); diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp index eb4832c0c92..fadd7605681 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.cpp +++ b/dbms/src/Storages/System/StorageSystemEvents.cpp @@ -18,7 +18,7 @@ void StorageSystemEvents::fillData(MutableColumns & res_columns, const Context & { for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) { - UInt64 value = ProfileEvents::counters[i]; + UInt64 value = ProfileEvents::global_counters[i]; if (0 != value) { diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 367a2639870..b834285e0d6 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -53,13 +53,13 @@ StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multi BlockInputStreams StorageSystemNumbers::read( const Names & column_names, const SelectQueryInfo &, - const Context &, - QueryProcessingStage::Enum & processed_stage, + const Context & context, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) { check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); if (limit && limit < max_block_size) { diff --git a/dbms/src/Storages/System/StorageSystemNumbers.h b/dbms/src/Storages/System/StorageSystemNumbers.h index 30c68cbd853..a23137fa976 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.h +++ b/dbms/src/Storages/System/StorageSystemNumbers.h @@ -29,7 +29,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index 7714a0ea53c..4a79d493e44 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -20,13 +20,13 @@ StorageSystemOne::StorageSystemOne(const std::string & name_) BlockInputStreams StorageSystemOne::read( const Names & column_names, const SelectQueryInfo &, - const Context &, - QueryProcessingStage::Enum & processed_stage, + const Context & context, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); return BlockInputStreams(1, std::make_shared( Block{ColumnWithTypeAndName( diff --git a/dbms/src/Storages/System/StorageSystemOne.h b/dbms/src/Storages/System/StorageSystemOne.h index 346f7a86982..721684b7802 100644 --- a/dbms/src/Storages/System/StorageSystemOne.h +++ b/dbms/src/Storages/System/StorageSystemOne.h @@ -25,7 +25,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index e9f95d5d6df..2fb3db219af 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -237,13 +237,12 @@ BlockInputStreams StorageSystemPartsBase::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { bool has_state_column = hasStateColumn(column_names); - - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); StoragesInfoStream stream(query_info, context, has_state_column); diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.h b/dbms/src/Storages/System/StorageSystemPartsBase.h index 11e6cde0d02..920ae184e94 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.h +++ b/dbms/src/Storages/System/StorageSystemPartsBase.h @@ -27,7 +27,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index ca18e9835d0..6f5d27fef12 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -1,8 +1,14 @@ #include #include -#include +#include #include #include +#include +#include +#include +#include +#include +#include namespace DB @@ -48,13 +54,19 @@ NamesAndTypesList StorageSystemProcesses::getNamesAndTypes() {"memory_usage", std::make_shared()}, {"peak_memory_usage", std::make_shared()}, {"query", std::make_shared()}, + + { "thread_numbers", std::make_shared(std::make_shared()) }, + { "ProfileEvents.Names", std::make_shared(std::make_shared()) }, + { "ProfileEvents.Values", std::make_shared(std::make_shared()) }, + { "Settings.Names", std::make_shared(std::make_shared()) }, + { "Settings.Values", std::make_shared(std::make_shared()) }, }; } void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - ProcessList::Info info = context.getProcessList().getInfo(); + ProcessList::Info info = context.getProcessList().getInfo(true, true, true); for (const auto & process : info) { @@ -89,6 +101,40 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex res_columns[i++]->insert(process.memory_usage); res_columns[i++]->insert(process.peak_memory_usage); res_columns[i++]->insert(process.query); + + { + Array threads_array; + threads_array.reserve(process.thread_numbers.size()); + for (const UInt32 thread_number : process.thread_numbers) + threads_array.emplace_back(UInt64(thread_number)); + res_columns[i++]->insert(threads_array); + } + + { + IColumn * column_profile_events_names = res_columns[i++].get(); + IColumn * column_profile_events_values = res_columns[i++].get(); + + if (process.profile_counters) + ProfileEvents::dumpToArrayColumns(*process.profile_counters, column_profile_events_names, column_profile_events_values, true); + else + { + column_profile_events_names->insertDefault(); + column_profile_events_values->insertDefault(); + } + } + + { + IColumn * column_settings_names = res_columns[i++].get(); + IColumn * column_settings_values = res_columns[i++].get(); + + if (process.query_settings) + process.query_settings->dumpToArrayColumns(column_settings_names, column_settings_values, true); + else + { + column_settings_names->insertDefault(); + column_settings_values->insertDefault(); + } + } } } diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index c81b923b7d6..ada3b95ff1d 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -55,12 +55,12 @@ BlockInputStreams StorageSystemReplicas::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { check(column_names); - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); /// We collect a set of replicated tables. std::map> replicated_tables; diff --git a/dbms/src/Storages/System/StorageSystemReplicas.h b/dbms/src/Storages/System/StorageSystemReplicas.h index 465b6baa581..53ba5ebd9bd 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.h +++ b/dbms/src/Storages/System/StorageSystemReplicas.h @@ -22,7 +22,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index c6b73b3e413..a6f2a8efc96 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -60,11 +60,11 @@ BlockInputStreams StorageSystemTables::read( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, const size_t /*max_block_size*/, const unsigned /*num_streams*/) { - processed_stage = QueryProcessingStage::FetchColumns; + checkQueryProcessingStage(processed_stage, context); check(column_names); diff --git a/dbms/src/Storages/System/StorageSystemTables.h b/dbms/src/Storages/System/StorageSystemTables.h index d66ca54be68..e646b21ff89 100644 --- a/dbms/src/Storages/System/StorageSystemTables.h +++ b/dbms/src/Storages/System/StorageSystemTables.h @@ -22,7 +22,7 @@ public: const Names & column_names, const SelectQueryInfo & query_info, const Context & context, - QueryProcessingStage::Enum & processed_stage, + QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/src/Storages/tests/hit_log.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index 435789a56b0..d2ac4cd9b70 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -73,7 +73,7 @@ try column_names.push_back("a"); column_names.push_back("b"); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(Context::createGlobal()); BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0]; diff --git a/dbms/src/Storages/tests/system_numbers.cpp b/dbms/src/Storages/tests/system_numbers.cpp index 8f0eb01572a..92d1f9d65e2 100644 --- a/dbms/src/Storages/tests/system_numbers.cpp +++ b/dbms/src/Storages/tests/system_numbers.cpp @@ -26,7 +26,7 @@ try WriteBufferFromOStream out_buf(std::cout); - QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum stage = table->getQueryProcessingStage(Context::createGlobal()); auto context = Context::createGlobal(); LimitBlockInputStream input(table->read(column_names, {}, context, stage, 10, 1)[0], 10, 96); diff --git a/dbms/src/TableFunctions/CMakeLists.txt b/dbms/src/TableFunctions/CMakeLists.txt index d61802020a4..08112e0b6cc 100644 --- a/dbms/src/TableFunctions/CMakeLists.txt +++ b/dbms/src/TableFunctions/CMakeLists.txt @@ -6,13 +6,3 @@ list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFuncti add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) target_link_libraries(clickhouse_table_functions clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY}) - -if (USE_POCO_SQLODBC) - target_link_libraries (clickhouse_table_functions ${Poco_SQLODBC_LIBRARY}) - target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS}) -endif () - -if (USE_POCO_DATAODBC) - target_link_libraries (clickhouse_table_functions ${Poco_DataODBC_LIBRARY}) - target_include_directories (clickhouse_table_functions SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS}) -endif () diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp index 94c2a1f3a25..f32c90d0654 100644 --- a/dbms/src/TableFunctions/TableFunctionODBC.cpp +++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp @@ -1,24 +1,22 @@ #include - -#if USE_POCO_SQLODBC || USE_POCO_DATAODBC #include #include #include #include +#include +#include #include #include +#include #include #include #include #include #include +#include #include -#include -#include -#include - namespace DB { @@ -28,33 +26,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -DataTypePtr getDataType(SQLSMALLINT type) -{ - const auto & factory = DataTypeFactory::instance(); - - switch (type) - { - case SQL_INTEGER: - return factory.get("Int32"); - case SQL_SMALLINT: - return factory.get("Int16"); - case SQL_FLOAT: - return factory.get("Float32"); - case SQL_REAL: - return factory.get("Float32"); - case SQL_DOUBLE: - return factory.get("Float64"); - case SQL_DATETIME: - return factory.get("DateTime"); - case SQL_TYPE_TIMESTAMP: - return factory.get("DateTime"); - case SQL_TYPE_DATE: - return factory.get("Date"); - default: - return factory.get("String"); - } -} - StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Context & context) const { const ASTFunction & args_func = typeid_cast(*ast_function); @@ -73,41 +44,19 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con std::string connection_string = static_cast(*args[0]).value.safeGet(); std::string table_name = static_cast(*args[1]).value.safeGet(); + const auto & config = context.getConfigRef(); + ODBCBridgeHelper helper(config, context.getSettingsRef().http_receive_timeout.value, connection_string); + helper.startODBCBridgeSync(); - Poco::Data::ODBC::SessionImpl session(connection_string, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); - SQLHDBC hdbc = session.dbc().handle(); + Poco::URI columns_info_uri = helper.getColumnsInfoURI(); + columns_info_uri.addQueryParameter("connection_string", connection_string); + columns_info_uri.addQueryParameter("table", table_name); - SQLHSTMT hstmt = nullptr; + ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, nullptr); - if (Poco::Data::ODBC::Utility::isError(SQLAllocStmt(hdbc, &hstmt))) - throw Poco::Data::ODBC::ODBCException("Could not allocate connection handle."); - - SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP)); - - /// TODO Why not do SQLColumns instead? - std::string query = "SELECT * FROM " + table_name + " WHERE 1 = 0"; - if (Poco::Data::ODBC::Utility::isError(Poco::Data::ODBC::SQLPrepare(hstmt, reinterpret_cast(&query[0]), query.size()))) - throw Poco::Data::ODBC::DescriptorException(session.dbc()); - - if (Poco::Data::ODBC::Utility::isError(SQLExecute(hstmt))) - throw Poco::Data::ODBC::StatementException(hstmt); - - SQLSMALLINT cols = 0; - if (Poco::Data::ODBC::Utility::isError(SQLNumResultCols(hstmt, &cols))) - throw Poco::Data::ODBC::StatementException(hstmt); - - /// TODO cols not checked - - NamesAndTypesList columns; - for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol) - { - SQLSMALLINT type = 0; - /// TODO Why 301? - SQLCHAR column_name[301]; - /// TODO Result is not checked. - Poco::Data::ODBC::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), NULL, &type, NULL, NULL, NULL); - columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); - } + std::string columns_info; + readStringBinary(columns_info, buf); + NamesAndTypesList columns = NamesAndTypesList::parse(columns_info); auto result = StorageODBC::create(table_name, connection_string, "", table_name, ColumnsDescription{columns}, context); result->startup(); @@ -120,5 +69,3 @@ void registerTableFunctionODBC(TableFunctionFactory & factory) factory.registerFunction(); } } - -#endif diff --git a/dbms/src/TableFunctions/TableFunctionODBC.h b/dbms/src/TableFunctions/TableFunctionODBC.h index ce0ded30555..9417ac0e972 100644 --- a/dbms/src/TableFunctions/TableFunctionODBC.h +++ b/dbms/src/TableFunctions/TableFunctionODBC.h @@ -1,16 +1,12 @@ #pragma once #include -#if USE_POCO_SQLODBC || USE_POCO_DATAODBC - #include namespace DB { /* odbc (odbc connect string, table) - creates a temporary StorageODBC. - * The structure of the table is taken from the mysql query "SELECT * FROM table WHERE 1=0". - * If there is no such table, an exception is thrown. */ class TableFunctionODBC : public ITableFunction { @@ -24,5 +20,3 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override; }; } - -#endif diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index e8e1f940bff..47f3d93f32e 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -73,7 +73,7 @@ def main(args): base_dir = os.path.abspath(args.queries) tmp_dir = os.path.abspath(args.tmp) - # Keep same default values as in queries/0_stateless/00000_sh_lib.sh + # Keep same default values as in queries/shell_config.sh os.environ.setdefault("CLICKHOUSE_BINARY", args.binary) os.environ.setdefault("CLICKHOUSE_CLIENT", args.client) os.environ.setdefault("CLICKHOUSE_CONFIG", args.configserver) @@ -81,6 +81,12 @@ def main(args): os.environ.setdefault("CLICKHOUSE_CONFIG_CLIENT", args.configclient) os.environ.setdefault("CLICKHOUSE_TMP", tmp_dir) + # Force to print server warnings in stderr + # Shell scripts could change logging level + server_logs_level = "warning" + os.environ.setdefault("CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL", server_logs_level) + + # TODO ! use clickhouse-extract-from-config here: if args.zookeeper is None: code, out = commands.getstatusoutput(args.binary + "-extract-from-config --try --config " + args.configserver + ' --key zookeeper | grep . | wc -l') try: @@ -151,12 +157,12 @@ def main(args): break case_file = os.path.join(suite_dir, case) - if os.path.isfile(case_file) and (case.endswith('.sh') or case.endswith('.py') or case.endswith('.sql')): - (name, ext) = os.path.splitext(case) + (name, ext) = os.path.splitext(case) + + if os.path.isfile(case_file) and (ext == '.sql' or ext == '.sh' or ext == '.py'): report_testcase = et.Element("testcase", attrib = {"name": name}) try: - print "{0:70}".format(name + ": "), sys.stdout.flush() @@ -194,9 +200,9 @@ def main(args): stderr_file = os.path.join(suite_tmp_dir, name) + '.stderr' if ext == '.sql': - command = "{0} --testmode --multiquery < {1} > {2} 2> {3}".format(args.client, case_file, stdout_file, stderr_file) + command = "{0} --send_logs_level={1} --testmode --multiquery < {2} > {3} 2> {4}".format(args.client, server_logs_level, case_file, stdout_file, stderr_file) else: - command = "{0} > {1} 2> {2}".format(case_file, stdout_file, stderr_file) + command = "{} > {} 2> {}".format(case_file, stdout_file, stderr_file) proc = Popen(command, shell = True) start_time = datetime.now() diff --git a/dbms/tests/queries/0_stateless/00002_system_numbers.sql b/dbms/tests/queries/0_stateless/00002_system_numbers.sql index d6ebf8e89ed..8f1580e9127 100644 --- a/dbms/tests/queries/0_stateless/00002_system_numbers.sql +++ b/dbms/tests/queries/0_stateless/00002_system_numbers.sql @@ -1,3 +1,5 @@ +SET send_logs_level = 'none'; + SELECT * FROM system.numbers LIMIT 3; SELECT sys_num.number FROM system.numbers AS sys_num WHERE number > 2 LIMIT 2; SELECT number FROM system.numbers WHERE number >= 5 LIMIT 2; diff --git a/dbms/tests/queries/0_stateless/00155_long_merges.sh b/dbms/tests/queries/0_stateless/00155_long_merges.sh index d711a18a4b9..a5a636a678a 100755 --- a/dbms/tests/queries/0_stateless/00155_long_merges.sh +++ b/dbms/tests/queries/0_stateless/00155_long_merges.sh @@ -49,7 +49,7 @@ function test { $CLICKHOUSE_CLIENT --query="SELECT count() = $MAX, sum(s) = $SUM FROM test.summing" echo $CLICKHOUSE_CLIENT --query="SELECT count() = $SUM, sum(s) = $SUM FROM test.collapsing" - $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE test.collapsing" + $CLICKHOUSE_CLIENT --query="OPTIMIZE TABLE test.collapsing" --server_logs_file='/dev/null'; $CLICKHOUSE_CLIENT --query="SELECT count() = $MAX, sum(s) = $MAX FROM test.collapsing" echo $CLICKHOUSE_CLIENT --query="SELECT count() = $SUM, sumMerge(s) = $SUM FROM test.aggregating" diff --git a/dbms/tests/queries/0_stateless/00158_buffer_and_nonexistent_table.sql b/dbms/tests/queries/0_stateless/00158_buffer_and_nonexistent_table.sql index 4d3265d7903..af211314328 100644 --- a/dbms/tests/queries/0_stateless/00158_buffer_and_nonexistent_table.sql +++ b/dbms/tests/queries/0_stateless/00158_buffer_and_nonexistent_table.sql @@ -1,6 +1,7 @@ CREATE DATABASE IF NOT EXISTS test2; DROP TABLE IF EXISTS test2.mt_buffer; CREATE TABLE test2.mt_buffer (d Date DEFAULT today(), x UInt64) ENGINE = Buffer(test2, mt, 16, 100, 100, 1000000, 1000000, 1000000000, 1000000000); +SET send_logs_level = 'none'; -- Supress "Destination table test2.mt doesn't exist. Block of data is discarded." INSERT INTO test2.mt_buffer (x) SELECT number AS x FROM system.numbers LIMIT 100000; INSERT INTO test2.mt_buffer (x) SELECT number AS x FROM system.numbers LIMIT 1000000; DROP DATABASE test2; diff --git a/dbms/tests/queries/0_stateless/00183_skip_unavailable_shards.sql b/dbms/tests/queries/0_stateless/00183_skip_unavailable_shards.sql index 36768985c0f..db7cb24dfd8 100644 --- a/dbms/tests/queries/0_stateless/00183_skip_unavailable_shards.sql +++ b/dbms/tests/queries/0_stateless/00183_skip_unavailable_shards.sql @@ -1 +1,2 @@ +SET send_logs_level = 'none'; SELECT count() FROM remote('{127,1}.0.0.{2,3}', system.one) SETTINGS skip_unavailable_shards = 1; diff --git a/dbms/tests/queries/0_stateless/00336_shard_stack_trace.sh b/dbms/tests/queries/0_stateless/00336_shard_stack_trace.sh index 941cd5bbe15..f2a2556d328 100755 --- a/dbms/tests/queries/0_stateless/00336_shard_stack_trace.sh +++ b/dbms/tests/queries/0_stateless/00336_shard_stack_trace.sh @@ -8,6 +8,6 @@ ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?stacktrace=0" -d 'SELECT a' | wc -l [[ $(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?stacktrace=1" -d 'SELECT a' | wc -l) -gt 3 ]] && echo 'Ok' || echo 'Fail' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT intDiv(number, 0) FROM remote('127.0.0.{2,3}', system.numbers)" | wc -l -$CLICKHOUSE_CLIENT --query="SELECT a" 2>&1 | wc -l -[[ $($CLICKHOUSE_CLIENT --query="SELECT a" --stacktrace 2>&1 | wc -l) -gt 3 ]] && echo 'Ok' || echo 'Fail' -$CLICKHOUSE_CLIENT --query="SELECT intDiv(number, 0) FROM remote('127.0.0.{2,3}', system.numbers)" 2>&1 | wc -l +$CLICKHOUSE_CLIENT --query="SELECT a" --server_logs_file=/dev/null 2>&1 | wc -l +[[ $($CLICKHOUSE_CLIENT --query="SELECT a" --server_logs_file=/dev/null --stacktrace 2>&1 | wc -l) -gt 3 ]] && echo 'Ok' || echo 'Fail' +$CLICKHOUSE_CLIENT --query="SELECT intDiv(number, 0) FROM remote('127.0.0.{2,3}', system.numbers)" --server_logs_file=/dev/null 2>&1 | wc -l diff --git a/dbms/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh b/dbms/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh index 3b171439b5f..c8453119eb9 100755 --- a/dbms/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh +++ b/dbms/tests/queries/0_stateless/00443_optimize_final_vertical_merge.sh @@ -70,12 +70,12 @@ number AS di10, [hex(number), hex(number+1)] AS \`n.s\` FROM system.numbers LIMIT $res_rows" -while [[ `get_num_parts` -ne 1 ]] ; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE $name PARTITION 197001"; done +while [[ `get_num_parts` -ne 1 ]] ; do $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE $name PARTITION 197001" --server_logs_file=/dev/null; done $CLICKHOUSE_CLIENT -q "ALTER TABLE $name ADD COLUMN n.a Array(String)" $CLICKHOUSE_CLIENT -q "ALTER TABLE $name ADD COLUMN da Array(String) DEFAULT ['def']" -$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE $name PARTITION 197001 FINAL" +$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE $name PARTITION 197001 FINAL" --server_logs_file=/dev/null $CLICKHOUSE_CLIENT -q "ALTER TABLE $name MODIFY COLUMN n.a Array(String) DEFAULT ['zzz']" $CLICKHOUSE_CLIENT -q "ALTER TABLE $name MODIFY COLUMN da Array(String) DEFAULT ['zzz']" diff --git a/dbms/tests/queries/0_stateless/00474_readonly_settings.sh b/dbms/tests/queries/0_stateless/00474_readonly_settings.sh index 3cb5e025901..1907d51e273 100755 --- a/dbms/tests/queries/0_stateless/00474_readonly_settings.sh +++ b/dbms/tests/queries/0_stateless/00474_readonly_settings.sh @@ -6,8 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=0 | grep value $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=1 | grep value -$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.' -$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.' +$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.' +$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o 'value\|Cannot execute SET query in readonly mode.' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" | grep value ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}?query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" | grep value diff --git a/dbms/tests/queries/0_stateless/00492_drop_temporary_table.sql b/dbms/tests/queries/0_stateless/00492_drop_temporary_table.sql index dceb4dc2267..06ec5ea9805 100644 --- a/dbms/tests/queries/0_stateless/00492_drop_temporary_table.sql +++ b/dbms/tests/queries/0_stateless/00492_drop_temporary_table.sql @@ -2,7 +2,9 @@ DROP TEMPORARY TABLE IF EXISTS temp_tab; CREATE TEMPORARY TABLE temp_tab (number UInt64); INSERT INTO temp_tab SELECT number FROM system.numbers LIMIT 1; SELECT number FROM temp_tab; +SET send_logs_level = 'none'; DROP TABLE temp_tab; +SET send_logs_level = 'warning'; CREATE TEMPORARY TABLE temp_tab (number UInt64); SELECT number FROM temp_tab; DROP TEMPORARY TABLE temp_tab; diff --git a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.lib b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.lib index 531b54a64c0..892581ab298 100755 --- a/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.lib +++ b/dbms/tests/queries/0_stateless/00534_long_functions_bad_arguments.lib @@ -1,8 +1,5 @@ #!/usr/bin/env bash -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - function test_variant { perl -E "say \$_ for map {chomp; (qq{$1})} qx{$CLICKHOUSE_CLIENT -q 'SELECT name FROM system.functions ORDER BY name;'}" | $CLICKHOUSE_CLIENT -n --ignore-error >/dev/null 2>&1 $CLICKHOUSE_CLIENT -q "SELECT 'Still alive'" diff --git a/dbms/tests/queries/0_stateless/00550_join_insert_select.sh b/dbms/tests/queries/0_stateless/00550_join_insert_select.sh index 3e78942f989..3fbad66e7c2 100755 --- a/dbms/tests/queries/0_stateless/00550_join_insert_select.sh +++ b/dbms/tests/queries/0_stateless/00550_join_insert_select.sh @@ -20,6 +20,6 @@ INSERT INTO test.test1 SELECT id, name FROM test.test2 ANY LEFT OUTER JOIN test. DROP TABLE test.test1; DROP TABLE test.test2; DROP TABLE test.test3; -" 2>&1 | grep -F "Number of columns doesn't match" | wc -l +" --server_logs_file=/dev/null 2>&1 | grep -F "Number of columns doesn't match" | wc -l $CLICKHOUSE_CLIENT --query="SELECT 1"; diff --git a/dbms/tests/queries/0_stateless/00553_buff_exists_materlized_column.sql b/dbms/tests/queries/0_stateless/00553_buff_exists_materlized_column.sql index 49aff2aa184..9cef126ca49 100644 --- a/dbms/tests/queries/0_stateless/00553_buff_exists_materlized_column.sql +++ b/dbms/tests/queries/0_stateless/00553_buff_exists_materlized_column.sql @@ -15,5 +15,5 @@ INSERT INTO test.nums_buf (n) VALUES (5); SELECT n,m FROM test.nums ORDER BY n; SELECT n,m FROM test.nums_buf ORDER BY n; -DROP TABLE IF EXISTS test.nums; DROP TABLE IF EXISTS test.nums_buf; +DROP TABLE IF EXISTS test.nums; diff --git a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh b/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh index 7db122dcbb7..56e11138e2f 100755 --- a/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh +++ b/dbms/tests/queries/0_stateless/00568_compile_catch_throw.sh @@ -2,7 +2,7 @@ [ -z "$CLICKHOUSE_CLIENT" ] && CLICKHOUSE_CLIENT="clickhouse-client" -SETTINGS="--compile=1 --min_count_to_compile=0 --max_threads=1 --max_memory_usage=8000000" +SETTINGS="--compile=1 --min_count_to_compile=0 --max_threads=1 --max_memory_usage=8000000 --server_logs_file=/dev/null" output=$($CLICKHOUSE_CLIENT -q "SELECT length(groupArray(number)) FROM (SELECT * FROM system.numbers LIMIT 1000000)" $SETTINGS 2>&1) [[ $? -eq 0 ]] && echo "Expected non-zero RC" diff --git a/dbms/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh b/dbms/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh index 08c7f8ef298..d199096815c 100755 --- a/dbms/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh +++ b/dbms/tests/queries/0_stateless/00575_illegal_column_exception_when_drop_depen_column.sh @@ -10,7 +10,7 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test (dt Date DEFAULT now(), id UInt32, id2 UInt32 DEFAULT id + 1) ENGINE = MergeTree(dt, dt, 8192);" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test(dt,id) VALUES ('2018-02-22',3), ('2018-02-22',4), ('2018-02-22',5);" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test.test ORDER BY id;" -echo `${CLICKHOUSE_CLIENT} --query "ALTER TABLE test.test DROP COLUMN id;" 2>&1 | grep -c "$exception_pattern"` +echo `${CLICKHOUSE_CLIENT} --query "ALTER TABLE test.test DROP COLUMN id;" --server_logs_file=/dev/null 2>&1 | grep -c "$exception_pattern"` ${CLICKHOUSE_CLIENT} --query "ALTER TABLE test.test DROP COLUMN id2;" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test.test ORDER BY id;" ${CLICKHOUSE_CLIENT} --query "ALTER TABLE test.test DROP COLUMN id;" diff --git a/dbms/tests/queries/0_stateless/00595_insert_into_view.sh b/dbms/tests/queries/0_stateless/00595_insert_into_view.sh index 030f1931928..ae3e7d40835 100755 --- a/dbms/tests/queries/0_stateless/00595_insert_into_view.sh +++ b/dbms/tests/queries/0_stateless/00595_insert_into_view.sh @@ -11,7 +11,7 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_view;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test (s String) ENGINE = Log;" ${CLICKHOUSE_CLIENT} --query "CREATE VIEW test.test_view AS SELECT * FROM test.test;" -echo `${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_view VALUES('test_string');" 2>&1 | grep -c "$exception_pattern"` +(( `${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_view VALUES('test_string');" 2>&1 | grep -c "$exception_pattern"` >= 1 )) && echo 1 || echo "NO MATCH" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES('test_string');" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test.test;" diff --git a/dbms/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh b/dbms/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh index 85d13cbdb47..3cbbdefb3c0 100755 --- a/dbms/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh +++ b/dbms/tests/queries/0_stateless/00596_limit_on_expanded_ast.sh @@ -5,6 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) exception_pattern="too big" -${CLICKHOUSE_CLIENT} --max_expanded_ast_elements=500000 --query=" +${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --max_expanded_ast_elements=500000 --query=" select 1 as a, a+a as b, b+b as c, c+c as d, d+d as e, e+e as f, f+f as g, g+g as h, h+h as i, i+i as j, j+j as k, k+k as l, l+l as m, m+m as n, n+n as o, o+o as p, p+p as q, q+q as r, r+r as s, s+s as t, t+t as u, u+u as v, v+v as w, w+w as x, x+x as y, y+y as z " 2>&1 | grep -c "$exception_pattern" diff --git a/dbms/tests/queries/0_stateless/00597_push_down_predicate.reference b/dbms/tests/queries/0_stateless/00597_push_down_predicate.reference index 4de5516a033..c3cb55cdac6 100644 --- a/dbms/tests/queries/0_stateless/00597_push_down_predicate.reference +++ b/dbms/tests/queries/0_stateless/00597_push_down_predicate.reference @@ -1,7 +1,6 @@ 1 1 1 -1 2000-01-01 1 test string 1 1 1 1 @@ -10,9 +9,7 @@ 2000-01-01 1 2000-01-01 1 test string 1 1 2000-01-01 1 test string 1 1 -1 test string 1 -1 test string 1 +1 test string 1 1 test string 1 +1 test string 1 1 test string 1 test string 1 1 1 test string 1 1 1 -1 -1 diff --git a/dbms/tests/queries/0_stateless/00597_push_down_predicate.sh b/dbms/tests/queries/0_stateless/00597_push_down_predicate.sh deleted file mode 100755 index 33c686d29d7..00000000000 --- a/dbms/tests/queries/0_stateless/00597_push_down_predicate.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. $CURDIR/../shell_config.sh - -exception_pattern="Code: 277.*is not used and setting 'force_primary_key' is set.." - -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_union_1;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_union_2;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_join_1;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_join_2;" - - -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192);" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test_union_1(date_1 Date, id_1 Int8, name_1 String, value_1 Int64) ENGINE = MergeTree(date_1, (id_1, date_1), 8192);" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test_union_2(date_2 Date, id_2 Int8, name_2 String, value_2 Int64) ENGINE = MergeTree(date_2, (id_2, date_2), 8192);" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test_join_1(date_1 Date, id_1 Int8, name_1 String, value_1 Int64) ENGINE = MergeTree(date_1, (id_1, date_1), 8192);" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test_join_2(date_2 Date, id_2 Int8, name_2 String, value_2 Int64) ENGINE = MergeTree(date_2, (id_2, date_2), 8192);" - - -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES('2000-01-01', 1, 'test string 1', 1);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES('2000-01-01', 2, 'test string 2', 2);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_union_1 VALUES('2000-01-01', 1, 'test string 1', 1);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_union_1 VALUES('2000-01-01', 2, 'test string 2', 2);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_union_2 VALUES('2000-01-01', 1, 'test string 1', 1);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_union_2 VALUES('2000-01-01', 2, 'test string 2', 2);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_join_1 VALUES('2000-01-01', 1, 'test string 1', 1);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_join_1 VALUES('2000-01-01', 2, 'test string 2', 2);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_join_2 VALUES('2000-01-01', 1, 'test string 1', 1);" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test_join_2 VALUES('2000-01-01', 2, 'test string 2', 2);" - -# Queries that previously worked but now don't work. -echo `${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query 'SELECT * FROM (SELECT 1) WHERE \`1\` = 1;' 2>&1 | grep -c "Unknown identifier: 1."` - -# Not need push down, but it works. -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT 1;" -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT 1 AS id WHERE id = 1;" -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT arrayJoin([1,2,3]) AS id WHERE id = 1;" -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT * FROM test.test) WHERE id = 1;" - -# Need push down -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT arrayJoin([1, 2, 3]) AS id) WHERE id = 1;" -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT id FROM (SELECT arrayJoin([1, 2, 3]) AS id) WHERE id = 1;" -${CLICKHOUSE_CLIENT} --enable_optimize_predicate_expression 1 --query "SELECT date, id, name, value FROM (SELECT date, name, value,min(id) AS id FROM test.test GROUP BY date, name, value) WHERE id = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT date, id, name, value FROM (SELECT date, id, name, value FROM test.test) WHERE id = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT date, id FROM (SELECT id, date, min(value) FROM test.test GROUP BY id, date) WHERE id = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT date_1, id_1, name_1, value_1 FROM (SELECT date_1, id_1, name_1, value_1 FROM test.test_union_1 UNION ALL SELECT date_2, id_2, name_2, value_2 FROM test.test_union_2) WHERE id_1 = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT id_1, name_1 AS name FROM test.test_join_1) ANY LEFT JOIN (SELECT id_2, name_2 AS name FROM test.test_join_2) USING name WHERE id_1 = 1 AND id_2 = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT id_1, name_1 AS name FROM test.test_join_1) ANY LEFT JOIN (SELECT id_2, name_2 AS name FROM test.test_union_2 UNION ALL SELECT id_1, name_1 AS name FROM test.test_union_1) USING name WHERE id_1 = 1 AND id_2 = 1;" -${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT name_1,id_1 AS id_1, id_1 AS id_2 FROM test.test_union_1 UNION ALL (SELECT name,id_1,id_2 FROM (SELECT name_1 AS name, id_1 FROM test.test_join_1) ANY INNER JOIN (SELECT name_2 AS name, id_2 FROM test.test_join_2) USING (name))) WHERE id_1 = 1 AND id_2 = 1;" -echo `${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT * FROM (SELECT * FROM test.test) WHERE id = 1;" 2>&1 | grep -c "$exception_pattern"` -echo `${CLICKHOUSE_CLIENT} --force_primary_key 1 --enable_optimize_predicate_expression 1 --query "SELECT id FROM (SELECT min(id) AS id FROM test.test) WHERE id = 1;" 2>&1 | grep -c "$exception_pattern"` - -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_union_1;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_union_2;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_join_1;" -${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test_join_2;" diff --git a/dbms/tests/queries/0_stateless/00597_push_down_predicate.sql b/dbms/tests/queries/0_stateless/00597_push_down_predicate.sql new file mode 100644 index 00000000000..b884d9feae7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00597_push_down_predicate.sql @@ -0,0 +1,63 @@ +SET send_logs_level = 'none'; + +DROP TABLE IF EXISTS test.test; +DROP TABLE IF EXISTS test.test_union_1; +DROP TABLE IF EXISTS test.test_union_2; +DROP TABLE IF EXISTS test.test_join_1; +DROP TABLE IF EXISTS test.test_join_2; + + +CREATE TABLE test.test(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); +CREATE TABLE test.test_union_1(date_1 Date, id_1 Int8, name_1 String, value_1 Int64) ENGINE = MergeTree(date_1, (id_1, date_1), 8192); +CREATE TABLE test.test_union_2(date_2 Date, id_2 Int8, name_2 String, value_2 Int64) ENGINE = MergeTree(date_2, (id_2, date_2), 8192); +CREATE TABLE test.test_join_1(date_1 Date, id_1 Int8, name_1 String, value_1 Int64) ENGINE = MergeTree(date_1, (id_1, date_1), 8192); +CREATE TABLE test.test_join_2(date_2 Date, id_2 Int8, name_2 String, value_2 Int64) ENGINE = MergeTree(date_2, (id_2, date_2), 8192); + + +INSERT INTO test.test VALUES('2000-01-01', 1, 'test string 1', 1); +INSERT INTO test.test VALUES('2000-01-01', 2, 'test string 2', 2); +INSERT INTO test.test_union_1 VALUES('2000-01-01', 1, 'test string 1', 1); +INSERT INTO test.test_union_1 VALUES('2000-01-01', 2, 'test string 2', 2); +INSERT INTO test.test_union_2 VALUES('2000-01-01', 1, 'test string 1', 1); +INSERT INTO test.test_union_2 VALUES('2000-01-01', 2, 'test string 2', 2); +INSERT INTO test.test_join_1 VALUES('2000-01-01', 1, 'test string 1', 1); +INSERT INTO test.test_join_1 VALUES('2000-01-01', 2, 'test string 2', 2); +INSERT INTO test.test_join_2 VALUES('2000-01-01', 1, 'test string 1', 1); +INSERT INTO test.test_join_2 VALUES('2000-01-01', 2, 'test string 2', 2); + + +SET enable_optimize_predicate_expression = 1; + +-- Query that previously worked but now doesn't work. +SELECT * FROM (SELECT 1) WHERE `1` = 1; -- { serverError 47 } + +SELECT 1; -- Not need push down, but it works. +SELECT 1 AS id WHERE id = 1; +SELECT arrayJoin([1,2,3]) AS id WHERE id = 1; +SELECT * FROM (SELECT * FROM test.test) WHERE id = 1; + +-- Need push down +SELECT * FROM (SELECT arrayJoin([1, 2, 3]) AS id) WHERE id = 1; +SELECT id FROM (SELECT arrayJoin([1, 2, 3]) AS id) WHERE id = 1; +SELECT date, id, name, value FROM (SELECT date, name, value,min(id) AS id FROM test.test GROUP BY date, name, value) WHERE id = 1; + + +SET force_primary_key = 1; + +SELECT date, id, name, value FROM (SELECT date, id, name, value FROM test.test) WHERE id = 1; +SELECT date, id FROM (SELECT id, date, min(value) FROM test.test GROUP BY id, date) WHERE id = 1; +SELECT date_1, id_1, name_1, value_1 FROM (SELECT date_1, id_1, name_1, value_1 FROM test.test_union_1 UNION ALL SELECT date_2, id_2, name_2, value_2 FROM test.test_union_2) WHERE id_1 = 1; +SELECT * FROM (SELECT id_1, name_1 AS name FROM test.test_join_1) ANY LEFT JOIN (SELECT id_2, name_2 AS name FROM test.test_join_2) USING name WHERE id_1 = 1 AND id_2 = 1; +SELECT * FROM (SELECT id_1, name_1 AS name FROM test.test_join_1) ANY LEFT JOIN (SELECT id_2, name_2 AS name FROM test.test_union_2 UNION ALL SELECT id_1, name_1 AS name FROM test.test_union_1) USING name WHERE id_1 = 1 AND id_2 = 1; +SELECT * FROM (SELECT name_1,id_1 AS id_1, id_1 AS id_2 FROM test.test_union_1 UNION ALL (SELECT name,id_1,id_2 FROM (SELECT name_1 AS name, id_1 FROM test.test_join_1) ANY INNER JOIN (SELECT name_2 AS name, id_2 FROM test.test_join_2) USING (name))) WHERE id_1 = 1 AND id_2 = 1; + +-- TODO This should work: +SELECT * FROM (SELECT * FROM test.test) WHERE id = 1; -- { serverError 277 } + +SELECT id FROM (SELECT min(id) AS id FROM test.test) WHERE id = 1; -- { serverError 277 } + +DROP TABLE IF EXISTS test.test; +DROP TABLE IF EXISTS test.test_union_1; +DROP TABLE IF EXISTS test.test_union_2; +DROP TABLE IF EXISTS test.test_join_1; +DROP TABLE IF EXISTS test.test_join_2; diff --git a/dbms/tests/queries/0_stateless/00602_throw_if.sh b/dbms/tests/queries/0_stateless/00602_throw_if.sh index a63c109ffaf..8dae5033978 100755 --- a/dbms/tests/queries/0_stateless/00602_throw_if.sh +++ b/dbms/tests/queries/0_stateless/00602_throw_if.sh @@ -5,5 +5,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) exception_pattern="Value passed to 'throwIf' function is non zero" -${CLICKHOUSE_CLIENT} --query="SELECT throwIf(number = 1000000) FROM system.numbers" 2>&1 | grep -cF "$exception_pattern" -${CLICKHOUSE_CLIENT} --query="SELECT sum(x = 0) FROM (SELECT throwIf(number = 1000000) AS x FROM numbers(1000000))" 2>&1 +${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT throwIf(number = 1000000) FROM system.numbers" 2>&1 | grep -cF "$exception_pattern" +${CLICKHOUSE_CLIENT} --server_logs_file /dev/null --query="SELECT sum(x = 0) FROM (SELECT throwIf(number = 1000000) AS x FROM numbers(1000000))" 2>&1 diff --git a/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.reference b/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.reference new file mode 100644 index 00000000000..16357b19ff1 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.reference @@ -0,0 +1,2 @@ +2018-01-21 15:12:13 +2018-01-21 15:12:13 diff --git a/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql b/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql new file mode 100644 index 00000000000..a79383f2939 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00614_shard_same_header_for_local_and_remote_node_in_distributed_query.sql @@ -0,0 +1,7 @@ +create database if not exists test; +drop table if exists test.tab; +create table test.tab (date Date, time DateTime, data String) ENGINE = MergeTree(date, (time, data), 8192); +insert into test.tab values ('2018-01-21','2018-01-21 15:12:13','test'); +select time FROM remote('127.0.0.{1,2}', test.tab) WHERE date = '2018-01-21' limit 2; +drop table test.tab; + diff --git a/dbms/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh b/dbms/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh index 3b6d7581b17..7f9ae704b98 100755 --- a/dbms/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh +++ b/dbms/tests/queries/0_stateless/00623_truncate_table_throw_exception.sh @@ -15,7 +15,7 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_truncate.test_view_depend VALUES( ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_truncate.test_view;" ${CLICKHOUSE_CLIENT} --query "SELECT '========Execute Truncate========';" -echo `${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE test_truncate.test_view;" 2>&1 | grep -c "Code: 48.*Truncate is not supported by storage View"` +echo `${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE test_truncate.test_view;" --server_logs_file=/dev/null 2>&1 | grep -c "Code: 48.*Truncate is not supported by storage View"` ${CLICKHOUSE_CLIENT} --query "SELECT '========After Truncate========';" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_truncate.test_view;" diff --git a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table.sql b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table.sql index daf9950e9d5..397b459142a 100644 --- a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table.sql +++ b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table.sql @@ -43,7 +43,7 @@ SELECT count(), sum(d) FROM test.dst; INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database='test' AND table='dst' AND active AND name LIKE '1_%'; SELECT (max(m) - min(m) > 1) AS new_block_is_generated FROM test_block_numbers; -DROP TABLE test_block_numbers; +DROP TEMPORARY TABLE test_block_numbers; SELECT 'ATTACH FROM'; diff --git a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sql b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sql index 88a8dd1da35..208e06b80db 100644 --- a/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sql @@ -55,7 +55,7 @@ SELECT count(), sum(d) FROM test.dst_r2; INSERT INTO test_block_numbers SELECT max(max_block_number) AS m FROM system.parts WHERE database='test' AND table='dst_r1' AND active AND name LIKE '1_%'; SELECT (max(m) - min(m) > 1) AS new_block_is_generated FROM test_block_numbers; -DROP TABLE test_block_numbers; +DROP TEMPORARY TABLE test_block_numbers; SELECT 'ATTACH FROM'; diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.reference b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.reference new file mode 100644 index 00000000000..9f61a33663e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.reference @@ -0,0 +1,5 @@ +1000 +0 +1 1 1 +0 +1 1 1 diff --git a/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.sh b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.sh new file mode 100755 index 00000000000..7ac29a7e10f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00634_performance_introspection_and_logging_shard.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -e + +# Get all server logs +export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL="trace" + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +cur_name=${BASH_SOURCE[0]} +server_logs_file=$cur_name"_server.logs" +server_logs="--server_logs_file=$server_logs_file" +rm -f "$server_logs_file" + +settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" + + +# Test insert logging on each block and checkPacket() method + +$CLICKHOUSE_CLIENT $settings -n -q " +DROP TABLE IF EXISTS test.null; +CREATE TABLE test.null (i UInt8) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple();" + +head -c 1000 /dev/zero | $CLICKHOUSE_CLIENT $settings --max_insert_block_size=10 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -q "INSERT INTO test.null FORMAT RowBinary" + +$CLICKHOUSE_CLIENT $settings -n -q " +SELECT count() FROM test.null; +DROP TABLE test.null;" + +(( `cat "$server_logs_file" | wc -l` >= 110 )) || echo Fail + + +# Check ProfileEvents in query_log + +heavy_cpu_query="SELECT ignore(sum(sipHash64(hex(sipHash64(hex(sipHash64(hex(number)))))))) FROM (SELECT * FROM system.numbers_mt LIMIT 1000000)" +$CLICKHOUSE_CLIENT $settings --max_threads=1 -q "$heavy_cpu_query" +$CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH SYSTEM TABLES" +$CLICKHOUSE_CLIENT $settings -q " +WITH + any(query_duration_ms*1000) AS duration, + sumIf(PV, PN = 'RealTimeMicroseconds') AS threads_realtime, + sumIf(PV, PN IN ('UserTimeMicroseconds', 'SystemTimeMicroseconds', 'OSIOWaitMicroseconds', 'OSCPUWaitMicroseconds')) AS threads_time_user_system_io +SELECT + -- duration, threads_realtime, threads_time_user_system_io, + threads_realtime >= 0.99 * duration, + threads_realtime >= threads_time_user_system_io, + any(length(thread_numbers)) >= 1 + FROM + (SELECT * FROM system.query_log PREWHERE query='$heavy_cpu_query' WHERE event_date >= today()-1 AND type=2 ORDER BY event_time DESC LIMIT 1) + ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV" + + +# Check ProfileEvents in query_thread_log + +$CLICKHOUSE_CLIENT $settings --max_threads=3 -q "$heavy_cpu_query" +$CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH SYSTEM TABLES" +query_id=`$CLICKHOUSE_CLIENT $settings -q "SELECT query_id FROM system.query_log WHERE event_date >= today()-1 AND type=2 AND query='$heavy_cpu_query' ORDER BY event_time DESC LIMIT 1"` +query_elapsed=`$CLICKHOUSE_CLIENT $settings -q "SELECT query_duration_ms*1000 FROM system.query_log WHERE event_date >= today()-1 AND type=2 AND query_id='$query_id' ORDER BY event_time DESC LIMIT 1"` +threads=`$CLICKHOUSE_CLIENT $settings -q "SELECT length(thread_numbers) FROM system.query_log WHERE event_date >= today()-1 AND type=2 AND query_id='$query_id' ORDER BY event_time DESC LIMIT 1"` + +$CLICKHOUSE_CLIENT $settings -q " +SELECT + -- max(thread_realtime), $query_elapsed, max(thread_time_user_system_io), + 0.9 * $query_elapsed <= max(thread_realtime) AND max(thread_realtime) <= 1.1 * $query_elapsed, + 0.7 * $query_elapsed <= max(thread_time_user_system_io) AND max(thread_time_user_system_io) <= 1.3 * $query_elapsed, + uniqExact(thread_number) = $threads +FROM +( + SELECT + thread_number, + sumIf(PV, PN = 'RealTimeMicroseconds') AS thread_realtime, + sumIf(PV, PN IN ('UserTimeMicroseconds', 'SystemTimeMicroseconds', 'OSIOWaitMicroseconds', 'OSCPUWaitMicroseconds')) AS thread_time_user_system_io + FROM + (SELECT * FROM system.query_thread_log PREWHERE query_id='$query_id' WHERE event_date >= today()-1) + ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV + GROUP BY thread_number +) +" + +# Check per-thread and per-query ProfileEvents consistency + +$CLICKHOUSE_CLIENT $settings -q " +SELECT PN, PVq, PVt FROM +( + SELECT PN, sum(PV) AS PVt + FROM system.query_thread_log + ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV + WHERE event_date >= today()-1 AND query_id='$query_id' + GROUP BY PN +) +ANY INNER JOIN +( + SELECT PN, PV AS PVq + FROM system.query_log + ARRAY JOIN ProfileEvents.Names AS PN, ProfileEvents.Values AS PV + WHERE event_date >= today()-1 AND query_id='$query_id' +) +USING PN +WHERE + NOT PN IN ('ContextLock') AND + NOT (PVq <= PVt AND PVt <= 1.1 * PVq) +" + + +# Check that logs from remote servers are passed from client + +# SELECT +> "$server_logs_file" +$CLICKHOUSE_CLIENT $settings -q "SELECT 1 FROM system.one FORMAT Null" +lines_one_server=`cat "$server_logs_file" | wc -l` + +> "$server_logs_file" +$CLICKHOUSE_CLIENT $settings -q "SELECT 1 FROM remote('127.0.0.2,127.0.0.3', system, one) FORMAT Null" +lines_two_servers=`cat "$server_logs_file" | wc -l` + +(( $lines_two_servers >= 2 * $lines_one_server )) || echo "Fail: $lines_two_servers $lines_one_server" + +# INSERT +$CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS test.null" +$CLICKHOUSE_CLIENT $settings -q "CREATE TABLE test.null (i Int8) ENGINE = Null" + +> "$server_logs_file" +$CLICKHOUSE_CLIENT $settings -q "INSERT INTO test.null VALUES (0)" +lines_one_server=`cat "$server_logs_file" | wc -l` + +> "$server_logs_file" +$CLICKHOUSE_CLIENT $settings -q "INSERT INTO TABLE FUNCTION remote('127.0.0.2', 'test', 'null') VALUES (0)" +lines_two_servers=`cat "$server_logs_file" | wc -l` + +$CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS test.null" +(( $lines_two_servers > $lines_one_server )) || echo "Fail: $lines_two_servers $lines_one_server" + + +# Clean +rm "$server_logs_file" diff --git a/dbms/tests/queries/0_stateless/00687_top_and_offset.reference b/dbms/tests/queries/0_stateless/00687_top_and_offset.reference index 5cd83db0ff1..29c5fedbb5d 100644 --- a/dbms/tests/queries/0_stateless/00687_top_and_offset.reference +++ b/dbms/tests/queries/0_stateless/00687_top_and_offset.reference @@ -4,3 +4,5 @@ 2 3 4 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00687_top_and_offset.sh b/dbms/tests/queries/0_stateless/00687_top_and_offset.sh new file mode 100755 index 00000000000..47b20f343eb --- /dev/null +++ b/dbms/tests/queries/0_stateless/00687_top_and_offset.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test.test;" + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test.test(val Int64) engine = Memory;" + +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (1);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (2);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (3);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (4);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (5);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (6);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (7);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (8);" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test.test VALUES (9);" + +${CLICKHOUSE_CLIENT} --query "SELECT TOP 2 * FROM test.test ORDER BY val;" +${CLICKHOUSE_CLIENT} --query "SELECT TOP (2) * FROM test.test ORDER BY val;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test.test ORDER BY val LIMIT 2 OFFSET 2;" + +echo `${CLICKHOUSE_CLIENT} --query "SELECT TOP 2 * FROM test.test ORDER BY val LIMIT 2;" 2>&1 | grep -c "Code: 406"` +echo `${CLICKHOUSE_CLIENT} --query "SELECT * FROM test.test ORDER BY val LIMIT 2,3 OFFSET 2;" 2>&1 | grep -c "Code: 62"` + +${CLICKHOUSE_CLIENT} --query "DROP TABLE test.test;" diff --git a/dbms/tests/queries/0_stateless/00687_top_and_offset.sql b/dbms/tests/queries/0_stateless/00687_top_and_offset.sql deleted file mode 100644 index 0edff8c7539..00000000000 --- a/dbms/tests/queries/0_stateless/00687_top_and_offset.sql +++ /dev/null @@ -1,21 +0,0 @@ -DROP TABLE IF EXISTS test.test; - -CREATE TABLE test.test(val Int64) engine = Memory; - -INSERT INTO test.test VALUES (1); -INSERT INTO test.test VALUES (2); -INSERT INTO test.test VALUES (3); -INSERT INTO test.test VALUES (4); -INSERT INTO test.test VALUES (5); -INSERT INTO test.test VALUES (6); -INSERT INTO test.test VALUES (7); -INSERT INTO test.test VALUES (8); -INSERT INTO test.test VALUES (9); - -SELECT TOP 2 * FROM test.test ORDER BY val; -SELECT TOP (2) * FROM test.test ORDER BY val; -SELECT * FROM test.test ORDER BY val LIMIT 2 OFFSET 2; -SELECT TOP 2 * FROM test.test ORDER BY val LIMIT 2; -- { clientError 406 } -SELECT * FROM test.test ORDER BY val LIMIT 2,3 OFFSET 2; -- { clientError 62 } - -DROP TABLE test.test; diff --git a/dbms/tests/queries/0_stateless/00688_aggregation_retention.reference b/dbms/tests/queries/0_stateless/00688_aggregation_retention.reference new file mode 100644 index 00000000000..95a6127a23f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00688_aggregation_retention.reference @@ -0,0 +1,4 @@ +80 +80 50 +80 60 +80 50 60 diff --git a/dbms/tests/queries/0_stateless/00688_aggregation_retention.sql b/dbms/tests/queries/0_stateless/00688_aggregation_retention.sql new file mode 100644 index 00000000000..578f91833d0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00688_aggregation_retention.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS retention_test; + +CREATE TABLE retention_test(date Date, uid Int32)ENGINE = Memory; +INSERT INTO retention_test SELECT '2018-08-06', number FROM numbers(80); +INSERT INTO retention_test SELECT '2018-08-07', number FROM numbers(50); +INSERT INTO retention_test SELECT '2018-08-08', number FROM numbers(60); + +SELECT sum(r[1]) as r1 FROM (SELECT uid, retention(date = '2018-08-06') AS r FROM retention_test WHERE date IN ('2018-08-06') GROUP BY uid); +SELECT sum(r[1]) as r1, sum(r[2]) as r2 FROM (SELECT uid, retention(date = '2018-08-06', date = '2018-08-07') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-07') GROUP BY uid); +SELECT sum(r[1]) as r1, sum(r[2]) as r2 FROM (SELECT uid, retention(date = '2018-08-06', date = '2018-08-08') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-08') GROUP BY uid); +SELECT sum(r[1]) as r1, sum(r[2]) as r2, sum(r[3]) as r3 FROM (SELECT uid, retention(date = '2018-08-06', date = '2018-08-07', date = '2018-08-08') AS r FROM retention_test WHERE date IN ('2018-08-06', '2018-08-07', '2018-08-08') GROUP BY uid); + +DROP TABLE retention_test; diff --git a/dbms/tests/queries/0_stateless/00689_catboost_pool_files.reference b/dbms/tests/queries/0_stateless/00689_catboost_pool_files.reference new file mode 100644 index 00000000000..e965047ad7c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00689_catboost_pool_files.reference @@ -0,0 +1 @@ +Hello diff --git a/dbms/tests/queries/0_stateless/00689_catboost_pool_files.sh b/dbms/tests/queries/0_stateless/00689_catboost_pool_files.sh new file mode 100755 index 00000000000..c366949d695 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00689_catboost_pool_files.sh @@ -0,0 +1,30 @@ +SCRIPT_PATH="`dirname \"$0\"`" +SCRIPT_PATH="`( cd \"$SCRIPT_PATH\" && pwd )`" + +${CLICKHOUSE_CLIENT} --query="drop table if exists test.catboost_pool_desc;" +${CLICKHOUSE_CLIENT} --query="drop table if exists test.catboost_pool_vals;" +${CLICKHOUSE_CLIENT} --query="create table test.catboost_pool_desc (id String, type String) engine = File(TSV);" +${CLICKHOUSE_CLIENT} --query="insert into test.catboost_pool_desc select '0', 'Categ';" +${CLICKHOUSE_CLIENT} --query="create table test.catboost_pool_vals (str String) engine = File(TSV);" +${CLICKHOUSE_CLIENT} --query="insert into test.catboost_pool_vals select 'Hello';" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', 'data/test/catboost_pool_vals/data.TSV');" + +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('data/test/catboost_pool_desc/data.TSV', '../../../../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" + +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('${SCRIPT_PATH}/00689_file.txt', '${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../${SCRIPT_PATH}/00689_file.txt', '../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../../${SCRIPT_PATH}/00689_file.txt', '../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../../../${SCRIPT_PATH}/00689_file.txt', '../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../../../../${SCRIPT_PATH}/00689_file.txt', '../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../../../../../${SCRIPT_PATH}/00689_file.txt', '../../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" +${CLICKHOUSE_CLIENT} --query="select * from catBoostPool('../../../../../../${SCRIPT_PATH}/00689_file.txt', '../../../../../../${SCRIPT_PATH}/00689_file.txt');" 2>&1 | grep -o "Data" + +${CLICKHOUSE_CLIENT} --query="drop table if exists test.catboost_pool_desc;" +${CLICKHOUSE_CLIENT} --query="drop table if exists test.catboost_pool_vals;" diff --git a/dbms/tests/queries/0_stateless/00689_file.txt b/dbms/tests/queries/0_stateless/00689_file.txt new file mode 100644 index 00000000000..0b7088ec633 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00689_file.txt @@ -0,0 +1 @@ +Data diff --git a/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql b/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql index f685c5ee37d..f4d11ea7431 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql @@ -1,4 +1,6 @@ -SET allow_experimental_decimal_type=1; +SET allow_experimental_decimal_type = 1; +SET send_logs_level = 'none'; + CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql index a86490ef78b..e4ea6eb9608 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql @@ -1,4 +1,6 @@ -SET allow_experimental_decimal_type=1; +SET allow_experimental_decimal_type = 1; +SET send_logs_level = 'none'; + CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_compare.sql b/dbms/tests/queries/0_stateless/00700_decimal_compare.sql index 848e4df357f..0aefac6eac4 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_compare.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_compare.sql @@ -1,4 +1,6 @@ -SET allow_experimental_decimal_type=1; +SET send_logs_level = 'none'; +SET allow_experimental_decimal_type = 1; + CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/shell_config.sh b/dbms/tests/queries/shell_config.sh index 3ca8c728530..0078c3432ad 100644 --- a/dbms/tests/queries/shell_config.sh +++ b/dbms/tests/queries/shell_config.sh @@ -1,6 +1,8 @@ export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="${CLICKHOUSE_BINARY}-client"} +export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} +export CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}" export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:="/etc/clickhouse-server/config.xml"} diff --git a/debian/changelog b/debian/changelog index 7207e9268b1..8d7ec0bc431 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (18.10.3) unstable; urgency=low +clickhouse (18.11.0) unstable; urgency=low * Modified source code - -- Mon, 13 Aug 2018 12:42:01 +0300 + -- Alexey Milovidov Thu, 16 Aug 2018 01:43:16 +0300 diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index e92b3e281df..85cf9c9e72a 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -164,6 +164,8 @@ start() else mkdir -p $CLICKHOUSE_PIDDIR chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR + # Set net_admin capabilities to support ClickHouse better introspection + su -s setcap cap_net_admin=+ep "$BINDIR/$PROGRAM" initdb if ! is_running; then # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition. diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index 1e57ebef0a5..6f0690ec3bd 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -13,6 +13,7 @@ ExecStartPre=-/bin/chown clickhouse:clickhouse -R /etc/clickhouse-server ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml LimitCORE=infinity LimitNOFILE=500000 +CapabilityBoundingSet=CAP_NET_ADMIN [Install] WantedBy=multi-user.target diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 47e300a5f8a..8519f639368 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.10.3 +ARG version=18.11.0 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 2b183c3924b..2d0ba93b75c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.10.3 +ARG version=18.11.0 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 24fd55a7580..d84b2df138b 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=18.10.3 +ARG version=18.11.0 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docs/en/query_language/agg_functions/parametric_functions.md b/docs/en/query_language/agg_functions/parametric_functions.md index 6def6191371..5881bed4c90 100644 --- a/docs/en/query_language/agg_functions/parametric_functions.md +++ b/docs/en/query_language/agg_functions/parametric_functions.md @@ -85,6 +85,36 @@ ORDER BY level Simply, the level value could only be 0, 1, 2, 3, it means the maxium event action stage that one user could reach. +## retention(cond1, cond2, ...) + +Retention refers to the ability of a company or product to retain its customers over some specified periods. + +`cond1`, `cond2` ... is from one to 32 arguments of type UInt8 that indicate whether a certain condition was met for the event + +Example: + +Consider you are doing a website analytics, intend to calculate the retention of customers + +This could be easily calculate by `retention` + +``` +SELECT + sum(r[1]) AS r1, + sum(r[2]) AS r2, + sum(r[3]) AS r3 +FROM +( + SELECT + uid, + retention(date = '2018-08-10', date = '2018-08-11', date = '2018-08-12') AS r + FROM events + WHERE date IN ('2018-08-10', '2018-08-11', '2018-08-12') + GROUP BY uid +) +``` + +Simply, `r1` means the number of unique visitors who met the `cond1` condition, `r2` means the number of unique visitors who met `cond1` and `cond2` conditions, `r3` means the number of unique visitors who met `cond1` and `cond3` conditions. + ## uniqUpTo(N)(x) diff --git a/docs/en/query_language/dicts/external_dicts_dict_layout.md b/docs/en/query_language/dicts/external_dicts_dict_layout.md index d6802c4ccbf..e5c1b65dce3 100644 --- a/docs/en/query_language/dicts/external_dicts_dict_layout.md +++ b/docs/en/query_language/dicts/external_dicts_dict_layout.md @@ -72,7 +72,7 @@ Configuration example: ### hashed -The dictionary is completely stored in memory in the form of a hash table. The dictionary can contain any number of elements with any identifiers In practice, the number of keys can reach tens of millions of items. +The dictionary is completely stored in memory in the form of a hash table. The dictionary can contain any number of elements with any identifiers. In practice, the number of keys can reach tens of millions of items. All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety. diff --git a/libs/libdaemon/CMakeLists.txt b/libs/libdaemon/CMakeLists.txt index a0e4e7d2733..1bd7731664a 100644 --- a/libs/libdaemon/CMakeLists.txt +++ b/libs/libdaemon/CMakeLists.txt @@ -1,11 +1,17 @@ add_library (daemon ${SPLIT_SHARED} src/BaseDaemon.cpp src/GraphiteWriter.cpp + src/ExtendedLogChannel.cpp src/OwnPatternFormatter.cpp + src/OwnFormattingChannel.cpp + src/OwnSplitChannel.cpp include/daemon/BaseDaemon.h include/daemon/GraphiteWriter.h + include/daemon/ExtendedLogChannel.h include/daemon/OwnPatternFormatter.h + include/daemon/OwnFormattingChannel.h + include/daemon/OwnSplitChannel.h ) if (USE_UNWIND) diff --git a/libs/libdaemon/include/daemon/ExtendedLogChannel.h b/libs/libdaemon/include/daemon/ExtendedLogChannel.h new file mode 100644 index 00000000000..b90494273f2 --- /dev/null +++ b/libs/libdaemon/include/daemon/ExtendedLogChannel.h @@ -0,0 +1,38 @@ +#pragma once +#include +#include + + +namespace DB +{ + +/// Poco::Message with more ClickHouse-specific info +/// NOTE: Poco::Message is not polymorphic class, so we can't use inheritance in couple with dynamic_cast<>() +class ExtendedLogMessage +{ +public: + explicit ExtendedLogMessage(const Poco::Message & base) : base(base) {} + + /// Attach additional data to the message + static ExtendedLogMessage getFrom(const Poco::Message & base); + + // Do not copy for efficiency reasons + const Poco::Message & base; + + UInt32 time_seconds = 0; + UInt32 time_microseconds = 0; + + UInt32 thread_number = 0; + std::string query_id; +}; + + +/// Interface extension of Poco::Channel +class ExtendedLogChannel +{ +public: + virtual void logExtended(const ExtendedLogMessage & msg) = 0; + virtual ~ExtendedLogChannel() = default; +}; + +} diff --git a/libs/libdaemon/include/daemon/OwnFormattingChannel.h b/libs/libdaemon/include/daemon/OwnFormattingChannel.h new file mode 100644 index 00000000000..e206debb8e5 --- /dev/null +++ b/libs/libdaemon/include/daemon/OwnFormattingChannel.h @@ -0,0 +1,55 @@ +#pragma once +#include +#include +#include +#include + + +class OwnPatternFormatter; + + +namespace DB +{ + + +// Like Poco::FormattingChannel but supports the extended logging interface and log level filter +class OwnFormattingChannel : public Poco::Channel, public ExtendedLogChannel +{ +public: + explicit OwnFormattingChannel(Poco::AutoPtr pFormatter_ = nullptr, Poco::AutoPtr pChannel_ = nullptr) + : pFormatter(std::move(pFormatter_)), pChannel(std::move(pChannel_)) {} + + void setChannel(Poco::AutoPtr pChannel_) + { + pChannel = std::move(pChannel_); + } + + void setLevel(Poco::Message::Priority priority_) + { + priority = priority_; + } + + void open() override + { + if (pChannel) + pChannel->open(); + } + + void close() override + { + if (pChannel) + pChannel->close(); + } + + void log(const Poco::Message & msg) override; + void logExtended(const ExtendedLogMessage & msg) override; + + ~OwnFormattingChannel() override; + +private: + Poco::AutoPtr pFormatter; + Poco::AutoPtr pChannel; + Poco::Message::Priority priority = Poco::Message::PRIO_TRACE; +}; + +} diff --git a/libs/libdaemon/include/daemon/OwnPatternFormatter.h b/libs/libdaemon/include/daemon/OwnPatternFormatter.h index 63e14a34d84..e90734c21e8 100644 --- a/libs/libdaemon/include/daemon/OwnPatternFormatter.h +++ b/libs/libdaemon/include/daemon/OwnPatternFormatter.h @@ -2,6 +2,7 @@ #include +#include /** Форматирует по своему. @@ -23,6 +24,7 @@ class BaseDaemon; class OwnPatternFormatter : public Poco::PatternFormatter { public: + /// ADD_LAYER_TAG is needed only for Yandex.Metrika, that share part of ClickHouse code. enum Options { @@ -30,9 +32,10 @@ public: ADD_LAYER_TAG = 1 << 0 }; - OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING) : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {} + OwnPatternFormatter(const BaseDaemon * daemon_, Options options_ = ADD_NOTHING); void format(const Poco::Message & msg, std::string & text) override; + void formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text); private: const BaseDaemon * daemon; diff --git a/libs/libdaemon/include/daemon/OwnSplitChannel.h b/libs/libdaemon/include/daemon/OwnSplitChannel.h new file mode 100644 index 00000000000..0f0702c2ab5 --- /dev/null +++ b/libs/libdaemon/include/daemon/OwnSplitChannel.h @@ -0,0 +1,35 @@ +#pragma once +#include +#include +#include +#include + + +namespace DB +{ + +/// Works as Poco::SplitterChannel, but performs additional work: +/// passes logs to Client via TCP interface +/// tries to use extended logging interface of child for more comprehensive logging +class OwnSplitChannel : public Poco::Channel +{ +public: + OwnSplitChannel() = default; + + /// Makes an extended message from msg and passes it to the client logs queue and child (if possible) + void log(const Poco::Message & msg) override; + + /// Adds a child channel + void addChannel(Poco::AutoPtr channel); + + ~OwnSplitChannel() = default; + +private: + + using ChannelPtr = Poco::AutoPtr; + /// Handler and its pointer casted to extended interface + using ExtendedChannelPtrPair = std::pair; + std::vector channels; +}; + +} diff --git a/libs/libdaemon/src/BaseDaemon.cpp b/libs/libdaemon/src/BaseDaemon.cpp index 8e6a885cf05..a4a5e85cd03 100644 --- a/libs/libdaemon/src/BaseDaemon.cpp +++ b/libs/libdaemon/src/BaseDaemon.cpp @@ -1,6 +1,9 @@ #include +#include +#include #include +#include #include #include @@ -65,6 +68,7 @@ #include #include #include +#include #include @@ -706,8 +710,9 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) bool is_daemon = config.getBool("application.runAsDaemon", false); - // Split log and error log. - Poco::AutoPtr split = new SplitterChannel; + /// Split logs to ordinary log, error log, syslog and console. + /// Use extended interface of Channel for more comprehensive logging. + Poco::AutoPtr split = new DB::OwnSplitChannel; auto log_level = config.getString("logger.level", "trace"); const auto log_path = config.getString("logger.log", ""); @@ -717,10 +722,7 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) std::cerr << "Logging " << log_level << " to " << log_path << std::endl; // Set up two channel chains. - Poco::AutoPtr pf = new OwnPatternFormatter(this); - pf->setProperty("times", "local"); - Poco::AutoPtr log = new FormattingChannel(pf); - log_file = new FileChannel; + Poco::AutoPtr log_file = new FileChannel; log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(log_path).absolute().toString()); log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); @@ -728,9 +730,12 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); - log->setChannel(log_file); - split->addChannel(log); log_file->open(); + + Poco::AutoPtr pf = new OwnPatternFormatter(this); + + Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, log_file); + split->addChannel(log); } const auto errorlog_path = config.getString("logger.errorlog", ""); @@ -738,12 +743,8 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) { createDirectory(errorlog_path); std::cerr << "Logging errors to " << errorlog_path << std::endl; - Poco::AutoPtr level = new Poco::LevelFilterChannel; - level->setLevel(Message::PRIO_NOTICE); - Poco::AutoPtr pf = new OwnPatternFormatter(this); - pf->setProperty("times", "local"); - Poco::AutoPtr errorlog = new FormattingChannel(pf); - error_log_file = new FileChannel; + + Poco::AutoPtr error_log_file = new FileChannel; error_log_file->setProperty(Poco::FileChannel::PROP_PATH, Poco::Path(errorlog_path).absolute().toString()); error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M")); error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); @@ -751,10 +752,13 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1")); error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true")); error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false")); - errorlog->setChannel(error_log_file); - level->setChannel(errorlog); - split->addChannel(level); + + Poco::AutoPtr pf = new OwnPatternFormatter(this); + + Poco::AutoPtr errorlog = new DB::OwnFormattingChannel(pf, error_log_file); + errorlog->setLevel(Message::PRIO_NOTICE); errorlog->open(); + split->addChannel(errorlog); } /// "dynamic_layer_selection" is needed only for Yandex.Metrika, that share part of ClickHouse code. @@ -762,10 +766,6 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) if (config.getBool("logger.use_syslog", false) || config.getBool("dynamic_layer_selection", false)) { - Poco::AutoPtr pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG); - pf->setProperty("times", "local"); - Poco::AutoPtr log = new FormattingChannel(pf); - const std::string & cmd_name = commandName(); if (config.has("logger.syslog.address")) @@ -787,19 +787,17 @@ void BaseDaemon::buildLoggers(Poco::Util::AbstractConfiguration & config) syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID")); syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON")); } - - log->setChannel(syslog_channel); - split->addChannel(log); syslog_channel->open(); + + Poco::AutoPtr pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_LAYER_TAG); + + Poco::AutoPtr log = new DB::OwnFormattingChannel(pf, syslog_channel); + split->addChannel(log); } if (config.getBool("logger.console", false) || (!config.hasProperty("logger.console") && !is_daemon && (isatty(STDIN_FILENO) || isatty(STDERR_FILENO)))) { - Poco::AutoPtr file = new ConsoleChannel; - Poco::AutoPtr pf = new OwnPatternFormatter(this); - pf->setProperty("times", "local"); - Poco::AutoPtr log = new FormattingChannel(pf); - log->setChannel(file); + Poco::AutoPtr log = new DB::OwnFormattingChannel(new OwnPatternFormatter(this), new Poco::ConsoleChannel); logger().warning("Logging " + log_level + " to console"); split->addChannel(log); } @@ -1067,6 +1065,7 @@ void BaseDaemon::initialize(Application & self) initializeTerminationAndSignalProcessing(); + DB::CurrentThread::get(); /// TODO Why do we need this? logRevision(); for (const auto & key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) diff --git a/libs/libdaemon/src/ExtendedLogChannel.cpp b/libs/libdaemon/src/ExtendedLogChannel.cpp new file mode 100644 index 00000000000..94413f8b8ed --- /dev/null +++ b/libs/libdaemon/src/ExtendedLogChannel.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +ExtendedLogMessage ExtendedLogMessage::getFrom(const Poco::Message & base) +{ + ExtendedLogMessage msg_ext(base); + + ::timeval tv; + if (0 != gettimeofday(&tv, nullptr)) + DB::throwFromErrno("Cannot gettimeofday"); + + msg_ext.time_seconds = static_cast(tv.tv_sec); + msg_ext.time_microseconds = static_cast(tv.tv_usec); + msg_ext.query_id = CurrentThread::getCurrentQueryID(); + msg_ext.thread_number = Poco::ThreadNumber::get(); + + return msg_ext; +} + +} diff --git a/libs/libdaemon/src/OwnFormattingChannel.cpp b/libs/libdaemon/src/OwnFormattingChannel.cpp new file mode 100644 index 00000000000..a91f96bf650 --- /dev/null +++ b/libs/libdaemon/src/OwnFormattingChannel.cpp @@ -0,0 +1,33 @@ +#include +#include + + +namespace DB +{ + +void OwnFormattingChannel::logExtended(const ExtendedLogMessage & msg) +{ + if (pChannel && priority >= msg.base.getPriority()) + { + if (pFormatter) + { + std::string text; + pFormatter->formatExtended(msg, text); + pChannel->log(Poco::Message(msg.base, text)); + } + else + { + pChannel->log(msg.base); + } + } +} + +void OwnFormattingChannel::log(const Poco::Message & msg) +{ + logExtended(ExtendedLogMessage::getFrom(msg)); +} + +OwnFormattingChannel::~OwnFormattingChannel() = default; + + +} diff --git a/libs/libdaemon/src/OwnPatternFormatter.cpp b/libs/libdaemon/src/OwnPatternFormatter.cpp index fbb3d0c2061..3bc706634d9 100644 --- a/libs/libdaemon/src/OwnPatternFormatter.cpp +++ b/libs/libdaemon/src/OwnPatternFormatter.cpp @@ -1,18 +1,27 @@ #include -#include #include #include +#include +#include + +#include #include #include #include #include -void OwnPatternFormatter::format(const Poco::Message & msg, std::string & text) +OwnPatternFormatter::OwnPatternFormatter(const BaseDaemon * daemon_, OwnPatternFormatter::Options options_) + : Poco::PatternFormatter(""), daemon(daemon_), options(options_) {} + + +void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext, std::string & text) { DB::WriteBufferFromString wb(text); + const Poco::Message & msg = msg_ext.base; + /// For syslog: tag must be before message and first whitespace. if (options & ADD_LAYER_TAG && daemon) { @@ -25,28 +34,37 @@ void OwnPatternFormatter::format(const Poco::Message & msg, std::string & text) } } - /// Output time with microsecond resolution. - timeval tv; - if (0 != gettimeofday(&tv, nullptr)) - DB::throwFromErrno("Cannot gettimeofday"); - /// Change delimiters in date for compatibility with old logs. - DB::writeDateTimeText<'.', ':'>(tv.tv_sec, wb); + DB::writeDateTimeText<'.', ':'>(msg_ext.time_seconds, wb); DB::writeChar('.', wb); - DB::writeChar('0' + ((tv.tv_usec / 100000) % 10), wb); - DB::writeChar('0' + ((tv.tv_usec / 10000) % 10), wb); - DB::writeChar('0' + ((tv.tv_usec / 1000) % 10), wb); - DB::writeChar('0' + ((tv.tv_usec / 100) % 10), wb); - DB::writeChar('0' + ((tv.tv_usec / 10) % 10), wb); - DB::writeChar('0' + ((tv.tv_usec / 1) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 100000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 10000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 1000) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 100) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 10) % 10), wb); + DB::writeChar('0' + ((msg_ext.time_microseconds / 1) % 10), wb); writeCString(" [ ", wb); - DB::writeIntText(Poco::ThreadNumber::get(), wb); - writeCString(" ] <", wb); + DB::writeIntText(msg_ext.thread_number, wb); + writeCString(" ] ", wb); + + if (!msg_ext.query_id.empty()) + { + writeCString("{", wb); + DB::writeString(msg_ext.query_id, wb); + writeCString("} ", wb); + } + + writeCString("<", wb); DB::writeString(getPriorityName(static_cast(msg.getPriority())), wb); writeCString("> ", wb); DB::writeString(msg.getSource(), wb); writeCString(": ", wb); DB::writeString(msg.getText(), wb); } + +void OwnPatternFormatter::format(const Poco::Message & msg, std::string & text) +{ + formatExtended(DB::ExtendedLogMessage::getFrom(msg), text); +} diff --git a/libs/libdaemon/src/OwnSplitChannel.cpp b/libs/libdaemon/src/OwnSplitChannel.cpp new file mode 100644 index 00000000000..e6b2cb22013 --- /dev/null +++ b/libs/libdaemon/src/OwnSplitChannel.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + + +namespace DB +{ + + +void OwnSplitChannel::log(const Poco::Message & msg) +{ + auto logs_queue = CurrentThread::getInternalTextLogsQueue(); + + if (channels.empty() && (logs_queue == nullptr || msg.getPriority() > logs_queue->max_priority)) + return; + + ExtendedLogMessage msg_ext = ExtendedLogMessage::getFrom(msg); + + /// Log data to child channels + for (auto & channel : channels) + { + if (channel.second) + channel.second->logExtended(msg_ext); // extended child + else + channel.first->log(msg); // ordinary child + } + + /// Log to "TCP queue" if message is not too noisy + if (logs_queue && msg.getPriority() <= logs_queue->max_priority) + { + MutableColumns columns = InternalTextLogsQueue::getSampleColumns(); + + size_t i = 0; + columns[i++]->insert(static_cast(msg_ext.time_seconds)); + columns[i++]->insert(static_cast(msg_ext.time_microseconds)); + columns[i++]->insert(DNSResolver::instance().getHostName()); + columns[i++]->insert(msg_ext.query_id); + columns[i++]->insert(static_cast(msg_ext.thread_number)); + columns[i++]->insert(static_cast(msg.getPriority())); + columns[i++]->insert(msg.getSource()); + columns[i++]->insert(msg.getText()); + + logs_queue->emplace(std::move(columns)); + } + + /// TODO: Also log to system.internal_text_log table +} + +void OwnSplitChannel::addChannel(Poco::AutoPtr channel) +{ + channels.emplace_back(std::move(channel), dynamic_cast(channel.get())); +} + + +} diff --git a/libs/libmysqlxx/cmake/find_mysqlclient.cmake b/libs/libmysqlxx/cmake/find_mysqlclient.cmake index 2ebb3938b04..85a6275f1ff 100644 --- a/libs/libmysqlxx/cmake/find_mysqlclient.cmake +++ b/libs/libmysqlxx/cmake/find_mysqlclient.cmake @@ -1,5 +1,5 @@ option (ENABLE_MYSQL "Enable MySQL" ${OS_LINUX}) -if (OS_LINUX_X86_64) +if (OS_LINUX) option (USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ${NOT_UNBUNDLED}) else () option (USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" OFF) diff --git a/libs/libpocoext/include/Poco/Ext/LevelFilterChannel.h b/libs/libpocoext/include/Poco/Ext/LevelFilterChannel.h index ea85571e05f..a03b84168b2 100644 --- a/libs/libpocoext/include/Poco/Ext/LevelFilterChannel.h +++ b/libs/libpocoext/include/Poco/Ext/LevelFilterChannel.h @@ -2,55 +2,54 @@ #include "Poco/Foundation.h" #include "Poco/Channel.h" -#include "Poco/Mutex.h" #include "Poco/Message.h" #include -namespace Poco { +namespace Poco +{ - -class Foundation_API LevelFilterChannel: public Channel - /// This channel sends messages only higher then specified level +/// This channel sends messages only higher then specified level +class Foundation_API LevelFilterChannel : public Channel { public: - void log(const Message& msg); - /// Sends the given Message to all - /// attaches channels. + /// Sends the given Message to all + /// attaches channels. + void log(const Message & msg); - void setProperty(const std::string& name, const std::string& value); - /// Sets or changes a configuration property. - /// - /// Only the "level" property is supported, which allows setting desired level + /// Sets or changes a configuration property. + /// + /// Only the "level" property is supported, which allows setting desired level + void setProperty(const std::string & name, const std::string & value); - void setChannel(Channel* pChannel); - /// Sets the destination channel to which the formatted - /// messages are passed on. + /// Sets the destination channel to which the formatted + /// messages are passed on. + void setChannel(Channel * channel_); - Channel* getChannel() const; - /// Returns the channel to which the formatted - /// messages are passed on. + /// Returns the channel to which the formatted + /// messages are passed on. + Channel * getChannel() const; + /// Opens the attached channel. void open(); - /// Opens the attached channel. + /// Closes the attached channel. void close(); - /// Closes the attached channel. + /// Sets the Logger's log level. void setLevel(Message::Priority); - /// Sets the Logger's log level. - void setLevel(const std::string& value); - /// Sets the Logger's log level using a symbolic value. + /// Sets the Logger's log level using a symbolic value. + void setLevel(const std::string & value); + + /// Returns the Logger's log level. Message::Priority getLevel() const; - /// Returns the Logger's log level. protected: ~LevelFilterChannel(); private: - Channel* _channel = nullptr; - Message::Priority _priority = Message::PRIO_ERROR; + Channel * channel = nullptr; + Message::Priority priority = Message::PRIO_ERROR; }; - } diff --git a/libs/libpocoext/src/LevelFilterChannel.cpp b/libs/libpocoext/src/LevelFilterChannel.cpp index ae07a266a61..8839aa33e5d 100644 --- a/libs/libpocoext/src/LevelFilterChannel.cpp +++ b/libs/libpocoext/src/LevelFilterChannel.cpp @@ -1,90 +1,57 @@ -// -// LevelFilterChannel.cpp -// -// $Id$ -// -// Library: Ext -// Package: Logging -// Module: LevelFilterChannel -// -// Copyright (c) 2004-2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// Permission is hereby granted, free of charge, to any person or organization -// obtaining a copy of the software and accompanying documentation covered by -// this license (the "Software") to use, reproduce, display, distribute, -// execute, and transmit the Software, and to prepare derivative works of the -// Software, and to permit third-parties to whom the Software is furnished to -// do so, all subject to the following: -// -// The copyright notices in the Software and this entire statement, including -// the above license grant, this restriction and the following disclaimer, -// must be included in all copies of the Software, in whole or in part, and -// all derivative works of the Software, unless such copies or derivative -// works are solely in the form of machine-executable object code generated by -// a source language processor. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// - +// Copyright 2008, Yandex LLC. +// See the license of ClickHouse. #include "Poco/Ext/LevelFilterChannel.h" #include "Poco/LoggingRegistry.h" #include "Poco/String.h" -namespace Poco { - +namespace Poco +{ LevelFilterChannel::~LevelFilterChannel() { - if (_channel) - _channel->release(); + if (channel) + channel->release(); } -void LevelFilterChannel::setChannel(Channel* channel) +void LevelFilterChannel::setChannel(Channel * channel_) { - if (_channel) - _channel->release(); - _channel = channel; - if (_channel) - _channel->duplicate(); + if (channel) + channel->release(); + channel = channel_; + if (channel) + channel->duplicate(); } -Channel* LevelFilterChannel::getChannel() const +Channel * LevelFilterChannel::getChannel() const { - return _channel; + return channel; } void LevelFilterChannel::open() { - if (_channel) - _channel->open(); + if (channel) + channel->open(); } void LevelFilterChannel::close() { - if (_channel) - _channel->close(); + if (channel) + channel->close(); } -void LevelFilterChannel::setLevel(Message::Priority priority) +void LevelFilterChannel::setLevel(Message::Priority priority_) { - _priority = priority; + priority = priority_; } -void LevelFilterChannel::setLevel(const std::string& value) +void LevelFilterChannel::setLevel(const std::string & value) { if (icompare(value, "fatal") == 0) setLevel(Message::PRIO_FATAL); @@ -109,11 +76,11 @@ void LevelFilterChannel::setLevel(const std::string& value) Message::Priority LevelFilterChannel::getLevel() const { - return _priority; + return priority; } -void LevelFilterChannel::setProperty(const std::string& name, const std::string& value) +void LevelFilterChannel::setProperty(const std::string & name, const std::string & value) { if (icompare(name, "level") == 0) setLevel(value); @@ -126,9 +93,8 @@ void LevelFilterChannel::setProperty(const std::string& name, const std::string& void LevelFilterChannel::log(const Message& msg) { - if ((_priority >= msg.getPriority()) && _channel) - _channel->log(msg); + if ((priority >= msg.getPriority()) && channel) + channel->log(msg); } - }